brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aleds...@apache.org
Subject [3/6] git commit: (failing) test for split-brain scenario
Date Thu, 26 Jun 2014 11:27:58 GMT
(failing) test for split-brain scenario


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/2ed4c77e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/2ed4c77e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/2ed4c77e

Branch: refs/heads/master
Commit: 2ed4c77e864f43d0d95ef35ddb03cb39867032f1
Parents: be3d9d2
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Wed Jun 25 18:37:07 2014 +0100
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Thu Jun 26 02:43:19 2014 +0100

----------------------------------------------------------------------
 .../ha/HighAvailabilityManagerImpl.java         |   6 +-
 .../rebind/persister/InMemoryObjectStore.java   |  10 +-
 .../rebind/persister/ListeningObjectStore.java  |  54 +++-
 .../HighAvailabilityManagerSplitBrainTest.java  | 244 +++++++++++++++++++
 .../ha/HighAvailabilityManagerTestFixture.java  |   2 +-
 5 files changed, 299 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/2ed4c77e/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
index 2b38160..41cbe51 100644
--- a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
+++ b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
@@ -279,7 +279,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
     }
     
     /** invoked manually when initializing, and periodically thereafter */
-    protected synchronized void publishAndCheck(boolean initializing) {
+    @VisibleForTesting
+    public synchronized void publishAndCheck(boolean initializing) {
         publishHealth();
         checkMaster(initializing);
     }
@@ -404,7 +405,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
         URI newMasterNodeUri = (newMasterRecord == null) ? null : newMasterRecord.getUri();
         boolean newMasterIsSelf = ownNodeId.equals(newMasterNodeId);
         
-        if (LOG.isDebugEnabled())
+        if (LOG.isDebugEnabled()) {
             LOG.debug("Management node master-promotion required: newMaster={}; oldMaster={};
plane={}, self={}; heartbeatTimeout={}", 
                 new Object[] {
                     (newMasterRecord == null ? "<none>" : newMasterRecord.toVerboseString()),
@@ -413,6 +414,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager
{
                     ownNodeMemento.toVerboseString(), 
                     heartbeatTimeout
                 });
+        }
         if (!initializing) {
             LOG.warn("HA subsystem detected change of master from " 
                 + masterNodeId + " (" + (masterNodeMemento==null ? "?" : masterNodeMemento.getRemoteTimestamp())
+ ")"

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/2ed4c77e/core/src/test/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
b/core/src/test/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
index 41233b6..5ffadea 100644
--- a/core/src/test/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
+++ b/core/src/test/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
@@ -18,11 +18,17 @@ public class InMemoryObjectStore implements PersistenceObjectStore {
 
     private static final Logger log = LoggerFactory.getLogger(InMemoryObjectStore.class);
 
-    Map<String,String> filesByName = MutableMap.of();
-    Map<String, Date> fileModTimesByName = MutableMap.of();
+    final Map<String,String> filesByName;
+    final Map<String, Date> fileModTimesByName;
     boolean prepared = false;
     
     public InMemoryObjectStore() {
+        this(MutableMap.<String,String>of(), MutableMap.<String,Date>of());
+    }
+    
+    public InMemoryObjectStore(Map<String,String> map, Map<String, Date> fileModTimesByName)
{
+        filesByName = map;
+        this.fileModTimesByName = fileModTimesByName;
         log.info("Using memory-based objectStore");
     }
     

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/2ed4c77e/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
b/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
index 568490c..2d98aa1 100644
--- a/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
+++ b/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
@@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory;
 
 import brooklyn.management.ManagementContext;
 import brooklyn.management.ha.HighAvailabilityMode;
+import brooklyn.util.collections.MutableList;
 import brooklyn.util.text.Strings;
 import brooklyn.util.time.CountdownTimer;
 import brooklyn.util.time.Duration;
@@ -20,7 +21,8 @@ import com.google.common.base.Preconditions;
 public class ListeningObjectStore implements PersistenceObjectStore {
 
     protected final PersistenceObjectStore delegate;
-    protected final ObjectStoreTransactionListener listener;
+    protected final List<ObjectStoreTransactionListener> listeners = MutableList.of();
+    private boolean writesFailSilently = false;
 
     public static interface ObjectStoreTransactionListener {
         public void recordQueryOut(String summary, int size);
@@ -102,9 +104,10 @@ public class ListeningObjectStore implements PersistenceObjectStore {
         }
     }
 
-    public ListeningObjectStore(PersistenceObjectStore delegate, ObjectStoreTransactionListener
listener) {
+    public ListeningObjectStore(PersistenceObjectStore delegate, ObjectStoreTransactionListener
...listeners) {
         this.delegate = Preconditions.checkNotNull(delegate);
-        this.listener = Preconditions.checkNotNull(listener);
+        for (ObjectStoreTransactionListener listener: listeners)
+            this.listeners.add(listener);
     }
 
     @Override
@@ -124,15 +127,23 @@ public class ListeningObjectStore implements PersistenceObjectStore
{
 
     @Override
     public void createSubPath(String subPath) {
-        listener.recordQueryOut("creating path "+subPath, 1+subPath.length());
+        if (writesFailSilently)
+            return;
+        
+        for (ObjectStoreTransactionListener listener: listeners)
+            listener.recordQueryOut("creating path "+subPath, 1+subPath.length());
         delegate.createSubPath(subPath);
     }
 
     @Override
     public List<String> listContentsWithSubPath(String subPath) {
-        listener.recordQueryOut("requesting list "+subPath, 1+subPath.length());
+        for (ObjectStoreTransactionListener listener: listeners)
+            listener.recordQueryOut("requesting list "+subPath, 1+subPath.length());
+        
         List<String> result = delegate.listContentsWithSubPath(subPath);
-        listener.recordDataIn("receiving list "+subPath, result.toString().length());
+        
+        for (ObjectStoreTransactionListener listener: listeners)
+            listener.recordDataIn("receiving list "+subPath, result.toString().length());
         return result;
     }
 
@@ -153,7 +164,8 @@ public class ListeningObjectStore implements PersistenceObjectStore {
 
     @Override
     public void deleteCompletely() {
-        listener.recordDataOut("deleting completely", 1);
+        for (ObjectStoreTransactionListener listener: listeners)
+            listener.recordDataOut("deleting completely", 1);
         delegate.deleteCompletely();
     }
 
@@ -172,24 +184,39 @@ public class ListeningObjectStore implements PersistenceObjectStore
{
         }
         @Override
         public void put(String val) {
-            listener.recordDataOut("writing "+path, val.length());
+            if (writesFailSilently)
+                return;
+
+            for (ObjectStoreTransactionListener listener: listeners)
+                listener.recordDataOut("writing "+path, val.length());
             delegate.put(val);
         }
         @Override
         public void append(String s) {
-            listener.recordDataOut("appending "+path, s.length());
+            if (writesFailSilently)
+                return;
+
+            for (ObjectStoreTransactionListener listener: listeners)
+                listener.recordDataOut("appending "+path, s.length());
             delegate.append(s);
         }
         @Override
         public void delete() {
-            listener.recordQueryOut("deleting "+path, path.length());
+            if (writesFailSilently)
+                return;
+
+            for (ObjectStoreTransactionListener listener: listeners)
+                listener.recordQueryOut("deleting "+path, path.length());
             delegate.delete();
         }
         @Override
         public String get() {
-            listener.recordQueryOut("requesting "+path, path.length());
+            for (ObjectStoreTransactionListener listener: listeners)
+                listener.recordQueryOut("requesting "+path, path.length());
             String result = delegate.get();
-            listener.recordDataIn("reading "+path, result.length());
+            
+            for (ObjectStoreTransactionListener listener: listeners)
+                listener.recordDataIn("reading "+path, (result==null ? 0 : result.length()));
             return result;
         }
         @Override
@@ -198,4 +225,7 @@ public class ListeningObjectStore implements PersistenceObjectStore {
         }
     }
 
+    public void setWritesFailSilently(boolean writesFailSilently) {
+        this.writesFailSilently = writesFailSilently;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/2ed4c77e/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerSplitBrainTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerSplitBrainTest.java
b/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerSplitBrainTest.java
new file mode 100644
index 0000000..46fec19
--- /dev/null
+++ b/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerSplitBrainTest.java
@@ -0,0 +1,244 @@
+package brooklyn.management.ha;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import brooklyn.entity.basic.Entities;
+import brooklyn.entity.rebind.persister.BrooklynMementoPersisterToObjectStore;
+import brooklyn.entity.rebind.persister.InMemoryObjectStore;
+import brooklyn.entity.rebind.persister.ListeningObjectStore;
+import brooklyn.entity.rebind.persister.PersistMode;
+import brooklyn.entity.rebind.persister.PersistenceObjectStore;
+import brooklyn.management.ha.HighAvailabilityManagerTestFixture.RecordingPromotionListener;
+import brooklyn.management.internal.ManagementContextInternal;
+import brooklyn.test.entity.LocalManagementContextForTests;
+import brooklyn.util.collections.MutableList;
+import brooklyn.util.collections.MutableMap;
+import brooklyn.util.time.Duration;
+
+import com.google.common.base.Ticker;
+
+@Test
+public class HighAvailabilityManagerSplitBrainTest {
+
+    private static final Logger log = LoggerFactory.getLogger(HighAvailabilityManagerSplitBrainTest.class);
+    
+    private List<HaMgmtNode> nodes = new MutableList<HighAvailabilityManagerSplitBrainTest.HaMgmtNode>();
+    Map<String,String> sharedBackingStore = MutableMap.of();
+    Map<String,Date> sharedBackingStoreDates = MutableMap.of();
+    private AtomicLong sharedTime; // used to set the ticker's return value
+    
+    public class HaMgmtNode {
+        
+        private ManagementPlaneSyncRecordPersister persister;
+        private ManagementContextInternal managementContext;
+        private String ownNodeId;
+        private HighAvailabilityManagerImpl manager;
+        private Ticker ticker;
+        private AtomicLong currentTime; // used to set the ticker's return value
+        private RecordingPromotionListener promotionListener;
+        private ClassLoader classLoader = getClass().getClassLoader();
+        private ListeningObjectStore objectStore;
+        private String nodeName;
+
+        @BeforeMethod(alwaysRun=true)
+        public void setUp() throws Exception {
+            if (sharedTime==null)
+                currentTime = new AtomicLong(System.currentTimeMillis());
+            
+            ticker = new Ticker() {
+                // strictly not a ticker because returns millis UTC, but it works fine even
so
+                @Override public long read() {
+                    if (sharedTime!=null) return sharedTime.get();
+                    return currentTime.get();
+                }
+            };
+            
+            nodeName = "node "+nodes.size();
+            promotionListener = new RecordingPromotionListener();
+            managementContext = newLocalManagementContext();
+            ownNodeId = managementContext.getManagementNodeId();
+            objectStore = new ListeningObjectStore(newPersistenceObjectStore());
+            objectStore.injectManagementContext(managementContext);
+            objectStore.prepareForSharedUse(PersistMode.CLEAN, HighAvailabilityMode.DISABLED);
+            persister = new ManagementPlaneSyncRecordPersisterToObjectStore(managementContext,
objectStore, classLoader);
+            ((ManagementPlaneSyncRecordPersisterToObjectStore)persister).allowRemoteTimestampInMemento();
+            BrooklynMementoPersisterToObjectStore persisterObj = new BrooklynMementoPersisterToObjectStore(objectStore,
classLoader);
+            managementContext.getRebindManager().setPersister(persisterObj);
+            manager = new HighAvailabilityManagerImpl(managementContext)
+                .setPollPeriod(Duration.PRACTICALLY_FOREVER)
+                .setHeartbeatTimeout(Duration.THIRTY_SECONDS)
+                .setPromotionListener(promotionListener)
+                .setLocalTicker(ticker)
+                .setRemoteTicker(ticker)
+                .setPersister(persister);
+            log.info("Created "+nodeName+" "+ownNodeId);
+        }
+        
+        public void tearDown() throws Exception {
+            if (manager != null) manager.stop();
+            if (managementContext != null) Entities.destroyAll(managementContext);
+            if (objectStore != null) objectStore.deleteCompletely();
+        }
+        
+        private long tickerCurrentMillis() {
+            return ticker.read();
+        }
+        
+        private long tickerAdvance(Duration duration) {
+            if (sharedTime!=null)
+                throw new IllegalStateException("Using shared ticker; cannot advance private
node clock");
+            currentTime.addAndGet(duration.toMilliseconds());
+            return tickerCurrentMillis();
+        }
+        
+        @Override
+        public String toString() {
+            return nodeName+" "+ownNodeId;
+        }
+    }
+    
+    @BeforeMethod(alwaysRun=true)
+    public void setUp() throws Exception {
+        nodes.clear();
+        sharedBackingStore.clear();
+    }
+    
+    public HaMgmtNode newNode() throws Exception {
+        HaMgmtNode node = new HaMgmtNode();
+        node.setUp();
+        nodes.add(node);
+        return node;
+    }
+
+    @AfterMethod(alwaysRun=true)
+    public void tearDown() throws Exception {
+        for (HaMgmtNode n: nodes)
+            n.tearDown();
+    }
+
+    private void sharedTickerAdvance(Duration duration) {
+        if (sharedTime==null) {
+            for (HaMgmtNode n: nodes)
+                n.tickerAdvance(duration);
+        } else {
+            sharedTime.addAndGet(duration.toMilliseconds());
+        }
+    }
+    
+    private long sharedTickerCurrentMillis() {
+        return sharedTime.get();
+    }
+    
+    protected void useSharedTime() {
+        if (!nodes.isEmpty())
+            throw new IllegalStateException("shared time must be set up before any nodes
created");
+        sharedTime = new AtomicLong(System.currentTimeMillis());
+    }
+
+    protected ManagementContextInternal newLocalManagementContext() {
+        return new LocalManagementContextForTests();
+    }
+
+    protected PersistenceObjectStore newPersistenceObjectStore() {
+        return new InMemoryObjectStore(sharedBackingStore, sharedBackingStoreDates);
+    }
+    
+    @Test
+    public void testTwoNodes() throws Exception {
+        useSharedTime();
+        HaMgmtNode n1 = newNode();
+        HaMgmtNode n2 = newNode();
+        
+        n1.manager.start(HighAvailabilityMode.AUTO);
+        ManagementPlaneSyncRecord memento1 = n1.manager.getManagementPlaneSyncState();
+        
+        log.info(n1+" HA: "+memento1);
+        assertEquals(memento1.getMasterNodeId(), n1.ownNodeId);
+        Long time0 = sharedTickerCurrentMillis();
+        assertEquals(memento1.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time0);
+        assertEquals(memento1.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+
+        n2.manager.start(HighAvailabilityMode.AUTO);
+        ManagementPlaneSyncRecord memento2 = n2.manager.getManagementPlaneSyncState();
+        
+        log.info(n2+" HA: "+memento2);
+        assertEquals(memento2.getMasterNodeId(), n1.ownNodeId);
+        assertEquals(memento2.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+        assertEquals(memento2.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.STANDBY);
+        assertEquals(memento2.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time0);
+        assertEquals(memento2.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(),
time0);
+        
+        n1.objectStore.setWritesFailSilently(true);
+        log.info(n1+" writes off");
+        sharedTickerAdvance(Duration.ONE_MINUTE);
+        
+        n2.manager.publishAndCheck(false);
+        ManagementPlaneSyncRecord memento2b = n2.manager.getManagementPlaneSyncState();
+        log.info(n2+" HA now: "+memento2b);
+        Long time1 = sharedTickerCurrentMillis();
+        
+        // n2 infers n1 as failed 
+        assertEquals(memento2b.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.FAILED);
+        assertEquals(memento2b.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+        assertEquals(memento2b.getMasterNodeId(), n2.ownNodeId);
+        assertEquals(memento2b.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time0);
+        assertEquals(memento2b.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(),
time1);
+        
+        n1.objectStore.setWritesFailSilently(false);
+        log.info(n1+" writes on");
+        
+        sharedTickerAdvance(Duration.ONE_SECOND);
+        Long time2 = sharedTickerCurrentMillis();
+        
+        n1.manager.publishAndCheck(false);
+        ManagementPlaneSyncRecord memento1b = n1.manager.getManagementPlaneSyncState();
+        log.info(n1+" HA now: "+memento1b);
+        
+//        // n1 comes back and sees himself as master, but with both masters 
+//        assertEquals(memento1b.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+//        assertEquals(memento1b.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+//        assertEquals(memento1b.getMasterNodeId(), n1.ownNodeId);
+//        assertEquals(memento1b.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time2);
+//        assertEquals(memento1b.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(),
time1);
+//        
+//        // n2 sees itself as master, but again with both masters
+//        ManagementPlaneSyncRecord memento2c = n2.manager.getManagementPlaneSyncState();
+//        log.info(n2+" HA now: "+memento2c);
+//        assertEquals(memento2c.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+//        assertEquals(memento2c.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+//        assertEquals(memento2c.getMasterNodeId(), n2.ownNodeId);
+//        assertEquals(memento2c.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time2);
+//        assertEquals(memento2c.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(),
time2);
+
+        // current (unwanted) state is above, desired state below
+        
+        // n1 comes back and demotes himself 
+        assertEquals(memento1b.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.STANDBY);
+        assertEquals(memento1b.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+        assertEquals(memento1b.getMasterNodeId(), n2.ownNodeId);
+        assertEquals(memento1b.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time2);
+        assertEquals(memento1b.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(),
time1);
+        
+        // n2 now sees itself as master, with n1 in standby again
+        ManagementPlaneSyncRecord memento2c = n2.manager.getManagementPlaneSyncState();
+        log.info(n2+" HA now: "+memento2c);
+        assertEquals(memento2c.getManagementNodes().get(n1.ownNodeId).getStatus(), ManagementNodeState.STANDBY);
+        assertEquals(memento2c.getManagementNodes().get(n2.ownNodeId).getStatus(), ManagementNodeState.MASTER);
+        assertEquals(memento2c.getMasterNodeId(), n2.ownNodeId);
+        assertEquals(memento2c.getManagementNodes().get(n1.ownNodeId).getRemoteTimestamp(),
time2);
+        assertEquals(memento2c.getManagementNodes().get(n2.ownNodeId).getRemoteTimestamp(),
time2);
+
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/2ed4c77e/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerTestFixture.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerTestFixture.java
b/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerTestFixture.java
index 1ccf136..ce8dab5 100644
--- a/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerTestFixture.java
+++ b/core/src/test/java/brooklyn/management/ha/HighAvailabilityManagerTestFixture.java
@@ -163,7 +163,7 @@ public abstract class HighAvailabilityManagerTestFixture {
                 .node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY))
                 .node(newManagerMemento("zzzzzzz_node1", ManagementNodeState.STANDBY))
                 .build());
-        ManagementPlaneSyncRecord record = persister.loadSyncRecord();
+        persister.loadSyncRecord();
         long zzzTime = tickerCurrentMillis();
         tickerAdvance(Duration.FIVE_SECONDS);
         


Mime
View raw message