ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject ignite git commit: ignite-gg-12389
Date Tue, 04 Jul 2017 13:33:29 GMT
Repository: ignite
Updated Branches:
  refs/heads/ignite-gg-12389 af966135b -> 57d73160a


ignite-gg-12389


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/57d73160
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/57d73160
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/57d73160

Branch: refs/heads/ignite-gg-12389
Commit: 57d73160a1445dd6b860a35cebe7d3bfb39ed587
Parents: af96613
Author: sboikov <sboikov@gridgain.com>
Authored: Tue Jul 4 14:36:42 2017 +0300
Committer: sboikov <sboikov@gridgain.com>
Committed: Tue Jul 4 16:33:02 2017 +0300

----------------------------------------------------------------------
 .../processors/cache/CacheGroupContext.java     |   2 +
 .../processors/cache/ClusterCachesInfo.java     |  72 ++++-
 .../GridCachePartitionExchangeManager.java      |  38 ++-
 .../processors/cache/GridCacheProcessor.java    |   4 +-
 .../dht/preloader/GridDhtPartitionDemander.java |   2 +
 .../IgniteClusterActivateDeactivateTest.java    | 292 +++++++++++++++++--
 .../IgniteChangeGlobalStateAbstractTest.java    |  17 +-
 .../IgniteChangeGlobalStateFailOverTest.java    |  20 +-
 8 files changed, 376 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index c3ddc5f..c72deed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -683,6 +683,8 @@ public class CacheGroupContext {
 
         aff.cancelFutures(err);
 
+        preldr.onKernalStop();
+
         offheapMgr.stop();
 
         ctx.io().removeCacheGroupHandlers(grpId);

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 6ff239c..94e313b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -332,18 +332,39 @@ class ClusterCachesInfo {
      * @return {@code True} if minor topology version should be increased.
      */
     boolean onCacheChangeRequested(DynamicCacheChangeBatch batch, AffinityTopologyVersion
topVer) {
-        // TODO GG-12389 fail if inactive (for others msgs too).
-        ExchangeActions exchangeActions = new ExchangeActions();
+        DiscoveryDataClusterState state = ctx.state().clusterState();
 
-        final CacheChangeProcessResult res = processCacheChangeRequests(exchangeActions,
batch.requests(), topVer, false);
+        if (state.active() && !state.transition()) {
+            ExchangeActions exchangeActions = new ExchangeActions();
 
-        if (res.needExchange) {
-            assert !exchangeActions.empty() : exchangeActions;
+            CacheChangeProcessResult res = processCacheChangeRequests(exchangeActions,
+                batch.requests(),
+                topVer,
+                false);
+
+            if (res.needExchange) {
+                assert !exchangeActions.empty() : exchangeActions;
+
+                batch.exchangeActions(exchangeActions);
+            }
 
-            batch.exchangeActions(exchangeActions);
+            return res.needExchange;
         }
+        else {
+            IgniteCheckedException err = new IgniteCheckedException("Failed to start/stop
cache, cluster state change " +
+                "is in progress.");
+
+            for (DynamicCacheChangeRequest req : batch.requests()) {
+                if (req.template()) {
+                    ctx.cache().completeTemplateAddFuture(req.startCacheConfiguration().getName(),
+                        req.deploymentId());
+                }
+                else
+                    ctx.cache().completeCacheStartFuture(req, false, err);
+            }
 
-        return res.needExchange;
+            return false;
+        }
     }
 
     /**
@@ -994,15 +1015,15 @@ class ClusterCachesInfo {
      * @param firstNode {@code True} if first node in cluster starts.
      */
     private void initStartCachesForLocalJoin(boolean firstNode) {
-        assert locJoinStartCaches == null;
+        assert F.isEmpty(locJoinStartCaches) : locJoinStartCaches;
 
-        if (joinDiscoData != null) {
-            if (ctx.state().clusterState().transition()) {
-                joinOnTransition = true;
+        if (ctx.state().clusterState().transition()) {
+            joinOnTransition = true;
 
-                return;
-            }
+            return;
+        }
 
+        if (joinDiscoData != null) {
             locJoinStartCaches = new ArrayList<>();
             locCfgsForActivation = new HashMap<>();
 
@@ -1506,6 +1527,7 @@ class ClusterCachesInfo {
      */
     void onDisconnect() {
         cachesOnDisconnect = new CachesOnDisconnect(
+            ctx.state().clusterState(),
             new HashMap<>(registeredCacheGrps),
             new HashMap<>(registeredCaches));
 
@@ -1518,15 +1540,18 @@ class ClusterCachesInfo {
 
     /**
      * @param active {@code True} if reconnected to active cluster.
+     * @param transition {@code True} if reconnected while state transition in progress.
      * @return Information about stopped caches and cache groups.
      */
-    ClusterCachesReconnectResult onReconnected(boolean active) {
+    ClusterCachesReconnectResult onReconnected(boolean active, boolean transition) {
         assert disconnectedState();
 
         Set<String> stoppedCaches = new HashSet<>();
         Set<Integer> stoppedCacheGrps = new HashSet<>();
 
         if (!active) {
+            joinOnTransition = transition;
+
             if (F.isEmpty(locCfgsForActivation)) {
                 locCfgsForActivation = new HashMap<>();
 
@@ -1587,6 +1612,9 @@ class ClusterCachesInfo {
                 if (stopped)
                     stoppedCaches.add(cacheName);
             }
+
+            if (!cachesOnDisconnect.state.active())
+                initStartCachesForLocalJoin(false);
         }
 
         if (clientReconnectReqs != null) {
@@ -1646,19 +1674,33 @@ class ClusterCachesInfo {
      */
     private static class CachesOnDisconnect {
         /** */
+        final DiscoveryDataClusterState state;
+
+        /** */
         final Map<Integer, CacheGroupDescriptor> cacheGrps;
 
         /** */
         final Map<String, DynamicCacheDescriptor> caches;
 
         /**
+         * @param state Cluster state.
          * @param cacheGrps Cache groups.
          * @param caches Caches.
          */
-        CachesOnDisconnect(Map<Integer, CacheGroupDescriptor> cacheGrps, Map<String,
DynamicCacheDescriptor> caches) {
+        CachesOnDisconnect(DiscoveryDataClusterState state,
+            Map<Integer, CacheGroupDescriptor> cacheGrps,
+            Map<String, DynamicCacheDescriptor> caches) {
+            this.state = state;
             this.cacheGrps = cacheGrps;
             this.caches = caches;
         }
+
+        /**
+         * @return {@code True} if cluster was in active state.
+         */
+        boolean clusterActive() {
+            return state.active() && !state.transition();
+        }
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 779e589..6777351 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -326,17 +326,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
                 else
                     processEventInactive(evt, cache);
 
-                if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) {
-                    final ClusterNode n = evt.eventNode();
-
-                    assert cctx.discovery().node(n.id()) == null;
-
-                    for (GridDhtPartitionsExchangeFuture f : exchFuts.values())
-                        f.onNodeLeft(n);
-
-                    // Notify indexing engine about node leave so that we can re-map coordinator
accordingly.
-                    exchWorker.addCustomTask(new SchemaNodeLeaveExchangeWorkerTask(evt.eventNode()));
-                }
+                notifyNodeFail(evt);
             }
             finally {
                 leaveBusy();
@@ -344,10 +334,26 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
         }
     };
 
+    /**
+     * @param evt Event.
+     */
+    private void notifyNodeFail(DiscoveryEvent evt) {
+        if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) {
+            final ClusterNode n = evt.eventNode();
+
+            assert cctx.discovery().node(n.id()) == null;
+
+            for (GridDhtPartitionsExchangeFuture f : exchFuts.values())
+                f.onNodeLeft(n);
+        }
+    }
+
+    /**
+     * @param evt Event.
+     * @param cache Discovery data cache.
+     */
     private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
         log.info("Ignore event: " + evt);
-
-        // TODO GG-12389: finish operations with error.
    }
 
     /**
@@ -443,6 +449,12 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
             if (log.isDebugEnabled())
                 log.debug("Do not start exchange for discovery event: " + evt);
         }
+
+        notifyNodeFail(evt);
+
+        // Notify indexing engine about node leave so that we can re-map coordinator accordingly.
+        if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+            exchWorker.addCustomTask(new SchemaNodeLeaveExchangeWorkerTask(evt.eventNode()));
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index cf91ea0..4d2af50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -878,8 +878,6 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
         for (int i = 0, size = syncFuts.size(); i < size; i++)
             syncFuts.get(i).get();
-
-        assert ctx.config().isDaemon() || caches.containsKey(CU.UTILITY_CACHE_NAME) : "Utility
cache should be started";
     }
 
     /**
@@ -1076,7 +1074,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
 
         boolean active = state.active() && !state.transition();
 
-        ClusterCachesReconnectResult reconnectRes = cachesInfo.onReconnected(active);
+        ClusterCachesReconnectResult reconnectRes = cachesInfo.onReconnected(active, state.transition());
 
         final List<GridCacheAdapter> stoppedCaches = new ArrayList<>();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 774f0ce..e7e95b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -161,6 +161,8 @@ public class GridDhtPartitionDemander {
         lastExchangeFut = null;
 
         lastTimeoutObj.set(null);
+
+        syncFut.onDone();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index a84a0ca..bb6d23b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ThreadLocalRandom;
@@ -36,10 +38,11 @@ import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -78,6 +81,9 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     /** */
     private boolean testReconnectSpi;
 
+    /** */
+    private Class[] testSpiRecord;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws
Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -118,8 +124,14 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
             cfg.setPersistentStoreConfiguration(pCfg);
         }
 
-        if (testSpi)
-            cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+        if (testSpi) {
+            TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi();
+
+            if (testSpiRecord != null)
+                spi.record(testSpiRecord);
+
+            cfg.setCommunicationSpi(spi);
+        }
 
         return cfg;
     }
@@ -281,7 +293,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
      * @throws Exception If failed.
      */
     private void joinWhileActivate1(final boolean startClient, final boolean withNewCache)
throws Exception {
-        IgniteInternalFuture<?> activeFut = startNodesAndBlockStatusChange(2, 0, false);
+        IgniteInternalFuture<?> activeFut = startNodesAndBlockStatusChange(2, 0, 0,
false);
 
         IgniteInternalFuture<?> startFut = GridTestUtils.runAsync(new Callable<Void>()
{
             @Override public Void call() throws Exception {
@@ -332,40 +344,48 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     /**
      * @param srvs Number of servers.
      * @param clients Number of clients.
+     * @param stateChangeFrom Index of node initiating changes.
      * @param initiallyActive If {@code true} start cluster in active state (otherwise in
inactive).
+     * @param blockMsgNodes Nodes whcis block exchange messages.
      * @return State change future.
      * @throws Exception If failed.
      */
     private IgniteInternalFuture<?> startNodesAndBlockStatusChange(int srvs,
         int clients,
-        final boolean initiallyActive) throws Exception {
+        final int stateChangeFrom,
+        final boolean initiallyActive,
+        int... blockMsgNodes) throws Exception {
         active = initiallyActive;
         testSpi = true;
 
-        for (int i = 0; i < srvs + clients; i++) {
-            client = i >= srvs;
-
-            ccfgs = cacheConfigurations1();
-
-            startGrid(i);
-        }
+        startWithCaches1(srvs, clients);
 
         if (initiallyActive && persistenceEnabled())
             ignite(0).active(true);
 
-        TestRecordingCommunicationSpi spi1 = TestRecordingCommunicationSpi.spi(ignite(1));
+        if (blockMsgNodes.length == 0)
+            blockMsgNodes = new int[]{1};
 
         final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(srvs
+ clients, 1);
 
-        blockExchangeSingleMessage(spi1, STATE_CHANGE_TOP_VER);
+        List<TestRecordingCommunicationSpi> spis = new ArrayList<>();
+
+        for (int idx : blockMsgNodes) {
+            TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(ignite(idx));
+
+            spis.add(spi);
+
+            blockExchangeSingleMessage(spi, STATE_CHANGE_TOP_VER);
+        }
 
         IgniteInternalFuture<?> stateChangeFut = GridTestUtils.runAsync(new Runnable()
{
             @Override public void run() {
-                ignite(0).active(!initiallyActive);
+                ignite(stateChangeFrom).active(!initiallyActive);
             }
         });
 
-        spi1.waitForBlocked();
+        for (TestRecordingCommunicationSpi spi : spis)
+            spi.waitForBlocked();
 
         U.sleep(500);
 
@@ -420,7 +440,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
      * @throws Exception If failed.
      */
     private void joinWhileDeactivate1(final boolean startClient, final boolean withNewCache)
throws Exception {
-        IgniteInternalFuture<?> activeFut = startNodesAndBlockStatusChange(2, 0, true);
+        IgniteInternalFuture<?> activeFut = startNodesAndBlockStatusChange(2, 0, 0,
true);
 
         IgniteInternalFuture<?> startFut = GridTestUtils.runAsync(new Callable<Void>()
{
             @Override public Void call() throws Exception {
@@ -643,7 +663,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     /**
      * @throws Exception If failed.
      */
-    public void testClientReconnectActive() throws Exception {
+    public void testClientReconnectClusterActive() throws Exception {
         testReconnectSpi = true;
 
         ccfgs = cacheConfigurations1();
@@ -681,7 +701,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     /**
      * @throws Exception If failed.
      */
-    public void testClientReconnectInactive() throws Exception {
+    public void testClientReconnectClusterInactive() throws Exception {
         testReconnectSpi = true;
 
         active = false;
@@ -883,7 +903,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
             }
         });
 
-        checkCache(client, CACHE_NAME_PREFIX + 0, false);
+        checkCache(client, CACHE_NAME_PREFIX + 0, !transition);
 
         if (transition) {
             assertFalse(stateFut.get().isDone());
@@ -915,6 +935,232 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
     }
 
     /**
+     * @throws Exception If failed.
+     */
+    public void testInvactiveTopologyChanges() throws Exception {
+        testSpi = true;
+
+        testSpiRecord = new Class[]{GridDhtPartitionsSingleMessage.class, GridDhtPartitionsFullMessage.class};
+
+        active = false;
+
+        final int SRVS = 4;
+        final int CLIENTS = 4;
+
+        startWithCaches1(SRVS, CLIENTS);
+
+        checkRecordedMessages(false);
+
+        for (int i = 0; i < 2; i++) {
+            stopGrid(i);
+
+            client = false;
+
+            startGrid(i);
+        }
+
+        checkRecordedMessages(false);
+
+        for (int i = 0; i < 2; i++) {
+            stopGrid(SRVS + i);
+
+            client = true;
+
+            startGrid(SRVS + i);
+        }
+
+        checkRecordedMessages(false);
+
+        ignite(0).active(true);
+
+        checkCaches1(SRVS + CLIENTS);
+
+        checkRecordedMessages(true);
+
+        client = false;
+
+        startGrid(SRVS + CLIENTS);
+
+        client = true;
+
+        startGrid(SRVS + CLIENTS + 1);
+
+        checkRecordedMessages(true);
+
+        checkCaches1(SRVS + CLIENTS + 2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testActivateFailover1() throws Exception {
+        stateChangeFailover1(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeactivateFailover1() throws Exception {
+        stateChangeFailover1(false);
+    }
+
+    /**
+     * @param activate If {@code true} tests activation, otherwise deactivation.
+     * @throws Exception If failed.
+     */
+    private void stateChangeFailover1(boolean activate) throws Exception {
+        // Nodes 1 and 4 do not reply to coordinator.
+        IgniteInternalFuture<?> fut = startNodesAndBlockStatusChange(4, 4, 3, !activate,
1, 4);
+
+        client = false;
+
+        // Start one more node while transition is in progress.
+        IgniteInternalFuture startFut = GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                startGrid(8);
+
+                return null;
+            }
+        }, "start-node");
+
+        U.sleep(500);
+
+        stopGrid(getTestIgniteInstanceName(1), true, false);
+        stopGrid(getTestIgniteInstanceName(4), true, false);
+
+        fut.get();
+
+        startFut.get();
+
+        client = false;
+
+        startGrid(1);
+
+        client = true;
+
+        startGrid(4);
+
+        if (!activate) {
+            checkNoCaches(9);
+
+            ignite(0).active(true);
+        }
+
+        checkCaches1(9);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testActivateFailover2() throws Exception {
+        stateChangeFailover2(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeactivateFailover2() throws Exception {
+        stateChangeFailover2(false);
+    }
+
+    /**
+     * @param activate If {@code true} tests activation, otherwise deactivation.
+     * @throws Exception If failed.
+     */
+    private void stateChangeFailover2(boolean activate) throws Exception {
+        // Nodes 1 and 4 do not reply to coordinator.
+        IgniteInternalFuture<?> fut = startNodesAndBlockStatusChange(4, 4, 3, !activate,
1, 4);
+
+        client = false;
+
+        // Start one more nodes while transition is in progress.
+        IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                startGrid(8);
+
+                return null;
+            }
+        }, "start-node1");
+        IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                startGrid(9);
+
+                return null;
+            }
+        }, "start-node2");
+
+        U.sleep(500);
+
+        // Stop coordinator.
+        stopGrid(0);
+
+        stopGrid(getTestIgniteInstanceName(1), true, false);
+        stopGrid(getTestIgniteInstanceName(4), true, false);
+
+        fut.get();
+
+        startFut1.get();
+        startFut2.get();
+
+        client = false;
+
+        startGrid(0);
+        startGrid(1);
+
+        client = true;
+
+        startGrid(4);
+
+        if (!activate) {
+            checkNoCaches(10);
+
+            ignite(0).active(true);
+        }
+
+        checkCaches1(10);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testActivateFailover3() throws Exception {
+        stateChangeFailover3(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeactivateFailover3() throws Exception {
+        stateChangeFailover3(false);
+    }
+
+    /**
+     * @param activate If {@code true} tests activation, otherwise deactivation.
+     * @throws Exception If failed.
+     */
+    private void stateChangeFailover3(boolean activate) throws Exception {
+        // Nodes 1 and 4 do not reply to coordinator.
+        IgniteInternalFuture<?> fut = startNodesAndBlockStatusChange(4, 4, 3, !activate,
1, 4);
+
+
+    }
+
+    /**
+     * @param exp If {@code true} there should be recorded messages.
+     */
+    private void checkRecordedMessages(boolean exp) {
+        for (Ignite node : G.allGrids()) {
+            List<Object> recorded =
+                TestRecordingCommunicationSpi.spi(node).recordedMessages(false);
+
+            if (exp)
+                assertFalse(F.isEmpty(recorded));
+            else
+                assertTrue(F.isEmpty(recorded));
+        }
+    }
+
+    /**
      * @param nodes Expected nodes number.
      */
     private void checkCaches1(int nodes) {
@@ -971,9 +1217,9 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
         GridCacheAdapter cache = ((IgniteKernal)node).context().cache().internalCache(cacheName);
 
         if (exp)
-            assertNotNull(cache);
+            assertNotNull("Cache not found: " + cacheName, cache);
         else
-            assertNull(cache);
+            assertNull("Unexpected cache found: " + cacheName, cache);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
index a932035..dcca1ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
@@ -139,7 +139,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
     /**
      * @param idx idx.
      */
-    protected Ignite backUpClient(int idx) {
+    Ignite backUpClient(int idx) {
         return nodes.get("node" + idx + backUpSuffix + clientSuffix);
     }
 
@@ -147,7 +147,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
      * @param cnt Count.
      * @throws Exception If failed.
      */
-    protected void startPrimaryNodes(int cnt) throws Exception {
+    private void startPrimaryNodes(int cnt) throws Exception {
         for (int i = 0; i < cnt; i++)
             startPrimary(i);
 
@@ -159,7 +159,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
      * @param idx Index.
      * @throws Exception If failed.
      */
-    protected void startPrimary(int idx) throws Exception {
+    private void startPrimary(int idx) throws Exception {
         String node = "node" + idx;
 
         String name = node + primarySuffix;
@@ -177,7 +177,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
      * @param cnt Count.
      * @throws Exception If failed.
      */
-    protected void startBackUpNodes(int cnt) throws Exception {
+    private void startBackUpNodes(int cnt) throws Exception {
         for (int i = 0; i < cnt; i++)
             startBackUp(i);
     }
@@ -186,7 +186,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
      * @param idx Index.
      * @throws Exception If failed.
      */
-    protected void startBackUp(int idx) throws Exception {
+    void startBackUp(int idx) throws Exception {
         String node = "node" + idx;
 
         String name = node + backUpSuffix;
@@ -261,11 +261,10 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
     }
 
     /**
-     *
+     * @param includeClient If {@code true} then allow to return client.
+     * @return Random node from backup topology.
      */
-    protected Ignite randomBackUp(boolean includeClient) {
-        int nodes = 0;
-
+    Ignite randomBackUp(boolean includeClient) {
         List<Ignite> igs = new ArrayList<>();
 
         for (String name : this.nodes.keySet())

http://git-wip-us.apache.org/repos/asf/ignite/blob/57d73160/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
index f5a5ac0..92d1f21 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
@@ -25,6 +25,7 @@ import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static java.lang.Thread.sleep;
 import static org.apache.ignite.testframework.GridTestUtils.runAsync;
@@ -168,9 +169,9 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
             final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>()
{
                 @Override public Void call() throws Exception {
                     while (!stop.get()) {
-                        Ignite ig = randomBackUp(false);
-
                         if (canAct.get()) {
+                            Ignite ig = randomBackUp(false);
+
                             long start = System.currentTimeMillis();
 
                             ig.active(true);
@@ -184,6 +185,8 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
 
                             canAct.set(false);
                         }
+                        else
+                            U.sleep(100);
 
                     }
                     return null;
@@ -193,9 +196,9 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
             final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>()
{
                 @Override public Void call() throws Exception {
                     while (!stop.get()) {
-                        Ignite ig = randomBackUp(false);
-
                         if (!canAct.get()) {
+                            Ignite ig = randomBackUp(false);
+
                             long start = System.currentTimeMillis();
 
                             ig.active(false);
@@ -209,7 +212,8 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
 
                             canAct.set(true);
                         }
-
+                        else
+                            U.sleep(100);
                     }
                     return null;
                 }
@@ -243,9 +247,9 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
             jf2.get();
         }
         finally {
-            log.info("total started nodes: " + (seqIdx.get() - backUpNodes()));
+            log.info("Total started nodes: " + (seqIdx.get() - backUpNodes()));
 
-            log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:
"
+            log.info("Total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:
"
                 + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()))
             );
         }
@@ -348,7 +352,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
             df.get();
         }
         finally {
-            log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:"
+            log.info("Total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:"
                 + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()) + " nodes:
" + backUpNodes()));
         }
     }


Mime
View raw message