qpid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acon...@apache.org
Subject svn commit: r884612 - in /qpid/trunk/qpid: cpp/src/qpid/cluster/Cluster.cpp cpp/src/qpid/cluster/InitialStatusMap.cpp cpp/src/qpid/cluster/StoreStatus.cpp cpp/src/qpid/cluster/StoreStatus.h cpp/src/tests/cluster_tests.py python/qpid/brokertest.py
Date Thu, 26 Nov 2009 15:56:46 GMT
Author: aconway
Date: Thu Nov 26 15:56:46 2009
New Revision: 884612

URL: http://svn.apache.org/viewvc?rev=884612&view=rev
Log:
Cluster consistency: check for no clean store condition.

Modified:
    qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h
    qpid/trunk/qpid/cpp/src/tests/cluster_tests.py
    qpid/trunk/qpid/python/qpid/brokertest.py

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp Thu Nov 26 15:56:46 2009
@@ -260,6 +260,7 @@
         store.load();
         if (store.getClusterId())
             clusterId = store.getClusterId(); // Use stored ID if there is one.
+        QPID_LOG(notice, "Cluster store state: " << store)
     }
 
     cpg.join(name);

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp Thu Nov 26 15:56:46 2009
@@ -148,39 +148,49 @@
         return map.begin()->second->getClusterId(); // Youngest newcomer in node-id
order
 }
 
+void checkId(Uuid& expect, const Uuid& actual, const string& msg) {
+    if (!expect) expect = actual;
+    assert(expect);
+    if (expect != actual)
+        throw Exception(msg);
+}
+
 void InitialStatusMap::checkConsistent() {
     assert(isComplete());
-    bool persistent = (map.begin()->second->getStoreState() != STORE_STATE_NO_STORE);
+    int clean = 0;
+    int dirty = 0;
+    int empty = 0;
+    int none = 0;
+    int active = 0;
     Uuid clusterId;
+    Uuid shutdownId;
+
     for (Map::iterator i = map.begin(); i != map.end(); ++i) {
-        // Must not mix transient and persistent members.
-        if (persistent != (i->second->getStoreState() != STORE_STATE_NO_STORE))
-            throw Exception("Mixing transient and persistent brokers in a cluster");
-        // Members with non-empty stores must have same cluster-id
+        if (i->second->getActive()) ++active;
         switch (i->second->getStoreState()) {
-          case STORE_STATE_NO_STORE:
-          case STORE_STATE_EMPTY_STORE:
-            break;               
+          case STORE_STATE_NO_STORE: ++none; break;
+          case STORE_STATE_EMPTY_STORE: ++empty; break;
           case STORE_STATE_DIRTY_STORE:
+            ++dirty;
+            checkId(clusterId, i->second->getClusterId(),
+                    "Cluster-ID mismatch. Stores belong to different clusters.");
+            break;
           case STORE_STATE_CLEAN_STORE:
-            if (!clusterId) clusterId = i->second->getClusterId();
-            assert(clusterId);
-            if (clusterId != i->second->getClusterId())
-                throw Exception("Cluster-id mismatch, brokers belonged to different clusters.");
-        }
-    }
-    // If this is a newly forming cluster, clean stores must have same shutdown-id
-    if (find_if(map.begin(), map.end(), &isActive) == map.end()) {
-        Uuid shutdownId;
-        for (Map::iterator i = map.begin(); i != map.end(); ++i) {
-            if (i->second->getStoreState() == STORE_STATE_CLEAN_STORE) {
-                if (!shutdownId) shutdownId = i->second->getShutdownId();
-                assert(shutdownId);
-                if (shutdownId != i->second->getShutdownId())
-                    throw Exception("Shutdown-id mismatch, brokers were not shut down together.");
-            }
+            ++clean;
+            checkId(clusterId, i->second->getClusterId(),
+                    "Cluster-ID mismatch. Stores belong to different clusters.");
+            checkId(shutdownId, i->second->getShutdownId(),
+                    "Shutdown-ID mismatch. Stores were not shut down together");
+            break;
         }
     }
+    // Can't mix transient and persistent members.
+    if (none && (clean+dirty+empty))
+        throw Exception("Mixing transient and persistent brokers in a cluster");
+    // If there are no active members and there are dirty stores there
+    // must be at least one clean store.
+    if (!active && dirty && !clean)
+        throw Exception("Cannot recover, no clean store");
 }
 
 

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp Thu Nov 26 15:56:46 2009
@@ -31,6 +31,7 @@
 using framing::Uuid;
 using namespace framing::cluster;
 using namespace boost::filesystem;
+using std::ostream;
 
 StoreStatus::StoreStatus(const std::string& d)
     : state(STORE_STATE_NO_STORE), dataDir(d)
@@ -90,5 +91,20 @@
     save();
 }
 
+ostream& operator<<(ostream& o, const StoreStatus& s) {
+    switch (s.getState()) {
+      case STORE_STATE_NO_STORE: o << "no store"; break;
+      case STORE_STATE_EMPTY_STORE: o << "empty store"; break;
+      case STORE_STATE_DIRTY_STORE:
+        o << "dirty store, cluster-id=" << s.getClusterId();
+        break;
+      case STORE_STATE_CLEAN_STORE:
+        o << "clean store, cluster-id=" << s.getClusterId()
+          << " shutdown-id=" << s.getShutdownId();
+        break;
+    }
+    return o;
+}
+
 }} // namespace qpid::cluster
 

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h Thu Nov 26 15:56:46 2009
@@ -24,6 +24,7 @@
 
 #include "qpid/framing/Uuid.h"
 #include "qpid/framing/enum.h"
+#include <iosfwd>
 
 namespace qpid {
 namespace cluster {
@@ -56,6 +57,8 @@
     Uuid clusterId, shutdownId;
     std::string dataDir;
 };
+
+std::ostream& operator<<(std::ostream&, const StoreStatus&);
 }} // namespace qpid::cluster
 
 #endif  /*!QPID_CLUSTER_STORESTATE_H*/

Modified: qpid/trunk/qpid/cpp/src/tests/cluster_tests.py
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/tests/cluster_tests.py?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/tests/cluster_tests.py (original)
+++ qpid/trunk/qpid/cpp/src/tests/cluster_tests.py Thu Nov 26 15:56:46 2009
@@ -18,7 +18,7 @@
 # under the License.
 #
 
-import os, signal, sys, time, imp
+import os, signal, sys, time, imp, re
 from qpid import datatypes, messaging
 from qpid.brokertest import *
 from qpid.harness import Skipped
@@ -189,4 +189,17 @@
         a = cluster.start("a", expect=EXPECT_EXIT_FAIL, wait=False)
         b = cluster.start("b", expect=EXPECT_EXIT_FAIL, wait=False)
 
-
+    def test_total_failure(self):
+        # Verify we abort with sutiable error message if no clean stores.
+        cluster = self.cluster(0, args=self.args()+["--cluster-size=2"])
+        a = cluster.start("a", expect=EXPECT_EXIT_FAIL, wait=False)
+        b = cluster.start("b", expect=EXPECT_EXIT_FAIL, wait=True)
+        a.kill()
+        b.kill()
+        a = cluster.start("a", expect=EXPECT_EXIT_FAIL, wait=False)
+        b = cluster.start("b", expect=EXPECT_EXIT_FAIL, wait=False)
+        assert a.wait() != 0
+        assert b.wait() != 0
+        msg = re.compile("critical.*no clean store")
+        assert msg.search(file(a.log).read())
+        assert msg.search(file(b.log).read())

Modified: qpid/trunk/qpid/python/qpid/brokertest.py
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/python/qpid/brokertest.py?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/python/qpid/brokertest.py (original)
+++ qpid/trunk/qpid/python/qpid/brokertest.py Thu Nov 26 15:56:46 2009
@@ -157,8 +157,8 @@
         if (self._port is None):
             try: self._port = int(self.stdout.readline())
             except ValueError, e:
-                raise Exception("Can't get port for broker %s (%s)" %
-                                (self.name, self.pname))
+                raise Exception("Can't get port for broker %s (%s)\n    %s" %
+                                (self.name, self.pname, file(self.log).readlines()[-1]))
         return self._port
 
     def unexpected(self,msg):



---------------------------------------------------------------------
Apache Qpid - AMQP Messaging Implementation
Project:      http://qpid.apache.org
Use/Interact: mailto:commits-subscribe@qpid.apache.org


Mime
View raw message