Author: aconway
Date: Thu Nov 26 15:56:46 2009
New Revision: 884612
URL: http://svn.apache.org/viewvc?rev=884612&view=rev
Log:
Cluster consistency: check for no clean store condition.
Modified:
qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp
qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp
qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp
qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h
qpid/trunk/qpid/cpp/src/tests/cluster_tests.py
qpid/trunk/qpid/python/qpid/brokertest.py
Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp Thu Nov 26 15:56:46 2009
@@ -260,6 +260,7 @@
store.load();
if (store.getClusterId())
clusterId = store.getClusterId(); // Use stored ID if there is one.
+ QPID_LOG(notice, "Cluster store state: " << store)
}
cpg.join(name);
Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/InitialStatusMap.cpp Thu Nov 26
15:56:46 2009
@@ -148,39 +148,49 @@
return map.begin()->second->getClusterId(); // Youngest newcomer in
node-id order
}
+void checkId(Uuid& expect, const Uuid& actual, const string& msg) {
+ if (!expect) expect = actual;
+ assert(expect);
+ if (expect != actual)
+ throw Exception(msg);
+}
+
void InitialStatusMap::checkConsistent() {
assert(isComplete());
- bool persistent = (map.begin()->second->getStoreState() !=
STORE_STATE_NO_STORE);
+ int clean = 0;
+ int dirty = 0;
+ int empty = 0;
+ int none = 0;
+ int active = 0;
Uuid clusterId;
+ Uuid shutdownId;
+
for (Map::iterator i = map.begin(); i != map.end(); ++i) {
- // Must not mix transient and persistent members.
- if (persistent != (i->second->getStoreState() != STORE_STATE_NO_STORE))
- throw Exception("Mixing transient and persistent brokers in a
cluster");
- // Members with non-empty stores must have same cluster-id
+ if (i->second->getActive()) ++active;
switch (i->second->getStoreState()) {
- case STORE_STATE_NO_STORE:
- case STORE_STATE_EMPTY_STORE:
- break;
+ case STORE_STATE_NO_STORE: ++none; break;
+ case STORE_STATE_EMPTY_STORE: ++empty; break;
case STORE_STATE_DIRTY_STORE:
+ ++dirty;
+ checkId(clusterId, i->second->getClusterId(),
+ "Cluster-ID mismatch. Stores belong to different
clusters.");
+ break;
case STORE_STATE_CLEAN_STORE:
- if (!clusterId) clusterId = i->second->getClusterId();
- assert(clusterId);
- if (clusterId != i->second->getClusterId())
- throw Exception("Cluster-id mismatch, brokers belonged to
different clusters.");
- }
- }
- // If this is a newly forming cluster, clean stores must have same
shutdown-id
- if (find_if(map.begin(), map.end(), &isActive) == map.end()) {
- Uuid shutdownId;
- for (Map::iterator i = map.begin(); i != map.end(); ++i) {
- if (i->second->getStoreState() == STORE_STATE_CLEAN_STORE) {
- if (!shutdownId) shutdownId = i->second->getShutdownId();
- assert(shutdownId);
- if (shutdownId != i->second->getShutdownId())
- throw Exception("Shutdown-id mismatch, brokers were not
shut down together.");
- }
+ ++clean;
+ checkId(clusterId, i->second->getClusterId(),
+ "Cluster-ID mismatch. Stores belong to different
clusters.");
+ checkId(shutdownId, i->second->getShutdownId(),
+ "Shutdown-ID mismatch. Stores were not shut down
together");
+ break;
}
}
+ // Can't mix transient and persistent members.
+ if (none && (clean+dirty+empty))
+ throw Exception("Mixing transient and persistent brokers in a
cluster");
+ // If there are no active members and there are dirty stores there
+ // must be at least one clean store.
+ if (!active && dirty && !clean)
+ throw Exception("Cannot recover, no clean store");
}
Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.cpp Thu Nov 26 15:56:46
2009
@@ -31,6 +31,7 @@
using framing::Uuid;
using namespace framing::cluster;
using namespace boost::filesystem;
+using std::ostream;
StoreStatus::StoreStatus(const std::string& d)
: state(STORE_STATE_NO_STORE), dataDir(d)
@@ -90,5 +91,20 @@
save();
}
+ostream& operator<<(ostream& o, const StoreStatus& s) {
+ switch (s.getState()) {
+ case STORE_STATE_NO_STORE: o << "no store"; break;
+ case STORE_STATE_EMPTY_STORE: o << "empty store"; break;
+ case STORE_STATE_DIRTY_STORE:
+ o << "dirty store, cluster-id=" << s.getClusterId();
+ break;
+ case STORE_STATE_CLEAN_STORE:
+ o << "clean store, cluster-id=" << s.getClusterId()
+ << " shutdown-id=" << s.getShutdownId();
+ break;
+ }
+ return o;
+}
+
}} // namespace qpid::cluster
Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/StoreStatus.h Thu Nov 26 15:56:46 2009
@@ -24,6 +24,7 @@
#include "qpid/framing/Uuid.h"
#include "qpid/framing/enum.h"
+#include <iosfwd>
namespace qpid {
namespace cluster {
@@ -56,6 +57,8 @@
Uuid clusterId, shutdownId;
std::string dataDir;
};
+
+std::ostream& operator<<(std::ostream&, const StoreStatus&);
}} // namespace qpid::cluster
#endif /*!QPID_CLUSTER_STORESTATE_H*/
Modified: qpid/trunk/qpid/cpp/src/tests/cluster_tests.py
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/tests/cluster_tests.py?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/tests/cluster_tests.py (original)
+++ qpid/trunk/qpid/cpp/src/tests/cluster_tests.py Thu Nov 26 15:56:46 2009
@@ -18,7 +18,7 @@
# under the License.
#
-import os, signal, sys, time, imp
+import os, signal, sys, time, imp, re
from qpid import datatypes, messaging
from qpid.brokertest import *
from qpid.harness import Skipped
@@ -189,4 +189,17 @@
a = cluster.start("a", expect=EXPECT_EXIT_FAIL, wait=False)
b = cluster.start("b", expect=EXPECT_EXIT_FAIL, wait=False)
-
+ def test_total_failure(self):
+ # Verify we abort with sutiable error message if no clean stores.
+ cluster = self.cluster(0, args=self.args()+["--cluster-size=2"])
+ a = cluster.start("a", expect=EXPECT_EXIT_FAIL, wait=False)
+ b = cluster.start("b", expect=EXPECT_EXIT_FAIL, wait=True)
+ a.kill()
+ b.kill()
+ a = cluster.start("a", expect=EXPECT_EXIT_FAIL, wait=False)
+ b = cluster.start("b", expect=EXPECT_EXIT_FAIL, wait=False)
+ assert a.wait() != 0
+ assert b.wait() != 0
+ msg = re.compile("critical.*no clean store")
+ assert msg.search(file(a.log).read())
+ assert msg.search(file(b.log).read())
Modified: qpid/trunk/qpid/python/qpid/brokertest.py
URL:
http://svn.apache.org/viewvc/qpid/trunk/qpid/python/qpid/brokertest.py?rev=884612&r1=884611&r2=884612&view=diff
==============================================================================
--- qpid/trunk/qpid/python/qpid/brokertest.py (original)
+++ qpid/trunk/qpid/python/qpid/brokertest.py Thu Nov 26 15:56:46 2009
@@ -157,8 +157,8 @@
if (self._port is None):
try: self._port = int(self.stdout.readline())
except ValueError, e:
- raise Exception("Can't get port for broker %s (%s)" %
- (self.name, self.pname))
+ raise Exception("Can't get port for broker %s (%s)\n %s" %
+ (self.name, self.pname,
file(self.log).readlines()[-1]))
return self._port
def unexpected(self,msg):
---------------------------------------------------------------------
Apache Qpid - AMQP Messaging Implementation
Project: http://qpid.apache.org
Use/Interact: mailto:commits-subscr...@qpid.apache.org
