Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[7624] branches
|
From: <tho...@us...> - 2013-12-09 16:04:11
|
Revision: 7624
http://bigdata.svn.sourceforge.net/bigdata/?rev=7624&view=rev
Author: thompsonbry
Date: 2013-12-09 16:04:03 +0000 (Mon, 09 Dec 2013)
Log Message:
-----------
Added unit tests to the HA CI test suite for forceRemoveService(). See #779.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java
branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java
branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java 2013-12-09 15:36:35 UTC (rev 7623)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java 2013-12-09 16:04:03 UTC (rev 7624)
@@ -72,6 +72,7 @@
import com.bigdata.ha.HAGlue;
import com.bigdata.ha.HAStatusEnum;
+import com.bigdata.ha.IndexManagerCallable;
import com.bigdata.ha.RunState;
import com.bigdata.ha.msg.HARootBlockRequest;
import com.bigdata.ha.msg.HASnapshotDigestRequest;
@@ -1100,6 +1101,34 @@
}
+ /**
+ * Debug class to explicitly ask one service to remove another.
+ *
+ * This emulates the behaviour of the service in receiving correct notification
+ * of a target service failure -for example after a wire pull or sure kill.
+ *
+ */
+ protected static class ForceRemoveService extends IndexManagerCallable<Void> {
+
+ private static final long serialVersionUID = 1L;
+ private final UUID service;
+
+ ForceRemoveService(final UUID service) {
+ this.service = service;
+ }
+
+ @Override
+ public Void call() throws Exception {
+
+ final HAJournal ha = (HAJournal) this.getIndexManager();
+
+ ha.getQuorum().getActor().forceRemoveService(service);
+
+ return null;
+ }
+
+ }
+
private void safeShutdown(final HAGlue haGlue, final File serviceDir,
final ServiceListener serviceListener) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java 2013-12-09 15:36:35 UTC (rev 7623)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java 2013-12-09 16:04:03 UTC (rev 7624)
@@ -44,6 +44,7 @@
import com.bigdata.journal.jini.ha.HAJournalServer.RunStateEnum;
import com.bigdata.journal.jini.ha.HAJournalTest.HAGlueTest;
import com.bigdata.journal.jini.ha.HAJournalTest.SpuriousTestException;
+import com.bigdata.quorum.QuorumActor;
import com.bigdata.quorum.zk.ZKQuorumImpl;
import com.bigdata.rdf.sail.webapp.client.RemoteRepository;
import com.bigdata.util.ClocksNotSynchronizedException;
@@ -705,7 +706,7 @@
}
}
- public void doBounceFollower() throws Exception {
+ private void doBounceFollower() throws Exception {
final HAGlue serverA = startA();
final HAGlue serverB = startB();
@@ -995,6 +996,295 @@
// }
/**
+ * Test of {@link QuorumActor#forceRemoveService(UUID)}. Start A + B. Once
+ * the quorum meets, we figure out which service is the leader. The leader
+ * then forces the other service out of the quorum.
+ */
+ public void test_AB_forceRemoveService_B() throws Exception {
+
+ final HAGlue serverA = startA();
+ final HAGlue serverB = startB();
+
+ final long token1 = quorum.awaitQuorum(awaitQuorumTimeout, TimeUnit.MILLISECONDS);
+
+ doNSSStatusRequest(serverA);
+ doNSSStatusRequest(serverB);
+
+ // Await initial commit point (KB create).
+ awaitCommitCounter(1L, serverA, serverB);
+
+ // Await [A] up and running as leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(serverA));
+
+ // Await [B] up and running as follower.
+ assertEquals(HAStatusEnum.Follower, awaitNSSAndHAReady(serverB));
+
+ // Verify self-reporting by RMI in their respective roles.
+ awaitHAStatus(serverA, HAStatusEnum.Leader);
+ awaitHAStatus(serverB, HAStatusEnum.Follower);
+
+ // Verify binary equality on the journal files.
+ assertDigestsEquals(new HAGlue[] { serverA, serverB });
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper before quorum break:\n" + dumpZoo());
+ }
+
+ /*
+ * Force the follower out of the quorum. Verify quorum meets again and
+ * that we can read on all services.
+ */
+ {
+
+ final HAGlue leader = quorum.getClient().getLeader(token1);
+
+ if (leader.equals(serverA)) {
+
+ leader.submit(new ForceRemoveService(getServiceBId()), true).get();
+
+ } else {
+
+ leader.submit(new ForceRemoveService(getServiceAId()), true).get();
+
+ }
+
+ // Thread.sleep(100000); // sleep to allow thread dump for analysis
+ // Okay, is the problem that the quorum doesn't break?
+ // assertFalse(quorum.isQuorumMet());
+
+ // Right so the Quorum is not met, but the follower deosn't seem to know it's broken
+
+ // Wait for the quorum to break and then meet again.
+ final long token2 = awaitNextQuorumMeet(token1);
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper after quorum meet:\n" + dumpZoo());
+ }
+
+ /*
+ * Bouncing the connection broke the quorun, so verify that the
+ * quorum token was advanced.
+ */
+ assertEquals(token1 + 1, token2);
+
+ // The leader MAY have changed (since the quorum broke).
+ final HAGlue leader2 = quorum.getClient().getLeader(token2);
+
+ // Verify leader self-reports in new role.
+ awaitHAStatus(leader2, HAStatusEnum.Leader);
+
+// final UUID leaderId2 = leader2.getServiceId();
+//
+// assertFalse(leaderId1.equals(leaderId2));
+
+ /*
+ * Verify we can read on the KB on both nodes.
+ *
+ * Note: It is important to test the reads for the first commit on
+ * both the leader and the follower.
+ */
+ for (HAGlue service : new HAGlue[] { serverA, serverB }) {
+
+ awaitNSSAndHAReady(service);
+
+ final RemoteRepository repo = getRemoteRepository(service);
+
+ // Should be empty.
+ assertEquals(
+ 0L,
+ countResults(repo.prepareTupleQuery(
+ "SELECT * {?a ?b ?c} LIMIT 10").evaluate()));
+
+ }
+
+ }
+
+ }
+
+ /**
+ * Test of {@link QuorumActor#forceRemoveService(UUID)}. Start A + B + C in
+ * strict order. Wait until the quorum is fully met and the initial KB
+ * create transaction is done. The leader then forces B out of the quorum.
+ * We verify that the quorum fully meets again, that B is now the last
+ * service in the pipeline order, and that the quorum did not break (same
+ * token).
+ */
+ public void test_ABC_forceRemoveService_B() throws Exception {
+
+ final ABC services = new ABC(true/*sequential*/);
+ final HAGlue serverA = services.serverA;
+ final HAGlue serverB = services.serverB;
+ final HAGlue serverC = services.serverC;
+
+ final long token1 = awaitFullyMetQuorum();
+
+ doNSSStatusRequest(serverA);
+ doNSSStatusRequest(serverB);
+ doNSSStatusRequest(serverC);
+
+ // Await initial commit point (KB create).
+ awaitCommitCounter(1L, serverA, serverB, serverC);
+
+ // Await [A] up and running as leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(serverA));
+
+ // Await [B] up and running as follower.
+ assertEquals(HAStatusEnum.Follower, awaitNSSAndHAReady(serverB));
+
+ // Verify self-reporting by RMI in their respective roles.
+ awaitHAStatus(serverA, HAStatusEnum.Leader);
+ awaitHAStatus(serverB, HAStatusEnum.Follower);
+ awaitHAStatus(serverC, HAStatusEnum.Follower);
+
+ // Verify binary equality on the journal files.
+ assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper before forcing service remove:\n" + dumpZoo());
+ }
+
+ /*
+ * Bounce the 1st follower out of the quorum. Verify quorum meets again
+ * and that we can read on all services.
+ */
+ {
+
+ serverA.submit(new ForceRemoveService(getServiceBId()), true).get();
+
+ // Wait for the quorum to fully meet again.
+ final long token2 = awaitFullyMetQuorum();
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper after quorum fully met again:\n" + dumpZoo());
+ }
+
+ /*
+ * The quorum did not break. The token is unchanged.
+ */
+ assertEquals(token1, token2);
+
+ /*
+ * Service B came back in at the end of the pipeline.
+ */
+ awaitPipeline(new HAGlue[] { serverA, serverC, serverB });
+
+ /*
+ * Verify we can read on the KB on all nodes.
+ *
+ * Note: It is important to test the reads for the first commit on
+ * both the leader and the follower.
+ */
+ for (HAGlue service : new HAGlue[] { serverA, serverB, serverC }) {
+
+ awaitNSSAndHAReady(service);
+
+ final RemoteRepository repo = getRemoteRepository(service);
+
+ // Should be empty.
+ assertEquals(
+ 0L,
+ countResults(repo.prepareTupleQuery(
+ "SELECT * {?a ?b ?c} LIMIT 10").evaluate()));
+
+ }
+
+ }
+
+ }
+
+ /**
+ * Test of {@link QuorumActor#forceRemoveService(UUID)}. Start A + B + C in
+ * strict order. Wait until the quorum is fully met and the initial KB
+ * create transaction is done. The leader then forces C out of the quorum.
+ * We verify that the quorum fully meets again, that C is again the last
+ * service in the pipeline order, and that the quorum did not break (same
+ * token).
+ */
+ public void test_ABC_forceRemoveService_C() throws Exception {
+
+ final ABC services = new ABC(true/*sequential*/);
+ final HAGlue serverA = services.serverA;
+ final HAGlue serverB = services.serverB;
+ final HAGlue serverC = services.serverC;
+
+ final long token1 = awaitFullyMetQuorum();
+
+ doNSSStatusRequest(serverA);
+ doNSSStatusRequest(serverB);
+ doNSSStatusRequest(serverC);
+
+ // Await initial commit point (KB create).
+ awaitCommitCounter(1L, serverA, serverB, serverC);
+
+ // Await [A] up and running as leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(serverA));
+
+ // Await [B] up and running as follower.
+ assertEquals(HAStatusEnum.Follower, awaitNSSAndHAReady(serverB));
+
+ // Verify self-reporting by RMI in their respective roles.
+ awaitHAStatus(serverA, HAStatusEnum.Leader);
+ awaitHAStatus(serverB, HAStatusEnum.Follower);
+ awaitHAStatus(serverC, HAStatusEnum.Follower);
+
+ // Verify binary equality on the journal files.
+ assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper before forcing service remove:\n" + dumpZoo());
+ }
+
+ /*
+ * Bounce the 1st follower out of the quorum. Verify quorum meets again
+ * and that we can read on all services.
+ */
+ {
+
+ serverA.submit(new ForceRemoveService(getServiceCId()), true).get();
+
+ // Wait for the quorum to fully meet again.
+ final long token2 = awaitFullyMetQuorum();
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper after quorum fully met again:\n" + dumpZoo());
+ }
+
+ /*
+ * The quorum did not break. The token is unchanged.
+ */
+ assertEquals(token1, token2);
+
+ /*
+ * Service C came back in at the end of the pipeline (i.e., the
+ * pipeline is unchanged).
+ */
+ awaitPipeline(new HAGlue[] { serverA, serverB, serverC });
+
+ /*
+ * Verify we can read on the KB on all nodes.
+ *
+ * Note: It is important to test the reads for the first commit on
+ * both the leader and the follower.
+ */
+ for (HAGlue service : new HAGlue[] { serverA, serverB, serverC }) {
+
+ awaitNSSAndHAReady(service);
+
+ final RemoteRepository repo = getRemoteRepository(service);
+
+ // Should be empty.
+ assertEquals(
+ 0L,
+ countResults(repo.prepareTupleQuery(
+ "SELECT * {?a ?b ?c} LIMIT 10").evaluate()));
+
+ }
+
+ }
+
+ }
+
+ /**
* Verify ability to stop and restart the zookeeper process under test
* control.
*
Modified: branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java
===================================================================
--- branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java 2013-12-09 15:36:35 UTC (rev 7623)
+++ branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java 2013-12-09 16:04:03 UTC (rev 7624)
@@ -1,21 +1,12 @@
package com.bigdata.journal.jini.ha;
-import java.util.UUID;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
-import com.bigdata.ha.HAGlue;
-import com.bigdata.ha.IndexManagerCallable;
-import com.bigdata.ha.QuorumService;
-import com.bigdata.journal.AbstractJournal;
-import com.bigdata.journal.IIndexManager;
-import com.bigdata.journal.jini.ha.AbstractHA3JournalServerTestCase.ABC;
-import com.bigdata.journal.jini.ha.AbstractHA3JournalServerTestCase.LargeLoadTask;
-import com.bigdata.quorum.Quorum;
-
import net.jini.config.Configuration;
-import junit.framework.TestCase;
+import com.bigdata.ha.HAGlue;
+
public class TestHA3JustKills extends AbstractHA3JournalServerTestCase {
@@ -78,7 +69,7 @@
// FIXME: in the face of no implemented error propagation we can explicitly
// tell the leader to remove the killed service!
- startup.serverA.submit(new ForceRemoveService(getServiceCId()), true);
+ startup.serverA.submit(new ForceRemoveService(getServiceCId()), true).get();
awaitPipeline(20, TimeUnit.SECONDS, new HAGlue[] {startup.serverA, startup.serverB});
@@ -143,7 +134,7 @@
kill(startup.serverB);
// FIXME: temporary call to explicitly remove the service prior to correct protocol
- startup.serverA.submit(new ForceRemoveService(getServiceBId()), true);
+ startup.serverA.submit(new ForceRemoveService(getServiceBId()), true).get();
awaitPipeline(10, TimeUnit.SECONDS, new HAGlue[] {startup.serverA, startup.serverC});
Modified: branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java
===================================================================
--- branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java 2013-12-09 15:36:35 UTC (rev 7623)
+++ branches/MGC_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHAJournalServerOverride.java 2013-12-09 16:04:03 UTC (rev 7624)
@@ -44,6 +44,7 @@
import com.bigdata.journal.jini.ha.HAJournalServer.RunStateEnum;
import com.bigdata.journal.jini.ha.HAJournalTest.HAGlueTest;
import com.bigdata.journal.jini.ha.HAJournalTest.SpuriousTestException;
+import com.bigdata.quorum.QuorumActor;
import com.bigdata.quorum.zk.ZKQuorumImpl;
import com.bigdata.rdf.sail.webapp.client.RemoteRepository;
import com.bigdata.util.ClocksNotSynchronizedException;
@@ -705,7 +706,7 @@
}
}
- public void doBounceFollower() throws Exception {
+ private void doBounceFollower() throws Exception {
final HAGlue serverA = startA();
final HAGlue serverB = startB();
@@ -995,6 +996,295 @@
// }
/**
+ * Test of {@link QuorumActor#forceRemoveService(UUID)}. Start A + B. Once
+ * the quorum meets, we figure out which service is the leader. The leader
+ * then forces the other service out of the quorum.
+ */
+ public void test_AB_forceRemoveService_B() throws Exception {
+
+ final HAGlue serverA = startA();
+ final HAGlue serverB = startB();
+
+ final long token1 = quorum.awaitQuorum(awaitQuorumTimeout, TimeUnit.MILLISECONDS);
+
+ doNSSStatusRequest(serverA);
+ doNSSStatusRequest(serverB);
+
+ // Await initial commit point (KB create).
+ awaitCommitCounter(1L, serverA, serverB);
+
+ // Await [A] up and running as leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(serverA));
+
+ // Await [B] up and running as follower.
+ assertEquals(HAStatusEnum.Follower, awaitNSSAndHAReady(serverB));
+
+ // Verify self-reporting by RMI in their respective roles.
+ awaitHAStatus(serverA, HAStatusEnum.Leader);
+ awaitHAStatus(serverB, HAStatusEnum.Follower);
+
+ // Verify binary equality on the journal files.
+ assertDigestsEquals(new HAGlue[] { serverA, serverB });
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper before quorum break:\n" + dumpZoo());
+ }
+
+ /*
+ * Force the follower out of the quorum. Verify quorum meets again and
+ * that we can read on all services.
+ */
+ {
+
+ final HAGlue leader = quorum.getClient().getLeader(token1);
+
+ if (leader.equals(serverA)) {
+
+ leader.submit(new ForceRemoveService(getServiceBId()), true).get();
+
+ } else {
+
+ leader.submit(new ForceRemoveService(getServiceAId()), true).get();
+
+ }
+
+ // Thread.sleep(100000); // sleep to allow thread dump for analysis
+ // Okay, is the problem that the quorum doesn't break?
+ // assertFalse(quorum.isQuorumMet());
+
+ // Right so the Quorum is not met, but the follower deosn't seem to know it's broken
+
+ // Wait for the quorum to break and then meet again.
+ final long token2 = awaitNextQuorumMeet(token1);
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper after quorum meet:\n" + dumpZoo());
+ }
+
+ /*
+ * Bouncing the connection broke the quorun, so verify that the
+ * quorum token was advanced.
+ */
+ assertEquals(token1 + 1, token2);
+
+ // The leader MAY have changed (since the quorum broke).
+ final HAGlue leader2 = quorum.getClient().getLeader(token2);
+
+ // Verify leader self-reports in new role.
+ awaitHAStatus(leader2, HAStatusEnum.Leader);
+
+// final UUID leaderId2 = leader2.getServiceId();
+//
+// assertFalse(leaderId1.equals(leaderId2));
+
+ /*
+ * Verify we can read on the KB on both nodes.
+ *
+ * Note: It is important to test the reads for the first commit on
+ * both the leader and the follower.
+ */
+ for (HAGlue service : new HAGlue[] { serverA, serverB }) {
+
+ awaitNSSAndHAReady(service);
+
+ final RemoteRepository repo = getRemoteRepository(service);
+
+ // Should be empty.
+ assertEquals(
+ 0L,
+ countResults(repo.prepareTupleQuery(
+ "SELECT * {?a ?b ?c} LIMIT 10").evaluate()));
+
+ }
+
+ }
+
+ }
+
+ /**
+ * Test of {@link QuorumActor#forceRemoveService(UUID)}. Start A + B + C in
+ * strict order. Wait until the quorum is fully met and the initial KB
+ * create transaction is done. The leader then forces B out of the quorum.
+ * We verify that the quorum fully meets again, that B is now the last
+ * service in the pipeline order, and that the quorum did not break (same
+ * token).
+ */
+ public void test_ABC_forceRemoveService_B() throws Exception {
+
+ final ABC services = new ABC(true/*sequential*/);
+ final HAGlue serverA = services.serverA;
+ final HAGlue serverB = services.serverB;
+ final HAGlue serverC = services.serverC;
+
+ final long token1 = awaitFullyMetQuorum();
+
+ doNSSStatusRequest(serverA);
+ doNSSStatusRequest(serverB);
+ doNSSStatusRequest(serverC);
+
+ // Await initial commit point (KB create).
+ awaitCommitCounter(1L, serverA, serverB, serverC);
+
+ // Await [A] up and running as leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(serverA));
+
+ // Await [B] up and running as follower.
+ assertEquals(HAStatusEnum.Follower, awaitNSSAndHAReady(serverB));
+
+ // Verify self-reporting by RMI in their respective roles.
+ awaitHAStatus(serverA, HAStatusEnum.Leader);
+ awaitHAStatus(serverB, HAStatusEnum.Follower);
+ awaitHAStatus(serverC, HAStatusEnum.Follower);
+
+ // Verify binary equality on the journal files.
+ assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper before forcing service remove:\n" + dumpZoo());
+ }
+
+ /*
+ * Bounce the 1st follower out of the quorum. Verify quorum meets again
+ * and that we can read on all services.
+ */
+ {
+
+ serverA.submit(new ForceRemoveService(getServiceBId()), true).get();
+
+ // Wait for the quorum to fully meet again.
+ final long token2 = awaitFullyMetQuorum();
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper after quorum fully met again:\n" + dumpZoo());
+ }
+
+ /*
+ * The quorum did not break. The token is unchanged.
+ */
+ assertEquals(token1, token2);
+
+ /*
+ * Service B came back in at the end of the pipeline.
+ */
+ awaitPipeline(new HAGlue[] { serverA, serverC, serverB });
+
+ /*
+ * Verify we can read on the KB on all nodes.
+ *
+ * Note: It is important to test the reads for the first commit on
+ * both the leader and the follower.
+ */
+ for (HAGlue service : new HAGlue[] { serverA, serverB, serverC }) {
+
+ awaitNSSAndHAReady(service);
+
+ final RemoteRepository repo = getRemoteRepository(service);
+
+ // Should be empty.
+ assertEquals(
+ 0L,
+ countResults(repo.prepareTupleQuery(
+ "SELECT * {?a ?b ?c} LIMIT 10").evaluate()));
+
+ }
+
+ }
+
+ }
+
+ /**
+ * Test of {@link QuorumActor#forceRemoveService(UUID)}. Start A + B + C in
+ * strict order. Wait until the quorum is fully met and the initial KB
+ * create transaction is done. The leader then forces C out of the quorum.
+ * We verify that the quorum fully meets again, that C is again the last
+ * service in the pipeline order, and that the quorum did not break (same
+ * token).
+ */
+ public void test_ABC_forceRemoveService_C() throws Exception {
+
+ final ABC services = new ABC(true/*sequential*/);
+ final HAGlue serverA = services.serverA;
+ final HAGlue serverB = services.serverB;
+ final HAGlue serverC = services.serverC;
+
+ final long token1 = awaitFullyMetQuorum();
+
+ doNSSStatusRequest(serverA);
+ doNSSStatusRequest(serverB);
+ doNSSStatusRequest(serverC);
+
+ // Await initial commit point (KB create).
+ awaitCommitCounter(1L, serverA, serverB, serverC);
+
+ // Await [A] up and running as leader.
+ assertEquals(HAStatusEnum.Leader, awaitNSSAndHAReady(serverA));
+
+ // Await [B] up and running as follower.
+ assertEquals(HAStatusEnum.Follower, awaitNSSAndHAReady(serverB));
+
+ // Verify self-reporting by RMI in their respective roles.
+ awaitHAStatus(serverA, HAStatusEnum.Leader);
+ awaitHAStatus(serverB, HAStatusEnum.Follower);
+ awaitHAStatus(serverC, HAStatusEnum.Follower);
+
+ // Verify binary equality on the journal files.
+ assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper before forcing service remove:\n" + dumpZoo());
+ }
+
+ /*
+ * Bounce the 1st follower out of the quorum. Verify quorum meets again
+ * and that we can read on all services.
+ */
+ {
+
+ serverA.submit(new ForceRemoveService(getServiceCId()), true).get();
+
+ // Wait for the quorum to fully meet again.
+ final long token2 = awaitFullyMetQuorum();
+
+ if (log.isInfoEnabled()) {
+ log.info("Zookeeper after quorum fully met again:\n" + dumpZoo());
+ }
+
+ /*
+ * The quorum did not break. The token is unchanged.
+ */
+ assertEquals(token1, token2);
+
+ /*
+ * Service C came back in at the end of the pipeline (i.e., the
+ * pipeline is unchanged).
+ */
+ awaitPipeline(new HAGlue[] { serverA, serverB, serverC });
+
+ /*
+ * Verify we can read on the KB on all nodes.
+ *
+ * Note: It is important to test the reads for the first commit on
+ * both the leader and the follower.
+ */
+ for (HAGlue service : new HAGlue[] { serverA, serverB, serverC }) {
+
+ awaitNSSAndHAReady(service);
+
+ final RemoteRepository repo = getRemoteRepository(service);
+
+ // Should be empty.
+ assertEquals(
+ 0L,
+ countResults(repo.prepareTupleQuery(
+ "SELECT * {?a ?b ?c} LIMIT 10").evaluate()));
+
+ }
+
+ }
+
+ }
+
+ /**
* Verify ability to stop and restart the zookeeper process under test
* control.
*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
