Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[7600] branches/PIPELINE_RESYNC/bigdata-jini/src/test/ com/bigdata/journal/jini/ha
|
From: <mar...@us...> - 2013-11-27 14:24:42
|
Revision: 7600
http://bigdata.svn.sourceforge.net/bigdata/?rev=7600&view=rev
Author: martyncutcher
Date: 2013-11-27 14:24:34 +0000 (Wed, 27 Nov 2013)
Log Message:
-----------
Adding missing Unit tests!
Modified Paths:
--------------
branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JournalServer.java
Added Paths:
-----------
branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3ChangeLeader.java
branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java
branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestStressKill.java
Added: branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3ChangeLeader.java
===================================================================
--- branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3ChangeLeader.java (rev 0)
+++ branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3ChangeLeader.java 2013-11-27 14:24:34 UTC (rev 7600)
@@ -0,0 +1,191 @@
+package com.bigdata.journal.jini.ha;
+
+import java.util.Random;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+
+import com.bigdata.ha.HAGlue;
+import com.bigdata.journal.jini.ha.AbstractHA3JournalServerTestCase.LargeLoadTask;
+import com.bigdata.journal.jini.ha.HAJournalTest.HAGlueTest;
+
+public class TestHA3ChangeLeader extends AbstractHA3JournalServerTestCase {
+
+ public TestHA3ChangeLeader() {
+ }
+
+ public TestHA3ChangeLeader(String name) {
+ super(name);
+ }
+
+ /**
+ * We have seen problems with updates when the leader changes, this test reconstructs
+ * this simple scenario, with and update transaction, change of leader and then a
+ * second update transaction.
+ *
+ * @throws Exception
+ */
+ public void testStartABC_ChangeLeader() throws Exception {
+
+ doStartABC_ChangeLeader(1);
+ }
+
+ public void testStartABC_ChangeLeader_3Trans() throws Exception {
+
+ doStartABC_ChangeLeader(2);
+ }
+
+ public void testStartABC_ChangeLeader_RandomTrans() throws Exception {
+ final Random r = new Random();
+ final int ntrans = r.nextInt(900);
+ try {
+ doStartABC_ChangeLeader(ntrans);
+ } catch (Exception e) {
+ System.err.println("Problem with " + ntrans + " transactions");
+ throw e;
+ }
+ }
+
+ public void doStartABC_ChangeLeader(final int ntrans) throws Exception {
+
+ // Start 3 services
+ final HAGlue serverA = startA();
+ final HAGlue serverB = startB();
+ final HAGlue serverC = startC();
+
+ // Wait for a quorum meet.
+ final long token1 = awaitFullyMetQuorum();
+
+ // await pipeline
+ awaitPipeline(20, TimeUnit.SECONDS, new HAGlue[] { serverA, serverB,
+ serverC });
+
+ awaitCommitCounter(1L, new HAGlue[] { serverA, serverB, serverC });
+
+ /*
+ * Now go through sevearl commit points with a met quorum. The HALog
+ * files should be retained at the final commit point.
+ */
+ for (int t = 0; t < ntrans; t++) {
+ simpleTransaction();
+ }
+
+ shutdownA();
+
+ final long token2 = awaitNextQuorumMeet(token1);
+
+ // let's commit several transactions with the new leader
+ for (int t = 0; t < 20; t++) {
+ simpleTransaction();
+ // Check store states
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+ }
+
+ // And again verify binary equality of ALL journals.
+ assertDigestsEquals(new HAGlue[] { serverB, serverC });
+
+ assertTrue(token2 == quorum.token());
+ }
+
+ public void testStartABC_KillLeader_RandomTrans() throws Exception {
+ final Random r = new Random();
+ final int ntrans = r.nextInt(900);
+ try {
+ doStartABC_KillLeader(ntrans);
+ } catch (Exception e) {
+ System.err.println("Problem with " + ntrans + " transactions");
+ throw e;
+ }
+ }
+
+ public void doStartABC_KillLeader(final int ntrans) throws Exception {
+
+ // Start 3 services
+ final HAGlue serverA = startA();
+ final HAGlue serverB = startB();
+ final HAGlue serverC = startC();
+
+ // Wait for a quorum meet.
+ final long token1 = awaitFullyMetQuorum();
+
+ // await pipeline
+ awaitPipeline(20, TimeUnit.SECONDS, new HAGlue[] { serverA, serverB,
+ serverC });
+
+ awaitCommitCounter(1L, new HAGlue[] { serverA, serverB, serverC });
+
+ /*
+ * Now go through sevearl commit points with a met quorum. The HALog
+ * files should be retained at the final commit point.
+ */
+ for (int t = 0; t < ntrans; t++) {
+ simpleTransaction();
+
+ // Check store states
+ assertStoreStates(new HAGlueTest[] { (HAGlueTest) serverA, (HAGlueTest) serverB, (HAGlueTest) serverC });
+ }
+
+ kill(serverA);
+
+ final long token2 = awaitNextQuorumMeet(token1);
+
+ // let's commit several transactions with the new leader
+ for (int t = 0; t < 20; t++) {
+ simpleTransaction();
+ }
+
+ // And again verify binary equality of ALL journals.
+ assertDigestsEquals(new HAGlue[] { serverB, serverC });
+
+ assertStoreStates(new HAGlueTest[] { (HAGlueTest) serverB, (HAGlueTest) serverC });
+ assertTrue(token2 == quorum.token());
+ }
+ /*
+ * Similar to ChangeLeader but with a LargeLoad
+ */
+ public void testStartABC_StressChangeLeader() throws Exception {
+
+ // Start 3 services
+ final HAGlue serverA = startA();
+ final HAGlue serverB = startB();
+ final HAGlue serverC = startC();
+
+ // Wait for a quorum meet.
+ final long token1 = awaitFullyMetQuorum();
+
+ // await pipeline
+ awaitPipeline(20, TimeUnit.SECONDS, new HAGlue[] { serverA, serverB,
+ serverC });
+
+ awaitCommitCounter(1L, new HAGlue[] { serverA, serverB, serverC });
+
+ /*
+ * LOAD data on leader.
+ */
+ for (int i = 0; i < 100; i++) {
+ final FutureTask<Void> ft = new FutureTask<Void>(new LargeLoadTask(
+ token1, true/* reallyLargeLoad */, false/*dropAll*/));
+
+ // Start LOAD.
+ executorService.submit(ft);
+
+ // Await LOAD, but with a timeout.
+ ft.get(longLoadTimeoutMillis, TimeUnit.MILLISECONDS);
+ }
+
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+
+ shutdownA();
+
+ final long token2 = awaitNextQuorumMeet(token1);
+
+ simpleTransaction();
+
+ // And again verify binary equality of ALL journals.
+ assertDigestsEquals(new HAGlue[] { serverB, serverC });
+
+ assertStoreStates(new HAGlue[] { serverB, serverC });
+
+ assertTrue(token2 == quorum.token());
+
+ }
+}
Modified: branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JournalServer.java
===================================================================
--- branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JournalServer.java 2013-11-27 14:23:29 UTC (rev 7599)
+++ branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JournalServer.java 2013-11-27 14:24:34 UTC (rev 7600)
@@ -194,7 +194,9 @@
awaitHAStatus(serverB, HAStatusEnum.Follower);
awaitHAStatus(serverC, HAStatusEnum.Follower);
- // The commit counter has not changed.
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+
+ // The commit counter has not changed.
assertEquals(
lastCommitCounter,
serverA.getRootBlock(
@@ -543,9 +545,15 @@
// Verify binary equality of ALL journals.
assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+ // Check transient store states after resync
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+
// Now force further commit when fully met to remove log files
simpleTransaction();
+ // Check transient store states after commit
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+
// And again verify binary equality of ALL journals.
assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
@@ -559,6 +567,46 @@
}
/**
+ * We have seen problems with updates when the leader changes, this test reconstructs
+ * this simple scenario, with and update transaction, change of leader and then a
+ * second update transaction.
+ *
+ * @throws Exception
+ */
+ public void testStartABC_ChangeLeader() throws Exception {
+
+ // Start 3 services
+ final HAGlue serverA = startA();
+ final HAGlue serverB = startB();
+ final HAGlue serverC = startC();
+
+ // Wait for a quorum meet.
+ final long token1 = quorum.awaitQuorum(awaitQuorumTimeout,
+ TimeUnit.MILLISECONDS);
+
+ // await pipeline
+ awaitPipeline(20, TimeUnit.SECONDS, new HAGlue[] {serverA, serverB, serverC});
+
+ /*
+ * Now go through a commit point with a met quorum. The HALog
+ * files should be retained at that commit point.
+ */
+ simpleTransaction();
+
+ shutdownA();
+
+ final long token2 = quorum.awaitQuorum(awaitQuorumTimeout,
+ TimeUnit.MILLISECONDS);
+ assertFalse(token1 == token2);
+
+ simpleTransaction();
+
+ // And again verify binary equality of ALL journals.
+ assertDigestsEquals(new HAGlue[] { serverB, serverC });
+
+ }
+
+ /**
* Unit test of the ability to go through a simultaneous restart of all
* services once those services are no longer at commit point 0. Two
* services will meet on the lastCommitTime. The third will need to RESYNC
@@ -883,6 +931,8 @@
HAStatusEnum.Follower, HAStatusEnum.Follower },
new HAGlue[] { serverA, serverB, serverC });
+ // Check store states
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
/*
* Verify binary equality of ALL journals.
*
@@ -893,7 +943,7 @@
* could result in the leader not including the newly joined
* follower in the 2-phase commit.
*/
- assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
// Now force further commit when fully met to remove log files
simpleTransaction();
@@ -901,7 +951,10 @@
// And again verify binary equality of ALL journals.
// assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
- // Now verify no HALog files since fully met quorum @ commit.
+ // Check store states
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+
+ // Now verify no HALog files since fully met quorum @ commit.
final long lastCommitCounter3 = leader
.getRootBlock(new HARootBlockRequest(null/* storeUUID */))
.getRootBlock().getCommitCounter();
@@ -1019,6 +1072,8 @@
// And again verify binary equality of ALL journals.
assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
+
log.info("ALL GOOD!");
}
@@ -1291,6 +1346,8 @@
assertHALogNotFound(0L/* firstCommitCounter */, lastCommitCounter3,
new HAGlue[] { serverA, serverB, serverC });
+ // Check store states
+ assertStoreStates(new HAGlueTest[] { (HAGlueTest) serverA, (HAGlueTest) serverB, (HAGlueTest) serverC });
}
/**
@@ -1639,6 +1696,7 @@
// Verify binary equality of ALL journals.
assertDigestsEquals(new HAGlue[] { serverA, serverB, serverC });
+ assertStoreStates(new HAGlue[] { serverA, serverB, serverC });
/*
* Now go through a commit point with a fully met quorum. The HALog
* files should be purged at that commit point.
@@ -1676,6 +1734,8 @@
// ...and with original token
assertTrue(token == quorum.token());
+
+ assertStoreStates(new HAGlue[] { serverA, serverB});
}
/**
@@ -1712,6 +1772,7 @@
assertTrue(leader.equals(startup.serverB)
|| leader.equals(startup.serverC));
+ assertStoreStates(new HAGlue[] { startup.serverB, startup.serverC });
}
/**
@@ -1748,6 +1809,7 @@
assertTrue(leader.equals(startup.serverB)
|| leader.equals(startup.serverC));
+ assertStoreStates(new HAGlue[] { startup.serverB, startup.serverC });
}
/**
@@ -2352,6 +2414,7 @@
assertDigestsEquals(new HAGlue[] { services.serverA, services.serverB, services.serverC });
+ assertStoreStates(new HAGlue[] { services.serverA, services.serverB, services.serverC });
} finally {
destroyAll();
@@ -2858,6 +2921,9 @@
// Verify quorum becomes fully met now that LOAD is done.
assertEquals(token, awaitFullyMetQuorum());
+
+ // Check store states
+ assertStoreStates(new HAGlue[] { startup.serverA, startup.serverB, serverC2 });
}
/**
@@ -3033,6 +3099,8 @@
// Verify quorum becomes fully met now that LOAD is done.
assertEquals(token, awaitFullyMetQuorum());
+ // Check store states
+ assertStoreStates(new HAGlue[] { startup.serverA, serverB2, startup.serverC });
}
/**
@@ -3085,6 +3153,8 @@
// Verify fully met.
assertTrue(quorum.isQuorumFullyMet(token));
+ serverB2.awaitHAReady(awaitQuorumTimeout, TimeUnit.MILLISECONDS);
+
// Await LOAD, but with a timeout.
ft.get(longLoadTimeoutMillis, TimeUnit.MILLISECONDS);
@@ -3108,7 +3178,8 @@
// log.warn("Result Leader: " + resultLeader.next().getBinding("count"));
// }
- // assertDigestsEquals(new HAGlue[] { startup.serverA, serverB2, startup.serverC });
+ // Check store states after load and final transaction
+ assertStoreStates(new HAGlue[] { startup.serverA, serverB2, startup.serverC });
}
/**
@@ -3480,7 +3551,7 @@
}
- public void _testStressQuorumABC_HAStatusUpdatesWithFailovers()
+ public void testStressQuorumABC_HAStatusUpdatesWithFailovers()
throws Exception {
for (int i = 1; i <= 20; i++) {
try {
Added: branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java
===================================================================
--- branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java (rev 0)
+++ branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA3JustKills.java 2013-11-27 14:24:34 UTC (rev 7600)
@@ -0,0 +1,151 @@
+package com.bigdata.journal.jini.ha;
+
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+
+import com.bigdata.ha.HAGlue;
+import com.bigdata.journal.jini.ha.AbstractHA3JournalServerTestCase.ABC;
+import com.bigdata.journal.jini.ha.AbstractHA3JournalServerTestCase.LargeLoadTask;
+
+import net.jini.config.Configuration;
+import junit.framework.TestCase;
+
+public class TestHA3JustKills extends AbstractHA3JournalServerTestCase {
+
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Note: This overrides some {@link Configuration} values for the
+ * {@link HAJournalServer} in order to establish conditions suitable for
+ * testing the {@link ISnapshotPolicy} and {@link IRestorePolicy}.
+ */
+ @Override
+ protected String[] getOverrides() {
+
+ return new String[]{
+// "com.bigdata.journal.HAJournal.properties=" +TestHA3JournalServer.getTestHAJournalProperties(com.bigdata.journal.HAJournal.properties),
+ "com.bigdata.journal.jini.ha.HAJournalServer.restorePolicy=new com.bigdata.journal.jini.ha.DefaultRestorePolicy(0L,1,0)",
+ "com.bigdata.journal.jini.ha.HAJournalServer.snapshotPolicy=new com.bigdata.journal.jini.ha.NoSnapshotPolicy()",
+// "com.bigdata.journal.jini.ha.HAJournalServer.HAJournalClass=\""+HAJournalTest.class.getName()+"\"",
+ "com.bigdata.journal.jini.ha.HAJournalServer.onlineDisasterRecovery=true",
+ };
+
+ }
+
+ public TestHA3JustKills() {
+ }
+
+ public TestHA3JustKills(String name) {
+ super(name);
+ }
+
+ /**
+ * Start A+B+C in strict sequence. Wait until the quorum fully meets. Start
+ * a long running LOAD. While the LOAD is running, sure kill C (the last
+ * follower). Verify that the LOAD completes successfully with the remaining
+ * services (A+B).
+ */
+ public void testABC_LiveLoadRemainsMet_kill_C() throws Exception {
+
+ // enforce join order
+ final ABC startup = new ABC(true /*sequential*/);
+
+ final long token = awaitFullyMetQuorum();
+
+ // start concurrent task loads that continue until fully met
+ final FutureTask<Void> ft = new FutureTask<Void>(new LargeLoadTask(
+ token));
+
+ executorService.submit(ft);
+
+ // allow load head start
+ Thread.sleep(300/* ms */);
+
+ // Verify load is still running.
+ assertFalse(ft.isDone());
+
+ // Dump Zookeeper
+ log.warn("ZOOKEEPER\n" + dumpZoo());
+
+ kill(startup.serverC);
+
+ awaitPipeline(20, TimeUnit.SECONDS, new HAGlue[] {startup.serverA, startup.serverB});
+
+ // token must remain unchanged to indicate same quorum
+ assertEquals(token, awaitMetQuorum());
+
+ awaitMembers(new HAGlue[] {startup.serverA, startup.serverB});
+ awaitJoined(new HAGlue[] {startup.serverA, startup.serverB});
+
+ // token must remain unchanged to indicate same quorum
+ assertEquals(token, awaitMetQuorum());
+
+ // Await LOAD, but with a timeout.
+ ft.get(longLoadTimeoutMillis, TimeUnit.MILLISECONDS);
+
+ // token must remain unchanged to indicate same quorum
+ assertEquals(token, awaitMetQuorum());
+
+ }
+
+ public void testStressABC_LiveLoadRemainsMet_kill_C() throws Exception {
+ for (int i = 0; i < 5; i++) {
+ try {
+ testABC_LiveLoadRemainsMet_kill_C();
+ } catch (Throwable t) {
+ fail("Run " + i, t);
+ } finally {
+ Thread.sleep(1000);
+ destroyAll();
+ }
+ }
+ }
+
+ /**
+ * Start A+B+C in strict sequence. Wait until the quorum fully meets. Start
+ * a long running LOAD. While the LOAD is running, sure kill B (the first
+ * follower). Verify that the LOAD completes successfully with the remaining
+ * services (A+C), after the leader re-orders the pipeline.
+ */
+ public void testABC_LiveLoadRemainsMet_kill_B() throws Exception {
+
+ // enforce join order
+ final ABC startup = new ABC(true /*sequential*/);
+
+ final long token = awaitFullyMetQuorum();
+
+ // start concurrent task loads that continue until fully met
+ final FutureTask<Void> ft = new FutureTask<Void>(new LargeLoadTask(
+ token));
+
+ executorService.submit(ft);
+
+ // allow load head start
+ Thread.sleep(300/* ms */);
+
+ // Verify load is still running.
+ assertFalse(ft.isDone());
+
+ // Dump Zookeeper
+ log.warn("ZOOKEEPER\n" + dumpZoo());
+
+ kill(startup.serverB);
+
+ awaitPipeline(10, TimeUnit.SECONDS, new HAGlue[] {startup.serverA, startup.serverC});
+
+ // also check members and joined
+ awaitMembers(new HAGlue[] {startup.serverA, startup.serverC});
+ awaitJoined(new HAGlue[] {startup.serverA, startup.serverC});
+
+ // token must remain unchanged to indicate same quorum
+ assertEquals(token, awaitMetQuorum());
+
+ // Await LOAD, but with a timeout.
+ ft.get(longLoadTimeoutMillis, TimeUnit.MILLISECONDS);
+
+ // token must remain unchanged to indicate same quorum
+ assertEquals(token, awaitMetQuorum());
+
+ }
+}
Added: branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestStressKill.java
===================================================================
--- branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestStressKill.java (rev 0)
+++ branches/PIPELINE_RESYNC/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestStressKill.java 2013-11-27 14:24:34 UTC (rev 7600)
@@ -0,0 +1,22 @@
+package com.bigdata.journal.jini.ha;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+public class TestStressKill extends TestCase {
+
+ public static Test suite()
+ {
+
+ final TestSuite suite = new TestSuite("TestStressKill");
+
+ // commitTime => (HALog|Snapshot)Record test suites.
+ for (int i = 0; i < 100; i++) {
+ suite.addTestSuite(TestHA3JustKills.class);
+ }
+
+ return suite;
+ }
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
