Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[8123] branches/BIGDATA_RELEASE_1_3_0
|
From: <tho...@us...> - 2014-04-15 17:08:46
|
Revision: 8123
http://sourceforge.net/p/bigdata/code/8123
Author: thompsonbry
Date: 2014-04-15 17:08:37 +0000 (Tue, 15 Apr 2014)
Log Message:
-----------
Merge of the HA1/HA5 branch back to the main development branch. This closes out #722 (HA1). #723 remains open with one test that fails in CI.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/BufferedWrite.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCache.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RWStrategy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WORMStrategy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWWriteCacheService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestRWWriteCacheService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestWORMWriteCacheService.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/HAJournalServer.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/SnapshotManager.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3BackupTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHAJournalServerTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/StressTestHA3JournalServer.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestAll.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/zkClient.config
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/SD.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA5JournalServerTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/HAJournal-D.config
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/HAJournal-E.config
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1JournalServer.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1SnapshotPolicy.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA1SnapshotPolicy2.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA5JournalServer.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/TestHA5JournalServerWithHALogs.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-A.properties
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-B.properties
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-C.properties
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-D.properties
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-E.properties
Removed Paths:
-------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-A.properties
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-B.properties
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/log4j-template-C.properties
Property Changed:
----------------
branches/BIGDATA_RELEASE_1_3_0/
branches/BIGDATA_RELEASE_1_3_0/bigdata/lib/jetty/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/aggregate/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/util/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/htree/raba/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/jsr166/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/joinGraph/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/util/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/jsr166/
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/util/httpd/
branches/BIGDATA_RELEASE_1_3_0/bigdata-compatibility/
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/attr/
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/disco/
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/util/config/
branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/
branches/BIGDATA_RELEASE_1_3_0/bigdata-perf/
branches/BIGDATA_RELEASE_1_3_0/bigdata-perf/btc/
branches/BIGDATA_RELEASE_1_3_0/bigdata-perf/btc/src/resources/
branches/BIGDATA_RELEASE_1_3_0/bigdata-perf/lubm/
branches/BIGDATA_RELEASE_1_3_0/bigdata-perf/uniprot/
branches/BIGDATA_RELEASE_1_3_0/bigdata-perf/uniprot/src/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/changesets/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/error/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/relation/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/samples/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/aggregate/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/
branches/BIGDATA_RELEASE_1_3_0/bigdata-rdf/src/test/com/bigdata/rdf/relation/
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/bench/
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/LEGAL/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/lib/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/java/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/java/it/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/java/it/unimi/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/test/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/test/it/unimi/
branches/BIGDATA_RELEASE_1_3_0/dsi-utils/src/test/it/unimi/dsi/
branches/BIGDATA_RELEASE_1_3_0/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/BIGDATA_RELEASE_1_3_0/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/BIGDATA_RELEASE_1_3_0/osgi/
branches/BIGDATA_RELEASE_1_3_0/src/resources/bin/config/
Index: branches/BIGDATA_RELEASE_1_3_0
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE:6769-6785
/branches/BIGDATA_RELEASE_1_2_0:6766-7380
/branches/BTREE_BUFFER_BRANCH:2004-2045
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/lib/jetty
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/lib/jetty 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/lib/jetty 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/lib/jetty
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/lib/jetty:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/lib/jetty:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/lib/jetty:6766-7380
/branches/INT64_BRANCH/bigdata/lib/jetty:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/aggregate
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/aggregate 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/aggregate 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/aggregate
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/java/com/bigdata/bop/aggregate:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/java/com/bigdata/bop/aggregate:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/aggregate:6766-7380
/branches/INT64_BRANCH/bigdata/src/java/com/bigdata/bop/aggregate:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/java/com/bigdata/bop/joinGraph:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/java/com/bigdata/bop/joinGraph:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/joinGraph:6766-7380
/branches/INT64_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/util
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/util 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/util 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/util
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/java/com/bigdata/bop/util:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/java/com/bigdata/bop/util:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/util:6766-7380
/branches/INT64_BRANCH/bigdata/src/java/com/bigdata/bop/util:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/htree/raba
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/htree/raba 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/htree/raba 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/htree/raba
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/java/com/bigdata/htree/raba:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/java/com/bigdata/htree/raba:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/htree/raba:6766-7380
/branches/INT64_BRANCH/bigdata/src/java/com/bigdata/htree/raba:4486-4522
\ No newline at end of property
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/BufferedWrite.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/BufferedWrite.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/BufferedWrite.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -137,6 +137,13 @@
}
+ /**
+ * Used to zero pad slots in buffered writes.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/721#comment:10"> HA1 </a>
+ */
+ static private final byte[] s_zeros = new byte[256];
+
/**
* Buffer a write.
*
@@ -188,6 +195,19 @@
}
// copy the caller's record into the buffer.
m_data.put(data);
+
+ // if data_len < slot_len then clear remainder of buffer
+ int padding = slot_len - data_len;
+ while (padding > 0) {
+ if (padding > s_zeros.length) {
+ m_data.put(s_zeros);
+ padding -= s_zeros.length;
+ } else {
+ m_data.put(s_zeros, 0, padding);
+ break;
+ }
+ }
+
// update the file offset by the size of the allocation slot
m_endAddr += slot_len;
// update the buffer position by the size of the allocation slot.
@@ -250,8 +270,9 @@
final ByteBuffer m_data = tmp.buffer();
// reset the buffer state.
- m_data.position(0);
- m_data.limit(m_data.capacity());
+ //m_data.position(0);
+ //m_data.limit(m_data.capacity());
+ m_data.clear();
m_startAddr = -1;
m_endAddr = 0;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCache.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCache.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -310,7 +310,7 @@
/**
* When a record is used as a read cache then the readCount is
- * maintained as a metric on its access. �This could be used to
+ * maintained as a metric on its access. This could be used to
* determine eviction/compaction.
* <p>
* Note: volatile to guarantee visibility of updates. Might do better
@@ -509,7 +509,8 @@
* @param isHighlyAvailable
* when <code>true</code> the whole record checksum is maintained
* for use when replicating the write cache along the write
- * pipeline.
+ * pipeline. This needs to be <code>true</code> for HA1 as well
+ * since we need to write the HALog.
* @param bufferHasData
* when <code>true</code> the caller asserts that the buffer has
* data (from a replicated write), in which case the position
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -1151,6 +1151,7 @@
done = WriteCache.transferTo(cache/* src */,
curCompactingCache/* dst */, serviceMap, 0/*threshold*/);
if (done) {
+ // Everything was compacted. Send just the address metadata (empty cache block).
sendAddressMetadata(cache);
if (log.isDebugEnabled())
@@ -1231,7 +1232,7 @@
* been allocated on the leader in the same order in which the leader
* made those allocations. This information is used to infer the order
* in which the allocators for the different allocation slot sizes are
- * created. This method will synchronous send those address notices and
+ * created. This method will synchronously send those address notices and
* and also makes sure that the followers see the recycled addresses
* records so they can keep both their allocators and the actual
* allocations synchronized with the leader.
@@ -1244,13 +1245,15 @@
* @throws InterruptedException
* @throws ExecutionException
* @throws IOException
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/721"> HA1 </a>
*/
private void sendAddressMetadata(final WriteCache cache)
throws IllegalStateException, InterruptedException,
ExecutionException, IOException {
- if (quorum == null || !quorum.isHighlyAvailable()
- || !quorum.getClient().isLeader(quorumToken)) {
+ if (quorum == null) { //|| !quorum.isHighlyAvailable()
+// || !quorum.getClient().isLeader(quorumToken)) {
return;
}
@@ -1344,20 +1347,15 @@
private void writeCacheBlock(final WriteCache cache)
throws InterruptedException, ExecutionException, IOException {
- /*
- * IFF HA
+ /**
+ * IFF HA and this is the quorum leader.
*
- * TODO isHA should be true even if the quorum is not highly
- * available since there still could be other services in the write
- * pipeline (e.g., replication to an offline HAJournalServer prior
- * to changing over into an HA3 quorum or off-site replication). The
- * unit tests need to be updated to specify [isHighlyAvailable] for
- * ALL quorum based test runs.
+ * Note: This is true for HA1 as well. The code path enabled by this
+ * is responsible for writing the HALog files.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/721"> HA1 </a>
*/
- final boolean isHA = quorum != null && quorum.isHighlyAvailable();
-
- // IFF HA and this is the quorum leader.
- final boolean isHALeader = isHA
+ final boolean isHALeader = quorum != null
&& quorum.getClient().isLeader(quorumToken);
/*
@@ -1438,15 +1436,25 @@
* then clean up the documentation here (see the commented
* out version of this line below).
*/
- quorumMember.logWriteCacheBlock(pkg.getMessage(), pkg.getData().duplicate());
-
- // ASYNC MSG RMI + NIO XFER.
- remoteWriteFuture = quorumMember.replicate(null/* req */, pkg.getMessage(),
- pkg.getData().duplicate());
-
- counters.get().nsend++;
+ quorumMember.logWriteCacheBlock(pkg.getMessage(), pkg.getData().duplicate());
/*
+ * TODO Do we want to always support the replication code path
+ * when a quorum exists (that is, also for HA1) in case there
+ * are pipeline listeners that are not HAJournalServer
+ * instances? E.g., for offsite replication?
+ */
+ if (quorum.replicationFactor() > 1) {
+
+ // ASYNC MSG RMI + NIO XFER.
+ remoteWriteFuture = quorumMember.replicate(null/* req */,
+ pkg.getMessage(), pkg.getData().duplicate());
+
+ counters.get().nsend++;
+
+ }
+
+ /*
* The quorum leader logs the write cache block here. For the
* followers, the write cache blocks are currently logged by
* HAJournalServer.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -2473,18 +2473,18 @@
}
- /**
- * Return <code>true</code> if the journal is configured for high
- * availability.
- *
- * @see QuorumManager#isHighlyAvailable()
- */
- public boolean isHighlyAvailable() {
+// /**
+// * Return <code>true</code> if the journal is configured for high
+// * availability.
+// *
+// * @see Quorum#isHighlyAvailable()
+// */
+// public boolean isHighlyAvailable() {
+//
+// return quorum == null ? false : quorum.isHighlyAvailable();
+//
+// }
- return quorum == null ? false : quorum.isHighlyAvailable();
-
- }
-
/**
* {@inheritDoc}
* <p>
@@ -3428,8 +3428,10 @@
if (quorum == null)
return;
- if (!quorum.isHighlyAvailable())
+ if (!quorum.isHighlyAvailable()) {
+ // Gather and 2-phase commit are not used in HA1.
return;
+ }
/**
* CRITICAL SECTION. We need obtain a distributed consensus for the
@@ -3542,6 +3544,25 @@
// reload the commit record from the new root block.
store._commitRecord = store._getCommitRecord();
+ if (quorum != null) {
+ /**
+ * Write the root block on the HALog file, closing out that
+ * file.
+ *
+ * @see <a href="http://trac.bigdata.com/ticket/721"> HA1 </a>
+ */
+ final QuorumService<HAGlue> localService = quorum.getClient();
+ if (localService != null) {
+ // Quorum service not asynchronously closed.
+ try {
+ // Write the closing root block on the HALog file.
+ localService.logRootBlock(newRootBlock);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
if (txLog.isInfoEnabled())
txLog.info("COMMIT: commitTime=" + commitTime);
@@ -3846,9 +3867,9 @@
// Prepare the new root block.
cs.newRootBlock();
- if (quorum == null) {
+ if (quorum == null || quorum.replicationFactor() == 1) {
- // Non-HA mode.
+ // Non-HA mode (including HA1).
cs.commitSimple();
} else {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RWStrategy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RWStrategy.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/RWStrategy.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -146,6 +146,7 @@
}
+ @Override
public ByteBuffer read(final long addr) {
try {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WORMStrategy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WORMStrategy.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/journal/WORMStrategy.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -272,7 +272,7 @@
* which use this flag to conditionally track the checksum of the entire
* write cache buffer).
*/
- private final boolean isHighlyAvailable;
+ private final boolean isQuorumUsed;
/**
* The {@link UUID} which identifies the journal (this is the same for each
@@ -970,11 +970,11 @@
com.bigdata.journal.Options.HALOG_COMPRESSOR,
com.bigdata.journal.Options.DEFAULT_HALOG_COMPRESSOR);
- isHighlyAvailable = quorum != null && quorum.isHighlyAvailable();
+ isQuorumUsed = quorum != null; // && quorum.isHighlyAvailable();
final boolean useWriteCacheService = fileMetadata.writeCacheEnabled
&& !fileMetadata.readOnly && fileMetadata.closeTime == 0L
- || isHighlyAvailable;
+ || isQuorumUsed;
if (useWriteCacheService) {
/*
@@ -1049,7 +1049,7 @@
final long fileExtent)
throws InterruptedException {
- super(baseOffset, buf, useChecksum, isHighlyAvailable,
+ super(baseOffset, buf, useChecksum, isQuorumUsed,
bufferHasData, opener, fileExtent);
}
@@ -1379,6 +1379,7 @@
* to get the data from another node based on past experience for that
* record.
*/
+ @Override
public ByteBuffer read(final long addr) {
try {
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/jsr166
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/jsr166 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/jsr166 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/jsr166
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/java/com/bigdata/jsr166:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/java/com/bigdata/jsr166:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/jsr166:6766-7380
/branches/INT64_BRANCH/bigdata/src/java/com/bigdata/jsr166:4486-4522
\ No newline at end of property
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -839,6 +839,7 @@
m_statsBucket.allocate(size);
}
+
return value;
} else {
StringBuilder sb = new StringBuilder();
@@ -1300,4 +1301,33 @@
return count;
}
+ /**
+ * Determines if the provided physical address is within an allocated slot
+ * @param addr
+ * @return
+ */
+ public boolean verifyAllocatedAddress(long addr) {
+ if (log.isTraceEnabled())
+ log.trace("Checking Allocator " + m_index + ", size: " + m_size);
+
+ final Iterator<AllocBlock> blocks = m_allocBlocks.iterator();
+ final long range = m_size * m_bitSize * 32;
+ while (blocks.hasNext()) {
+ final int startAddr = blocks.next().m_addr;
+ if (startAddr != 0) {
+ final long start = RWStore.convertAddr(startAddr);
+ final long end = start + range;
+
+ if (log.isTraceEnabled())
+ log.trace("Checking " + addr + " between " + start + " - " + end);
+
+ if (addr >= start && addr < end)
+ return true;
+ } else {
+ break;
+ }
+ }
+ return false;
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -688,7 +688,7 @@
throws InterruptedException {
super(buf, useChecksum, m_quorum != null
- && m_quorum.isHighlyAvailable(), bufferHasData, opener,
+ /*&& m_quorum.isHighlyAvailable()*/, bufferHasData, opener,
fileExtent,
m_bufferedWrite);
@@ -1080,16 +1080,17 @@
private RWWriteCacheService newWriteCacheService() {
try {
- final boolean highlyAvailable = m_quorum != null
- && m_quorum.isHighlyAvailable();
+// final boolean highlyAvailable = m_quorum != null
+// && m_quorum.isHighlyAvailable();
- final boolean prefixWrites = highlyAvailable;
+ final boolean prefixWrites = m_quorum != null; // highlyAvailable
return new RWWriteCacheService(m_writeCacheBufferCount,
m_minCleanListSize, m_readCacheBufferCount, prefixWrites, m_compactionThreshold, m_hotCacheSize, m_hotCacheThreshold,
convertAddr(m_fileSize), m_reopener, m_quorum, this) {
-
+
+ @Override
@SuppressWarnings("unchecked")
public WriteCache newWriteCache(final IBufferAccess buf,
final boolean useChecksum,
@@ -6962,7 +6963,7 @@
if (log.isDebugEnabled())
log.debug("writeRaw: " + offset);
-
+
// Guard IO against concurrent file extension.
final Lock lock = m_extensionLock.readLock();
@@ -7067,6 +7068,22 @@
}
}
+ /**
+ * Can be used to determine if an address is within an allocated slot.
+ *
+ * @param addr
+ * @return whether addr is within slot allocated area
+ */
+ public boolean verifyAllocatedAddress(final long addr) {
+ for (int index = 0; index < m_allocs.size(); index++) {
+ final FixedAllocator xfa = m_allocs.get(index);
+ if (xfa.verifyAllocatedAddress(addr))
+ return true;
+ }
+
+ return false;
+ }
+
public StoreState getStoreState() {
final RWStoreState ret = new RWStoreState(this);
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWWriteCacheService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWWriteCacheService.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWWriteCacheService.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -80,8 +80,9 @@
final long fileExtent)
throws InterruptedException {
- final boolean highlyAvailable = getQuorum() != null
- && getQuorum().isHighlyAvailable();
+// final boolean highlyAvailable = getQuorum() != null
+// && getQuorum().isHighlyAvailable();
+ final boolean highlyAvailable = getQuorum() != null;
return new FileChannelScatteredWriteCache(buf, true/* useChecksum */,
highlyAvailable,
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/joinGraph
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/joinGraph 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/joinGraph 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/joinGraph
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/test/com/bigdata/bop/joinGraph:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/test/com/bigdata/bop/joinGraph:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/joinGraph:6766-7380
/branches/INT64_BRANCH/bigdata/src/test/com/bigdata/bop/joinGraph:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/util
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/util 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/util 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/util
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/test/com/bigdata/bop/util:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/test/com/bigdata/bop/util:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/util:6766-7380
/branches/INT64_BRANCH/bigdata/src/test/com/bigdata/bop/util:4486-4522
\ No newline at end of property
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestRWWriteCacheService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestRWWriteCacheService.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestRWWriteCacheService.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -35,6 +35,7 @@
import java.util.HashMap;
import java.util.Map.Entry;
import java.util.Random;
+import java.util.concurrent.TimeUnit;
import junit.framework.AssertionFailedError;
@@ -48,7 +49,6 @@
import com.bigdata.quorum.QuorumActor;
import com.bigdata.rwstore.RWWriteCacheService;
import com.bigdata.util.ChecksumUtility;
-import com.bigdata.util.InnerCause;
/**
* Test suite for the {@link WriteCacheService} using scattered writes on a
@@ -138,6 +138,19 @@
actor.castVote(0);
fixture.awaitDeque();
+ // Await quorum meet.
+ assertCondition(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ assertEquals(0L, quorum.token());
+ } catch (Exception e) {
+ fail();
+ }
+ }
+
+ }, 5000/*timeout*/, TimeUnit.MILLISECONDS);
+
file = File.createTempFile(getName(), ".rw.tmp");
opener = new ReopenFileChannel(file, "rw");
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestWORMWriteCacheService.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestWORMWriteCacheService.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/io/writecache/TestWORMWriteCacheService.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -521,7 +521,7 @@
final int nbuffers = 1;
final boolean useChecksums = false;
- final boolean isHighlyAvailable = false;
+ final boolean isHighlyAvailable = true;
// No write pipeline.
final int k = 1;
@@ -574,7 +574,7 @@
*/
final double largeRecordRate = 0d;
final boolean useChecksums = false;
- final boolean isHighlyAvailable = false;
+ final boolean isHighlyAvailable = true;
final int k = 1;
final long lastCommitTime = 0L;
@@ -619,7 +619,7 @@
final int nbuffers = 2;
final boolean useChecksums = false;
- final boolean isHighlyAvailable = false;
+ final boolean isHighlyAvailable = true;
// No write pipeline.
final int k = 1;
@@ -672,7 +672,7 @@
*/
final double largeRecordRate = 0d;
final boolean useChecksums = false;
- final boolean isHighlyAvailable = false;
+ final boolean isHighlyAvailable = true;
// No write pipeline.
final int k = 1;
@@ -681,8 +681,7 @@
final String logicalServiceId = "logicalService_"+getName();
final MockQuorum<HAPipelineGlue, MyMockQuorumMember<HAPipelineGlue>> quorum = new MockQuorum<HAPipelineGlue, MyMockQuorumMember<HAPipelineGlue>>(
k, fixture);
- try {
-
+ try {
fixture.start();
quorum.start(new MyMockQuorumMember<HAPipelineGlue>(fixture,logicalServiceId));
@@ -718,7 +717,7 @@
final int nbuffers = 6;
final boolean useChecksums = true;
- final boolean isHighlyAvailable = false;
+ final boolean isHighlyAvailable = true;
// No write pipeline.
final int k = 1;
@@ -771,7 +770,7 @@
*/
final double largeRecordRate = 0d;
final boolean useChecksums = true;
- final boolean isHighlyAvailable = false;
+ final boolean isHighlyAvailable = true;
// No write pipeline.
final int k = 1;
@@ -2120,6 +2119,19 @@
+ ", isHighlyAvailable=" + isHighlyAvailable);
}
+ // Await quorum meet.
+ assertCondition(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ assertEquals(0L, quorum.token());
+ } catch (Exception e) {
+ fail();
+ }
+ }
+
+ }, 5000/*timeout*/, TimeUnit.MILLISECONDS);
+
File file = null;
ReopenFileChannel opener = null;
WriteCacheService writeCacheService = null;
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/jsr166
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/jsr166 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/jsr166 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/jsr166
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/test/com/bigdata/jsr166:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/test/com/bigdata/jsr166:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/jsr166:6766-7380
/branches/INT64_BRANCH/bigdata/src/test/com/bigdata/jsr166:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/util/httpd
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/util/httpd 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/util/httpd 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/util/httpd
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata/src/test/com/bigdata/util/httpd:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata/src/test/com/bigdata/util/httpd:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/util/httpd:6766-7380
/branches/INT64_BRANCH/bigdata/src/test/com/bigdata/util/httpd:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata-compatibility
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-compatibility 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-compatibility 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata-compatibility
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata-compatibility:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata-compatibility:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata-compatibility:6766-7380
/branches/INT64_BRANCH/bigdata-compatibility:4486-4522
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/attr
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/attr 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/attr 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/attr
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata-jini/src/java/com/bigdata/attr:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata-jini/src/java/com/bigdata/attr:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata-jini/src/java/com/bigdata/attr:6766-7380
/branches/BTREE_BUFFER_BRANCH/bigdata-jini/src/java/com/bigdata/attr:2004-2045
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/disco
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/disco 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/disco 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/disco
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata-jini/src/java/com/bigdata/disco:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata-jini/src/java/com/bigdata/disco:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata-jini/src/java/com/bigdata/disco:6766-7380
/branches/BTREE_BUFFER_BRANCH/bigdata-jini/src/java/com/bigdata/disco:2004-2045
\ No newline at end of property
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/HAJournalServer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/HAJournalServer.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/HAJournalServer.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -4659,7 +4659,7 @@
throw new IllegalStateException("Server is not running");
return tmp.getConnectors()[0].getLocalPort();
-
+
}
/**
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/SnapshotManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/SnapshotManager.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/journal/jini/ha/SnapshotManager.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -1096,6 +1096,8 @@
+ haLogBytesOnDisk//
+ ", journalSize="
+ journalSize//
+ + ", thresholdPercentLogSize="
+ + thresholdPercentLogSize//
+ ", percentLogSize="
+ actualPercentLogSize//
+ "%, takeSnapshot=" + takeSnapshot //
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/util/config
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/util/config 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/util/config 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/java/com/bigdata/util/config
___________________________________________________________________
Modified: svn:mergeinfo
## -1,3 +1,4 ##
+/branches/BIGDATA_MGC_HA1_HA5/bigdata-jini/src/java/com/bigdata/util/config:8025-8122
/branches/BIGDATA_OPENRDF_2_6_9_UPDATE/bigdata-jini/src/java/com/bigdata/util/config:6769-6785
/branches/BIGDATA_RELEASE_1_2_0/bigdata-jini/src/java/com/bigdata/util/config:6766-7380
/branches/BTREE_BUFFER_BRANCH/bigdata-jini/src/java/com/bigdata/util/config:2004-2045
\ No newline at end of property
Index: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha 2014-04-15 17:08:37 UTC (rev 8123)
Property changes on: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha
___________________________________________________________________
Modified: svn:ignore
## -1,3 +1,4 ##
log4j.properties
logging.properties
results.txt
+TestRWStoreAddress.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3BackupTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3BackupTestCase.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3BackupTestCase.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -43,6 +43,7 @@
import com.bigdata.ha.msg.HARootBlockRequest;
import com.bigdata.journal.CommitCounterUtility;
import com.bigdata.journal.IHABufferStrategy;
+import com.bigdata.journal.IRootBlockView;
import com.bigdata.journal.Journal;
import com.bigdata.rdf.sail.webapp.client.ConnectOptions;
import com.bigdata.rdf.sail.webapp.client.RemoteRepository;
@@ -62,6 +63,13 @@
super(name);
}
+ @Override
+ protected int replicationFactor() {
+
+ return 3;
+
+ }
+
/**
* Issue HTTP request to a service to take a snapshot.
*
@@ -146,10 +154,12 @@
* The current commit counter on the server. This is the commit point
* that should be restored.
*/
- final long commitCounterM = serverA
- .getRootBlock(new HARootBlockRequest(null/* storeUUID */))
- .getRootBlock().getCommitCounter();
+
+ final IRootBlockView serverARootBlock = serverA.getRootBlock(
+ new HARootBlockRequest(null/* storeUUID */)).getRootBlock();
+ final long commitCounterM = serverARootBlock.getCommitCounter();
+
final File snapshotFile = SnapshotManager.getSnapshotFile(
getSnapshotDirA(), commitCounterN);
@@ -198,10 +208,29 @@
*/
rest.restore(false/* listCommitPoints */, Long.MAX_VALUE/* haltingCommitCounter */);
+ /*
+ * FIXME For some reason, we need to close and reopen the
+ * journal before it can be used. See HARestore.
+ */
+ if (true) {
+ jnl.close();
+
+ // reopen.
+ jnl = new Journal(p);
+ }
+
+ // Verify can dump journal after restore.
+ dumpJournal(jnl);
+
// Verify journal now at the expected commit point.
assertEquals(commitCounterM, jnl.getRootBlockView()
.getCommitCounter());
+ if (!serverARootBlock.equals(jnl.getRootBlockView())) {
+ fail("Root blocks differ: serverA=" + serverARootBlock
+ + ", restored=" + jnl.getRootBlockView());
+ }
+
/*
* Compute digest of the restored journal. The digest should
* agree with the digest of the Journal on A since we rolled
@@ -214,14 +243,17 @@
new HADigestRequest(null/* storeUUID */))
.getDigest();
- final MessageDigest digest = MessageDigest
- .getInstance("MD5");
+ final byte[] digest2;
+ {
+ final MessageDigest digest = MessageDigest
+ .getInstance("MD5");
- // digest of restored journal.
- ((IHABufferStrategy) (jnl.getBufferStrategy()))
- .computeDigest(null/* snapshot */, digest);
+ // digest of restored journal.
+ ((IHABufferStrategy) (jnl.getBufferStrategy()))
+ .computeDigest(null/* snapshot */, digest);
- final byte[] digest2 = digest.digest();
+ digest2 = digest.digest();
+ }
if (!BytesUtil.bytesEqual(digestA, digest2)) {
@@ -238,20 +270,6 @@
}
- /*
- * FIXME For some reason, we need to close and reopen the
- * journal before it can be used. See HARestore.
- */
- if (true) {
- jnl.close();
-
- // reopen.
- jnl = new Journal(p);
- }
-
- // Verify can dump journal after restore.
- dumpJournal(jnl);
-
} finally {
if (jnl != null) {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java 2014-04-15 14:16:12 UTC (rev 8122)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA3JournalServerTestCase.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -87,7 +87,6 @@
import com.bigdata.jini.util.JiniUtil;
import com.bigdata.journal.IRootBlockView;
import com.bigdata.journal.StoreState;
-import com.bigdata.journal.jini.ha.HAJournalServer.ConfigurationOptions;
import com.bigdata.journal.jini.ha.HAJournalTest.HAGlueTest;
import com.bigdata.quorum.AbstractQuorumClient;
import com.bigdata.quorum.AsynchronousQuorumCloseException;
@@ -110,7 +109,7 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
-public class AbstractHA3JournalServerTestCase extends
+public abstract class AbstractHA3JournalServerTestCase extends
AbstractHAJournalServerTestCase implements DiscoveryListener {
/** Quorum client used to monitor (or act on) the logical service quorum. */
@@ -133,7 +132,7 @@
* Implementation listens for the death of the child process and can be used
* to decide when the child process is no longer executing.
*/
- private static class ServiceListener implements IServiceListener {
+ static class ServiceListener implements IServiceListener {
private volatile HAGlue haGlue;
private volatile ProcessHelper processHelper;
@@ -152,13 +151,14 @@
this.haGlue = haGlue;
}
- @SuppressWarnings("unused")
- public HAGlue getHAGlue() {
+// @SuppressWarnings("unused")
+// public HAGlue getHAGlue() {
+//
+// return haGlue;
+//
+// }
- return haGlue;
-
- }
-
+ @Override
public void add(final ProcessHelper processHelper) {
if (processHelper == null)
@@ -218,31 +218,41 @@
* The {@link Remote} interfaces for these services (if started and
* successfully discovered).
*/
- private HAGlue serverA = null, serverB = null, serverC = null;
+ protected HAGlue serverA = null;
+ protected HAGlue serverB = null;
+
+ protected HAGlue serverC = null;
+
/**
* {@link UUID}s for the {@link HAJournalServer}s.
*/
- private UUID serverAId = UUID.randomUUID(), serverBId = UUID.randomUUID(),
- serverCId = UUID.randomUUID();
+ private UUID serverAId = UUID.randomUUID();
+ private UUID serverBId = UUID.randomUUID();
+
+ private UUID serverCId = UUID.randomUUID();
+
/**
* The HTTP ports at which the services will respond.
*
* @see <a href="http://trac.bigdata.com/ticket/730" > Allow configuration
* of embedded NSS jetty server using jetty-web.xml </a>
*/
- private final int A_JETTY_PORT = 8090, B_JETTY_PORT = A_JETTY_PORT + 1,
- C_JETTY_PORT = B_JETTY_PORT + 1;
+ protected final int A_JETTY_PORT = 8090;
+ protected final int B_JETTY_PORT = A_JETTY_PORT + 1;
+ protected final int C_JETTY_PORT = B_JETTY_PORT + 1;
/**
* These {@link IServiceListener}s are used to reliably detect that the
* corresponding process starts and (most importantly) that it is really
* dies once it has been shutdown or destroyed.
*/
- private ServiceListener serviceListenerA = null, serviceListenerB = null;
+ protected ServiceListener serviceListenerA = null;
- private ServiceListener serviceListenerC = null;
+ protected ServiceListener serviceListenerB = null;
+
+ protected ServiceListener serviceListenerC = null;
private LookupDiscoveryManager lookupDiscoveryManager = null;
@@ -1143,14 +1153,14 @@
}
- private void safeShutdown(final HAGlue haGlue, final File serviceDir,
+ void safeShutdown(final HAGlue haGlue, final File serviceDir,
final ServiceListener serviceListener) {
safeShutdown(haGlue, serviceDir, serviceListener, false/* now */);
}
- private void safeShutdown(final HAGlue haGlue, final File serviceDir,
+ protected void safeShutdown(final HAGlue haGlue, final File serviceDir,
final ServiceListener serviceListener, final boolean now) {
if (haGlue == null)
@@ -1369,6 +1379,32 @@
}
/**
+ * Return the zookeeper client configuration file.
+ */
+ final protected String getZKConfigFile() {
+
+ return "zkClient.config";
+
+ }
+
+ /**
+ * The as-configured replication factor.
+ * <p>
+ * Note: This is defined in the HAJournal.config file, which is where the
+ * {@link HAJournalServer} gets the correct value. We also need to have the
+ * replicationFactor on hand for the test suite so we can setup the quorum
+ * in the test fixture correctly. However, it is difficult to reach the
+ * appropriate HAJournal.config file from the text fixture during
+ * {@link #setUp()}. Therefore, for the test setup, this is achieved by
+ * overriding this abstract method in the test class.
+ */
+ protected int replicationFactor() {
+
+ return 3;
+
+ }
+
+ /**
* Return Zookeeper quorum that can be used to reflect (or act on) the
* distributed quorum state for the logical service.
*
@@ -1382,7 +1418,7 @@
KeeperException, IOException {
final Configuration config = ConfigurationProvider
- .getInstance(new String[] { SRC_PATH + "zkClient.config" });
+ .getInstance(new String[] { SRC_PATH + getZKConfigFile() });
zkClientConfig = new ZookeeperClientConfig(config);
@@ -1393,7 +1429,7 @@
// Note: Save reference.
this.zookeeper = new ZooKeeper(zoohosts, sessionTimeout, new Watcher() {
@Override
- public void process(WatchedEvent event) {
+ public void process(final WatchedEvent event) {
if (log.isInfoEnabled())
log.info(event);
}
@@ -1443,9 +1479,19 @@
logicalServiceZPath = logicalServiceZPathPrefix + "/"
+ logicalServiceId;
- final int replicationFactor = (Integer) config.getEntry(
- ZookeeperClientConfig.Options.NAMESPACE,
- ConfigurationOptions.REPLICATION_FACTOR, Integer.TYPE);
+ /**
+ * Note: This is defined in the HAJournal.config file, which is where
+ * the HAJournalServer gets the correct value.
+ *
+ * However, we also need to have the replicationFactor on hand for the
+ * test suite so we can setup the quorum in the test fixture correctly.
+ */
+ final int replicationFactor = replicationFactor();
+// {
+// replicationFactor = (Integer) config.getEntry(
+// ConfigurationOptions.COMPONENT,
+// ConfigurationOptions.REPLICATION_FACTOR, Integer.TYPE);
+// }
// if (!zka.awaitZookeeperConnected(10, TimeUnit.SECONDS)) {
//
@@ -1551,7 +1597,7 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*/
- abstract private class StartServerTask implements Callable<HAGlue> {
+ abstract class StartServerTask implements Callable<HAGlue> {
private final String name;
private final String configName;
Copied: branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA5JournalServerTestCase.java (from rev 8122, branches/BIGDATA_MGC_HA1_HA5/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA5JournalServerTestCase.java)
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA5JournalServerTestCase.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-jini/src/test/com/bigdata/journal/jini/ha/AbstractHA5JournalServerTestCase.java 2014-04-15 17:08:37 UTC (rev 8123)
@@ -0,0 +1,489 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+package com.bigdata.journal.jini.ha;
+
+import java.io.File;
+import java.io.IOException;
+import java.rmi.Remote;
+import java.security.DigestException;
+import java.security.NoSuchAlgorithmException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent...
[truncated message content] |
