Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2010-08-06 15:59:49
|
Revision: 3424
http://bigdata.svn.sourceforge.net/bigdata/?rev=3424&view=rev
Author: thompsonbry
Date: 2010-08-06 15:59:39 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Merged from trunk [r3407:r3423].
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/QUADS_QUERY_BRANCH/build.xml
branches/QUADS_QUERY_BRANCH/src/resources/config/README
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
branches/QUADS_QUERY_BRANCH/src/resources/scripts/dumpFed.sh
branches/QUADS_QUERY_BRANCH/src/resources/scripts/nanoSparqlServer.sh
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/dsi-utils/LEGAL/
branches/QUADS_QUERY_BRANCH/dsi-utils/lib/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/osgi/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -420,7 +420,7 @@
}
}
-
+
/**
* Note: A commit is required in order for a read-committed view to have
* access to the registered indices. When running against an
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -2840,7 +2840,8 @@
* might also want to limit the maximum size of the reads.
*/
- final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+ final DirectBufferPool pool = DirectBufferPool.INSTANCE;
if (true
&& ((flags & REVERSE) == 0)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -644,7 +644,18 @@
this.lastCommitTime = lastCommitTime;
}
- private long lastCommitTime = 0L;// Until the first commit.
+
+ /**
+ * The lastCommitTime of the {@link Checkpoint} record from which the
+ * {@link BTree} was loaded.
+ * <p>
+ * Note: Made volatile on 8/2/2010 since it is not otherwise obvious what
+ * would guarantee visibility of this field, through I do seem to remember
+ * that visibility might be guaranteed by how the BTree class is discovered
+ * and returned to the class. Still, it does no harm to make this a volatile
+ * read.
+ */
+ volatile private long lastCommitTime = 0L;// Until the first commit.
/**
* Return the {@link IDirtyListener}.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -36,6 +36,7 @@
import org.apache.log4j.Logger;
import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf;
+import com.bigdata.io.DirectBufferPool;
import com.bigdata.journal.DumpJournal;
import com.bigdata.rawstore.IRawStore;
@@ -154,6 +155,16 @@
}
+ // multi-block scan of the index segment.
+ boolean multiBlockScan = false; // @todo command line option.
+ if (multiBlockScan) {
+
+ writeBanner("dump leaves using multi-block forward scan");
+
+ dumpLeavesMultiBlockForwardScan(store);
+
+ }
+
// dump the leaves using a fast reverse scan.
boolean fastReverseScan = true;// @todo command line option
if (fastReverseScan) {
@@ -524,6 +535,36 @@
}
+ /**
+ * Dump leaves using the {@link IndexSegmentMultiBlockIterator}.
+ *
+ * @param store
+ */
+ static void dumpLeavesMultiBlockForwardScan(final IndexSegmentStore store) {
+
+ final long begin = System.currentTimeMillis();
+
+ final IndexSegment seg = store.loadIndexSegment();
+
+ final ITupleIterator<?> itr = new IndexSegmentMultiBlockIterator(seg, DirectBufferPool.INSTANCE,
+ null/* fromKey */, null/* toKey */, IRangeQuery.DEFAULT/* flags */);
+
+ int nscanned = 0;
+
+ while(itr.hasNext()) {
+
+ itr.next();
+
+ nscanned++;
+
+ }
+
+ final long elapsed = System.currentTimeMillis() - begin;
+
+ System.out.println("Visited "+nscanned+" tuples using multi-block forward scan in "+elapsed+" ms");
+
+ }
+
static void writeBanner(String s) {
System.out.println(bar);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -218,12 +218,12 @@
*/
public final static DirectBufferPool INSTANCE;
- /**
- * A JVM-wide pool of direct {@link ByteBuffer}s with a default
- * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
- * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
- */
- public final static DirectBufferPool INSTANCE_10M;
+// /**
+// * A JVM-wide pool of direct {@link ByteBuffer}s with a default
+// * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
+// * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
+// */
+// public final static DirectBufferPool INSTANCE_10M;
/**
* An unbounded list of all {@link DirectBufferPool} instances.
@@ -251,11 +251,11 @@
bufferCapacity//
);
- INSTANCE_10M = new DirectBufferPool(//
- "10M",//
- Integer.MAX_VALUE, // poolCapacity
- 10 * Bytes.megabyte32 // bufferCapacity
- );
+// INSTANCE_10M = new DirectBufferPool(//
+// "10M",//
+// Integer.MAX_VALUE, // poolCapacity
+// 10 * Bytes.megabyte32 // bufferCapacity
+// );
/*
* This configuration will block if there is a concurrent demand for
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -7,6 +7,7 @@
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
+import com.bigdata.resources.StoreManager;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
@@ -171,16 +172,18 @@
* Delay between attempts reach the remote service (ms).
*/
final long delay = 10L;
-
- /**
- * #of attempts to reach the remote service.
- *
- * Note: delay*maxtries == 1000ms of trying before we give up.
- *
- * If this is not enough, then consider adding an optional parameter giving
- * the time the caller will wait and letting the StoreManager wait longer
- * during startup to discover the timestamp service.
- */
+
+ /**
+ * #of attempts to reach the remote service.
+ * <p>
+ * Note: delay*maxtries == 1000ms of trying before we give up, plus however
+ * long we are willing to wait for service discovery if the problem is
+ * locating the {@link ITransactionService}.
+ * <p>
+ * If this is not enough, then consider adding an optional parameter giving
+ * the time the caller will wait and letting the {@link StoreManager} wait
+ * longer during startup to discover the timestamp service.
+ */
final int maxtries = 100;
/**
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1752,11 +1752,11 @@
//
// }
- /**
- * Flag may be set to force overflow processing during the next group
- * commit. The flag is cleared once an overflow has occurred.
- */
- public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
+// /**
+// * Flag may be set to force overflow processing during the next group
+// * commit. The flag is cleared once an overflow has occurred.
+// */
+// public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
/**
* Return <code>true</code> if the pre-conditions for overflow processing
@@ -1765,7 +1765,8 @@
private boolean isShouldOverflow() {
return resourceManager.isOverflowEnabled()
- && (forceOverflow.get() || resourceManager.shouldOverflow());
+// && (forceOverflow.get() || resourceManager.shouldOverflow());
+ && resourceManager.shouldOverflow();
}
@@ -1815,10 +1816,10 @@
log.error("Overflow error: "+serviceName+" : "+t, t);
- } finally {
-
- // clear force flag.
- forceOverflow.set(false);
+// } finally {
+//
+// // clear force flag.
+// forceOverflow.set(false);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -582,9 +582,21 @@
}
/**
+ * The default implementation only logs the event.
+ */
+ public AbstractResource<E> init() {
+
+ if (log.isInfoEnabled())
+ log.info(toString());
+
+ return this;
+
+ }
+
+ /**
*
* @todo Lock service supporting shared locks, leases and lease renewal,
- * excalation of shared locks to exclusive locks, deadlock detection,
+ * escalation of shared locks to exclusive locks, deadlock detection,
* and possibly a resource hierarchy. Leases should be Callable
* objects that are submitted by the client to its executor service so
* that they will renew automatically until cancelled (and will cancel
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -38,7 +38,10 @@
public interface IMutableResource<T> extends ILocatableResource<T> {
/**
- * Create any logically contained resources (relations, indices).
+ * Create any logically contained resources (relations, indices). There is
+ * no presumption that {@link #init()} is suitable for invocation from
+ * {@link #create()}. Instead, you are responsible for invoking {@link #init()}
+ * from this method IFF it is appropriate to reuse its initialization logic.
*/
void create();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -21,8 +21,8 @@
*/
public class RelationFusedView<E> implements IRelation<E> {
- private IRelation<E> relation1;
- private IRelation<E> relation2;
+ final private IRelation<E> relation1;
+ final private IRelation<E> relation2;
public IRelation<E> getRelation1() {
@@ -36,6 +36,13 @@
}
+ // NOP
+ public RelationFusedView<E> init() {
+
+ return this;
+
+ }
+
/**
*
* @param relation1
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -586,6 +586,8 @@
properties //
});
+ r.init();
+
if(INFO) {
log.info("new instance: "+r);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -45,6 +45,13 @@
*/
public interface ILocatableResource<T> {
+ /**
+ * Deferred initialization method is automatically invoked when the resource
+ * is materialized by the {@link IResourceLocator}. The implementation is
+ * encouraged to strengthen the return type.
+ */
+ public ILocatableResource<T> init();
+
/**
* The identifying namespace.
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1074,16 +1074,20 @@
final UUID sinkUUID = locator.getDataServiceUUID();
+ final IDataService dataService;
if (sinkUUID.equals(fed.getServiceUUID())) {
- /*
- * @todo As an optimization, special case when the downstream
- * data service is _this_ data service.
- */
+ /*
+ * As an optimization, special case when the downstream
+ * data service is _this_ data service.
+ */
+ dataService = (IDataService)fed.getService();
+ } else {
+
+ dataService = fed.getDataService(sinkUUID);
+
}
-
- final IDataService dataService = fed.getDataService(sinkUUID);
sink = new JoinTaskSink(fed, locator, this);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -310,6 +310,7 @@
private final OverflowActionEnum action;
private final ViewMetadata vmd;
+ private final boolean forceCompactingMerge;
private final AbstractTask<T> task;
/**
@@ -319,11 +320,17 @@
* @param vmd
* The {@link ViewMetadata} for the index partition for which
* that action will be taken.
+ * @param forceCompactingMerge
+ * if a compacting merge should be taken even if the view was
+ * simply copied to the new journal.
* @param task
* The task which implements that action.
*/
- public AtomicCallable(final OverflowActionEnum action,
- final ViewMetadata vmd, final AbstractTask<T> task) {
+ public AtomicCallable(final OverflowActionEnum action,//
+ final ViewMetadata vmd,//
+ final boolean forceCompactingMerge, //
+ final AbstractTask<T> task//
+ ) {
if (action == null)
throw new IllegalArgumentException();
@@ -337,6 +344,8 @@
this.action = action;
this.vmd = vmd;
+
+ this.forceCompactingMerge = forceCompactingMerge;
this.task = task;
@@ -407,110 +416,112 @@
}
- /**
- * Schedule a build for each shard and a merge for each shard with a
- * non-zero merge priority. Whether a build or a merge is performed for a
- * shard will depend on which action is initiated first. When an build or
- * merge action is initiated, that choice is atomically registered on the
- * {@link ViewMetadata} and any subsequent attempt (within this method
- * invocation) to start a build or merge for the same shard will be dropped.
- * Processing ends once all tasks scheduled on a "build" service are
- * complete.
- * <p>
- * After actions are considered for each shard for which a compacting merge
- * is executed. These after actions can cause a shard split, join, or move.
- * Deferring such actions until we have a compact view (comprised of one
- * journal and one index segment) greatly improves our ability to decide
- * whether a shard should be split or joined and simplifies the logic and
- * effort required to split, join or move a shard.
- * <p>
- * The following is a brief summary of some after actions on compact shards.
- * <dl>
- * <dt>split</dt>
- * <dd>A shard is split when its size on the disk exceeds the (adjusted)
- * nominal size of a shard (overflow). By waiting until the shard view is
- * compact we have exact information about the size of the shard (it is
- * contained in a single {@link IndexSegment}) and we are able to easily
- * select the separator key to split the shard.</dd>
- * <dt>tailSplit</dt>
- * <dd>A tail split may be selected for a shard which has a mostly append
- * access pattern. For such access patterns, a normal split would leave the
- * left sibling 50% full and the right sibling would quickly fill up with
- * continued writes on the tail of the key range. To compensate for this
- * access pattern, a tail split chooses a separator key near the end of the
- * key range of a shard. This results in a left sibling which is mostly full
- * and a right sibling which is mostly empty. If the pattern of heavy tail
- * append continues, then the left sibling will remain mostly full and the
- * new writes will flow mostly into the right sibling.</dd>
- * <dt>scatterSplit</dt>
- * <dd>A scatter split breaks the first shard for a new scale-out index into
- * N shards and scatters those shards across the data services in a
- * federation in order to improve the data distribution and potential
- * concurrency of the index. By waiting until the shard view is compact we
- * are able to quickly select appropriate separator keys for the shard
- * splits.</dd>
- * <dt>move</dt>
- * <dd>A move transfer a shard from this data service to another data
- * service in order to reduce the load on this data service. By waiting
- * until the shard view is compact we are able to rapidly transfer the bulk
- * of the data in the form of a single {@link IndexSegment}.</dd>
- * <dt>join</dt>
- * <dd>A join combines a shard which is under 50% of its (adjusted) nominal
- * maximum size on the disk (underflow) with its right sibling. Joins are
- * driven by deletes of tuples from a key range. Since deletes are handled
- * as writes where a delete marker is set on the tuple, neither the shard
- * size on the disk nor the range count of the shard will decrease until a
- * compacting merge. A join is indicated if the size on disk for the shard
- * has shrunk considerably since the last time a compacting merge was
- * performed for the view (this covers both the case of deletes, which
- * reduce the range count, and updates which replace the values in the
- * tuples with more compact data). <br>
- * There are actually three cases for a join.
- * <ol>
- * <li>If the right sibling is local, then the shard will be joined with its
- * right sibling.</li>
- * <li>If the right sibling is remote, then the shard will be moved to the
- * data service on which the right sibling is found.</li>
- * <li>If the right sibling does not exist, then nothing is done (the last
- * shard in a scale-out index does not have a right sibling). The right most
- * sibling will remain undercapacity until and unless its left sibling also
- * underflows, at which point the left sibling will cause itself to be
- * joined with the right sibling (this is done to simplify the logic which
- * searches for a sibling with which to join an undercapacity shard).</li>
- * </ol>
- * </dl>
- *
- * @param forceCompactingMerges
- * When <code>true</code> a compacting merge will be forced for
- * each non-compact view.
- *
- * @throws InterruptedException
- *
- * @todo The size of the merge queue (or its sum of priorities) may be an
- * indication of the load of the node which could be used to decide
- * that index partitions should be shed/moved.
- *
- * @todo For HA, this needs to be a shared priority queue using zk or the
- * like since any node in the failover set could do the merge (or
- * build). [Alternatively, nodes do the build/merge for the shards for
- * which they have the highest affinity out of the failover set.]
- *
- * FIXME tailSplits currently operate on the mutable BTree rather than
- * a compact view). This task does not require a compact view (at
- * least, not yet) and generating one for it might be a waste of time.
- * Instead it examines where the inserts are occurring in the index
- * and splits of the tail if the index is heavy for write append. It
- * probably could defer that choice until a compact view was some
- * percentage of a split (maybe .6?) So, probably an after action for
- * the mergeQ.
- *
- * FIXME joins must track metadata about the previous size on disk of
- * the compact view in order to decide when underflow has resulted. In
- * order to handle the change in the value of the acceleration factor,
- * this data should be stored as the percentage of an adjusted split
- * of the last compact view. We can update that metadata each time we
- * do a compacting merge.
- */
+ /**
+ * Schedule a build for each shard and a merge for each shard with a
+ * non-zero merge priority. Whether a build or a merge is performed for a
+ * shard will depend on which action is initiated first. When an build or
+ * merge action is initiated, that choice is atomically registered on the
+ * {@link ViewMetadata} and any subsequent attempt (within this method
+ * invocation) to start a build or merge for the same shard will be dropped.
+ * Processing ends once all tasks scheduled on a "build" service are
+ * complete.
+ * <p>
+ * After actions are considered for each shard for which a compacting merge
+ * is executed. These after actions can cause a shard split, join, or move.
+ * Deferring such actions until we have a compact view (comprised of one
+ * journal and one index segment) greatly improves our ability to decide
+ * whether a shard should be split or joined and simplifies the logic and
+ * effort required to split, join or move a shard.
+ * <p>
+ * The following is a brief summary of some after actions on compact shards.
+ * <dl>
+ * <dt>split</dt>
+ * <dd>A shard is split when its size on the disk exceeds the (adjusted)
+ * nominal size of a shard (overflow). By waiting until the shard view is
+ * compact we have exact information about the size of the shard (it is
+ * contained in a single {@link IndexSegment}) and we are able to easily
+ * select the separator key to split the shard.</dd>
+ * <dt>tailSplit</dt>
+ * <dd>A tail split may be selected for a shard which has a mostly append
+ * access pattern. For such access patterns, a normal split would leave the
+ * left sibling 50% full and the right sibling would quickly fill up with
+ * continued writes on the tail of the key range. To compensate for this
+ * access pattern, a tail split chooses a separator key near the end of the
+ * key range of a shard. This results in a left sibling which is mostly full
+ * and a right sibling which is mostly empty. If the pattern of heavy tail
+ * append continues, then the left sibling will remain mostly full and the
+ * new writes will flow mostly into the right sibling.</dd>
+ * <dt>scatterSplit</dt>
+ * <dd>A scatter split breaks the first shard for a new scale-out index into
+ * N shards and scatters those shards across the data services in a
+ * federation in order to improve the data distribution and potential
+ * concurrency of the index. By waiting until the shard view is compact we
+ * are able to quickly select appropriate separator keys for the shard
+ * splits.</dd>
+ * <dt>move</dt>
+ * <dd>A move transfer a shard from this data service to another data
+ * service in order to reduce the load on this data service. By waiting
+ * until the shard view is compact we are able to rapidly transfer the bulk
+ * of the data in the form of a single {@link IndexSegment}.</dd>
+ * <dt>join</dt>
+ * <dd>A join combines a shard which is under 50% of its (adjusted) nominal
+ * maximum size on the disk (underflow) with its right sibling. Joins are
+ * driven by deletes of tuples from a key range. Since deletes are handled
+ * as writes where a delete marker is set on the tuple, neither the shard
+ * size on the disk nor the range count of the shard will decrease until a
+ * compacting merge. A join is indicated if the size on disk for the shard
+ * has shrunk considerably since the last time a compacting merge was
+ * performed for the view (this covers both the case of deletes, which
+ * reduce the range count, and updates which replace the values in the
+ * tuples with more compact data). <br>
+ * There are actually three cases for a join.
+ * <ol>
+ * <li>If the right sibling is local, then the shard will be joined with its
+ * right sibling.</li>
+ * <li>If the right sibling is remote, then the shard will be moved to the
+ * data service on which the right sibling is found.</li>
+ * <li>If the right sibling does not exist, then nothing is done (the last
+ * shard in a scale-out index does not have a right sibling). The right most
+ * sibling will remain undercapacity until and unless its left sibling also
+ * underflows, at which point the left sibling will cause itself to be
+ * joined with the right sibling (this is done to simplify the logic which
+ * searches for a sibling with which to join an undercapacity shard).</li>
+ * </ol>
+ * </dl>
+ *
+ * @param forceCompactingMerges
+ * When <code>true</code> a compacting merge will be forced for
+ * each non-compact view. Compacting merges will be taken in
+ * priority order and will continue until finished or until the
+ * journal is nearing its nominal maximum extent.
+ *
+ * @throws InterruptedException
+ *
+ * @todo The size of the merge queue (or its sum of priorities) may be an
+ * indication of the load of the node which could be used to decide
+ * that index partitions should be shed/moved.
+ *
+ * @todo For HA, this needs to be a shared priority queue using zk or the
+ * like since any node in the failover set could do the merge (or
+ * build). [Alternatively, nodes do the build/merge for the shards for
+ * which they have the highest affinity out of the failover set.]
+ *
+ * FIXME tailSplits currently operate on the mutable BTree rather than
+ * a compact view). This task does not require a compact view (at
+ * least, not yet) and generating one for it might be a waste of time.
+ * Instead it examines where the inserts are occurring in the index
+ * and splits of the tail if the index is heavy for write append. It
+ * probably could defer that choice until a compact view was some
+ * percentage of a split (maybe .6?) So, probably an after action for
+ * the mergeQ.
+ *
+ * FIXME joins must track metadata about the previous size on disk of
+ * the compact view in order to decide when underflow has resulted. In
+ * order to handle the change in the value of the acceleration factor,
+ * this data should be stored as the percentage of an adjusted split
+ * of the last compact view. We can update that metadata each time we
+ * do a compacting merge.
+ */
private List<Future<?>> scheduleAndAwaitTasks(
final boolean forceCompactingMerges) throws InterruptedException {
@@ -554,21 +565,30 @@
if (log.isInfoEnabled())
log.info("was copied : " + vmd);
- continue;
+ } else {
+ buildList.add(new Priority<ViewMetadata>(vmd.buildPriority, vmd));
+
}
- buildList.add(new Priority<ViewMetadata>(vmd.buildPriority, vmd));
+ if (vmd.mergePriority > 0d || forceCompactingMerges) {
- if (vmd.mergePriority > 0d) {
+ /*
+ * Schedule a merge if the priority is non-zero or if compacting
+ * merges are being forced.
+ */
- mergeList
- .add(new Priority<ViewMetadata>(vmd.mergePriority, vmd));
+ mergeList
+ .add(new Priority<ViewMetadata>(vmd.mergePriority, vmd));
}
} // itr.hasNext()
+ if(log.isInfoEnabled()) {
+ log.info("Scheduling tasks: buildList="+buildList.size()+", mergeList="+mergeList.size());
+ }
+
/*
* Schedule build and merge tasks and await their futures. The tasks are
* submitted from a PriorityQueue, so the order in which the tasks are
@@ -606,18 +626,23 @@
resourceManager.mergeServiceCorePoolSize);
// Schedule merge tasks.
- if (!forceCompactingMerges) {
-
for (Priority<ViewMetadata> p : mergeList) {
final ViewMetadata vmd = p.v;
- if (vmd.mergePriority > 0) {
+ if (vmd.mergePriority > 0 || forceCompactingMerges) {
+ if(forceCompactingMerges && vmd.getAction().equals(OverflowActionEnum.Copy)) {
+
+ vmd.clearCopyAction();
+
+ }
+
// Schedule a compacting merge.
final FutureTask<?> ft = new FutureTask(
new AtomicCallable(OverflowActionEnum.Merge,
- vmd, new CompactingMergeTask(vmd)));
+ vmd, forceCompactingMerges,
+ new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
@@ -625,8 +650,6 @@
}
- }
-
// Schedule build tasks.
for (Priority<ViewMetadata> p : buildList) {
@@ -636,7 +659,8 @@
// Force a compacting merge.
final FutureTask<?> ft = new FutureTask(new AtomicCallable(
- OverflowActionEnum.Merge, vmd,
+ OverflowActionEnum.Merge, vmd,
+ forceCompactingMerges,
new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
@@ -646,6 +670,7 @@
// Schedule a build.
final FutureTask<?> ft = new FutureTask(new AtomicCallable(
OverflowActionEnum.Build, vmd,
+ forceCompactingMerges,
new IncrementalBuildTask(vmd)));
buildFutures.add(ft);
buildService.execute(ft);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -280,6 +280,25 @@
actionRef.set(action);
}
+
+ /**
+ * Used to force clear a {@link OverflowActionEnum#Copy} action
+ * when we will force a compacting merge. This allows us to do
+ * compacting merges on shard views which would otherwise simply
+ * be copied onto the new journal.
+ */
+ void clearCopyAction() {
+
+ lock.lock();
+ try {
+ if(actionRef.get().equals(OverflowActionEnum.Copy)) {
+ actionRef.set(null/*clear*/);
+ }
+ } finally {
+ lock.unlock();
+ }
+
+ }
/**
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1684,16 +1684,28 @@
final StringBuilder sb = new StringBuilder();
final AbstractJournal journal = getJournal(timestamp);
+
+ if (journal == null) {
+ /*
+ * This condition can occur if there are no shard views on the
+ * previous journal and the releaseAge is zero since the previous
+ * journal can be purged (deleted) before this method is invoked.
+ * This situation arises in a few of the unit tests which begin with
+ * an empty journal and copy everything onto the new journal such
+ * that the old journal can be immediately released.
+ */
+ return "No journal: timestamp=" + timestamp;
+ }
sb.append("timestamp="+timestamp+"\njournal="+journal.getResourceMetadata());
// historical view of Name2Addr as of that timestamp.
- final ITupleIterator itr = journal.getName2Addr(timestamp)
+ final ITupleIterator<?> itr = journal.getName2Addr(timestamp)
.rangeIterator();
while (itr.hasNext()) {
- final ITuple tuple = itr.next();
+ final ITuple<?> tuple = itr.next();
final Entry entry = EntrySerializer.INSTANCE
.deserialize(new DataInputBuffer(tuple.getValue()));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -287,6 +287,14 @@
*/
protected final AtomicBoolean asyncOverflowEnabled = new AtomicBoolean(true);
+ /**
+ * Flag may be set to force overflow processing during the next group
+ * commit. The flag is cleared by {@link #overflow()}.
+ *
+ * @see DataService#forceOverflow(boolean, boolean)
+ */
+ public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
+
/**
* A flag that may be set to force the next asynchronous overflow to perform
* a compacting merge for all indices that are not simply copied over to the
@@ -295,6 +303,8 @@
* made compact and SHOULD NOT be used for deployed federations</strong>).
* The state of the flag is cleared each time asynchronous overflow
* processing begins.
+ *
+ * @see DataService#forceOverflow(boolean, boolean)
*/
public final AtomicBoolean compactingMerge = new AtomicBoolean(false);
@@ -1849,6 +1859,19 @@
*/
public boolean shouldOverflow() {
+ if(forceOverflow.get()) {
+
+ /*
+ * Note: forceOverflow trumps everything else.
+ */
+
+ if (log.isInfoEnabled())
+ log.info("Forcing overflow.");
+
+ return true;
+
+ }
+
if (isTransient()) {
/*
@@ -1886,7 +1909,7 @@
return false;
}
-
+
/*
* Look for overflow condition on the "live" journal.
*/
@@ -1959,8 +1982,18 @@
*/
public Future<Object> overflow() {
- assert overflowAllowed.get();
+// assert overflowAllowed.get();
+ /*
+ * Atomically test and clear the flag. The local boolean is inspected
+ * below. When true, asynchronous overflow processing will occur unless
+ * an error occurs during synchronous overflow processing. This ensures
+ * that we can force a compacting merge on the shards of a data service
+ * even if that data service has not buffer sufficient writes to warrant
+ * a build on any of the index segments.
+ */
+ final boolean forceOverflow = this.forceOverflow.getAndSet(false/* newValue */);
+
final Event e = new Event(getFederation(), new EventResource(),
EventType.SynchronousOverflow).addDetail(
"synchronousOverflowCounter",
@@ -1982,7 +2015,12 @@
if (asyncOverflowEnabled.get()) {
- if (overflowMetadata.postProcess) {
+ /*
+ * Do overflow processing if overflow is being forced OR if we
+ * need to do a build for at least one index partition.
+ */
+
+ if (forceOverflow || overflowMetadata.postProcess) {
/*
* Post-processing SHOULD be performed.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1416,22 +1416,45 @@
* Verify that the concurrency manager has been set and wait a while
* it if is not available yet.
*/
- if (log.isInfoEnabled())
- log.info("Waiting for concurrency manager");
- for (int i = 0; i < 5; i++) {
- try {
- getConcurrencyManager(); break;
- } catch (IllegalStateException ex) {
- Thread.sleep(100/* ms */);
- }
+ {
+ int nwaits = 0;
+ while (true) {
+ try {
+ getConcurrencyManager();
+ break;
+ } catch (IllegalStateException ex) {
+ Thread.sleep(100/* ms */);
+ if (++nwaits % 50 == 0)
+ log.warn("Waiting for concurrency manager");
+ }
+ }
}
- getConcurrencyManager();
- if (Thread.interrupted())
- throw new InterruptedException();
- /*
- * Look for pre-existing data files.
- */
+ try {
+ final IBigdataFederation<?> fed = getFederation();
+ if (fed == null) {
+ /*
+ * Some of the unit tests do not start the txs until after
+ * the DataService. For those unit tests getFederation()
+ * will return null during startup() of the DataService. To
+ * have a common code path, we throw the exception here
+ * which is caught below.
+ */
+ throw new UnsupportedOperationException();
+ }
+ while (true) {
+ if (fed.getTransactionService() != null) {
+ break;
+ }
+ log.warn("Waiting for transaction service discovery");
+ }
+ } catch (UnsupportedOperationException ex) {
+ log.warn("Federation not available - running in test case?");
+ }
+
+ /*
+ * Look for pre-existing data files.
+ */
if (!isTransient) {
if (log.isInfoEnabled())
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -829,7 +829,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public T getService() {
@@ -840,7 +840,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public String getServiceName() {
@@ -851,7 +851,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public Class getServiceIface() {
@@ -862,7 +862,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public UUID getServiceUUID() {
@@ -873,7 +873,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public boolean isServiceReady() {
@@ -894,7 +894,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public void reattachDynamicCounters() {
@@ -905,7 +905,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public void didStart() {
@@ -916,7 +916,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public AbstractHTTPD newHttpd(final int httpdPort,
final CounterSet counterSet) throws IOException {
@@ -927,7 +927,10 @@
}
- public void serviceJoin(IService service, UUID serviceUUID) {
+ /**
+ * Delegated. {@inheritDoc}
+ */
+ public void serviceJoin(final IService service, final UUID serviceUUID) {
if (!isOpen()) return;
@@ -941,7 +944,10 @@
}
- public void serviceLeave(UUID serviceUUID) {
+ /**
+ * Delegated. {@inheritDoc}
+ */
+ public void serviceLeave(final UUID serviceUUID) {
if(!isOpen()) return;
@@ -1129,9 +1135,9 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
+
}
-
/**
* Setup sampling on the client's thread pool. This collects interesting
* statistics about the thread pool for reporting to the load balancer
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -47,6 +47,7 @@
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.IndexSegment;
import com.bigdata.journal.ITransactionService;
import com.bigdata.journal.ITx;
import com.bigdata.mdi.IMetadataIndex;
@@ -492,40 +493,56 @@
}
- /**
- * Force overflow of each data service in the scale-out federation (only
- * scale-out federations support overflow processing). This method is
- * synchronous. It will not return until all {@link DataService}s have
- * initiated and completed overflow processing. Any unused resources (as
- * determined by the {@link StoreManager}) will have been purged.
- *
- * @param truncateJournal
- * When <code>true</code>, the live journal will be truncated
- * to its minimum extent (all writes will be preserved but there
- * will be no free space left in the journal). This may be used
- * to force the {@link DataService} to its minimum possible
- * footprint.
- *
- * @todo when overflow processing is enabled for the {@link MetadataService}
- * we will have to modify this to also trigger overflow for those
- * services.
- */
- public void forceOverflow(final boolean truncateJournal) {
+ /**
+ * Force overflow of each data service in the scale-out federation (only
+ * scale-out federations support overflow processing). This method is
+ * synchronous. It will not return until all {@link DataService}s have
+ * initiated and completed overflow processing. Any unused resources (as
+ * determined by the {@link StoreManager}) will have been purged.
+ * <p>
+ * This is a relatively fast operation when
+ * <code>compactingMerge := false</code>. By specifying both
+ * <code>compactingMerge := false</code> and
+ * <code>truncateJournal := false</code> you can cause the data services to
+ * close out their current journals against further writes. While this is
+ * not a global synchronous operation, it can provide a basis to obtain a
+ * "near synchronous" snapshot from the federation consisting of all writes
+ * up to the point where overflow was triggered on each data service.
+ *
+ * @param compactingMerge
+ * When <code>true</code>, each shard on each
+ * {@link IDataService} will undergo a compacting merge.
+ * Synchronous parallel compacting merge of all shards is an
+ * expensive operation. This parameter shoudl normally be
+ * <code>false</code> unless you are requesting a compacting
+ * merge for specific purposes, such as benchmarking when all
+ * data is known to exist in one {@link IndexSegment} per shard.
+ * @param truncateJournal
+ * When <code>true</code>, the live journal will be truncated to
+ * its minimum extent (all writes will be preserved but there
+ * will be no free space left in the journal). This may be used
+ * to force the {@link DataService} to its minimum possible
+ * footprint.
+ *
+ * @todo when overflow processing is enabled for the {@link MetadataService}
+ * we will have to modify this to also trigger overflow for those
+ * services.
+ */
+ public void forceOverflow(final boolean compactingMerge, final boolean truncateJournal) {
// find UUID for each data service.
final UUID[] dataServiceUUIDs = getDataServiceUUIDs(0/* maxCount */);
final int ndataServices = dataServiceUUIDs.length;
- if(log.isInfoEnabled())
- log.info("#dataServices=" + ndataServices + ", now=" + new Date());
+ log.warn("Forcing overflow: #dataServices=" + ndataServices + ", now=" + new Date());
final List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(ndataServices);
for (UUID serviceUUID : dataServiceUUIDs) {
tasks.add(new ForceOverflowTask(getDataService(serviceUUID),
- truncateJournal));
+ compactingMerge, truncateJournal));
}
@@ -570,8 +587,7 @@
}
- if(log.isInfoEnabled())
- log.info("Did overflow: #ok=" + nok + ", #dataServices="
+ log.warn("Did overflow: #ok=" + nok + ", #dataServices="
+ ndataServices + ", now=" + new Date());
if (nok != tasks.size()) {
@@ -643,16 +659,19 @@
.getLogger(ForceOverflowTask.class);
private final IDataService dataService;
+ private final boolean compactingMerge;
private final boolean truncateJournal;
- public ForceOverflowTask(final IDataService dataService,
- final boolean truncateJournal) {
+ public ForceOverflowTask(final IDataService dataService,
+ final boolean compactingMerge, final boolean truncateJournal) {
if (dataService == null)
throw new IllegalArgumentException();
this.dataService = dataService;
+ this.compactingMerge = compactingMerge;
+
this.truncateJournal = truncateJournal;
}
@@ -663,8 +682,7 @@
log.info("dataService: " + dataService.getServiceName());
// returns once synchronous overflow is complete.
- dataService
- .forceOverflow(true/* immediate */, true/* compactingMerge */);
+ dataService.forceOverflow(true/* immediate */, compactingMerge);
if (log.isInfoEnabled())
log.info("Synchronous overflow is done: "
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -43,7 +43,6 @@
import org.apache.log4j.Logger;
-import com.bigdata.concurrent.LockManager;
import com.bigdata.config.LongValidator;
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
@@ -80,9 +79,9 @@
*/
protected static final Logger log = Logger.getLogger(AbstractTransactionService.class);
- protected static final boolean INFO = log.isInfoEnabled();
+// protected static final boolean INFO = log.isInfoEnabled();
- protected static final boolean DEBUG = log.isDebugEnabled();
+// protected static final boolean DEBUG = log.isDebugEnabled();
/**
* Options understood by this service.
@@ -91,29 +90,39 @@
* @version $Id$
*/
public interface Options {
-
- /**
- * How long you want to hold onto the database history (in milliseconds)
- * or {@link Long#MAX_VALUE} for an (effectively) immortal database. The
- * {@link ITransactionService} tracks the timestamp corresponding to the
- * earliest running transaction (if any). When such a transaction
- * exists, the actual release time is:
- *
- * <pre>
- * releaseTime = min(earliestRunningTx, now - minimumReleaseAge) - 1
- * </pre>
- *
- * This ensures that history in use by running transactions is not
- * released even when the minimumReleaseAge is ZERO (0).
- *
- * @see #DEFAULT_MIN_RELEASE_AGE
- * @see #MIN_RELEASE_AGE_1H
- * @see #MIN_RELEASE_AGE_1D
- * @see #MIN_RELEASE_AGE_1W
- * @see #MIN_RELEASE_AGE_NEVER
- *
- * @see AbstractTransactionService#updateReleaseTime(long)
- */
+
+ /**
+ * How long you want to hold onto the database history (in milliseconds)
+ * or {@link Long#MAX_VALUE} for an (effectively) immortal database. The
+ * {@link ITransactionService} tracks the timestamp corresponding to the
+ * earliest running transaction (if any). When such a transaction
+ * exists, the actual release time is:
+ *
+ * <pre>
+ * releaseTime = min(lastCommitTime - 1, min(earliestRunningTx, now - minimumReleaseAge))
+ * </pre>
+ *
+ * This ensures that history in use by running transactions is not
+ * released even when the minimumReleaseAge is ZERO (0).
+ * <p>
+ * When no transactions exist the actual release time is:
+ *
+ * <pre>
+ * releaseTime = min(commitTime - 1, now - minimumReleaseAge)
+ * </pre>
+ *
+ * This ensures that the the release time advances when no transactions
+ * are in use, but that the mi...
[truncated message content] |
|
From: <tho...@us...> - 2010-08-06 15:46:16
|
Revision: 3423
http://bigdata.svn.sourceforge.net/bigdata/?rev=3423&view=rev
Author: thompsonbry
Date: 2010-08-06 15:46:07 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Rationalized the bigdata cluster configuration files slightly and added one for a single node cluster (bigdataStandalone.config).
Modified build.xml to remove all the "standalone" targets and to state the new bigdataStandalone.config" file.
Modified Paths:
--------------
trunk/build.xml
trunk/src/resources/config/README
trunk/src/resources/config/bigdataCluster.config
trunk/src/resources/config/bigdataCluster16.config
Added Paths:
-----------
trunk/src/resources/config/bigdataStandalone.config
Modified: trunk/build.xml
===================================================================
--- trunk/build.xml 2010-08-06 15:14:23 UTC (rev 3422)
+++ trunk/build.xml 2010-08-06 15:46:07 UTC (rev 3423)
@@ -777,118 +777,8 @@
</java>
</target>
-
+
<!-- -->
-<!-- STANDALONE FEDERATION TARGETS -->
-<!-- (test/benchamarking) -->
-
-<target name="generateLookupStarterJar" unless="lookupStarterJarAvailable">
-<antcall target="testCompile" />
-</target>
-
-<target name="testLookupStarterJarAvailability">
-<property name="bigdata-test.lib" location="${bigdata.dir}/bigdata-test/lib" />
-<condition property="lookupStarterJarAvailable">
- <available file="${bigdata-test.lib}/lookupstarter.jar" />
-</condition>
-</target>
-
-<target name="standalone-setup" depends="testLookupStarterJarAvailability,generateLookupStarterJar" description="Setup properties used by standalone federation and LUS start/stop.">
-<property name="app.home" location="${bigdata.dir}" />
-<property name="test.codebase.port" value="23333" />
-<property name="test.codebase.dir" location="${bigdata.dir}/bigdata-jini/lib/jini/lib-dl" />
-<property name="dist.lib" location="${bigdata.dir}/bigdata-jini/lib/jini/lib" />
-<property name="dist.lib.dl" location="${bigdata.dir}/bigdata-jini/lib/jini/lib-dl" />
-<property name="test.codebase" value="http://${this.hostname}:${test.codebase.port}/jsk-dl.jar" />
-<property name="java.security.policy" value="${bigdata.dir}/policy.all" />
-<property name="log4j.configuration" value="resources/logging/log4j.properties" />
-<property name="java.net.preferIPv4Stack" value="true" />
-<property name="bigdata.fedname" value="${standalone.fed}" />
-</target>
-
-<!-- Note: You should 'nohup' this, e.g., "nohup ant standalone-start" to
- avoid taking down the ServicesManagerServer if you are disconnected
- from a terminal. -->
-<target name="standalone-start" depends="jar,standalone-setup" description="Start the standalone federation.">
-<!-- Start the lookup service. -->
-<antcall target="startHttpd" />
-<antcall target="startLookup" />
-<java classname="com.bigdata.jini.start.ServicesManagerServer" failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx200m" />
- <jvmarg value="-showversion" />
- <!-- The name of the federation instance. -->
- <jvmarg value="-Dbigdata.fedname=${standalone.fed}" />
- <jvmarg value="-Djava.security.policy=policy.all" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
- <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties" />
- <arg value="src/resources/config/standalone/bigdataStandalone.config" />
-</java>
-</target>
-
-<target name="standalone-stop" depends="jar,standalone-setup" description="Stop the standalone federation.">
-<java classname="com.bigdata.service.jini.util.ShutdownFederation" failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx200m" />
- <jvmarg value="-showversion" />
- <!-- The name of the federation instance. -->
- <jvmarg value="-Dbigdata.fedname=${standalone.fed}" />
- <jvmarg value="-Djava.security.policy=policy.all" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
- <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties" />
- <arg value="src/resources/config/standalone/bigdataStandalone.config" />
-</java>
-<!-- Then take down the lookup service as well. -->
-<antcall target="stopLookup" />
-<antcall target="stopHttpd" />
-</target>
-
-<target name="standalone-start-nano-server" depends="jar" description="Start a small http server fronting for a bigdata database instance.">
-<java classname="com.bigdata.rdf.sail.bench.NanoSparqlServer" fork="true" failonerror="true">
- <arg line="${standalone.nanoServerPort} ${standalone.namespace} src/resources/config/standalone/bigdataStandalone.config" />
- <jvmarg line="-server" />
- <jvmarg line="-Xmx200M" />
- <classpath refid="runtime.classpath" />
-</java>
-</target>
-
-<target name="standalone-stop-nano-server" depends="jar" description="Stop the small http server running at the configured port.">
-<java classname="com.bigdata.rdf.sail.bench.NanoSparqlServer" fork="true" failonerror="true">
- <arg line="${standalone.nanoServerPort} -stop" />
- <classpath refid="runtime.classpath" />
-</java>
-</target>
-
-<target name="standalone-bulk-load" depends="jar" description="Bulk load RDF data into the standalone federation.">
-<java classname="com.bigdata.rdf.load.MappedRDFDataLoadMaster" failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx200m" />
- <jvmarg value="-showversion" />
- <!-- The name of the federation instance. -->
- <jvmarg value="-Dbigdata.fedname=${standalone.fed}" />
- <jvmarg value="-Djava.security.policy=policy.all" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
- <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties" />
- <!-- -->
- <!-- Per job parameters -->
- <!-- -->
- <!-- The namespace of the target KB. -->
- <jvmarg value="-Dbigdata.rdf.namespace=${standalone.namespace}" />
- <!-- The job name (same as the KB namespace is a common default). -->
- <jvmarg value="-Dbigdata.rdf.job.name=bulk-load-kb-${standalone-namespace}" />
- <!-- The file or directory containing zero or more files to be loaded first. -->
- <jvmarg value="-Dbigdata.rdf.ontology=${standalone.bulkLoad.ontology}" />
- <!-- The file or directory containing RDF data to be loaded. -->
- <jvmarg value="-Dbigdata.rdf.data=${standalone.bulkLoad.data}" />
- <!-- The main configuration file. -->
- <arg value="src/resources/config/standalone/bigdataStandalone.config" />
-</java>
-</target>
-
-<!-- -->
<!-- MISC. UTILITY TARGETS -->
<!-- -->
@@ -1122,9 +1012,9 @@
<!-- Stage the bigdata Jini config files -->
+<copy file="${src.resources.config}/bigdataStandalone.config" todir="${dist.var.config.jini}" />
<copy file="${src.resources.config}/bigdataCluster.config" todir="${dist.var.config.jini}" />
<copy file="${src.resources.config}/bigdataCluster16.config" todir="${dist.var.config.jini}" />
-<copy file="${src.resources.config}/standalone/bigdataStandalone.config" todir="${dist.var.config.jini}" />
<!-- Stage the infrastructure service config files -->
Modified: trunk/src/resources/config/README
===================================================================
--- trunk/src/resources/config/README 2010-08-06 15:14:23 UTC (rev 3422)
+++ trunk/src/resources/config/README 2010-08-06 15:46:07 UTC (rev 3423)
@@ -3,8 +3,10 @@
bigdataStandalone.config - A sample configuration file for a workstation.
-bigdataCluster.config - A sample configuration file for a cluster.
+bigdataCluster.config - A sample configuration file for a 3-node cluster.
+bigdataCluster.config - A sample configuration file for a 16-node cluster.
+
log4j.properties - A default log4j configuration file for use by the bigdata
clients and services.
Modified: trunk/src/resources/config/bigdataCluster.config
===================================================================
--- trunk/src/resources/config/bigdataCluster.config 2010-08-06 15:14:23 UTC (rev 3422)
+++ trunk/src/resources/config/bigdataCluster.config 2010-08-06 15:46:07 UTC (rev 3423)
@@ -1200,15 +1200,6 @@
static private awaitDataServicesTimeout = 8000;
/* Multiplier for the scatter effect.
- *
- * Note: TERM2ID tends to grow more slowly than the other indices for two
- * reasons. First, there are many more distinct RDF Statements than RDF
- * Values for nearly any data set (except if statement identifiers are enabled,
- * in which case there are more terms than statements). Second, the keys of
- * the TERM2ID index compress nicely since long prefixes are very common.
- * Therefore it makes sense to use a smaller scatter factor for this index
- * UNLESS you have only 2-3 data services, in which case you will see hot
- * spots develop with this index unless it is more widely distributed.
*/
static private scatterFactor = 2;
static private scatterFactor_term2id = 2; // use 1 @ 4DS and up.
Modified: trunk/src/resources/config/bigdataCluster16.config
===================================================================
--- trunk/src/resources/config/bigdataCluster16.config 2010-08-06 15:14:23 UTC (rev 3422)
+++ trunk/src/resources/config/bigdataCluster16.config 2010-08-06 15:46:07 UTC (rev 3423)
@@ -1305,13 +1305,6 @@
static private awaitDataServicesTimeout = 8000;
/* Multiplier for the scatter effect.
- *
- * Note: TERM2ID tends to grow more slowly than the other indices for two
- * reasons. First, there are many more distinct RDF Statements than RDF
- * Values for nearly any data set (except if statement identifiers are enabled,
- * in which case there are more terms than statements). Second, the keys of
- * the TERM2ID index compress nicely since long prefixes are very common.
- * Therefore it makes sense to use a smaller scatter factor for this index.
*/
static private scatterFactor = 2;
static private scatterFactor_term2id = 1;
Added: trunk/src/resources/config/bigdataStandalone.config
===================================================================
--- trunk/src/resources/config/bigdataStandalone.config (rev 0)
+++ trunk/src/resources/config/bigdataStandalone.config 2010-08-06 15:46:07 UTC (rev 3423)
@@ -0,0 +1,1886 @@
+import net.jini.jeri.BasicILFactory;
+import net.jini.jeri.BasicJeriExporter;
+import net.jini.jeri.tcp.TcpServerEndpoint;
+
+import net.jini.discovery.LookupDiscovery;
+import net.jini.core.discovery.LookupLocator;
+import net.jini.core.entry.Entry;
+import net.jini.lookup.entry.Name;
+import net.jini.lookup.entry.Comment;
+import net.jini.lookup.entry.Address;
+import net.jini.lookup.entry.Location;
+import net.jini.lookup.entry.ServiceInfo;
+import net.jini.core.lookup.ServiceTemplate;
+
+import java.io.File;
+
+import com.bigdata.util.NV;
+import com.bigdata.journal.BufferMode;
+import com.bigdata.jini.lookup.entry.*;
+import com.bigdata.service.IBigdataClient;
+import com.bigdata.service.jini.*;
+import com.bigdata.service.jini.lookup.DataServiceFilter;
+import com.bigdata.service.jini.master.ServicesTemplate;
+import com.bigdata.jini.start.config.*;
+import com.bigdata.jini.util.ConfigMath;
+
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
+// imports for various options.
+import com.bigdata.btree.IndexMetadata;
+import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.rdf.sail.BigdataSail;
+import com.bigdata.rdf.spo.SPORelation;
+import com.bigdata.rdf.spo.SPOKeyOrder;
+import com.bigdata.rdf.lexicon.LexiconRelation;
+import com.bigdata.rdf.lexicon.LexiconKeyOrder;
+import com.bigdata.rawstore.Bytes;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeUnit.*;
+
+/*
+ * This is a sample configuration file for a bigdata federation.
+ *
+ * Note: The original file is a template. The template contains parameters
+ * of the form @XXX@. The values for those template parameters are specified
+ * in the build.properties file when you use ant to install bigdata.
+ *
+ * Note: This file uses the jini configuration mechanism. The syntax
+ * is a subset of Java. The properties for each component are grouped
+ * within the namespace for that component.
+ *
+ * See the net.jini.config.ConfigurationFile javadoc for more
+ * information.
+ */
+
+/*
+ * A namespace use for static entries referenced elsewhere in this
+ * ConfigurationFile.
+ */
+bigdata {
+
+ /**
+ * The name for this federation.
+ *
+ * Note: This is used to form the [zroot] (root node in zookeeper
+ * for the federation) and the [serviceDir] (path in the file
+ * system for persistent state for the federation).
+ *
+ * Note: If you will be running more than one federation, then you
+ * MUST use unicast discovery and specify the federation name in
+ * the [groups].
+ */
+ static private fedname = "@FED@";
+
+ /**
+ * Where to put all the persistent state.
+ */
+ static private serviceDir = new File("@LAS@");
+
+ /**
+ * Which JDK to use.
+ */
+ static private javaHome = new File("@JAVA_HOME@");
+
+ /**
+ * A common point to set the Zookeeper client's requested
+ * sessionTimeout and the jini lease timeout. The default lease
+ * renewal period for jini is 5 minutes while for zookeeper it is
+ * more like 5 seconds. This puts the two systems onto a similar
+ * timeout period so that a disconnected client is more likely to
+ * be noticed in roughly the same period of time for either
+ * system. A value larger than the zookeeper default helps to
+ * prevent client disconnects under sustained heavy load.
+ */
+ // jini
+ static private leaseTimeout = ConfigMath.m2ms(60);// 20s=20000; 5m=300000;
+ // zookeeper
+ static private sessionTimeout = (int)ConfigMath.m2ms(10);// was 5m 20s=20000; 5m=300000;
+
+ /*
+ * Example cluster configuration.
+ *
+ * Data services are load balanced. Index partitions will be
+ * moved around as necessary to ensure hosts running data
+ * service(s) are neither under nor over utilized. Data services
+ * can be very resource intensive processes. They heavily buffer
+ * both reads and writes, and they use RAM to do so. They also
+ * support high concurrency and can use up to one thread per index
+ * partition. How many cores they will consume is very much a
+ * function of the application.
+ *
+ * Zookeeper services use a quorum model. Always allocate an odd
+ * number. 3 gives you one failure. 5 gives you two failures.
+ * Zookeeper will sync the disk almost continuously while it is
+ * running. It really deserves its own local disk. Zookeeper
+ * also runs in memory. Since all operations are serialized, if
+ * it starts swapping then peformance will drop through the floor.
+ *
+ * Jini uses a peer model. Each service registers with each
+ * registrar that it discovers. Each client listeners to each
+ * registrar that it discovers. The default jini core services
+ * installation runs entirely in memory (no disk operations, at
+ * least not for service registration). A second instance of the
+ * jini core services provides a safety net. If you are using
+ * multicast then you can always add another instance.
+ */
+
+ /* Declare the hosts. This provides indirection for planning
+ * purposes.
+ *
+ * The summary notation is: cores@GHZ/cache x RAM x DISK
+ */
+ static private h0 = "192.168.1.50"; // 4@3ghz/1kb x 4GB x 263G
+ //static private h1 = "192.168.20.27"; // 4@3ghz/2kb x 4GB x 263G
+ //static private h2 = "192.168.20.28"; // 4@3ghz/1kb x 4GB x 64G
+
+ /* Note: this configuration puts things that are not disk intensive
+ * on the host with the least disk space and zookeeper.
+ */
+ static private lbs = h0; // specify as @LOAD_BALANCER_HOST@ ?
+ static private txs = h0;
+ static private mds = h0;
+
+ // 1+ jini servers
+ static private jini1 = h0;
+ //static private jini2 = h1;
+ static private jini = new String[]{ jini1 }; //,jini2};
+
+ // Either 1 or 3 zookeeper machines (one instance per).
+ // See the QuorumPeerMain and ZooKeeper configurations below.
+ static private zoo1 = h0;
+ //static private zoo2 = h1;
+ //static private zoo3 = h2;
+ static private zoo = new String[] { zoo1 }; // ,zoo2,zoo3};
+
+ // 1+ client service machines (1+ instance per host).
+ static private cs0 = h0;
+
+ // 1+ data service machines (1+ instance per host).
+ static private ds0 = h0;
+ static private ds1 = h1;
+
+ // client servers
+ static private cs = new String[] {
+ cs0 //, ...
+ };
+
+ // The target #of client servers.
+ static private clientServiceCount = 1;
+ static private maxClientServicePerHost = 1;
+
+ // data servers
+ static private ds = new String[]{
+ ds0//, ds1 //, ...
+ };
+
+ // The target #of data services.
+ static private dataServiceCount = 1;
+
+ // Maximum #of data services per host.
+ static private maxDataServicesPerHost = 1;
+
+ // @todo also specify k (replicationCount)
+
+ // Sets the initial and maximum journal extents.
+ static private journalExtent = ConfigMath.multiply(200, Bytes.megabyte);
+
+ /**
+ * A String[] whose values are the group(s) to be used for discovery
+ * (no default). Note that multicast discovery is always used if
+ * LookupDiscovery.ALL_GROUPS (a <code>null</code>) is specified.
+ */
+
+ // one federation, multicast discovery.
+ //static private groups = LookupDiscovery.ALL_GROUPS;
+
+ // unicast discovery or multiple federations, MUST specify groups.
+ static private groups = new String[]{bigdata.fedname};
+
+ /**
+ * One or more unicast URIs of the form <code>jini://host/</code>
+ * or <code>jini://host:port/</code> (no default).
+ *
+ * This MAY be an empty array if you want to use multicast
+ * discovery <strong>and</strong> you have specified the groups as
+ * LookupDiscovery.ALL_GROUPS (a <code>null</code>).
+ */
+ static private locators = new LookupLocator[] {
+
+ // runs jini on the localhost using unicast locators.
+ //new LookupLocator("jini://localhost/")
+
+ // runs jini on two hosts using unicast locators.
+ new LookupLocator("jini://"+jini1),
+ //new LookupLocator("jini://"+jini2),
+
+ };
+
+ /**
+ * The policy file that will be used to start services.
+ */
+ private static policy = "@POLICY_FILE@";
+
+ /**
+ * log4j configuration file (applies to bigdata and zookeeper).
+ *
+ * Note: The value is URI!
+ *
+ * Note: You should aggregate all of the log output to a single
+ * host. For example, using the log4j SocketAppender and the
+ * SimpleNodeServer.
+ */
+ log4j = "@LOG4J_CONFIG@";
+
+ /**
+ * java.util.logging configuration file (applies to jini as used
+ * within bigdata).
+ *
+ * Note: The value is a file path!
+ */
+ logging = "@LOGGING_CONFIG@";
+
+ /*
+ private static host = ConfigUtil.getHostName();
+ private static port = "8081";
+ private static jskdl = " http://" + host + ":" + port + "/jsk-dl.jar";
+ */
+
+ /**
+ * JVM argument may be used to enable the yourkit profiler agent on a
+ * service. Of course, yourkit must be installed at this location and
+ * you must have a licensed copy of the yourkit UI running either on a
+ * node of the cluster or on a machine routed to the cluster, e.g., via
+ * an ssh tunnel. The yourkit profiler uses ports in [10001:100010] by
+ * default on each node.
+ *
+ * See http://www.yourkit.com/docs/80/help/running_with_profiler.jsp
+ *
+ * See http://www.yourkit.com/docs/80/help/agent.jsp
+ *
+ * See http://www.yourkit.com/docs/80/help/additional_agent_options.jsp
+ *
+ * Note: Conditionally include ${profilerAgent} iff you want to enable
+ * profiling for some service class.
+ */
+
+ // linux-64 with all profiling options initially disabled.
+ profilerAgent="-agentpath:/usr/java/yjp-9.0.3/bin/linux-x86-64/libyjpagent.so=disableexceptiontelemetry,disablestacktelemetry";
+
+}
+
+/*
+ * Service configuration defaults. These can also be specified on a
+ * per service-type basis. When the property is an array type, the
+ * value here is concatenated with the optional array value on the per
+ * service-type configuration. Otherwise it is used iff no value is
+ * specified for the service-type configuration.
+ */
+com.bigdata.jini.start.config.ServiceConfiguration {
+
+ /*
+ * Default java command line arguments that will be used for all
+ * java-based services
+ *
+ * Note: [-Dcom.sun.jini.jeri.tcp.useNIO=true] enables NIO in
+ * combination with the [exporter] configured below.
+ */
+ defaultJavaArgs = new String[]{
+ "-server",
+ "-ea",
+ "-showversion",
+ //"-Xmx2G",
+ /* This is a workaround for a JVM bug which can result in a
+ * lost wakeup. This bug is fixed in JDK1.6.0_18. However,
+ * JDK1.6.0_18 has other problems which result in segfaults.
+ *
+ * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6822370
+ */
+ "-XX:+UseMembar",
+ "-Dcom.sun.jini.jeri.tcp.useNIO=@USE_NIO@",
+ "-Djava.security.policy="+bigdata.policy,
+ "-Djava.util.logging.config.file="+bigdata.logging,
+ "-Dcom.bigdata.counters.linux.sysstat.path=@SYSSTAT_HOME@",
+ //bigdata.profilerAgent,
+ };
+
+ /* Default path for service instances and their persistent
+ * data. This may be overriden on a per service-type basis.
+ *
+ * Note: For logical services that support failover, the concrete
+ * service directory is assigned dynamically when a physical
+ * service instance is created.
+ */
+ serviceDir = bigdata.serviceDir;
+
+ // The JVM to use.
+ javaHome = bigdata.javaHome;
+
+ /* The bigdata services default logging configuration (a URI!)
+ */
+ log4j = bigdata.log4j;
+
+ /*
+ * Set up some default properties values that will be inherited
+ * (copy by value) by all clients and services started using this
+ * configuration file.
+ */
+ properties = new NV[] {
+
+ /*
+ * Each JiniClient (and hence all bigdata services) can run an
+ * httpd that will expose performance counters for the service and
+ * the host on which it is running. This property specifies the
+ * port for that httpd service. Valid values are port number,
+ * zero (0) for a random open port, MINUS ONE (-1) to disable the
+ * httpd service.
+ */
+ //new NV(IBigdataClient.Options.HTTPD_PORT, "-1"),
+
+ /*
+ * Option to disable collection of performance counters for the
+ * host on which the client or service is running.
+ *
+ * Note: The load balancer relies on this information!
+ */
+ //new NV(IBigdataClient.Options.COLLECT_PLATFORM_STATISTICS,"false"),
+
+ /* Option to disable collection of performance counters on the
+ * queues used internally by the client or service.
+ *
+ * Note: The load balancer relies on this information!
+ */
+ //new NV(IBigdataClient.Options.COLLECT_QUEUE_STATISTICS,"false"),
+
+ /* Option controls how many times a client request will be
+ * reissued on receiving notice that an index partition locator is
+ * stale. Stale locators arise when an index partition is split,
+ * moved, or joined.
+ *
+ * Note: This option needs to be larger if we are aggressively
+ * driving journal overflows and index partitions splits during
+ * the "young" phase of a data service or scale-out index since a
+ * LOT of redirects will result.
+ */
+ new NV(IBigdataClient.Options.CLIENT_MAX_STALE_LOCATOR_RETRIES,"1000"),
+
+ };
+
+}
+
+/**
+ * JoinManager options.
+ *
+ * Note: These options must be copied into the service.config (to
+ * specify the service lease timeout) as well as used by the client
+ * (which uses this file directly).
+ */
+net.jini.lookup.JoinManager {
+
+ // The lease timeout for jini joins.
+ maxLeaseDuration = bigdata.leaseTimeout;
+
+}
+
+/**
+ * Jini service configuration.
+ */
+jini {
+
+ /* This sets command line arguments for the ServiceStarter which
+ * is used to run the jini services.
+ */
+ args = new String[] {
+
+ "-Xmx400m",
+ "-Djava.security.policy="+bigdata.policy,
+ "-Djava.util.logging.config.file="+bigdata.logging,
+ "-Dlog4j.configuration="+bigdata.log4j,
+ "-Dlog4j.primary.configuration="+bigdata.log4j,
+ "-DinitialMemberGroups="+bigdata.fedname
+
+ };
+
+ /**
+ * The main jini configuration file. This file contains a
+ * NonActivatableServiceDescriptor[]. The elements of that array
+ * describe how to start each of the jini services.
+ */
+ configFile = new File("@JINI_CONFIG@");
+
+ /**
+ * The #of instances to run.
+ *
+ * Note: A jini service instance may be started on a host if it is
+ * declared in [locators]. If locators is empty, then you are
+ * using multicast discovery. In this case an instance may be
+ * started on any host, unless [constraints] are imposed. In any
+ * case, no more than [serviceCount] jini services will be started
+ * at any given time. This is checked against the #of discovered
+ * instances.
+ */
+ serviceCount = 1;
+
+}
+
+/**
+ * Zookeeper server configuration.
+ */
+org.apache.zookeeper.server.quorum.QuorumPeerMain {
+
+ /* Directory for zookeeper's persistent state. The [id] will be
+ * appended as another path component automatically to keep
+ * instances separate.
+ */
+ dataDir = new File(bigdata.serviceDir,"zookeeper");
+
+ /* Optional directory for the zookeeper log files. The [id] will
+ * be appended as another path component automatically to keep
+ * instances separate.
+ *
+ * Note: A dedicated local storage device is highly recommended
+ * for the zookeeper transaction logs!
+ */
+ //dataLogDir=new File("/var/zookeeper-log");
+
+ // required.
+ clientPort=2181;
+
+ tickTime=2000;
+
+ initLimit=5;
+
+ syncLimit=2;
+
+ /* A comma delimited list of the known zookeeper servers together
+ * with their assigned "myid": {myid=host:port(:port)}+
+ *
+ * Note: You SHOULD specify the full list of servers that are
+ * available to the federation. An instance of zookeeper will be
+ * started automatically on each host running ServicesManager that
+ * is present in the [servers] list IF no instance is found
+ * running on that host at the specified [clientPort].
+ *
+ * Note: zookeeper interprets NO entries as the localhost with
+ * default peer and leader ports. This will work as long as the
+ * localhost is already running zookeeper. However, zookeeper
+ * WILL NOT automatically start zookeeper if you do not specify
+ * the [servers] property. You can also explicitly specify
+ * "localhost" as the hostname, but that only works for a single
+ * machine.
+ */
+ // standalone
+ //servers="1=localhost:2888:3888";
+ // ensemble
+ /**/
+ servers = "1="+bigdata.zoo1+":2888:3888"
+// + ",2="+bigdata.zoo2+":2888:3888"
+// + ",3="+bigdata.zoo3+":2888:3888"
+ ;
+
+ // This is all you need to run zookeeper.
+ classpath = new String[] {
+ "@LIB_DIR@/apache/zookeeper-3.2.1.jar",
+ "@LIB_DIR@/apache/log4j-1.2.15.jar"
+ };
+
+ /* Optional command line arguments for the JVM used to execute
+ * zookeeper.
+ *
+ * Note: swapping for zookeeper is especially bad since the
+ * operations are serialized, so if anything hits then disk then
+ * all operations in the queue will have that latency as well.
+ * However, bigdata places a very light load on
+ * zookeeper so a modest heap should be Ok. For example, I have
+ * observed a process size of only 94m after 10h on a 15-node
+ * cluster.
+ */
+ args = new String[]{
+ "-Xmx200m",
+ /*
+ * Enable JXM remote management.
+ *
+ "-Dcom.sun.management.jmxremote.port=9997",
+ "-Dcom.sun.management.jmxremote.authenticate=false",
+ "-Dcom.sun.management.jmxremote.ssl=false",
+ */
+};
+
+ // zookeeper server logging configuration (value is a URI!)
+ log4j = bigdata.log4j;
+
+}
+
+/*
+ * Zookeeper client configuration.
+ */
+org.apache.zookeeper.ZooKeeper {
+
+ /* Root znode for the federation instance. */
+ zroot = "/"+bigdata.fedname;
+
+ /* A comma separated list of host:port pairs, where the port is
+ * the CLIENT port for the zookeeper server instance.
+ */
+ // standalone.
+ // servers = "localhost:2181";
+ // ensemble
+ servers = bigdata.zoo1+":2181" // @TODO enable other instances.
+// + ","+bigdata.zoo2+":2181"
+// + ","+bigdata.zoo3+":2181"
+ ;
+
+ /* Session timeout (optional). */
+ sessionTimeout = bigdata.sessionTimeout;
+
+ /*
+ * ACL for the zookeeper nodes created by the bigdata federation.
+ *
+ * Note: zookeeper ACLs are not transmitted over secure channels
+ * and are placed into plain text Configuration files by the
+ * ServicesManagerServer.
+ */
+ acl = new ACL[] {
+
+ new ACL(ZooDefs.Perms.ALL, new Id("world", "anyone"))
+
+ };
+
+}
+
+/*
+ * Jini client configuration
+ */
+com.bigdata.service.jini.JiniClient {
+
+ /* Default Entry[] for jini services. Also used by the
+ * ServicesManagerService as is.
+ *
+ * Note: A Name attribute will be added automatically using the
+ * service type and the znode of the service instance. That Name
+ * will be canonical. It is best if additional service names are
+ * NOT specified as that might confuse somethings :-)
+ *
+ * Note: A Hostname attribute will be added dynamically.
+ */
+ entries = new Entry[] {
+ // Purely informative.
+ new Comment(bigdata.fedname),
+ };
+
+ groups = bigdata.groups;
+
+ locators = bigdata.locators;
+
+ // optional JiniClient properties.
+ // properties = new NV[] {};
+
+ /*
+ * Overrides for jini SERVICES (things which are started
+ * automatically) BUT NOT CLIENTs (things which you start by hand
+ * and which read this file directly).
+ *
+ * The difference here is whether or not a service.config file is
+ * being generated. When it is, the jiniOptions[] will be
+ * included in how that service is invoked and will operate as
+ * overrides for the parameters specified in the generated
+ * service.config file. However, normal clients directly consume
+ * this config file rather than the generated one and therefore
+ * you must either specify their overrides directly on the command
+ * line when you start the client or specify them explicitly in
+ * the appropriate component section within this configuration
+ * file.
+ *
+ * In practice, this means that you must specify some parameters
+ * both here and in the appropriate component configuration. E.g.,
+ * see the component section for "net.jini.lookup.JoinManager"
+ * elsewhere in this file.
+ */
+ jiniOptions = new String[] {
+
+ // The lease timeout for jini joins.
+ "net.jini.lookup.JoinManager.maxLeaseDuration="+bigdata.leaseTimeout,
+
+ };
+
+}
+
+/**
+ * Options for the bigdata services manager.
+ */
+com.bigdata.jini.start.ServicesManagerServer {
+
+ /*
+ * This object is used to export the service proxy. The choice
+ * here effects the protocol that will be used for communications
+ * between the clients and the service.
+ */
+ exporter = new BasicJeriExporter(TcpServerEndpoint.getInstance(0),
+ new BasicILFactory());
+
+ /*
+ * The data directory and the file on which the serviceID will be
+ * written.
+ *
+ * Note: These properties MUST be specified explicitly for the
+ * ServicesManager since it uses this as its Configuration file.
+ * For other services, it generates the Configuration file and
+ * will generate this property as well.
+ */
+
+ serviceDir = new File(bigdata.serviceDir,"ServicesManager");
+
+ serviceIdFile = new File(serviceDir,"service.id");
+
+ /* The services that will be started. For each service, there
+ * must be a corresponding component defined within this
+ * configuration file. For each "ManagedServiceConfiguration", an
+ * entry will be made in zookeeper and logical and physical
+ * service instances will be managed automatically. For unmanaged
+ * services, such as jini and zookeeper itself, instances will be
+ * started iff necessary by the services manager when it starts
+ * up.
+ */
+ services = new String[] {
+
+ "jini",
+ "org.apache.zookeeper.server.quorum.QuorumPeerMain",
+ "com.bigdata.service.jini.TransactionServer",
+ "com.bigdata.service.jini.MetadataServer",
+ "com.bigdata.service.jini.DataServer",
+ "com.bigdata.service.jini.LoadBalancerServer",
+ "com.bigdata.service.jini.ClientServer"
+
+ };
+
+ /*
+ * Additional properties passed through to the JiniClient or the
+ * service.
+ *
+ * Note: The services manager is used to collect statistics from the
+ * OS for each host so we have performance counters for hosts which
+ * are only running non-bigdata services, such as jini or zookeeper.
+ */
+ properties = new NV[]{
+
+ };
+
+ /* The services manager MUDT be run on every host so that it may
+ * start both bigdata and non-bigdata services (jini, zookeeper).
+ * This is also used to report per-host performance counters to
+ * the load balancer for hosts that are not running bigdata
+ * services.
+ */
+ constraints = new IServiceConstraint[] {
+
+ };
+
+}
+
+com.bigdata.service.jini.TransactionServer {
+
+ constraints = new IServiceConstraint[] {
+
+ new JiniRunningConstraint(),
+ new ZookeeperRunningConstraint(),
+
+ new HostAllowConstraint(bigdata.txs)
+
+ };
+
+ args = new String[]{
+
+ // Does not need much RAM.
+ "-Xmx200m"
+
+ };
+
+ properties = new NV[] {
+
+ /* The #of milliseconds that the database will retain history no
+ * longer required to support the earliest active transaction.
+ *
+ * A value of ZERO means that only the last commit point will
+ * be retained. The larger the value the more history will be
+ * retained. You can use a really big number if you never want
+ * to release history and you have lots of disk space :-)
+ *
+ * Note: The most recent committed state of the database is
+ * NEVER released.
+ */
+ new NV(TransactionServer.Options.MIN_RELEASE_AGE, "0"),
+
+ };
+
+}
+
+com.bigdata.service.jini.MetadataServer {
+
+ constraints = new IServiceConstraint[] {
+
+ new JiniRunningConstraint(),
+ new ZookeeperRunningConstraint(),
+ //new TXRunningConstraint(),
+
+ new HostAllowConstraint(bigdata.mds),
+
+ };
+
+ args = new String[]{
+
+ // Does not need much RAM.
+ "-Xmx200m"
+
+ };
+
+ properties = new NV[]{
+
+ /*
+ * The MDS does not support overflow at this time so
+ * overflow MUST be disabled for this service.
+ */
+ new NV(MetadataServer.Options.OVERFLOW_ENABLED,"false")
+
+ };
+
+}
+
+com.bigdata.service.jini.DataServer {
+
+ args = new String[]{
+ //bigdata.profilerAgent,
+ /*
+ * Grant lots of memory, but read on.
+ *
+ * Note: 32-bit JVMs have a 2G limit on the heap, but the practical limit
+ * is often much less - maybe 1400m. 64-bit JVMs can use much more RAM.
+ * However, the heap which you grant to java DOES NOT determine the total
+ * process heap. I have seen 64-bit java processes using an additional
+ * 3-4GB of heap beyond what is specified here. So, you need to consider
+ * the total RAM, subtract out enough for the other processes and the OS
+ * buffers, divide by the #of client/data services you plan to run on that
+ * host (generally 1-2) and then subtract out some more space for the JVM
+ * itself.
+ *
+ * For example, if you have 32G RAM and a 64-bit JVM and plan to run two
+ * CS/DS on the host, I would recommend 10G for the Java heap. You can
+ * expect to see Java grab another 4G per process over time. That makes
+ * the per CS/DS heap 14G. With two processes you have taken 28G leaving
+ * 4G for everything else.
+ *
+ * Here is another example: 4G RAM, 32-bit JVM, and 2 CS/DS per host. I
+ * would stick to 800m for the Java heap. You don't have a problem unless
+ * you see an OOM (OutOfMemoryException) or a process killed because GC is
+ * taking too much time.
+ *
+ * See http://www.ibm.com/developerworks/linux/library/j-nativememory-linux/index.html?ca=dgr-lnxw07Linux-JVM&S_TACT=105AGX59&S_CMP=grlnxw07
+ *
+ * Note: for linux, "sysctl -w vm.swappiness=0" will keep the RAM you do
+ * have for your applications!
+ */
+ "-Xmx4g",// was 800
+ /* Optionally, grab all/most of the max heap at once. This makes sense for
+ * DS but is less necessary for other bigdata services.
+ */
+ "-Xms2G", // 1/2 of the max heap is a good value.
+ /*
+ * This option will keep the JVM "alive" even when it is memory starved
+ * but perform of a memory starved JVM is terrible.
+ */
+ //"-XX:-UseGCOverheadLimit",
+ /* Configure GC for higher throughput. Together these options
+ * request parallel old generation collection using N threads.
+ * The application will be paused when this occurs, but GC will
+ * be faster. Hence throughput will be higher. However, be
+ * sure to use JDK 6u10+ (6676016 : ParallelOldGC leaks memory).
+ *
+ * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6676016
+ */
+ "-XX:+UseParallelOldGC",
+ //"-XX:ParallelGCThreads=8",
+ /*
+ * Enable JXM remote management for the data service.
+ *
+ * Note: This will not work if you have two data services on a host
+ * because it will assign the same port to each service. In order
+ * to work around that the argument would have to be specified by
+ * the service starter and then published in the Entry[] attributes.
+ *
+ * However, you can use ssh -X to open a tunnel with X
+ * forwarding and then run jconsole locally on the target host
+ * and bring up these data services without enabling remote
+ * JMX.
+ *
+ "-Dcom.sun.management.jmxremote.port=9999",
+ "-Dcom.sun.management.jmxremote.authenticate=false",
+ "-Dcom.sun.management.jmxremote.ssl=false",
+ */
+ /*
+ * Override the size of the default pool of direct (native) byte
+ * buffers. This was done to ensure that the nodes region for
+ * index segments remain fully buffered as the index partitions
+ * approach their maximum size before a split.
+ */
+ "-Dcom.bigdata.io.DirectBufferPool.bufferCapacity="+
+ ConfigMath.multiply(Bytes.kilobyte,1250),
+ };
+
+ serviceCount = bigdata.dataServiceCount;
+
+ // restrict where the data services can run.
+ constraints = new IServiceConstraint[] {
+
+ new JiniRunningConstraint(),
+ new ZookeeperRunningConstraint(),
+ //new TXRunningConstraint(),
+
+ new HostAllowConstraint(bigdata.ds),
+
+ new MaxDataServicesPerHostConstraint(bigdata.maxDataServicesPerHost),
+
+ };
+
+ /*
+ * Note: the [dataDir] will be filled in when a new service
+ * instance is created based on the [servicesDir], so don't set it
+ * here yourself.
+ */
+ properties = new NV[]{
+
+ new NV(DataServer.Options.BUFFER_MODE,
+ //""+com.bigdata.journal.BufferMode.Direct
+ ""+com.bigdata.journal.BufferMode.DiskWORM
+ ),
+
+ /* Option disables synchronous overflow after N times and
+ * configures the offset bits for the journal for a scale-up
+ * configuration so we may use very large journals.
+ */
+ //new NV(DataServer.Options.OVERFLOW_MAX_COUNT,"5"),
+ //new NV(DataServer.Options.OFFSET_BITS,""+com.bigdata.rawstore.WormAddressManager.SCALE_UP_OFFSET_BITS),
+
+ /* Synchronous overflow is triggered when the live journal is
+ * this full (the value is a percentage, expressed as a
+ * floating point number in [0:1]).
+ */
+ //new NV(DataServer.Options.OVERFLOW_THRESHOLD,".9"),
+
+ /* Override the initial and maximum extent so that they are more
+ * more suited to large data sets. Overflow will be triggered as
+ * the size of the journal approaches the maximum extent. The
+ * initial and maximum extent are configured up above.
+ */
+
+ new NV(DataServer.Options.INITIAL_EXTENT, "" + bigdata.journalExtent),
+ new NV(DataServer.Options.MAXIMUM_EXTENT, "" + bigdata.journalExtent),
+
+ /* Specify the queue capacity for the write service (unisolated
+ * write operations).
+ *
+ * 0 := SynchronousQueue.
+ * N := bounded queue of capacity N
+ * Integer.MAX_VALUE := unbounded queue.
+ *
+ * Note: The corePoolSize will never increase for an unbounded
+ * queue so the value specified for maximumPoolSize will
+ * essentially be ignored in this case.
+ *
+ * Note: A SynchronousQueue is a good choice here since it allows
+ * the #of threads to change in response to demand. The pool
+ * size should be unbounded when using a SynchronousQueue.
+ */
+ new NV(DataServer.Options.WRITE_SERVICE_QUEUE_CAPACITY,"0"), // synchronous queue.
+ new NV(DataServer.Options.WRITE_SERVICE_CORE_POOL_SIZE,"50"), //
+ new NV(DataServer.Options.WRITE_SERVICE_MAXIMUM_POOL_SIZE,""+Integer.MAX_VALUE),
+ new NV(DataServer.Options.WRITE_SERVICE_PRESTART_ALL_CORE_THREADS,"true"),
+
+ /*
+ * Options turns off overflow processing (debugging only).
+ * All writes will go onto the live journal, no index segments
+ * will be built, and indices will not be split, moved,
+ * joined, etc.
+ */
+ //new NV(DataServer.Options.OVERFLOW_ENABLED,"false"),
+
+ /* Maximum #of index partition moves per overflow.
+ */
+ new NV(DataServer.Options.MAXIMUM_MOVES,"1"),
+
+ /* Option controls how many index partitions may be moved onto
+ * any given target data service in a single overflow cycle
+ * and may be used to disable index partition moves (for
+ * debugging purposes).
+ */
+ new NV(DataServer.Options.MAXIMUM_MOVES_PER_TARGET,"1"),
+
+ /* The minimum CPU activity on a host before it will consider moving an
+ * index partition to shed some load.
+ *
+ * @todo A high threshold was chosen for the 3-node cluster since there
+ * are only 2 machines running data services. A "feature" in the load
+ * balancer allows moves between two heavily loaded hosts even when they
+ * are very close in their load, which is typically the case if you have
+ * only 2 machines running data services. The high threshold here is a
+ * workaround until the load balancer is modified to take into account
+ * whether or not a significant difference exists in the load between
+ * the source and possible target data service hosts.
+ */
+ new NV(DataServer.Options.MOVE_PERCENT_CPU_TIME_THRESHOLD,".99"),//was .7
+
+ /* Option limits the #of index segments in a view before a
+ * compacting merge is forced.
+ */
+ new NV(DataServer.Options.MAXIMUM_SEGMENTS_PER_VIEW,"5"), // default 6
+
+ /* Option limits the #of optional merges that are performed in each
+ * overflow cycle.
+ */
+ new NV(DataServer.Options.MAXIMUM_OPTIONAL_MERGES_PER_OVERFLOW,"1"),
+
+ /* Option effects how much splits are emphasized for a young
+ * scale-out index. If the index has fewer than this many
+ * partitions, then there will be a linear reduction in the
+ * target index partition size which will increase the likelyhood
+ * of an index split under heavy writes. This helps to distribute
+ * the index early in its life cycle.
+ */
+ new NV(DataServer.Options.ACCELERATE_SPLIT_THRESHOLD,"20"),//20//50
+
+ /* Options accelerates overflow for data services have fewer than
+ * the threshold #of bytes under management. Acceleration is
+ * accomplished by reducing the maximum extent of the live journal
+ * linearly, but with a minimum of a 10M maximum extent. When the
+ * maximum extent is reduced by this option, the initial and the
+ * maximum extent will always be set to the same value for that
+ * journal.
+ */
+ new NV(DataServer.Options.ACCELERATE_OVERFLOW_THRESHOLD,
+ //"0"
+ //""+com.bigdata.rawstore.Bytes.gigabyte
+ "2147483648" // 2G
+ ),
+
+ // #of threads for index segment builds (default 3).
+ new NV(DataServer.Options.BUILD_SERVICE_CORE_POOL_SIZE,"5"),
+
+ // #of threads for compacting merges (default 1).
+ new NV(DataServer.Options.MERGE_SERVICE_CORE_POOL_SIZE,"1"),
+
+// // Zero is full parallelism; otherwise #of threads in the pool.
+// new NV(DataServer.Options.OVERFLOW_TASKS_CONCURRENT,"5"),
+
+ /* Use Long.MAX_VALUE to always run overflow processing to
+ * completion (until no more data remains on the old journal).
+ */
+ new NV(DataServer.Options.OVERFLOW_TIMEOUT,""+Long.MAX_VALUE),
+
+ new NV(DataServer.Options.OVERFLOW_CANCELLED_WHEN_JOURNAL_FULL,"false"),
+
+ new NV(DataServer.Options.LIVE_INDEX_CACHE_CAPACITY,"10"), // was 60
+
+ new NV(DataServer.Options.HISTORICAL_INDEX_CACHE_CAPACITY,"10"), // was 60
+
+ /* The maximum #of clean indices that will be retained on the
+ * hard reference queue (default 20).
+ */
+ new NV(DataServer.Options.INDEX_CACHE_CAPACITY,"10"), // was 50
+
+ /* The timeout for unused index references before they are
+ * cleared from the hard reference queue (default is 1m).
+ * After this timeout the index reference is cleared from the
+ * queue and the index will be closed unless a hard reference
+ * exists to the index.
+ */
+// new NV(DataServer.Options.INDEX_CACHE_TIMEOUT,"1200000"), // 20m vs 1m
+
+ /* The maximum #of clean index segments that will be retained
+ * on the hard reference queue (default 60). Note that ALL
+ * index segments are clean (they are read-only).
+ */
+ new NV(DataServer.Options.INDEX_SEGMENT_CACHE_CAPACITY,"20"), // was 100
+
+ /* The timeout for unused index segment references before they
+ * are cleared from the hard reference queue (default is 1m).
+ * After this timeout the index segment reference is cleared
+ * from the queue and the index segment will be closed unless
+ * a hard reference exists to the index segment.
+ */
+// new NV(DataServer.Options.INDEX_SEGMENT_CACHE_TIMEOUT,"60000000"), // 10m vs 1m
+
+ /* The #of store files (journals and index segment stores)
+ * whose hard references will be maintained on a queue. The
+ * value should be slightly more than the index segment cache
+ * capacity since some journals also used by the views, but
+ * same journals are shared by all views so adding 3 is plenty..
+ */
+ new NV(DataServer.Options.STORE_CACHE_CAPACITY,"23"),// was 110
+
+// new NV(DataServer.Options.STORE_CACHE_TIMEOUT,"1200000"),//20m vs 1m.
+
+ };
+
+}
+
+/**
+ * Configuration options for the containers used to distribute application tasks
+ * across a federation.
+ *
+ * @todo There should be a means to tag certain client servers for one purpose
+ * or another. This could be handled by subclassing, but it really should be
+ * declarative.
+ */
+com.bigdata.service.jini.ClientServer {
+
+ args = new String[]{
+ //bigdata.profilerAgent,
+ /*
+ * Grant lots of memory, but read on.
+ *
+ * Note: 32-bit JVMs have a 2G limit on the heap, but the practical limit
+ * is often much less - maybe 1400m. 64-bit JVMs can use much more RAM.
+ * However, the heap which you grant to java DOES NOT determine the total
+ * process heap. I have seen 64-bit java processes using an additional
+ * 3-4GB of heap beyond what is specified here. So, you need to consider
+ * the total RAM, subtract out enough for the other processes and the OS
+ * buffers, divide by the #of client/data services you plan to run on that
+ * host (generally 1-2) and then subtract out some more space for the JVM
+ * itself.
+ *
+ * For example, if you have 32G RAM and a 64-bit JVM and plan to run two
+ * CS/DS on the host, I would recommend 10G for the Java heap. You can
+ * expect to see Java grab another 4G per process over time. That makes
+ * the per CS/DS heap 14G. With two processes you have taken 28G leaving
+ * 4G for everything else.
+ *
+ * Here is another example: 4G RAM, 32-bit JVM, and 2 CS/DS per host. I
+ * would stick to 800m for the Java heap. You don't have a problem unless
+ * you see an OOM (OutOfMemoryException) or a process killed because GC is
+ * taking too much time.
+ *
+ * See http://www.ibm.com/developerworks/linux/library/j-nativememory-linux/index.html?ca=dgr-lnxw07Linux-JVM&S_TACT=105AGX59&S_CMP=grlnxw07
+ *
+ * Note: for linux, "sysctl -w vm.swappiness=0" will keep the RAM you do
+ * have for your applications!
+ */
+ "-Xmx2g", // was 800m
+ /*
+ * This option will keep the JVM "alive" even when it is memory starved
+ * but perform of a memory starved JVM is terrible.
+ */
+ //"-XX:-UseGCOverheadLimit",
+ /* Configure GC for higher throughput. Together these options
+ * request parallel old generation collection using N threads.
+ * The application will be paused when this occurs, but GC will
+ * be faster. Hence throughput will be higher.
+ */
+ "-XX:+UseParallelOldGC",
+ //"-XX:ParallelGCThreads=8",
+ /*
+ * Enable JXM remote management for the data service.
+ *
+ * Note: This will not work if you have two such services on a host
+ * because it will assign the same port to each service. In order
+ * to work around that the argument would have to be specified by
+ * the service starter and then published in the Entry[] attributes.
+ *
+ * However, you can use ssh -X to open a tunnel with X
+ * forwarding and then run jconsole locally on the target host
+ * and bring up these data services without enabling remote
+ * JMX.
+ *
+ "-Dcom.sun.management.jmxremote.port=9996",
+ "-Dcom.sun.management.jmxremote.authenticate=false",
+ "-Dcom.sun.management.jmxremote.ssl=false",
+ */
+ };
+
+ serviceCount = bigdata.clientServiceCount;
+
+ constraints = new IServiceConstraint[] {
+
+ new JiniRunningConstraint(),
+ new ZookeeperRunningConstraint(),
+
+ new HostAllowConstraint(bigdata.cs),
+
+ new MaxClientServicesPerHostConstraint(bigdata.maxClientServicePerHost),
+
+ };
+
+ properties = new NV[] {
+
+ };
+
+}
+
+com.bigdata.service.jini.LoadBalancerServer {
+
+ constraints = new IServiceConstraint[] {
+
+ new JiniRunningConstraint(),
+ new ZookeeperRunningConstraint(),
+
+ new HostAllowConstraint(bigdata.lbs)
+
+ };
+
+ args = new String[]{
+ /*
+ * FIXME The load balancer is a big piggish on long runs because it
+ * keeps the performance counter histories in RAM. While those histories
+ * are bounded, it still uses more RAM than it should.
+ */
+ "-Xmx1G",
+ /*
+ * Enable JXM remote management for the data service.
+ *
+ * Note: This will not work if you have two data services on a host
+ * because it will assign the same port to each service. In order
+ * to work around that the argument would have to be specified by
+ * the service starter and then published in the Entry[] attributes.
+ *
+ "-Dcom.sun.management.jmxremote.port=9998",
+ "-Dcom.sun.management.jmxremote.authenticate=false",
+ "-Dcom.sun.management.jmxremote.ssl=false",
+ */
+ };
+
+ /*
+ * Override some properties.
+ */
+ properties = new NV[] {
+
+ /*
+ * Each JiniClient (and hence all bigdata services) can run an
+ * httpd that will expose performance counters for the service and
+ * the host on which it is running. This property specifies the
+ * port for that httpd service. Valid values are port number,
+ * zero (0) for a random open port, MINUS ONE (-1) to disable the
+ * httpd service.
+ *
+ * Note: The load balancer httpd normally uses a known port so
+ * that it is easy to find. This is where you will find all of
+ * the performance counters aggregated for the entire federation,
+ * including their history.
+ */
+ new NV(IBigdataClient.Options.HTTPD_PORT, "@LOAD_BALANCER_PORT@"),
+
+ /*
+ * Note: The load balancer SHOULD NOT collect platform statistics
+ * itself since that interfers with its ability to aggregate
+ * statistics about the host on which it is running. Instead it
+ * should rely on the presence of at least one other service
+ * running on the same host to report those statistics to the load
+ * balancer.
+ */
+ new NV(IBigdataClient.Options.COLLECT_PLATFORM_STATISTICS,"false"),
+
+ /*
+ * The directory where the aggregated statistics will be logged.
+ * The load balancer will write snapshots of the historical
+ * counters into this directory. See LoadBalancerService javadoc
+ * for configuration options which effect how frequently it will
+ * log its counters and how many snapshots will be preserved.
+ *
+ * Note: You only need to specify this option if you want to put
+ * the files into a well known location, e.g, on a shared volume.
+ */
+ //new NV(LoadBalancerServer.Options.LOG_DIR,"/opt2/var/log/bigdata"),
+
+ /* Option essentially turns off the load-based decision making for
+ * this many minutes and substitutes a round-robin policy for
+ * recommending the least utilized data services. The main reason
+ * to this is to force the initial allocation to be distributed as
+ * evenly as possible across the data services in the cluster.
+ */
+ new NV(LoadBalancerServer.Options.INITIAL_ROUND_ROBIN_UPDATE_COUNT,"10"),
+
+ };
+
+}
+
+/**
+ * Configuration options for the KB instance.
+ */
+lubm {
+
+ // The #of universities to generate.
+ // U8000 is 1.2B told triples
+ // U25000 is 3.4B told triples.
+ // U50000 is 6.7B told triples.
+ // U100000 is ~12B told triples.
+ static private univNum = 1000;
+
+ // the KB namespace (based on the #of universities by default).
+ static private namespace = "U"+univNum+"";
+
+ // minimum #of data services to run.
+ static private minDataServices = bigdata.dataServiceCount;
+
+ // How long the master will wait to discover the minimum #of data
+ // services that you specified (ms).
+ static private awaitDataServicesTimeout = 8000;
+
+ /* Multiplier for the scatter effect.
+ */
+ static private scatterFactor = 1;
+ static private scatterFactor_term2id = 1;
+
+ /* The #of index partitions to allocate on a scatter split. ZERO
+ * (0) means that 2 index partitions will be allocated per
+ * data service which partiticpates in the scatter split.
+ * Non-zero values directly give the #of index partitions to
+ * create.
+ */
+ static private scatterSplitIndexPartitionCount = ConfigMath.multiply
+ ( scatterFactor,
+ bigdata.dataServiceCount
+ );
+ static private scatterSplitIndexPartitionCount_term2id = ConfigMath.multiply
+ ( scatterFactor_term2id,
+ bigdata.dataServiceCount
+ );
+
+ // Use all discovered data services when scattering an index.
+ static private scatterSplitDataServiceCount = 0;
+
+ /* Scatter split trigger point. The scatter split will not be
+ * triggered until the initial index partition has reached
+ * this percentage of a nominal index partition in size.
+ */
+ static private scatterSplitPercentOfSplitThreshold = 0.5;//was .5
+
+ /*
+ * Multipliers that compensate for the consumer/producer ratio for
+ * the asynchronous index write API. These are empirical factors
+ * based on observing the ratio (chunkWritingTime/chunkWaitingTime).
+ * Assuming a constant chunk writing time, if the chunk size for each
+ * index is adjusted by its multiplier then this ratio would be 1:1.
+ * In practice, the chunk writing time is not a linear function of
+ * the chunk size, which is one reason why we prefer larger chunks
+ * and why the asynchronous write API is a win.
+ *
+ * Note: These factors were set relative to TERM2ID. However, when
+ * I reduced the scatterFactor for TERM2ID by 1/2, I doubled its
+ * chunk size to keep up the same throughput so it is now at 2.00
+ * rather than 1.00.
+ */
+ static private chunkSizeFactor_id2term = 1.79;
+ static private chunkSizeFactor_term2id = 2.00;
+ static private chunkSizeFactor_spo = 8.00; // was 3.89
+ static private chunkSizeFactor_pos = 8.00; // was 13.37
+ static private chunkSizeFactor_osp = 8.00; // was 27.35
+
+ /* The nominal sink chunk size. For each index, this is adjusted
+ * by the factor specified above.
+ */
+// static private sinkChunkSize = 10000;
+ static private sinkChunkSize = 1000;
+
+ /*
+ * Specify / override some triple store properties.
+ *
+ * Note: You must reference this object in the section for the
+ * component which will actually create the KB instance, e.g.,
+ * either the RDFDataLoadMaster or the LubmGeneratorMaster.
+ */
+ static private properties = new NV[] {
+
+ /*
+ * When "true", the store will perform incremental closure as
+ * the data are loaded. When "false", the closure will be
+ * computed after all data are loaded. (Actually, since we are
+ * not loading through the SAIL making this true does not
+ * cause incremental TM but it does disable closure, so
+ * "false" is what you need here).
+ */
+ new NV(BigdataSail.Options.TRUTH_MAINTENANCE, "false" ),
+
+ /*
+ * Enable rewrites of high-level queries into native rules (native JOIN
+ * execution). (Can be changed without re-loading the data to compare
+ * the performance of the Sesame query evaluation against using the
+ * native rules to perform query evaluation.)
+ */
+ new NV(BigdataSail.Options.NATIVE_JOINS, "true"),
+
+ /*
+ * May be used to turn off inference during query, but will
+ * cause ALL inferences to be filtered out when reading on the
+ * database.
+ */
+ // new NV(BigdataSail.Options.INCLUDE_INFERRED, "false"),
+
+ /*
+ * May be used to turn off query-time expansion of entailments such as
+ * (x rdf:type rdfs:Resource) and owl:sameAs even through those
+ * entailments were not materialized during forward closure (this
+ * disables the backchainer!)
+ */
+ new NV(BigdataSail.Options.QUERY_TIME_EXPANDER, "false"),
+
+ /*
+ * Option to restrict ourselves to RDFS only inference. This
+ * condition may be compared readily to many other stores.
+ *
+ * Note: While we can turn on some kinds of owl processing
+ * (e.g., TransitiveProperty, see below), we can not compute
+ * all the necessary entailments (only queries 11 and 13
+ * benefit).
+ *
+ * Note: There are no owl:sameAs assertions in LUBM.
+ *
+ * Note: lubm query does not benefit from owl:inverseOf.
+ *
+ * Note: lubm query does benefit from owl:TransitiveProperty
+ * (queries 11 and 13).
+ *
+ * Note: owl:Restriction (which we can not compute) plus
+ * owl:TransitiveProperty is required to get all the answers
+ * for LUBM.
+ */
+ new NV(BigdataSail.Options.AXIOMS_CLASS, "com.bigdata.rdf.axioms.RdfsAxioms"),
+ // new NV(BigdataSail.Options.AXIOMS_CLASS,"com.bigdata.rdf.axioms.NoAxioms"),
+
+ /*
+ * Produce a full closure (all entailments) so that the
+ * backward chainer is always a NOP. Note th...
[truncated message content] |
|
From: <tho...@us...> - 2010-08-06 15:14:29
|
Revision: 3422
http://bigdata.svn.sourceforge.net/bigdata/?rev=3422&view=rev
Author: thompsonbry
Date: 2010-08-06 15:14:23 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Made the private lastCommitTime field volatile to ensure visibility.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/btree/BTree.java
Modified: trunk/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-06 15:09:58 UTC (rev 3421)
+++ trunk/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-06 15:14:23 UTC (rev 3422)
@@ -644,7 +644,18 @@
this.lastCommitTime = lastCommitTime;
}
- private long lastCommitTime = 0L;// Until the first commit.
+
+ /**
+ * The lastCommitTime of the {@link Checkpoint} record from which the
+ * {@link BTree} was loaded.
+ * <p>
+ * Note: Made volatile on 8/2/2010 since it is not otherwise obvious what
+ * would guarantee visibility of this field, through I do seem to remember
+ * that visibility might be guaranteed by how the BTree class is discovered
+ * and returned to the class. Still, it does no harm to make this a volatile
+ * read.
+ */
+ volatile private long lastCommitTime = 0L;// Until the first commit.
/**
* Return the {@link IDirtyListener}.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-06 15:10:04
|
Revision: 3421
http://bigdata.svn.sourceforge.net/bigdata/?rev=3421&view=rev
Author: thompsonbry
Date: 2010-08-06 15:09:58 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Reconciled a merge conflict that I had missed in the comments block.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-06 15:02:44 UTC (rev 3420)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-06 15:09:58 UTC (rev 3421)
@@ -201,20 +201,6 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*
-<<<<<<< .working
-=======
- * @todo Checksums and/or record compression are currently handled on a per-
- * {@link BTree} or other persistence capable data structure basis. It is
- * nice to be able to choose for which indices and when ( {@link Journal}
- * vs {@link IndexSegment}) to apply these algorithms. However, it might
- * be nice to factor their application out a bit into a layered api - as
- * long as the right layering is correctly re-established on load of the
- * persistence data structure. In that view the {@link IRawStore} either
- * computes checksums or it does not and the checksums is stored in the
- * record, perhaps in the last 4 bytes. The checksum itself would not be
- * visible at the {@link IRawStore} API layer.
- *
->>>>>>> .merge-right.r3391
* @todo There are lots of annoying ways in which asynchronously closing the
* journal, e.g., using {@link #close()} or {@link #shutdown()} can cause
* exceptions to be thrown out of concurrent threads. It would be nice if
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-06 15:02:50
|
Revision: 3420
http://bigdata.svn.sourceforge.net/bigdata/?rev=3420&view=rev
Author: thompsonbry
Date: 2010-08-06 15:02:44 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Modified the pipeline join to NOT use RMI when pushing binding sets to the same data service.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
Modified: trunk/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-08-06 15:02:14 UTC (rev 3419)
+++ trunk/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-08-06 15:02:44 UTC (rev 3420)
@@ -1074,16 +1074,20 @@
final UUID sinkUUID = locator.getDataServiceUUID();
+ final IDataService dataService;
if (sinkUUID.equals(fed.getServiceUUID())) {
- /*
- * @todo As an optimization, special case when the downstream
- * data service is _this_ data service.
- */
+ /*
+ * As an optimization, special case when the downstream
+ * data service is _this_ data service.
+ */
+ dataService = (IDataService)fed.getService();
+ } else {
+
+ dataService = fed.getDataService(sinkUUID);
+
}
-
- final IDataService dataService = fed.getDataService(sinkUUID);
sink = new JoinTaskSink(fed, locator, this);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-06 15:02:26
|
Revision: 3419
http://bigdata.svn.sourceforge.net/bigdata/?rev=3419&view=rev
Author: thompsonbry
Date: 2010-08-06 15:02:14 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Bug fix to some unit tests which were failing due to the changed initialization in StoreManager.
Added the jgrapht dependency to the top-level build.xml file to fix some unit test failures.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
trunk/bigdata/src/java/com/bigdata/service/AbstractFederation.java
trunk/build.xml
Modified: trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 10:46:50 UTC (rev 3418)
+++ trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 15:02:14 UTC (rev 3419)
@@ -1430,16 +1430,31 @@
}
}
- while (true) {
- if (getFederation().getTransactionService() != null) {
- break;
+ try {
+ final IBigdataFederation<?> fed = getFederation();
+ if (fed == null) {
+ /*
+ * Some of the unit tests do not start the txs until after
+ * the DataService. For those unit tests getFederation()
+ * will return null during startup() of the DataService. To
+ * have a common code path, we throw the exception here
+ * which is caught below.
+ */
+ throw new UnsupportedOperationException();
}
- log.warn("Waiting for transaction service discovery");
+ while (true) {
+ if (fed.getTransactionService() != null) {
+ break;
+ }
+ log.warn("Waiting for transaction service discovery");
+ }
+ } catch (UnsupportedOperationException ex) {
+ log.warn("Federation not available - running in test case?");
}
-
- /*
- * Look for pre-existing data files.
- */
+
+ /*
+ * Look for pre-existing data files.
+ */
if (!isTransient) {
if (log.isInfoEnabled())
Modified: trunk/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 10:46:50 UTC (rev 3418)
+++ trunk/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 15:02:14 UTC (rev 3419)
@@ -829,7 +829,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public T getService() {
@@ -840,7 +840,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public String getServiceName() {
@@ -851,7 +851,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public Class getServiceIface() {
@@ -862,7 +862,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public UUID getServiceUUID() {
@@ -873,7 +873,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public boolean isServiceReady() {
@@ -894,7 +894,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public void reattachDynamicCounters() {
@@ -905,7 +905,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public void didStart() {
@@ -916,7 +916,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public AbstractHTTPD newHttpd(final int httpdPort,
final CounterSet counterSet) throws IOException {
@@ -927,7 +927,10 @@
}
- public void serviceJoin(IService service, UUID serviceUUID) {
+ /**
+ * Delegated. {@inheritDoc}
+ */
+ public void serviceJoin(final IService service, final UUID serviceUUID) {
if (!isOpen()) return;
@@ -941,7 +944,10 @@
}
- public void serviceLeave(UUID serviceUUID) {
+ /**
+ * Delegated. {@inheritDoc}
+ */
+ public void serviceLeave(final UUID serviceUUID) {
if(!isOpen()) return;
@@ -1129,9 +1135,9 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
+
}
-
/**
* Setup sampling on the client's thread pool. This collects interesting
* statistics about the thread pool for reporting to the load balancer
Modified: trunk/build.xml
===================================================================
--- trunk/build.xml 2010-08-06 10:46:50 UTC (rev 3418)
+++ trunk/build.xml 2010-08-06 15:02:14 UTC (rev 3419)
@@ -39,8 +39,8 @@
<target name="clean" description="cleans everything in [build.dir], but not the releases.">
<delete dir="${build.dir}" />
- <delete dir="${bigdata.dir}/bigdata-test" quiet="true" />
- <delete dir="${bigdata.dir}/dist" quiet="true" />
+ <delete dir="${bigdata.dir}/bigdata-test" quiet="true" />
+ <delete dir="${bigdata.dir}/dist" quiet="true" />
</target>
<target name="prepare">
@@ -53,7 +53,7 @@
<istrue value="${snapshot}" />
</condition>
<condition property="osgi.version" value="${build.ver.osgi}.${osgiDate}" else="${build.ver.osgi}.0">
- <istrue value="${snapshot}" />
+ <istrue value="${snapshot}" />
</condition>
<!--<echo message="today=${today}"/>-->
<echo message="version=${version}" />
@@ -69,10 +69,7 @@
<!-- I set the target to 1.5 to support deployment on non-1.6 JVMs. -->
<target name="compile" depends="prepare">
<mkdir dir="${build.dir}" />
- <javac destdir="${build.dir}/classes" classpathref="build.classpath"
- debug="${javac.debug}" debuglevel="${javac.debuglevel}" verbose="${javac.verbose}"
- encoding="${javac.encoding}"
- >
+ <javac destdir="${build.dir}/classes" classpathref="build.classpath" debug="${javac.debug}" debuglevel="${javac.debuglevel}" verbose="${javac.verbose}" encoding="${javac.encoding}">
<!-- note: must also specify -bootclasspath and -extdirs when cross-compiling -->
<!-- target="${javac.target}" source="${javac.source}" -->
<src path="${bigdata.dir}/bigdata/src/java" />
@@ -85,7 +82,7 @@
<src path="${bigdata.dir}/bigdata-rdf/src/test"/>
<src path="${bigdata.dir}/bigdata-sails/src/test"/>
-->
- <compilerarg value="-version"/>
+ <compilerarg value="-version" />
</javac>
<!-- copy resources. -->
<copy toDir="${build.dir}/classes">
@@ -107,15 +104,14 @@
<exclude name="**/package.html" />
</fileset>
<fileset dir="${bigdata.dir}/bigdata-sails/src/resources/sesame-server">
- <include name="META-INF/**"/>
+ <include name="META-INF/**" />
</fileset>
</copy>
</target>
<!-- Builds the bigdata JAR and bundles it together with all of its dependencies in the ${build.dir}/lib directory. -->
- <target name="bundleJar" depends="bundle, jar"
- description="Builds the bigdata JAR and bundles it together with all of its dependencies in the ${build.dir}/lib directory.">
- <copy file="${build.dir}/${version}.jar" todir="${build.dir}/lib"/>
+ <target name="bundleJar" depends="bundle, jar" description="Builds the bigdata JAR and bundles it together with all of its dependencies in the ${build.dir}/lib directory.">
+ <copy file="${build.dir}/${version}.jar" todir="${build.dir}/lib" />
<!--<property name="myclasspath" refid="runtime.classpath" />
<echo message="${myclasspath}"/>-->
</target>
@@ -124,114 +120,55 @@
See 'bundleJar'. -->
<target name="jar" depends="compile" description="Generates the jar (see also bundleJar).">
<jar destfile="${build.dir}/${version}.jar">
- <fileset dir="${build.dir}/classes"
- excludes="test/**" />
+ <fileset dir="${build.dir}/classes" excludes="test/**" />
<manifest>
<!--<attribute name="Main-Class" value="com/bigdata/rdf/rio/TestRioIntegration"/>-->
</manifest>
</jar>
</target>
-
-
+
+
<!-- This generates an osgi bundle jar, and does not bundled the dependencies.
See 'bundleJar'. -->
<target name="osgi" depends="compile, bundle" description="Generates the osgi bundle jar (see also bundleJar).">
- <taskdef resource="aQute/bnd/ant/taskdef.properties" classpath="bigdata/lib/bnd-0.0.384.jar"/>
+ <taskdef resource="aQute/bnd/ant/taskdef.properties" classpath="bigdata/lib/bnd-0.0.384.jar" />
<mkdir dir="${build.dir}/bundles" />
<jar destfile="${build.dir}/bundles/com.bigdata.source_${osgi.version}.jar">
- <manifest>
- <attribute name="Eclipse-SourceBundle" value='com.bigdata;version="${osgi.version}";roots="."'/>
- <attribute name="Bundle-Vendor" value="Systap"/>
- <attribute name="Bundle-Version" value="${build.ver.osgi}"/>
- <attribute name="Bundle-ManifestVersion" value="2"/>
- <attribute name="Bundle-SymbolicName" value="com.bigdata.source"/>
- <attribute name="Bundle-DocURL" value="http://www.bigdata.com"/>
- <attribute name="Bundle-Description" value="Bigdata Source"/>
- </manifest>
- <fileset dir="bigdata/src/java"/>
- <fileset dir="bigdata/src/java" />
- <fileset dir="bigdata-jini/src/java" />
- <fileset dir="bigdata-rdf/src/java" />
- <fileset dir="bigdata-sails/src/java" />
- </jar>
- <bnd
- output="${build.dir}/bundles/com.bigata-${osgi.version}.jar"
- classpath="${build.dir}/classes"
- eclipse="false"
- failok="false"
- exceptions="true"
- files="${basedir}/osgi/bigdata.bnd"/>
-
- <bndwrap
- jars="${build.dir}/lib/unimi/colt-1.2.0.jar"
- output="${build.dir}/bundles/colt-1.2.0.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/unimi/fastutil-5.1.5.jar"
- output="${build.dir}/bundles/fastutil-5.1.5.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/ctc_utils-5-4-2005.jar"
- output="${build.dir}/bundles/ctc_utils-5-4-2005.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/cweb-commons-1.1-b2-dev.jar"
- output="${build.dir}/bundles/cweb-commons-1.1.2.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/cweb-extser-0.1-b2-dev.jar"
- output="${build.dir}/bundles/cweb-extser-1.1.2.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/dsi-utils-1.0.6-020610.jar"
- output="${build.dir}/bundles/dsi-utils-1.0.6-020610.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/iris-0.58.jar"
- output="${build.dir}/bundles/iris-0.58.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/jgrapht-jdk1.5-0.7.1.jar"
- output="${build.dir}/bundles/jgrapht-jdk1.5-0.7.1.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/lgpl-utils-1.0.6-020610.jar"
- output="${build.dir}/bundles/lgpl-utils-1.0.6-020610.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/high-scale-lib-v1.1.2.jar"
- output="${build.dir}/bundles/high-scale-lib-v1.1.2.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/openrdf-sesame-2.3.0-onejar.jar"
- output="${build.dir}/bundles/openrdf-sesame-2.3.0.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/apache/zookeeper-3.2.1.jar"
- output="${build.dir}/bundles/zookeeper-3.2.1.jar"
- definitions="${basedir}/osgi/"
- />
- <bndwrap
- jars="${build.dir}/lib/nxparser-6-22-2010.jar"
- output="${build.dir}/bundles/nxparser-2010.6.22.jar"
- definitions="${basedir}/osgi/"
- />
+ <manifest>
+ <attribute name="Eclipse-SourceBundle" value='com.bigdata;version="${osgi.version}";roots="."' />
+ <attribute name="Bundle-Vendor" value="Systap" />
+ <attribute name="Bundle-Version" value="${build.ver.osgi}" />
+ <attribute name="Bundle-ManifestVersion" value="2" />
+ <attribute name="Bundle-SymbolicName" value="com.bigdata.source" />
+ <attribute name="Bundle-DocURL" value="http://www.bigdata.com" />
+ <attribute name="Bundle-Description" value="Bigdata Source" />
+ </manifest>
+ <fileset dir="bigdata/src/java" />
+ <fileset dir="bigdata/src/java" />
+ <fileset dir="bigdata-jini/src/java" />
+ <fileset dir="bigdata-rdf/src/java" />
+ <fileset dir="bigdata-sails/src/java" />
+ </jar>
+ <bnd output="${build.dir}/bundles/com.bigata-${osgi.version}.jar" classpath="${build.dir}/classes" eclipse="false" failok="false" exceptions="true" files="${basedir}/osgi/bigdata.bnd" />
+
+ <bndwrap jars="${build.dir}/lib/unimi/colt-1.2.0.jar" output="${build.dir}/bundles/colt-1.2.0.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/unimi/fastutil-5.1.5.jar" output="${build.dir}/bundles/fastutil-5.1.5.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/ctc_utils-5-4-2005.jar" output="${build.dir}/bundles/ctc_utils-5-4-2005.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/cweb-commons-1.1-b2-dev.jar" output="${build.dir}/bundles/cweb-commons-1.1.2.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/cweb-extser-0.1-b2-dev.jar" output="${build.dir}/bundles/cweb-extser-1.1.2.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/dsi-utils-1.0.6-020610.jar" output="${build.dir}/bundles/dsi-utils-1.0.6-020610.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/iris-0.58.jar" output="${build.dir}/bundles/iris-0.58.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/jgrapht-jdk1.5-0.7.1.jar" output="${build.dir}/bundles/jgrapht-jdk1.5-0.7.1.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/lgpl-utils-1.0.6-020610.jar" output="${build.dir}/bundles/lgpl-utils-1.0.6-020610.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/high-scale-lib-v1.1.2.jar" output="${build.dir}/bundles/high-scale-lib-v1.1.2.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/openrdf-sesame-2.3.0-onejar.jar" output="${build.dir}/bundles/openrdf-sesame-2.3.0.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/apache/zookeeper-3.2.1.jar" output="${build.dir}/bundles/zookeeper-3.2.1.jar" definitions="${basedir}/osgi/" />
+ <bndwrap jars="${build.dir}/lib/nxparser-6-22-2010.jar" output="${build.dir}/bundles/nxparser-2010.6.22.jar" definitions="${basedir}/osgi/" />
</target>
-
-
+
+
<!-- Note: the javadoc requires a LOT of RAM, but runs quickly on a
server class machine.
@@ -241,7 +178,7 @@
<target name="javadoc" depends="prepare" if="javadoc">
<mkdir dir="${build.dir}/docs/api" />
<javadoc destdir="${build.dir}/docs/api" defaultexcludes="yes" author="true" version="true" use="true" overview="../bigdata/overview.html" windowtitle="bigdata®" classpathref="build.classpath">
- <arg value="-J-Xmx1000m"/>
+ <arg value="-J-Xmx1000m" />
<packageset dir="${bigdata.dir}/bigdata/src/java" />
<packageset dir="${bigdata.dir}/bigdata-jini/src/java" />
<packageset dir="${bigdata.dir}/bigdata-rdf/src/java" />
@@ -413,7 +350,7 @@
</condition>
</fail>
<input message="username:" addproperty="ssh.username2" defaultValue="${ssh.username2}" />
- <!--
+<!--
<input message="password:" addproperty="ssh.password2" defaultValue="${ssh.password2}" />
-->
<exec executable="${ssh.scp}">
@@ -437,18 +374,17 @@
<target name="banner" depends="jar" description="Displays the banner (verifies runtime classpath).">
- <java classname="com.bigdata.Banner"
- failonerror="true" fork="false" logerror="true">
- <classpath refid="runtime.classpath" />
- </java>
+<java classname="com.bigdata.Banner" failonerror="true" fork="false" logerror="true">
+ <classpath refid="runtime.classpath" />
+</java>
</target>
- <!-- -->
- <!-- CLUSTER INSTALL TARGETS -->
- <!-- -->
-
+<!-- -->
+<!-- CLUSTER INSTALL TARGETS -->
+<!-- -->
+
<!--
This is the cluster-based install. You need to edit build.properties, decide
@@ -611,8 +547,8 @@
<!-- set execute bit for scripts in this directory (must be the last step). -->
<chmod perm="u+x,g+rx,o-rwx">
<fileset dir="${install.bin.dir}">
- <exclude name="README"/>
- <exclude name="POST-INSTALL"/>
+ <exclude name="README" />
+ <exclude name="POST-INSTALL" />
</fileset>
</chmod>
<!-- Setup the status file which will be read by the bigdata script and
@@ -628,20 +564,19 @@
<chmod perm="g+rw,o-rw" file="${stateFile}" />
<chmod perm="g+rw,o-rw" file="${stateLog}" />
<!-- Make sure that the entire shared directory structure is read/write for the group. -->
-<chmod perm="g+rwx" type="both" dir="${NAS}" verbose="true"/>
+<chmod perm="g+rwx" type="both" dir="${NAS}" verbose="true" />
<!-- Make sure that it is all accessible to the install group (ant 1.6+ plus extension module required).
<chown file="${NAS}" type="both" owner="${install.user}.${install.group}" verbose="true"/>
-->
<!-- Works for earlier versions of ant LT 1.6 which do not bundle "chown". -->
<apply executable="chown" description="set owner on NAS files" osfamily="unix">
- <arg value="-R"/>
- <arg value="${install.user}.${install.group}"/>
- <dirset dir="${NAS}"/>
+ <arg value="-R" />
+ <arg value="${install.user}.${install.group}" />
+ <dirset dir="${NAS}" />
</apply>
<!-- @todo check the installed configuration file (after parameter substitution). -->
<!-- @todo also check the installed jini configuration files. -->
-<java classname="com.bigdata.jini.util.CheckConfiguration"
- failonerror="true" fork="true" logerror="true">
+<java classname="com.bigdata.jini.util.CheckConfiguration" failonerror="true" fork="true" logerror="true">
<classpath refid="install.classpath" />
<arg value="${bigdata.config}" />
</java>
@@ -655,92 +590,89 @@
target platform and diagnose errors related to a missing or incomplete
sysstat install or other monitoring dependencies. -->
<target name="test-monitoring" depends="compile" description="Run the statistics collectors for the deployment platform.">
- <java classname="com.bigdata.counters.AbstractStatisticsCollector"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="install.classpath" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}"/>
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties"/>
- <arg value="1" /><!-- interval between reports. -->
- <arg value="10" /><!-- #of seconds to run. -->
- </java>
+<java classname="com.bigdata.counters.AbstractStatisticsCollector" failonerror="true" fork="true" logerror="true">
+ <classpath refid="install.classpath" />
+ <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties" />
+ <arg value="1" />
+ <!-- interval between reports. -->
+ <arg value="10" />
+ <!-- #of seconds to run. -->
+</java>
</target>
-
+
<!-- Note: we must fork the JVM to the jvmarg overrides applied. -->
<!-- Note: We disable registration of log4j MBeans since that requires policy file. -->
<!-- @todo add a target to launch the post-mortem counter set/events viewer. -->
<target name="analysis" depends="bundleJar" description="Extracts performance counters from logged XML files.">
- <java classname="com.bigdata.counters.query.CounterSetQuery"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx1500m" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties"/>
- <arg value="-outputDir" />
- <arg value="${analysis.out.dir}" />
- <arg value="-mimeType" />
- <arg value="text/plain" />
- <arg value="-queries" />
- <arg file="${analysis.queries}" />
- <arg file="${analysis.counters.dir}" />
- </java>
-
+<java classname="com.bigdata.counters.query.CounterSetQuery" failonerror="true" fork="true" logerror="true">
+ <classpath refid="runtime.classpath" />
+ <jvmarg value="-Xmx1500m" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties" />
+ <arg value="-outputDir" />
+ <arg value="${analysis.out.dir}" />
+ <arg value="-mimeType" />
+ <arg value="text/plain" />
+ <arg value="-queries" />
+ <arg file="${analysis.queries}" />
+ <arg file="${analysis.counters.dir}" />
+</java>
+
</target>
- <!-- -->
- <!-- LUBM TARGETS (OPTIONAL) -->
- <!-- -->
-
+<!-- -->
+<!-- LUBM TARGETS (OPTIONAL) -->
+<!-- -->
+
<target name="lubm-clean" description="Clean the lubm-integration from the build directory.">
- <delete dir="${build.dir}/lubm" />
+<delete dir="${build.dir}/lubm" />
</target>
<target name="lubm-prepare" description="Clean the lubm-integration from the build directory.">
- <mkdir dir="${build.dir}/lubm" />
- <mkdir dir="${build.dir}/lubm/classes" />
- <mkdir dir="${build.dir}/lubm/lib" />
+<mkdir dir="${build.dir}/lubm" />
+<mkdir dir="${build.dir}/lubm/classes" />
+<mkdir dir="${build.dir}/lubm/lib" />
</target>
<path id="lubm.build.classpath" description="The lubm build-time classpath (this expects to find the bigdata JAR already installed).">
- <fileset dir="${install.lib.dir}">
- <include name="**/*.jar" />
- </fileset>
+<fileset dir="${install.lib.dir}">
+ <include name="**/*.jar" />
+</fileset>
</path>
<!-- And now for something totally weird. If you compile against the bigdata.jar
rather than build.dir/classes then you will see some errors reported in
LubmGeneratorMaster.java which otherwise are not reported... -->
<target name="lubm-compile" depends="lubm-prepare" description="Compile the optional lubm integration.">
- <javac destdir="${build.dir}/lubm/classes" classpathref="runtime.classpath"
- debug="${javac.debug}" debuglevel="${javac.debuglevel}" verbose="${javac.verbose}"
- encoding="${javac.encoding}"
- >
- <!-- note: must also specify -bootclasspath and -extdirs when cross-compiling -->
- <!-- target="${javac.target}" source="${javac.source}" -->
- <src path="${bigdata.dir}/bigdata-lubm/src/java" />
- </javac>
- <!-- copy resources. -->
- <copy toDir="${build.dir}/lubm/classes">
- <fileset dir="${bigdata.dir}/bigdata-lubm/src/java">
- <exclude name="**/*.java" />
- </fileset>
- </copy>
+<javac destdir="${build.dir}/lubm/classes" classpathref="runtime.classpath" debug="${javac.debug}" debuglevel="${javac.debuglevel}" verbose="${javac.verbose}" encoding="${javac.encoding}">
+ <!-- note: must also specify -bootclasspath and -extdirs when cross-compiling -->
+ <!-- target="${javac.target}" source="${javac.source}" -->
+ <src path="${bigdata.dir}/bigdata-lubm/src/java" />
+</javac>
+<!-- copy resources. -->
+<copy toDir="${build.dir}/lubm/classes">
+ <fileset dir="${bigdata.dir}/bigdata-lubm/src/java">
+ <exclude name="**/*.java" />
+ </fileset>
+</copy>
</target>
<target name="lubm-jar" depends="lubm-compile" description="Generates the JAR containing the optional LUBM integration.">
- <jar destfile="${build.dir}/lubm/lib/bigdata-lubm.jar">
- <fileset dir="${build.dir}/lubm/classes" />
- </jar>
+<jar destfile="${build.dir}/lubm/lib/bigdata-lubm.jar">
+ <fileset dir="${build.dir}/lubm/classes" />
+</jar>
</target>
<!-- This explicitly enumerates the lubm scripts so we don't run
fixcrlf or set the execute bit on arbitrary files in the
install directory. -->
<fileset dir="${install.bin.dir}" id="lubm-scripts" description="The lubm scripts.">
- <include name="lubmMaster.sh" />
- <include name="lubmQuery.sh" />
- <include name="lubmGen.sh" />
+<include name="lubmMaster.sh" />
+<include name="lubmQuery.sh" />
+<include name="lubmGen.sh" />
</fileset>
<!-- While this installs the LUBM integration into the same place as the
@@ -751,287 +683,270 @@
a model for how to install and run your own software against a bigdata
federation that is already up and running. -->
<target name="lubm-install" depends="lubm-jar" description="Install the optional lubm integration which may be used for benchmarking the RDF database.">
- <mkdir dir="${install.lubm.dir}"/>
- <mkdir dir="${install.lubm.lib.dir}"/>
- <mkdir dir="${install.lubm.config.dir}"/>
- <!-- install JAR. -->
- <copy toDir="${install.lubm.lib.dir}" file="${build.dir}/lubm/lib/bigdata-lubm.jar"/>
- <!-- install ontology, configuration files, and query files. -->
- <copy toDir="${install.lubm.config.dir}">
- <fileset dir="bigdata-lubm/resources/config"/>
- </copy>
- <!-- install scripts. -->
- <copy toDir="${install.bin.dir}">
- <fileset dir="bigdata-lubm/resources/scripts"/>
- </copy>
- <!-- replace will only find those @XXX@ parameters which have not yet been
+<mkdir dir="${install.lubm.dir}" />
+<mkdir dir="${install.lubm.lib.dir}" />
+<mkdir dir="${install.lubm.config.dir}" />
+<!-- install JAR. -->
+<copy toDir="${install.lubm.lib.dir}" file="${build.dir}/lubm/lib/bigdata-lubm.jar" />
+<!-- install ontology, configuration files, and query files. -->
+<copy toDir="${install.lubm.config.dir}">
+ <fileset dir="bigdata-lubm/resources/config" />
+</copy>
+<!-- install scripts. -->
+<copy toDir="${install.bin.dir}">
+ <fileset dir="bigdata-lubm/resources/scripts" />
+</copy>
+<!-- replace will only find those @XXX@ parameters which have not yet been
transcribed out by the bigdata ant install. -->
- <replace dir="${install.bin.dir}" summary="true">
- <replacefilter token="@NAS@" value="${NAS}" />
- <replacefilter token="@BIN_DIR@" value="${install.bin.dir}" />
- <replacefilter token="@BIGDATA_CONFIG@" value="${bigdata.config}" />
- <replacefilter token="@LUBM_CLASS_SERVER_PORT@" value="${LUBM_CLASS_SERVER_PORT}" />
- <replacefilter token="@LUBM_CLASS_SERVER_HOSTNAME@" value="${LUBM_CLASS_SERVER_HOSTNAME}" />
- <replacefilter token="@LUBM_RMI_CODEBASE_URL@" value="${LUBM_RMI_CODEBASE_URL}" />
- <replacefilter token="@install.lubm.lib.dir@" value="${install.lubm.lib.dir}" />
- <replacefilter token="@install.lubm.config.dir@" value="${install.lubm.config.dir}" />
- </replace>
- <replace dir="${install.lubm.config.dir}" summary="true">
- <replacefilter token="@NAS@" value="${NAS}" />
- <replacefilter token="@BIN_DIR@" value="${install.bin.dir}" />
- <replacefilter token="@BIGDATA_CONFIG@" value="${bigdata.config}" />
- <replacefilter token="@LUBM_CLASS_SERVER_PORT@" value="${LUBM_CLASS_SERVER_PORT}" />
- <replacefilter token="@LUBM_CLASS_SERVER_HOSTNAME@" value="${LUBM_CLASS_SERVER_HOSTNAME}" />
- <replacefilter token="@LUBM_RMI_CODEBASE_URL@" value="${LUBM_RMI_CODEBASE_URL}" />
- <replacefilter token="@install.lubm.lib.dir@" value="${install.lubm.lib.dir}" />
- <replacefilter token="@install.lubm.config.dir@" value="${install.lubm.config.dir}" />
- </replace>
- <!-- fix newlines (otherwise substitutions cause things to break). -->
- <fixcrlf srcDir="${install.bin.dir}" >
- <!-- file set not supported. <fileset refid="scripts" /> -->
- </fixcrlf>
- <!-- set execute bit for scripts in this directory (must be the last step). -->
- <chmod perm="u+x,g+rx,o-rwx">
- <fileset refid="lubm-scripts" />
- </chmod>
- <!-- Make sure that it is all accessible to the install group (ant 1.6+ plus extension module required).
+<replace dir="${install.bin.dir}" summary="true">
+ <replacefilter token="@NAS@" value="${NAS}" />
+ <replacefilter token="@BIN_DIR@" value="${install.bin.dir}" />
+ <replacefilter token="@BIGDATA_CONFIG@" value="${bigdata.config}" />
+ <replacefilter token="@LUBM_CLASS_SERVER_PORT@" value="${LUBM_CLASS_SERVER_PORT}" />
+ <replacefilter token="@LUBM_CLASS_SERVER_HOSTNAME@" value="${LUBM_CLASS_SERVER_HOSTNAME}" />
+ <replacefilter token="@LUBM_RMI_CODEBASE_URL@" value="${LUBM_RMI_CODEBASE_URL}" />
+ <replacefilter token="@install.lubm.lib.dir@" value="${install.lubm.lib.dir}" />
+ <replacefilter token="@install.lubm.config.dir@" value="${install.lubm.config.dir}" />
+</replace>
+<replace dir="${install.lubm.config.dir}" summary="true">
+ <replacefilter token="@NAS@" value="${NAS}" />
+ <replacefilter token="@BIN_DIR@" value="${install.bin.dir}" />
+ <replacefilter token="@BIGDATA_CONFIG@" value="${bigdata.config}" />
+ <replacefilter token="@LUBM_CLASS_SERVER_PORT@" value="${LUBM_CLASS_SERVER_PORT}" />
+ <replacefilter token="@LUBM_CLASS_SERVER_HOSTNAME@" value="${LUBM_CLASS_SERVER_HOSTNAME}" />
+ <replacefilter token="@LUBM_RMI_CODEBASE_URL@" value="${LUBM_RMI_CODEBASE_URL}" />
+ <replacefilter token="@install.lubm.lib.dir@" value="${install.lubm.lib.dir}" />
+ <replacefilter token="@install.lubm.config.dir@" value="${install.lubm.config.dir}" />
+</replace>
+<!-- fix newlines (otherwise substitutions cause things to break). -->
+<fixcrlf srcDir="${install.bin.dir}">
+ <!-- file set not supported. <fileset refid="scripts" /> -->
+</fixcrlf>
+<!-- set execute bit for scripts in this directory (must be the last step). -->
+<chmod perm="u+x,g+rx,o-rwx">
+ <fileset refid="lubm-scripts" />
+</chmod>
+<!-- Make sure that it is all accessible to the install group (ant 1.6+ plus extension module required).
<chown file="${NAS}" type="both" owner="${install.user}.${install.group}" verbose="true"/>
-->
- <!-- Works for earlier versions of ant LT 1.6 which do not bundle "chown". -->
- <apply executable="chown" description="set owner on NAS files" osfamily="unix">
- <arg value="-R"/>
- <arg value="${install.user}.${install.group}"/>
- <dirset dir="${install.bin.dir}"/>
- </apply>
- <apply executable="chown" description="set owner on NAS files" osfamily="unix">
- <arg value="-R"/>
- <arg value="${install.user}.${install.group}"/>
- <dirset dir="${install.lubm.dir}"/>
- </apply>
+<!-- Works for earlier versions of ant LT 1.6 which do not bundle "chown". -->
+<apply executable="chown" description="set owner on NAS files" osfamily="unix">
+ <arg value="-R" />
+ <arg value="${install.user}.${install.group}" />
+ <dirset dir="${install.bin.dir}" />
+</apply>
+<apply executable="chown" description="set owner on NAS files" osfamily="unix">
+ <arg value="-R" />
+ <arg value="${install.user}.${install.group}" />
+ <dirset dir="${install.lubm.dir}" />
+</apply>
</target>
<!-- lubm runtime classpath w/o install. -->
<path id="lubm.runtime.classpath">
- <pathelement location="${build.dir}/lubm/classes"/>
- <pathelement location="${build.dir}/classes" />
- <path refid="build.classpath" />
+<pathelement location="${build.dir}/lubm/classes" />
+<pathelement location="${build.dir}/classes" />
+<path refid="build.classpath" />
</path>
<target name="lubm-load" depends="jar, lubm-compile" description="Load data into a configured lubm test harness, typically standalone.">
- <java classname="edu.lehigh.swat.bench.ubt.Test"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="lubm.runtime.classpath"/>
- <jvmarg value="-server" />
- <jvmarg value="-Xmx1024m" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties"/>
- <arg value="load" />
- <arg value="bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/config.kb.bigdata" />
- </java>
-
+<java classname="edu.lehigh.swat.bench.ubt.Test" failonerror="true" fork="true" logerror="true">
+ <classpath refid="lubm.runtime.classpath" />
+ <jvmarg value="-server" />
+ <jvmarg value="-Xmx1024m" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties" />
+ <arg value="load" />
+ <arg value="bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/config.kb.bigdata" />
+</java>
+
</target>
-
+
<target name="lubm-test" depends="jar, lubm-compile" description="Run queries against a configured lubm test harness, typically standalone.">
- <java classname="edu.lehigh.swat.bench.ubt.Test"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="lubm.runtime.classpath" />
- <jvmarg value="-server" />
- <jvmarg value="-Xmx1024m" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties"/>
- <arg value="query" />
- <arg value="bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/config.kb.bigdata" />
- <arg value="bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/config.query.sparql" />
- </java>
-
+<java classname="edu.lehigh.swat.bench.ubt.Test" failonerror="true" fork="true" logerror="true">
+ <classpath refid="lubm.runtime.classpath" />
+ <jvmarg value="-server" />
+ <jvmarg value="-Xmx1024m" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dlog4j.configuration=file:bigdata/src/resources/logging/log4j.properties" />
+ <arg value="query" />
+ <arg value="bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/config.kb.bigdata" />
+ <arg value="bigdata-lubm/src/java/edu/lehigh/swat/bench/ubt/bigdata/config.query.sparql" />
+</java>
+
</target>
- <!-- -->
- <!-- STANDALONE FEDERATION TARGETS -->
- <!-- (test/benchamarking) -->
+<!-- -->
+<!-- STANDALONE FEDERATION TARGETS -->
+<!-- (test/benchamarking) -->
<target name="generateLookupStarterJar" unless="lookupStarterJarAvailable">
- <antcall target="testCompile" />
+<antcall target="testCompile" />
</target>
<target name="testLookupStarterJarAvailability">
- <property name="bigdata-test.lib" location="${bigdata.dir}/bigdata-test/lib" />
- <condition property="lookupStarterJarAvailable">
- <available file="${bigdata-test.lib}/lookupstarter.jar"/>
- </condition>
+<property name="bigdata-test.lib" location="${bigdata.dir}/bigdata-test/lib" />
+<condition property="lookupStarterJarAvailable">
+ <available file="${bigdata-test.lib}/lookupstarter.jar" />
+</condition>
</target>
<target name="standalone-setup" depends="testLookupStarterJarAvailability,generateLookupStarterJar" description="Setup properties used by standalone federation and LUS start/stop.">
- <property name="app.home" location="${bigdata.dir}" />
- <property name="test.codebase.port" value="23333"/>
- <property name="test.codebase.dir" location="${bigdata.dir}/bigdata-jini/lib/jini/lib-dl"/>
- <property name="dist.lib" location="${bigdata.dir}/bigdata-jini/lib/jini/lib" />
- <property name="dist.lib.dl" location="${bigdata.dir}/bigdata-jini/lib/jini/lib-dl" />
- <property name="test.codebase" value="http://${this.hostname}:${test.codebase.port}/jsk-dl.jar"/>
- <property name="java.security.policy" value="${bigdata.dir}/policy.all"/>
- <property name="log4j.configuration" value="resources/logging/log4j.properties"/>
- <property name="java.net.preferIPv4Stack" value="true"/>
- <property name="bigdata.fedname" value="${standalone.fed}"/>
+<property name="app.home" location="${bigdata.dir}" />
+<property name="test.codebase.port" value="23333" />
+<property name="test.codebase.dir" location="${bigdata.dir}/bigdata-jini/lib/jini/lib-dl" />
+<property name="dist.lib" location="${bigdata.dir}/bigdata-jini/lib/jini/lib" />
+<property name="dist.lib.dl" location="${bigdata.dir}/bigdata-jini/lib/jini/lib-dl" />
+<property name="test.codebase" value="http://${this.hostname}:${test.codebase.port}/jsk-dl.jar" />
+<property name="java.security.policy" value="${bigdata.dir}/policy.all" />
+<property name="log4j.configuration" value="resources/logging/log4j.properties" />
+<property name="java.net.preferIPv4Stack" value="true" />
+<property name="bigdata.fedname" value="${standalone.fed}" />
</target>
<!-- Note: You should 'nohup' this, e.g., "nohup ant standalone-start" to
avoid taking down the ServicesManagerServer if you are disconnected
from a terminal. -->
<target name="standalone-start" depends="jar,standalone-setup" description="Start the standalone federation.">
- <!-- Start the lookup service. -->
- <antcall target="startHttpd" />
- <antcall target="startLookup" />
- <java classname="com.bigdata.jini.start.ServicesManagerServer"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx200m" />
- <jvmarg value="-showversion"/>
- <!-- The name of the federation instance. -->
- <jvmarg value="-Dbigdata.fedname=${standalone.fed}"/>
- <jvmarg value="-Djava.security.policy=policy.all"/>
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
- <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties"/>
- <arg value="src/resources/config/standalone/bigdataStandalone.config" />
- </java>
+<!-- Start the lookup service. -->
+<antcall target="startHttpd" />
+<antcall target="startLookup" />
+<java classname="com.bigdata.jini.start.ServicesManagerServer" failonerror="true" fork="true" logerror="true">
+ <classpath refid="runtime.classpath" />
+ <jvmarg value="-Xmx200m" />
+ <jvmarg value="-showversion" />
+ <!-- The name of the federation instance. -->
+ <jvmarg value="-Dbigdata.fedname=${standalone.fed}" />
+ <jvmarg value="-Djava.security.policy=policy.all" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
+ <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties" />
+ <arg value="src/resources/config/standalone/bigdataStandalone.config" />
+</java>
</target>
<target name="standalone-stop" depends="jar,standalone-setup" description="Stop the standalone federation.">
- <java classname="com.bigdata.service.jini.util.ShutdownFederation"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx200m" />
- <jvmarg value="-showversion"/>
- <!-- The name of the federation instance. -->
- <jvmarg value="-Dbigdata.fedname=${standalone.fed}"/>
- <jvmarg value="-Djava.security.policy=policy.all"/>
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
- <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties"/>
- <arg value="src/resources/config/standalone/bigdataStandalone.config" />
- </java>
- <!-- Then take down the lookup service as well. -->
- <antcall target="stopLookup" />
- <antcall target="stopHttpd" />
+<java classname="com.bigdata.service.jini.util.ShutdownFederation" failonerror="true" fork="true" logerror="true">
+ <classpath refid="runtime.classpath" />
+ <jvmarg value="-Xmx200m" />
+ <jvmarg value="-showversion" />
+ <!-- The name of the federation instance. -->
+ <jvmarg value="-Dbigdata.fedname=${standalone.fed}" />
+ <jvmarg value="-Djava.security.policy=policy.all" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
+ <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties" />
+ <arg value="src/resources/config/standalone/bigdataStandalone.config" />
+</java>
+<!-- Then take down the lookup service as well. -->
+<antcall target="stopLookup" />
+<antcall target="stopHttpd" />
</target>
<target name="standalone-start-nano-server" depends="jar" description="Start a small http server fronting for a bigdata database instance.">
- <java classname="com.bigdata.rdf.sail.bench.NanoSparqlServer" fork="true" failonerror="true">
- <arg line="${standalone.nanoServerPort} ${standalone.namespace} src/resources/config/standalone/bigdataStandalone.config" />
- <jvmarg line="-server"/>
- <jvmarg line="-Xmx200M"/>
- <classpath refid="runtime.classpath" />
- </java>
+<java classname="com.bigdata.rdf.sail.bench.NanoSparqlServer" fork="true" failonerror="true">
+ <arg line="${standalone.nanoServerPort} ${standalone.namespace} src/resources/config/standalone/bigdataStandalone.config" />
+ <jvmarg line="-server" />
+ <jvmarg line="-Xmx200M" />
+ <classpath refid="runtime.classpath" />
+</java>
</target>
<target name="standalone-stop-nano-server" depends="jar" description="Stop the small http server running at the configured port.">
- <java classname="com.bigdata.rdf.sail.bench.NanoSparqlServer" fork="true" failonerror="true">
- <arg line="${standalone.nanoServerPort} -stop" />
- <classpath refid="runtime.classpath" />
- </java>
+<java classname="com.bigdata.rdf.sail.bench.NanoSparqlServer" fork="true" failonerror="true">
+ <arg line="${standalone.nanoServerPort} -stop" />
+ <classpath refid="runtime.classpath" />
+</java>
</target>
<target name="standalone-bulk-load" depends="jar" description="Bulk load RDF data into the standalone federation.">
- <java classname="com.bigdata.rdf.load.MappedRDFDataLoadMaster"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx200m" />
- <jvmarg value="-showversion"/>
- <!-- The name of the federation instance. -->
- <jvmarg value="-Dbigdata.fedname=${standalone.fed}"/>
- <jvmarg value="-Djava.security.policy=policy.all"/>
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
- <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties"/>
- <!-- -->
- <!-- Per job parameters -->
- <!-- -->
- <!-- The namespace of the target KB. -->
- <jvmarg value="-Dbigdata.rdf.namespace=${standalone.namespace}"/>
- <!-- The job name (same as the KB namespace is a common default). -->
- <jvmarg value="-Dbigdata.rdf.job.name=bulk-load-kb-${standalone-namespace}"/>
- <!-- The file or directory containing zero or more files to be loaded first. -->
- <jvmarg value="-Dbigdata.rdf.ontology=${standalone.bulkLoad.ontology}"/>
- <!-- The file or directory containing RDF data to be loaded. -->
- <jvmarg value="-Dbigdata.rdf.data=${standalone.bulkLoad.data}"/>
- <!-- The main configuration file. -->
- <arg value="src/resources/config/standalone/bigdataStandalone.config" />
- </java>
+<java classname="com.bigdata.rdf.load.MappedRDFDataLoadMaster" failonerror="true" fork="true" logerror="true">
+ <classpath refid="runtime.classpath" />
+ <jvmarg value="-Xmx200m" />
+ <jvmarg value="-showversion" />
+ <!-- The name of the federation instance. -->
+ <jvmarg value="-Dbigdata.fedname=${standalone.fed}" />
+ <jvmarg value="-Djava.security.policy=policy.all" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dcom.bigdata.counters.linux.sysstat.path=${SYSSTAT_HOME}" />
+ <jvmarg value="-Dlog4j.configuration=file:src/resources/config/standalone/log4j.properties" />
+ <!-- -->
+ <!-- Per job parameters -->
+ <!-- -->
+ <!-- The namespace of the target KB. -->
+ <jvmarg value="-Dbigdata.rdf.namespace=${standalone.namespace}" />
+ <!-- The job name (same as the KB namespace is a common default). -->
+ <jvmarg value="-Dbigdata.rdf.job.name=bulk-load-kb-${standalone-namespace}" />
+ <!-- The file or directory containing zero or more files to be loaded first. -->
+ <jvmarg value="-Dbigdata.rdf.ontology=${standalone.bulkLoad.ontology}" />
+ <!-- The file or directory containing RDF data to be loaded. -->
+ <jvmarg value="-Dbigdata.rdf.data=${standalone.bulkLoad.data}" />
+ <!-- The main configuration file. -->
+ <arg value="src/resources/config/standalone/bigdataStandalone.config" />
+</java>
</target>
- <!-- -->
- <!-- MISC. UTILITY TARGETS -->
- <!-- -->
-
+<!-- -->
+<!-- MISC. UTILITY TARGETS -->
+<!-- -->
+
<target name="scale-out-sample" description="Run the scale-out sample code.">
- <javac destdir="${build.dir}/classes" classpathref="build.classpath"
- debug="${javac.debug}" debuglevel="${javac.debuglevel}" verbose="${javac.verbose}"
- encoding="${javac.encoding}"
- >
- <src path="${bigdata.dir}/bigdata-sails/src/samples" />
- </javac>
- <!-- copy resources. -->
- <copy toDir="${build.dir}/classes">
- <fileset dir="${bigdata.dir}/bigdata-sails/src/samples">
- <exclude name="**/*.java" />
- <exclude name="**/package.html" />
- </fileset>
- </copy>
- <java classname="com.bigdata.samples.ScaleOut"
- failonerror="true" fork="true" logerror="true">
- <classpath refid="runtime.classpath" />
- <jvmarg value="-Xmx1500m" />
- <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
- <jvmarg value="-Dlog4j.configuration=file:bigdata-sails/src/samples/com/bigdata/samples/log4j.properties"/>
- <arg value="${bigdata.config}" />
- </java>
-
+<javac destdir="${build.dir}/classes" classpathref="build.classpath" debug="${javac.debug}" debuglevel="${javac.debuglevel}" verbose="${javac.verbose}" encoding="${javac.encoding}">
+ <src path="${bigdata.dir}/bigdata-sails/src/samples" />
+</javac>
+<!-- copy resources. -->
+<copy toDir="${build.dir}/classes">
+ <fileset dir="${bigdata.dir}/bigdata-sails/src/samples">
+ <exclude name="**/*.java" />
+ <exclude name="**/package.html" />
+ </fileset>
+</copy>
+<java classname="com.bigdata.samples.ScaleOut" failonerror="true" fork="true" logerror="true">
+ <classpath refid="runtime.classpath" />
+ <jvmarg value="-Xmx1500m" />
+ <jvmarg value="-Dcom.bigdata.jmx.log4j.disable=true" />
+ <jvmarg value="-Dlog4j.configuration=file:bigdata-sails/src/samples/com/bigdata/samples/log4j.properties" />
+ <arg value="${bigdata.config}" />
+</java>
+
</target>
<target name="DataLoader" depends="compile" description="Loads RDF data into a local KB. You MUST edit this ant target before running it.">
- <java classname="com.bigdata.rdf.store.DataLoader"
- fork="true"
- failonerror="true"
- >
- <!-- usage: [-namespace namespace] propertyFile (fileOrDir)+ -->
- <!-- Where: -->
- <!-- [-namespace namespace] is the KB namespace (default is 'kb'). -->
- <!-- propertyFile is a properties file identifying the Journal and
+<java classname="com.bigdata.rdf.store.DataLoader" fork="true" failonerror="true">
+ <!-- usage: [-namespace namespace] propertyFile (fileOrDir)+ -->
+ <!-- Where: -->
+ <!-- [-namespace namespace] is the KB namespace (default is 'kb'). -->
+ <!-- propertyFile is a properties file identifying the Journal and
giving various Journal and/or kb configuration properties if one
or the other needs to be created. -->
- <!-- (fileOrDir)+ is a list of one or more RDF files or directories to
+ <!-- (fileOrDir)+ is a list of one or more RDF files or directories to
be loaded. zip and gz extensions are recognized, but only one file
is loaded per archive. -->
- <arg line="custom.properties fileOrDir"/>
- <jvmarg value="-server"/>
- <!-- Specify the maximum Java heap size. -->
- <jvmarg value="-Xmx10g"/>
- <!-- optionally enable yourkit profiler.
+ <arg line="custom.properties fileOrDir" />
+ <jvmarg value="-server" />
+ <!-- Specify the maximum Java heap size. -->
+ <jvmarg value="-Xmx10g" />
+ <!-- optionally enable yourkit profiler.
<jvmarg value="-DLD_LIBRARY_PATH=/nas/install/yjp-8.0.19/bin/linux-x86-64"/>
<jvmarg value="-agentpath:/nas/install/yjp-8.0.20/bin/linux-x86-64/libyjpagent.so"/>
<jvmarg value="-agentlib:yjpagent=disableexceptiontelemetry,disablestacktelemetry"/>
-->
- <jvmarg value="-XX:+UseParallelOldGC"/>
- <!-- Optional enable GC trace.
+ <jvmarg value="-XX:+UseParallelOldGC" />
+ <!-- Optional enable GC trace.
<jvmarg line="-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:jvm_gc.log"/>
-->
- <classpath>
- <path refid="runtime.classpath" />
- </classpath>
- </java>
+ <classpath>
+ <path refid="runtime.classpath" />
+ </classpath>
+</java>
</target>
-<target name="set-properties" depends="compile"
- description="Set or change properties for a kb instance. You MUST edit this target to specify the name of the journal. The new values are read from stdin.">
- <java classname="com.bigdata.rdf.sail.BigdataSailHelper"
- fork="true"
- failonerror="true"
- inputstring="com.bigdata.relation.rule.eval.DefaultRuleTaskFactory.nestedSubquery=false"
- >
-<!-- Various things you might want to change:
+<target name="set-properties" depends="compile" description="Set or change properties for a kb instance. You MUST edit this target to specify the name of the journal. The new values are read from stdin.">
+<java classname="com.bigdata.rdf.sail.BigdataSailHelper" fork="true" failonerror="true" inputstring="com.bigdata.relation.rule.eval.DefaultRuleTaskFactory.nestedSubquery=false">
+ <!-- Various things you might want to change:
Maximum #of threads for joins.
inputstring="com.bigdata.relation.rule.eval.ProgramTask.maxParallelSubqueries=5"
@@ -1040,530 +955,466 @@
inputstring="com.bigdata.relation.rule.eval.DefaultRuleTaskFactory.nestedSubquery=true"
-->
- <arg line="d:/LTS.U50.jnl LTS kb"/>
- <classpath>
- <path refid="runtime.classpath" />
- </classpath>
- </java>
+ <arg line="d:/LTS.U50.jnl LTS kb" />
+ <classpath>
+ <path refid="runtime.classpath" />
+ </classpath>
+</java>
</target>
- <!-- -->
- <!-- STAGING -->
- <!-- -->
- <target name="stage"
- description="stages resources (jar, config, policy, logging files) needed to package or execute the bigdata distribution."
- depends="jar">
+<!-- -->
+<!-- STAGING -->
+<!-- -->
+<target name="stage" description="stages resources (jar, config, policy, logging files) needed to package or execute the bigdata distribution." depends="jar">
- <!-- Create staging directories -->
- <property name="dist.dir" location="${bigdata.dir}/dist/bigdata" />
+<!-- Create staging directories -->
+<property name="dist.dir" location="${bigdata.dir}/dist/bigdata" />
- <property name="dist.bin" location="${dist.dir}/bin" />
- <property name="dist.bin.config" location="${dist.bin}/config" />
- <property name="dist.lib" location="${dist.dir}/lib" />
- <property name="dist.lib.dl" location="${dist.dir}/lib-dl" />
- <property name="dist.lib.ext" location="${dist.dir}/lib-ext" />
- <property name="dist.var" location="${dist.dir}/var" />
+<property name="dist.bin" location="${dist.dir}/bin" />
+<property name="dist.bin.config" location="${dist.bin}/config" />
+<property name="dist.lib" location="${dist.dir}/lib" />
+<property name="dist.lib.dl" location="${dist.dir}/lib-dl" />
+<property name="dist.lib.ext" location="${dist.dir}/lib-ext" />
+<property name="dist.var" location="${dist.dir}/var" />
- <property name="dist.var.config" location="${dist.var}/config" />
- <property name="dist.var.config.policy" location="${dist.var.config}/policy" />
- <property name="dist.var.config.logging" location="${dist.var.config}/logging" />
- <property name="dist.var.config.jini" location="${dist.var.config}/jini" />
+<property name="dist.var.config" location="${dist.var}/config" />
+<property name="dist.var.config.policy" location="${dist.var.config}/policy" />
+<property name="dist.var.config.logging" location="${dist.var.config}/logging" />
+<property name="dist.var.config.jini" location="${dist.var.config}/jini" />
- <delete dir="${dist.dir}" quiet="true"/>
- <mkdir dir="${dist.dir}"/>
- <mkdir dir="${dist.bin}"/>
- <mkdir dir="${dist.lib}"/>
- <mkdir dir="${dist.lib.dl}"/>
- <mkdir dir="${dist.lib.ext}"/>
- <mkdir dir="${dist.var}"/>
- <mkdir dir="${dist.var.config}"/>
- <mkdir dir="${dist.var.config.policy}"/>
- <mkdir dir="${dist.var.config.logging}"/>
- <mkdir dir="${dist.var.config.jini}"/>
+<delete dir="${dist.dir}" quiet="true" />
+<mkdir dir="${dist.dir}" />
+<mkdir dir="${dist.bin}" />
+<mkdir dir="${dist.lib}" />
+<mkdir dir="${dist.lib.dl}" />
+<mkdir dir="${dist.lib.ext}" />
+<mkdir dir="${dist.var}" />
+<mkdir dir="${dist.var.config}" />
+<mkdir dir="${dist.var.config.policy}" />
+<mkdir dir="${dist.var.config.logging}" />
+<mkdir dir="${dist.var.config.jini}" />
- <!-- Copy the jar files created by the jar target to -->
- <!-- an application-specific but non-version-specific -->
- <!-- jar file to either the lib or lib-dl staging -->
- <!-- directory. When a new version of a given application's -->
- <!-- jar file becomes available, the version-specific jar -->
- <!-- file name should be changed here. -->
+<!-- Copy the jar files created by the jar target to -->
+<!-- an application-specific but non-version-specific -->
+<!-- jar file to either the lib or lib-dl staging -->
+<!-- directory. When a new version of a given application's -->
+<!-- jar file becomes available, the version-specific jar -->
+<!-- file name should be changed here. -->
- <property name="bigdata.lib" location="${bigdata.dir}/bigdata/lib" />
- <property name="bigdata-jini.dir" location="${bigdata.dir}/bigdata-jini" />
- <property name="bigdata-jini.lib" location="${bigdata.dir}/bigdata-jini/lib/jini/lib" />
- <property name="bigdata-rdf.lib" location="${bigdata.dir}/bigdata-rdf/lib" />
- <property name="bigdata-sails.lib" location="${bigdata.dir}/bigdata-sails/lib" />
- <property name="bigdata-zookeeper.lib" location="${bigdata.dir}/bigdata-jini/lib/apache" />
+<property name="bigdata.lib" location="${bigdata.dir}/bigdata/lib" />
+<property name="bigdata-jini.dir" location="${bigdata.dir}/bigdata-jini" />
+<property name="bigdata-jini.lib" location="${bigdata.dir}/bigdata-jini/lib/jini/lib" />
+<property name="bigdata-rdf.lib" location="${bigdata.dir}/bigdata-rdf/lib" />
+<property name="bigdata-sails.lib" location="${bigdata.dir}/bigdata-sails/lib" />
+<property name="bigdata-zookeeper.lib" location="${bigdata.dir}/bigdata-jini/lib/apache" />
- <!-- Utility libraries -->
+<!-- Utility libraries -->
- <copy file="${bigdata.lib}/unimi/colt-1.2.0.jar"
- tofile="${dist.lib}/colt.jar"/>
- <copy file="${bigdata.lib}/ctc_utils-5-4-2005.jar"
- tofile="${dist.lib}/ctc_utils.jar"/>
- <copy file="${bigdata.lib}/cweb-commons-1.1-b2-dev.jar"
- tofile="${dist.lib}/cweb-commons.jar"/>
- <copy file="${bigdata.lib}/cweb-extser-0.1-b2-dev.jar"
- tofile="${dist.lib}/cweb-extser.jar"/>
- <copy file="${bigdata.lib}/high-scale-lib-v1.1.2.jar"
- tofile="${dist.lib}/highscalelib.jar"/>
- <copy file="${bigdata.lib}/dsi-utils-1.0.6-020610.jar"
- tofile="${dist.lib}/dsiutils.jar"/>
- <copy file="${bigdata.lib}/lgpl-utils-1.0.6-020610.jar"
- tofile="${dist.lib}/lgplutils.jar"/>
- <copy file="${bigdata.lib}/unimi/fastutil-5.1.5.jar"
- tofile="${dist.lib}/fastutil.jar"/>
- <copy file="${bigdata.lib}/icu/icu4j-3_6.jar"
- tofile="${dist.lib}/icu4j.jar"/>
- <copy file="${bigdata.lib}/apache/log4j-1.2.15.jar"
- tofile="${dist.lib}/log4j.jar"/>
- <copy file="${bigdata.lib}/lucene/lucene-analyzers-3.0.0.jar"
- tofile="${dist.lib}/lucene-analyzer.jar"/>
- <copy file="${bigdata.lib}/lucene/lucene-core-3.0.0.jar"
- tofile="${dist.lib}/lucene-core.jar"/>
+<copy file="${bigdata.lib}/unimi/colt-1.2.0.jar" tofile="${dist.lib}/colt.jar" />
+<copy file="${bigdata.lib}/ctc_utils-5-4-2005.jar" tofile="${dist.lib}/ctc_utils.jar" />
+<copy file="${bigdata.lib}/cweb-commons-1.1-b2-dev.jar" tofile="${dist.lib}/cweb-commons.jar" />
+<copy file="${bigdata.lib}/cweb-extser-0.1-b2-dev.jar" tofile="${dist.lib}/cweb-extser.jar" />
+<copy file="${bigdata.lib}/high-scale-lib-v1.1.2.jar" tofile="${dist.lib}/highscalelib.jar" />
+<copy file="${bigdata.lib}/dsi-utils-1.0.6-020610.jar" tofile="${dist.lib}/dsiutils.jar" />
+<copy file="${bigdata.lib}/lgpl-utils-1.0.6-020610.jar" tofile="${dist.lib}/lgplutils.jar" />
+<copy file="${bigdata.lib}/unimi/fastutil-5.1.5.jar" tofile="${dist.lib}/fastutil.jar" />
+<copy file="${bigdata.lib}/icu/icu4j-3_6.jar" tofile="${dist.lib}/icu4j.jar" />
+<copy file="${bigdata.lib}/apache/log4j-1.2.15.jar" tofile="${dist.lib}/log4j.jar" />
+<copy file="${bigdata.lib}/lucene/lucene-analyzers-3.0.0.jar" tofile="${dist.lib}/lucene-analyzer.jar" />
+<copy file="${bigdata.lib}/lucene/lucene-core-3.0.0.jar" tofile="${dist.lib}/lucene-core.jar" />
- <!-- RDF library -->
+<!-- RDF library -->
- <copy file="${bigdata-rdf.lib}/iris-0.58.jar"
- tofile="${dist.lib}/iris.jar"/>
- <copy file="${bigdata-rdf.lib}/openrdf-sesame-2.3.0-onejar.jar"
- tofile="${dist.lib}/openrdf-sesame.jar"/>
- <copy file="${bigdata-rdf.lib}/slf4j-api-1.4.3.jar"
- tofile="${dist.lib}/slf4j.jar"/>
- <copy file="${bigdata-rdf.lib}/slf4j-log4j12-1.4.3.jar"
- tofile="${dist.lib}/slf4j-log4j.jar"/>
+<copy file="${bigdata-rdf.lib}/iris-0.58.jar" tofile="${dist.lib}/iris.jar" />
+<copy file="${bigdata-rdf.lib}/jgrapht-jdk1.5-0.7.1.jar" tofile="${dist.lib}/jgrapht.jar" />
+<copy file="${bigdata-rdf.lib}/openrdf-sesame-2.3.0-onejar.jar" tofile="${dist.lib}/openrdf-sesame.jar" />
+<copy file="${bigdata-rdf.lib}/slf4j-api-1.4.3.jar" tofile="${dist.lib}/slf4j.jar" />
+<copy file="${bigdata-rdf.lib}/slf4j-log4j12-1.4.3.jar" tofile="${dist.lib}/slf4j-log4j.jar" />
- <!-- NxParser (RDF NQuads support) -->
- <copy file="${bigdata-rdf.lib}/nxparser-6-22-2010.jar"
- tofile="${dist.lib}/nxparser.jar"/>
-
- <!-- Zookeeper library -->
- <copy file="${bigdata-zookeeper.lib}/zookeeper-3.2.1.jar"
- tofile="${dist.lib}/zookeeper.jar"/>
+<!-- NxParser (RDF NQuads support) -->
+<copy file="${bigdata-rdf.lib}/nxparser-6-22-2010.jar" tofile="${dist.lib}/nxparser.jar" />
- <!-- Jini library -->
+<!-- Zookeeper library -->
+<copy file="${bigdata-zookeeper.lib}/zookeeper-3.2.1.jar" tofile="${dist.lib}/zookeeper.jar" />
- <copy file="${bigdata-jini.lib}/browser.jar"
- todir="${dist.lib}"/>
- <copy file="${bigdata-jini.lib}/classserver.jar"
- todir="${dist.lib}"/>
- <copy file="${bigdata-jini.lib}/jsk-lib.jar"
- todir="${dist.lib}"/>
- <copy file="${bigdata-jini.lib}/jsk-platform.jar"
- todir="${dist.lib}"/>
- <copy file="${bigdata-jini.lib}/jsk-resources.jar"
- todir="${dist.lib}"/>
- <copy file="${bigdat...
[truncated message content] |
|
From: <tho...@us...> - 2010-08-06 10:46:56
|
Revision: 3418
http://bigdata.svn.sourceforge.net/bigdata/?rev=3418&view=rev
Author: thompsonbry
Date: 2010-08-06 10:46:50 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
This implements a workaround for the problem reported by
https://sourceforge.net/apps/trac/bigdata/ticket/111 and https://sourceforge.net/apps/trac/bigdata/ticket/94. The underlying problem is described in https://sourceforge.net/apps/trac/bigdata/ticket/111 and is a problem in the rules-based service startup logic. The workaround removes the TXRunningConstraint from the configuration files and modifies the StoreManager to wait for the transaction service to be discovered before proceeding with its startup. The changes made by this workaround are appropriate and close out https://sourceforge.net/apps/trac/bigdata/ticket/94, but I am going to leave https://sourceforge.net/apps/trac/bigdata/ticket/111 open since it documents the underlying problem which has not been resolved.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
trunk/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
trunk/src/resources/config/bigdataCluster.config
trunk/src/resources/config/bigdataCluster16.config
Modified: trunk/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-05 20:04:28 UTC (rev 3417)
+++ trunk/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-06 10:46:50 UTC (rev 3418)
@@ -7,6 +7,7 @@
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
+import com.bigdata.resources.StoreManager;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
@@ -171,16 +172,18 @@
* Delay between attempts reach the remote service (ms).
*/
final long delay = 10L;
-
- /**
- * #of attempts to reach the remote service.
- *
- * Note: delay*maxtries == 1000ms of trying before we give up.
- *
- * If this is not enough, then consider adding an optional parameter giving
- * the time the caller will wait and letting the StoreManager wait longer
- * during startup to discover the timestamp service.
- */
+
+ /**
+ * #of attempts to reach the remote service.
+ * <p>
+ * Note: delay*maxtries == 1000ms of trying before we give up, plus however
+ * long we are willing to wait for service discovery if the problem is
+ * locating the {@link ITransactionService}.
+ * <p>
+ * If this is not enough, then consider adding an optional parameter giving
+ * the time the caller will wait and letting the {@link StoreManager} wait
+ * longer during startup to discover the timestamp service.
+ */
final int maxtries = 100;
/**
Modified: trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-05 20:04:28 UTC (rev 3417)
+++ trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 10:46:50 UTC (rev 3418)
@@ -1430,6 +1430,13 @@
}
}
+ while (true) {
+ if (getFederation().getTransactionService() != null) {
+ break;
+ }
+ log.warn("Waiting for transaction service discovery");
+ }
+
/*
* Look for pre-existing data files.
*/
Modified: trunk/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java 2010-08-05 20:04:28 UTC (rev 3417)
+++ trunk/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java 2010-08-06 10:46:50 UTC (rev 3418)
@@ -211,7 +211,7 @@
* Return an arbitrary service from the cache -or- <code>null</code> if
* there is no such service in the cache and a remote lookup times out.
*/
- @SuppressWarnings("unchecked")
+// @SuppressWarnings("unchecked")
final public S getService() {
return getService(filter);
@@ -227,16 +227,18 @@
* An optional filter. If given it will be applied in addition to
* the optional filter specified to the ctor.
*/
- @SuppressWarnings("unchecked")
final public S getService(final ServiceItemFilter filter) {
- ServiceItem item = getServiceItem(filter);
+ final ServiceItem item = getServiceItem(filter);
- if (item != null)
- return (S) item.service;
- else
- return null;
-
+ if (item == null)
+ return null;
+
+ @SuppressWarnings("unchecked")
+ final S service = (S)item.service;
+
+ return service;
+
}
/**
@@ -302,7 +304,7 @@
try {
- item = getServiceDiscoveryManager().lookup(template, filter,
+ item = serviceDiscoveryManager.lookup(template, filter,
cacheMissTimeout);
} catch (RemoteException ex) {
Modified: trunk/src/resources/config/bigdataCluster.config
===================================================================
--- trunk/src/resources/config/bigdataCluster.config 2010-08-05 20:04:28 UTC (rev 3417)
+++ trunk/src/resources/config/bigdataCluster.config 2010-08-06 10:46:50 UTC (rev 3418)
@@ -700,7 +700,7 @@
new JiniRunningConstraint(),
new ZookeeperRunningConstraint(),
- new TXRunningConstraint(),
+ //new TXRunningConstraint(),
new HostAllowConstraint(bigdata.mds),
@@ -758,6 +758,10 @@
* have for your applications!
*/
"-Xmx1600m",// was 800
+ /* Optionally, grab all/most of the max heap at once. This makes sense for
+ * DS but is less necessary for other bigdata services.
+ */
+ "-Xms800m", // 1/2 of the max heap is a good value.
/*
* This option will keep the JVM "alive" even when it is memory starved
* but perform of a memory starved JVM is terrible.
@@ -807,7 +811,7 @@
new JiniRunningConstraint(),
new ZookeeperRunningConstraint(),
- new TXRunningConstraint(),
+ //new TXRunningConstraint(),
new HostAllowConstraint(bigdata.ds),
Modified: trunk/src/resources/config/bigdataCluster16.config
===================================================================
--- trunk/src/resources/config/bigdataCluster16.config 2010-08-05 20:04:28 UTC (rev 3417)
+++ trunk/src/resources/config/bigdataCluster16.config 2010-08-06 10:46:50 UTC (rev 3418)
@@ -745,7 +745,7 @@
new JiniRunningConstraint(),
new ZookeeperRunningConstraint(),
- new TXRunningConstraint(),
+ //new TXRunningConstraint(),
new HostAllowConstraint(bigdata.mds),
@@ -814,7 +814,9 @@
*/
"-Xmx9G", // Note: out of 32 available!
/* Optionally, grab all/most of the max heap at once. This makes sense for
- * DS, but is less necessary for other bigdata services.
+ * DS, but is less necessary for other bigdata services. If the machine is
+ * dedicated to the DataService then use the maximum heap. Otherwise 1/2 of
+ * the maximum heap is a good value.
*/
"-Xms9G",
/*
@@ -888,7 +890,7 @@
new JiniRunningConstraint(),
new ZookeeperRunningConstraint(),
- new TXRunningConstraint(),
+ //new TXRunningConstraint(),
new HostAllowConstraint(bigdata.ds),
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-05 20:04:35
|
Revision: 3417
http://bigdata.svn.sourceforge.net/bigdata/?rev=3417&view=rev
Author: thompsonbry
Date: 2010-08-05 20:04:28 +0000 (Thu, 05 Aug 2010)
Log Message:
-----------
Added init() to ILocatableResource to allow one time initialization logic after the constructor invocation. This is a good pattern in general and was used to fix an initialization issue within the LexiconRelation and its inner ILexiconConfiguration object.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
trunk/bigdata/src/java/com/bigdata/relation/AbstractResource.java
trunk/bigdata/src/java/com/bigdata/relation/IMutableResource.java
trunk/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
trunk/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
trunk/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java
trunk/build.xml
Modified: trunk/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -420,7 +420,7 @@
}
}
-
+
/**
* Note: A commit is required in order for a read-committed view to have
* access to the registered indices. When running against an
Modified: trunk/bigdata/src/java/com/bigdata/relation/AbstractResource.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/relation/AbstractResource.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata/src/java/com/bigdata/relation/AbstractResource.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -582,9 +582,21 @@
}
/**
+ * The default implementation only logs the event.
+ */
+ public AbstractResource<E> init() {
+
+ if (log.isInfoEnabled())
+ log.info(toString());
+
+ return this;
+
+ }
+
+ /**
*
* @todo Lock service supporting shared locks, leases and lease renewal,
- * excalation of shared locks to exclusive locks, deadlock detection,
+ * escalation of shared locks to exclusive locks, deadlock detection,
* and possibly a resource hierarchy. Leases should be Callable
* objects that are submitted by the client to its executor service so
* that they will renew automatically until cancelled (and will cancel
Modified: trunk/bigdata/src/java/com/bigdata/relation/IMutableResource.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/relation/IMutableResource.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata/src/java/com/bigdata/relation/IMutableResource.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -38,7 +38,10 @@
public interface IMutableResource<T> extends ILocatableResource<T> {
/**
- * Create any logically contained resources (relations, indices).
+ * Create any logically contained resources (relations, indices). There is
+ * no presumption that {@link #init()} is suitable for invocation from
+ * {@link #create()}. Instead, you are responsible for invoking {@link #init()}
+ * from this method IFF it is appropriate to reuse its initialization logic.
*/
void create();
Modified: trunk/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/relation/RelationFusedView.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata/src/java/com/bigdata/relation/RelationFusedView.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -21,8 +21,8 @@
*/
public class RelationFusedView<E> implements IRelation<E> {
- private IRelation<E> relation1;
- private IRelation<E> relation2;
+ final private IRelation<E> relation1;
+ final private IRelation<E> relation2;
public IRelation<E> getRelation1() {
@@ -36,6 +36,13 @@
}
+ // NOP
+ public RelationFusedView<E> init() {
+
+ return this;
+
+ }
+
/**
*
* @param relation1
Modified: trunk/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -586,6 +586,8 @@
properties //
});
+ r.init();
+
if(INFO) {
log.info("new instance: "+r);
Modified: trunk/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -45,6 +45,13 @@
*/
public interface ILocatableResource<T> {
+ /**
+ * Deferred initialization method is automatically invoked when the resource
+ * is materialized by the {@link IResourceLocator}. The implementation is
+ * encouraged to strengthen the return type.
+ */
+ public ILocatableResource<T> init();
+
/**
* The identifying namespace.
*/
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -73,26 +73,18 @@
}
- public void initExtensions(final IDatatypeURIResolver resolver) {
+ public void initExtensions(final IDatatypeURIResolver resolver) {
+
+ xFactory.init(resolver);
+
+ for (IExtension extension : xFactory.getExtensions()) {
+ BigdataURI datatype = extension.getDatatype();
+ if (datatype == null)
+ continue;
+ termIds.put((TermId) datatype.getIV(), extension);
+ datatypes.put(datatype.stringValue(), extension);
+ }
- xFactory.init(resolver);
-
- /*
- * Hacky way to know we haven't been initialized yet without using
- * non-final variables.
- */
- if (termIds.size() == 0 && xFactory.getExtensions().length > 0) {
-
- for (IExtension extension : xFactory.getExtensions()) {
- BigdataURI datatype = extension.getDatatype();
- if (datatype == null)
- continue;
- termIds.put((TermId) datatype.getIV(), extension);
- datatypes.put(datatype.stringValue(), extension);
- }
-
- }
-
}
public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -45,6 +45,7 @@
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.log4j.Logger;
import org.omg.CORBA.portable.ValueFactory;
@@ -99,6 +100,7 @@
import com.bigdata.relation.AbstractRelation;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IElementFilter;
+import com.bigdata.relation.locator.ILocatableResource;
import com.bigdata.relation.locator.IResourceLocator;
import com.bigdata.relation.rule.IBindingSet;
import com.bigdata.relation.rule.IPredicate;
@@ -447,6 +449,22 @@
}
+ @Override
+ public LexiconRelation init() {
+
+ super.init();
+
+ /*
+ * Allow the extensions to resolve their datatype URIs into term
+ * identifiers.
+ */
+ lexiconConfiguration.initExtensions(this);
+
+ return this;
+
+ }
+
+ @Override
public void create() {
final IResourceLock resourceLock = acquireExclusiveLock();
@@ -487,6 +505,12 @@
//
// assert id2term != null;
+ /*
+ * Allow the extensions to resolve their datatype URIs into term
+ * identifiers.
+ */
+ lexiconConfiguration.initExtensions(this);
+
} finally {
unlock(resourceLock);
@@ -706,6 +730,9 @@
tmp = (ITextIndexer) gi.invoke(null/* object */,
getIndexManager(), getNamespace(),
getTimestamp(), getProperties());
+ if(tmp instanceof ILocatableResource<?>) {
+ ((ILocatableResource<?>)tmp).init();
+ }
// new FullTextIndex(getIndexManager(),
// getNamespace(), getTimestamp(), getProperties())
viewRef.set(tmp);
@@ -2673,15 +2700,6 @@
//
// }
- /*
- * Allow the extensions to resolve their datatype URIs into
- * term identifiers. Unfortunately no way to tell whether to call this
- * or not without using non-final variables. The configuration will
- * have to be responsible for determining whether they are initialized
- * or not (again using only final variables). Hacky.
- */
- lexiconConfiguration.initExtensions(this);
-
return lexiconConfiguration;
}
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -450,7 +450,8 @@
return new RelationFusedView<ISPO>(
//
(IRelation<ISPO>)resourceLocator.locate(database, timestamp0),
- (IRelation<ISPO>)resourceLocator.locate(focusStore, timestamp1));
+ (IRelation<ISPO>)resourceLocator.locate(focusStore, timestamp1))
+ .init();
}
// final IAccessPath accessPath = (focusStore == null //
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -734,7 +734,7 @@
final IRelation relation1 = (IRelation) resourceLocator.locate(
relationName1, readTimestamp);//timestamp1);
- relation = new RelationFusedView(relation0, relation1);
+ relation = new RelationFusedView(relation0, relation1).init();
} else {
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -1222,7 +1222,17 @@
}
}
+
+ @Override
+ public AbstractTripleStore init() {
+ super.init();
+
+ return this;
+
+ }
+
+ @Override
public void create() {
assertWritable();
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -158,7 +158,7 @@
/**
* Create or re-open a triple store using a local embedded database.
*/
- public LocalTripleStore(Properties properties) {
+ /* public */LocalTripleStore(final Properties properties) {
/*
* FIXME This should pass up the existing properties for the KB instance
@@ -189,6 +189,10 @@
create();
+ } else {
+
+ init();
+
}
}
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -22,11 +22,9 @@
public void init(final IDatatypeURIResolver resolver) {
- if (extensions.size() == 0) {
- extensions.add(new EpochExtension(resolver));
- extensions.add(new ColorsEnumExtension(resolver));
- extensionsArray = extensions.toArray(new IExtension[2]);
- }
+ extensions.add(new EpochExtension(resolver));
+ extensions.add(new ColorsEnumExtension(resolver));
+ extensionsArray = extensions.toArray(new IExtension[2]);
}
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -87,8 +87,8 @@
* @todo We should run this test suite against a CI cluster on a single machine using
* the full bigdata federation rather than EDS.
*/
- suite.addTest(com.bigdata.rdf.store.TestScaleOutTripleStoreWithEmbeddedFederation
- .suite());
+// suite.addTest(com.bigdata.rdf.store.TestScaleOutTripleStoreWithEmbeddedFederation
+// .suite());
// if (Boolean.parseBoolean(System.getProperty("maven.test.services.skip",
// "false"))
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -256,7 +256,7 @@
ITx.UNISOLATED,
store.getProperties()
// client.getProperties()
- );
+ ).init();
}
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java 2010-08-05 20:04:28 UTC (rev 3417)
@@ -240,7 +240,7 @@
// obtain view of the triple store.
return new ScaleOutTripleStore(helper.client.getFederation(),
- namespace, ITx.UNISOLATED, store.getProperties());
+ namespace, ITx.UNISOLATED, store.getProperties()).init();
}
Modified: trunk/build.xml
===================================================================
--- trunk/build.xml 2010-08-05 18:11:38 UTC (rev 3416)
+++ trunk/build.xml 2010-08-05 20:04:28 UTC (rev 3417)
@@ -1351,6 +1351,12 @@
<fileset dir="${bigdata.dir}/src/resources/scripts" />
</copy>
+ <!-- Copy the analysis tools. The queries directory is required for extractCounters.sh. -->
+ <mkdir dir="${build.dir}/src/resources/analysis" />
+ <copy toDir="${build.dir}/src/resources/analysis">
+ <fileset dir="${bigdata.dir}/src/resources/analysis" />
+ </copy>
+
<copy tofile="${build.dir}/build.properties" file="build.properties" />
<copy tofile="${build.dir}/build.xml" file="build.xml" />
<copy tofile="${build.dir}/LICENSE.txt" file="LICENSE.txt" />
@@ -1988,6 +1994,13 @@
<!-- changing the appropriate property value above is all -->
<!-- that should be required. -->
+ <!-- This is a workaround for a JVM bug which can result in a -->
+ <!-- lost wakeup. This bug is fixed in JDK1.6.0_18. However, -->
+ <!-- JDK1.6.0_18 has other problems which result in segfaults. -->
+ <!-- -->
+ <!-- http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6822370 -->
+ <jvmarg value="-XX:+UseMembar"/>
+
<sysproperty key="log4j.path" value="${bigdata.test.log4j.abs.path}"/>
<sysproperty key="app.home" value="${app.home}"/>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-05 18:11:45
|
Revision: 3416
http://bigdata.svn.sourceforge.net/bigdata/?rev=3416&view=rev
Author: thompsonbry
Date: 2010-08-05 18:11:38 +0000 (Thu, 05 Aug 2010)
Log Message:
-----------
The forceCompactingMerge option used for some benchmarking operations on the federation was not forcing a compacting merge when the shard view had simply been copied to the new journal.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
trunk/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
Modified: trunk/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-05 17:57:00 UTC (rev 3415)
+++ trunk/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-05 18:11:38 UTC (rev 3416)
@@ -310,6 +310,7 @@
private final OverflowActionEnum action;
private final ViewMetadata vmd;
+ private final boolean forceCompactingMerge;
private final AbstractTask<T> task;
/**
@@ -319,11 +320,17 @@
* @param vmd
* The {@link ViewMetadata} for the index partition for which
* that action will be taken.
+ * @param forceCompactingMerge
+ * if a compacting merge should be taken even if the view was
+ * simply copied to the new journal.
* @param task
* The task which implements that action.
*/
- public AtomicCallable(final OverflowActionEnum action,
- final ViewMetadata vmd, final AbstractTask<T> task) {
+ public AtomicCallable(final OverflowActionEnum action,//
+ final ViewMetadata vmd,//
+ final boolean forceCompactingMerge, //
+ final AbstractTask<T> task//
+ ) {
if (action == null)
throw new IllegalArgumentException();
@@ -337,6 +344,8 @@
this.action = action;
this.vmd = vmd;
+
+ this.forceCompactingMerge = forceCompactingMerge;
this.task = task;
@@ -407,110 +416,112 @@
}
- /**
- * Schedule a build for each shard and a merge for each shard with a
- * non-zero merge priority. Whether a build or a merge is performed for a
- * shard will depend on which action is initiated first. When an build or
- * merge action is initiated, that choice is atomically registered on the
- * {@link ViewMetadata} and any subsequent attempt (within this method
- * invocation) to start a build or merge for the same shard will be dropped.
- * Processing ends once all tasks scheduled on a "build" service are
- * complete.
- * <p>
- * After actions are considered for each shard for which a compacting merge
- * is executed. These after actions can cause a shard split, join, or move.
- * Deferring such actions until we have a compact view (comprised of one
- * journal and one index segment) greatly improves our ability to decide
- * whether a shard should be split or joined and simplifies the logic and
- * effort required to split, join or move a shard.
- * <p>
- * The following is a brief summary of some after actions on compact shards.
- * <dl>
- * <dt>split</dt>
- * <dd>A shard is split when its size on the disk exceeds the (adjusted)
- * nominal size of a shard (overflow). By waiting until the shard view is
- * compact we have exact information about the size of the shard (it is
- * contained in a single {@link IndexSegment}) and we are able to easily
- * select the separator key to split the shard.</dd>
- * <dt>tailSplit</dt>
- * <dd>A tail split may be selected for a shard which has a mostly append
- * access pattern. For such access patterns, a normal split would leave the
- * left sibling 50% full and the right sibling would quickly fill up with
- * continued writes on the tail of the key range. To compensate for this
- * access pattern, a tail split chooses a separator key near the end of the
- * key range of a shard. This results in a left sibling which is mostly full
- * and a right sibling which is mostly empty. If the pattern of heavy tail
- * append continues, then the left sibling will remain mostly full and the
- * new writes will flow mostly into the right sibling.</dd>
- * <dt>scatterSplit</dt>
- * <dd>A scatter split breaks the first shard for a new scale-out index into
- * N shards and scatters those shards across the data services in a
- * federation in order to improve the data distribution and potential
- * concurrency of the index. By waiting until the shard view is compact we
- * are able to quickly select appropriate separator keys for the shard
- * splits.</dd>
- * <dt>move</dt>
- * <dd>A move transfer a shard from this data service to another data
- * service in order to reduce the load on this data service. By waiting
- * until the shard view is compact we are able to rapidly transfer the bulk
- * of the data in the form of a single {@link IndexSegment}.</dd>
- * <dt>join</dt>
- * <dd>A join combines a shard which is under 50% of its (adjusted) nominal
- * maximum size on the disk (underflow) with its right sibling. Joins are
- * driven by deletes of tuples from a key range. Since deletes are handled
- * as writes where a delete marker is set on the tuple, neither the shard
- * size on the disk nor the range count of the shard will decrease until a
- * compacting merge. A join is indicated if the size on disk for the shard
- * has shrunk considerably since the last time a compacting merge was
- * performed for the view (this covers both the case of deletes, which
- * reduce the range count, and updates which replace the values in the
- * tuples with more compact data). <br>
- * There are actually three cases for a join.
- * <ol>
- * <li>If the right sibling is local, then the shard will be joined with its
- * right sibling.</li>
- * <li>If the right sibling is remote, then the shard will be moved to the
- * data service on which the right sibling is found.</li>
- * <li>If the right sibling does not exist, then nothing is done (the last
- * shard in a scale-out index does not have a right sibling). The right most
- * sibling will remain undercapacity until and unless its left sibling also
- * underflows, at which point the left sibling will cause itself to be
- * joined with the right sibling (this is done to simplify the logic which
- * searches for a sibling with which to join an undercapacity shard).</li>
- * </ol>
- * </dl>
- *
- * @param forceCompactingMerges
- * When <code>true</code> a compacting merge will be forced for
- * each non-compact view.
- *
- * @throws InterruptedException
- *
- * @todo The size of the merge queue (or its sum of priorities) may be an
- * indication of the load of the node which could be used to decide
- * that index partitions should be shed/moved.
- *
- * @todo For HA, this needs to be a shared priority queue using zk or the
- * like since any node in the failover set could do the merge (or
- * build). [Alternatively, nodes do the build/merge for the shards for
- * which they have the highest affinity out of the failover set.]
- *
- * FIXME tailSplits currently operate on the mutable BTree rather than
- * a compact view). This task does not require a compact view (at
- * least, not yet) and generating one for it might be a waste of time.
- * Instead it examines where the inserts are occurring in the index
- * and splits of the tail if the index is heavy for write append. It
- * probably could defer that choice until a compact view was some
- * percentage of a split (maybe .6?) So, probably an after action for
- * the mergeQ.
- *
- * FIXME joins must track metadata about the previous size on disk of
- * the compact view in order to decide when underflow has resulted. In
- * order to handle the change in the value of the acceleration factor,
- * this data should be stored as the percentage of an adjusted split
- * of the last compact view. We can update that metadata each time we
- * do a compacting merge.
- */
+ /**
+ * Schedule a build for each shard and a merge for each shard with a
+ * non-zero merge priority. Whether a build or a merge is performed for a
+ * shard will depend on which action is initiated first. When an build or
+ * merge action is initiated, that choice is atomically registered on the
+ * {@link ViewMetadata} and any subsequent attempt (within this method
+ * invocation) to start a build or merge for the same shard will be dropped.
+ * Processing ends once all tasks scheduled on a "build" service are
+ * complete.
+ * <p>
+ * After actions are considered for each shard for which a compacting merge
+ * is executed. These after actions can cause a shard split, join, or move.
+ * Deferring such actions until we have a compact view (comprised of one
+ * journal and one index segment) greatly improves our ability to decide
+ * whether a shard should be split or joined and simplifies the logic and
+ * effort required to split, join or move a shard.
+ * <p>
+ * The following is a brief summary of some after actions on compact shards.
+ * <dl>
+ * <dt>split</dt>
+ * <dd>A shard is split when its size on the disk exceeds the (adjusted)
+ * nominal size of a shard (overflow). By waiting until the shard view is
+ * compact we have exact information about the size of the shard (it is
+ * contained in a single {@link IndexSegment}) and we are able to easily
+ * select the separator key to split the shard.</dd>
+ * <dt>tailSplit</dt>
+ * <dd>A tail split may be selected for a shard which has a mostly append
+ * access pattern. For such access patterns, a normal split would leave the
+ * left sibling 50% full and the right sibling would quickly fill up with
+ * continued writes on the tail of the key range. To compensate for this
+ * access pattern, a tail split chooses a separator key near the end of the
+ * key range of a shard. This results in a left sibling which is mostly full
+ * and a right sibling which is mostly empty. If the pattern of heavy tail
+ * append continues, then the left sibling will remain mostly full and the
+ * new writes will flow mostly into the right sibling.</dd>
+ * <dt>scatterSplit</dt>
+ * <dd>A scatter split breaks the first shard for a new scale-out index into
+ * N shards and scatters those shards across the data services in a
+ * federation in order to improve the data distribution and potential
+ * concurrency of the index. By waiting until the shard view is compact we
+ * are able to quickly select appropriate separator keys for the shard
+ * splits.</dd>
+ * <dt>move</dt>
+ * <dd>A move transfer a shard from this data service to another data
+ * service in order to reduce the load on this data service. By waiting
+ * until the shard view is compact we are able to rapidly transfer the bulk
+ * of the data in the form of a single {@link IndexSegment}.</dd>
+ * <dt>join</dt>
+ * <dd>A join combines a shard which is under 50% of its (adjusted) nominal
+ * maximum size on the disk (underflow) with its right sibling. Joins are
+ * driven by deletes of tuples from a key range. Since deletes are handled
+ * as writes where a delete marker is set on the tuple, neither the shard
+ * size on the disk nor the range count of the shard will decrease until a
+ * compacting merge. A join is indicated if the size on disk for the shard
+ * has shrunk considerably since the last time a compacting merge was
+ * performed for the view (this covers both the case of deletes, which
+ * reduce the range count, and updates which replace the values in the
+ * tuples with more compact data). <br>
+ * There are actually three cases for a join.
+ * <ol>
+ * <li>If the right sibling is local, then the shard will be joined with its
+ * right sibling.</li>
+ * <li>If the right sibling is remote, then the shard will be moved to the
+ * data service on which the right sibling is found.</li>
+ * <li>If the right sibling does not exist, then nothing is done (the last
+ * shard in a scale-out index does not have a right sibling). The right most
+ * sibling will remain undercapacity until and unless its left sibling also
+ * underflows, at which point the left sibling will cause itself to be
+ * joined with the right sibling (this is done to simplify the logic which
+ * searches for a sibling with which to join an undercapacity shard).</li>
+ * </ol>
+ * </dl>
+ *
+ * @param forceCompactingMerges
+ * When <code>true</code> a compacting merge will be forced for
+ * each non-compact view. Compacting merges will be taken in
+ * priority order and will continue until finished or until the
+ * journal is nearing its nominal maximum extent.
+ *
+ * @throws InterruptedException
+ *
+ * @todo The size of the merge queue (or its sum of priorities) may be an
+ * indication of the load of the node which could be used to decide
+ * that index partitions should be shed/moved.
+ *
+ * @todo For HA, this needs to be a shared priority queue using zk or the
+ * like since any node in the failover set could do the merge (or
+ * build). [Alternatively, nodes do the build/merge for the shards for
+ * which they have the highest affinity out of the failover set.]
+ *
+ * FIXME tailSplits currently operate on the mutable BTree rather than
+ * a compact view). This task does not require a compact view (at
+ * least, not yet) and generating one for it might be a waste of time.
+ * Instead it examines where the inserts are occurring in the index
+ * and splits of the tail if the index is heavy for write append. It
+ * probably could defer that choice until a compact view was some
+ * percentage of a split (maybe .6?) So, probably an after action for
+ * the mergeQ.
+ *
+ * FIXME joins must track metadata about the previous size on disk of
+ * the compact view in order to decide when underflow has resulted. In
+ * order to handle the change in the value of the acceleration factor,
+ * this data should be stored as the percentage of an adjusted split
+ * of the last compact view. We can update that metadata each time we
+ * do a compacting merge.
+ */
private List<Future<?>> scheduleAndAwaitTasks(
final boolean forceCompactingMerges) throws InterruptedException {
@@ -554,21 +565,30 @@
if (log.isInfoEnabled())
log.info("was copied : " + vmd);
- continue;
+ } else {
+ buildList.add(new Priority<ViewMetadata>(vmd.buildPriority, vmd));
+
}
- buildList.add(new Priority<ViewMetadata>(vmd.buildPriority, vmd));
+ if (vmd.mergePriority > 0d || forceCompactingMerges) {
- if (vmd.mergePriority > 0d) {
+ /*
+ * Schedule a merge if the priority is non-zero or if compacting
+ * merges are being forced.
+ */
- mergeList
- .add(new Priority<ViewMetadata>(vmd.mergePriority, vmd));
+ mergeList
+ .add(new Priority<ViewMetadata>(vmd.mergePriority, vmd));
}
} // itr.hasNext()
+ if(log.isInfoEnabled()) {
+ log.info("Scheduling tasks: buildList="+buildList.size()+", mergeList="+mergeList.size());
+ }
+
/*
* Schedule build and merge tasks and await their futures. The tasks are
* submitted from a PriorityQueue, so the order in which the tasks are
@@ -606,18 +626,23 @@
resourceManager.mergeServiceCorePoolSize);
// Schedule merge tasks.
- if (!forceCompactingMerges) {
-
for (Priority<ViewMetadata> p : mergeList) {
final ViewMetadata vmd = p.v;
- if (vmd.mergePriority > 0) {
+ if (vmd.mergePriority > 0 || forceCompactingMerges) {
+ if(forceCompactingMerges && vmd.getAction().equals(OverflowActionEnum.Copy)) {
+
+ vmd.clearCopyAction();
+
+ }
+
// Schedule a compacting merge.
final FutureTask<?> ft = new FutureTask(
new AtomicCallable(OverflowActionEnum.Merge,
- vmd, new CompactingMergeTask(vmd)));
+ vmd, forceCompactingMerges,
+ new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
@@ -625,8 +650,6 @@
}
- }
-
// Schedule build tasks.
for (Priority<ViewMetadata> p : buildList) {
@@ -636,7 +659,8 @@
// Force a compacting merge.
final FutureTask<?> ft = new FutureTask(new AtomicCallable(
- OverflowActionEnum.Merge, vmd,
+ OverflowActionEnum.Merge, vmd,
+ forceCompactingMerges,
new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
@@ -646,6 +670,7 @@
// Schedule a build.
final FutureTask<?> ft = new FutureTask(new AtomicCallable(
OverflowActionEnum.Build, vmd,
+ forceCompactingMerges,
new IncrementalBuildTask(vmd)));
buildFutures.add(ft);
buildService.execute(ft);
Modified: trunk/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java 2010-08-05 17:57:00 UTC (rev 3415)
+++ trunk/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java 2010-08-05 18:11:38 UTC (rev 3416)
@@ -280,6 +280,25 @@
actionRef.set(action);
}
+
+ /**
+ * Used to force clear a {@link OverflowActionEnum#Copy} action
+ * when we will force a compacting merge. This allows us to do
+ * compacting merges on shard views which would otherwise simply
+ * be copied onto the new journal.
+ */
+ void clearCopyAction() {
+
+ lock.lock();
+ try {
+ if(actionRef.get().equals(OverflowActionEnum.Copy)) {
+ actionRef.set(null/*clear*/);
+ }
+ } finally {
+ lock.unlock();
+ }
+
+ }
/**
*
Modified: trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-05 17:57:00 UTC (rev 3415)
+++ trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-05 18:11:38 UTC (rev 3416)
@@ -1416,18 +1416,19 @@
* Verify that the concurrency manager has been set and wait a while
* it if is not available yet.
*/
- if (log.isInfoEnabled())
- log.info("Waiting for concurrency manager");
- for (int i = 0; i < 5; i++) {
- try {
- getConcurrencyManager(); break;
- } catch (IllegalStateException ex) {
- Thread.sleep(100/* ms */);
- }
+ {
+ int nwaits = 0;
+ while (true) {
+ try {
+ getConcurrencyManager();
+ break;
+ } catch (IllegalStateException ex) {
+ Thread.sleep(100/* ms */);
+ if (++nwaits % 50 == 0)
+ log.warn("Waiting for concurrency manager");
+ }
+ }
}
- getConcurrencyManager();
- if (Thread.interrupted())
- throw new InterruptedException();
/*
* Look for pre-existing data files.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-08-05 17:57:07
|
Revision: 3415
http://bigdata.svn.sourceforge.net/bigdata/?rev=3415&view=rev
Author: mrpersonick
Date: 2010-08-05 17:57:00 +0000 (Thu, 05 Aug 2010)
Log Message:
-----------
eliminated use of non-final variables and hacked in a late initialization
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -9,17 +9,12 @@
public DefaultExtensionFactory() {
- extensions = new IExtension[] {
- };
+ extensions = new IExtension[0];
}
- public void resolveDatatypes(final IDatatypeURIResolver resolver) {
+ public void init(final IDatatypeURIResolver resolver) {
- for (IExtension extension : extensions) {
- extension.resolveDatatype(resolver);
- }
-
}
public IExtension[] getExtensions() {
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -40,14 +40,14 @@
*/
public interface IExtension<V extends BigdataValue> {
- /**
- * This will be called very early in the IExtension lifecycle so that the
- * {@link TermId} for the datatype URI will be on hand when needed.
- *
- * @param resolver
- * the datatype URI resolver
- */
- void resolveDatatype(final IDatatypeURIResolver resolver);
+// /**
+// * This will be called very early in the IExtension lifecycle so that the
+// * {@link TermId} for the datatype URI will be on hand when needed.
+// *
+// * @param resolver
+// * the datatype URI resolver
+// */
+// void resolveDatatype(final IDatatypeURIResolver resolver);
/**
* Return the fully resolved datatype in the form of a {@link BigdataURI}
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -42,7 +42,7 @@
* @param resolver
* the datatype URI resolver
*/
- void resolveDatatypes(final IDatatypeURIResolver resolver);
+ void init(final IDatatypeURIResolver resolver);
/**
* Return the supported extensions.
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -64,6 +64,15 @@
V asValue(final ExtensionIV iv, final BigdataValueFactory vf);
/**
+ * Initialize the extensions, which need to resolve their datatype URIs
+ * into term ids.
+ *
+ * @param resolver
+ * the datatype URI resolver
+ */
+ void initExtensions(final IDatatypeURIResolver resolver);
+
+ /**
* <code>true</code> iff the <code>vte</code> and <code>dte</code>
* should be inlined.
*
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -30,6 +30,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
+import org.apache.log4j.Logger;
import org.openrdf.model.BNode;
import org.openrdf.model.Literal;
import org.openrdf.model.URI;
@@ -40,33 +41,58 @@
import com.bigdata.rdf.model.BigdataValueFactory;
/**
- * An object which describes which kinds of RDF Values are inlined into the statement indices and how other RDF Values are coded into the lexicon.
+ * An object which describes which kinds of RDF Values are inlined into the
+ * statement indices and how other RDF Values are coded into the lexicon.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public class LexiconConfiguration<V extends BigdataValue> implements ILexiconConfiguration<V> {
+public class LexiconConfiguration<V extends BigdataValue>
+ implements ILexiconConfiguration<V> {
+ protected static final Logger log =
+ Logger.getLogger(LexiconConfiguration.class);
+
private final boolean inlineLiterals, inlineBNodes;
+ private final IExtensionFactory xFactory;
+
private final Map<TermId, IExtension> termIds;
private final Map<String, IExtension> datatypes;
-
- public LexiconConfiguration(final boolean inlineLiterals, final boolean inlineBNodes, final IExtensionFactory xFactory) {
+
+ public LexiconConfiguration(final boolean inlineLiterals,
+ final boolean inlineBNodes, final IExtensionFactory xFactory) {
+
this.inlineLiterals = inlineLiterals;
this.inlineBNodes = inlineBNodes;
-
+ this.xFactory = xFactory;
+
termIds = new HashMap<TermId, IExtension>();
datatypes = new HashMap<String, IExtension>();
- for (IExtension extension : xFactory.getExtensions()) {
- BigdataURI datatype = extension.getDatatype();
- if (datatype == null)
- continue;
- termIds.put((TermId) datatype.getIV(), extension);
- datatypes.put(datatype.stringValue(), extension);
+
+ }
+
+ public void initExtensions(final IDatatypeURIResolver resolver) {
+
+ xFactory.init(resolver);
+
+ /*
+ * Hacky way to know we haven't been initialized yet without using
+ * non-final variables.
+ */
+ if (termIds.size() == 0 && xFactory.getExtensions().length > 0) {
+
+ for (IExtension extension : xFactory.getExtensions()) {
+ BigdataURI datatype = extension.getDatatype();
+ if (datatype == null)
+ continue;
+ termIds.put((TermId) datatype.getIV(), extension);
+ datatypes.put(datatype.stringValue(), extension);
+ }
+
}
-
+
}
public V asValue(final ExtensionIV iv, final BigdataValueFactory vf) {
@@ -92,10 +118,11 @@
if (datatypes.containsKey(datatype.stringValue())) {
+ final IExtension xFactory =
+ datatypes.get(datatype.stringValue());
+
try {
- final IExtension xFactory = datatypes.get(datatype.stringValue());
-
final IV iv = xFactory.createIV(value);
if (iv != null && value instanceof BigdataValue)
@@ -105,6 +132,9 @@
} catch (Exception ex) {
+ log.warn("problem creating inline internal value for " +
+ "extension datatype: " + value.stringValue());
+
/*
* Some sort of parse error in the literal value most
* likely. Resort to term identifiers.
@@ -172,6 +202,8 @@
// some dummy doesn't know how to format a number
// default to term identifier for this term
+ log.warn("number format exception: " + v);
+
}
if (iv != null && value instanceof BigdataValue)
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -397,12 +397,6 @@
determineExtensionFactoryClass();
final IExtensionFactory xFactory = xfc.newInstance();
- /*
- * Allow the extensions to resolve their datatype URIs into
- * term identifiers.
- */
- xFactory.resolveDatatypes(this);
-
lexiconConfiguration = new LexiconConfiguration(
inlineLiterals, inlineBNodes, xFactory);
@@ -2679,6 +2673,15 @@
//
// }
+ /*
+ * Allow the extensions to resolve their datatype URIs into
+ * term identifiers. Unfortunately no way to tell whether to call this
+ * or not without using non-final variables. The configuration will
+ * have to be responsible for determining whether they are initialized
+ * or not (again using only final variables). Hacky.
+ */
+ lexiconConfiguration.initExtensions(this);
+
return lexiconConfiguration;
}
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -44,18 +44,14 @@
*/
public transient static final URI COLOR = new URIImpl(BD.NAMESPACE + "Color");
- private BigdataURI color;
+ private final BigdataURI color;
- public ColorsEnumExtension() {
+ public ColorsEnumExtension(final IDatatypeURIResolver resolver) {
- }
-
- public void resolveDatatype(final IDatatypeURIResolver resolver) {
-
this.color = resolver.resolve(COLOR);
}
-
+
public BigdataURI getDatatype() {
return color;
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -46,17 +46,14 @@
*/
public transient static final URI EPOCH = new URIImpl(BD.NAMESPACE + "Epoch");
- private BigdataURI epoch;
+ private final BigdataURI epoch;
- public EpochExtension() {
- }
-
- public void resolveDatatype(final IDatatypeURIResolver resolver) {
-
+ public EpochExtension(final IDatatypeURIResolver resolver) {
+
this.epoch = resolver.resolve(EPOCH);
}
-
+
public BigdataURI getDatatype() {
return epoch;
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -1,5 +1,8 @@
package com.bigdata.rdf.internal;
+import java.util.Collection;
+import java.util.LinkedList;
+
/**
* Simple {@link IExtensionFactory} implementation that creates two
* {@link IExtension}s - the {@link EpochExtension} and the
@@ -7,28 +10,29 @@
*/
public class SampleExtensionFactory implements IExtensionFactory {
- private final IExtension[] extensions;
+ private final Collection<IExtension> extensions;
+ private volatile IExtension[] extensionsArray;
+
public SampleExtensionFactory() {
- extensions = new IExtension[] {
- new EpochExtension(),
- new ColorsEnumExtension()
- };
-
+ extensions = new LinkedList<IExtension>();
+
}
- public void resolveDatatypes(final IDatatypeURIResolver resolver) {
-
- for (IExtension extension : extensions) {
- extension.resolveDatatype(resolver);
+ public void init(final IDatatypeURIResolver resolver) {
+
+ if (extensions.size() == 0) {
+ extensions.add(new EpochExtension(resolver));
+ extensions.add(new ColorsEnumExtension(resolver));
+ extensionsArray = extensions.toArray(new IExtension[2]);
}
}
public IExtension[] getExtensions() {
- return extensions;
+ return extensionsArray;
}
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-08-05 16:29:44 UTC (rev 3414)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java 2010-08-05 17:57:00 UTC (rev 3415)
@@ -720,8 +720,8 @@
final BigdataValueFactory vf = BigdataValueFactoryImpl.getInstance("test");
- EpochExtension<BigdataValue> ext = new EpochExtension<BigdataValue>();
- ext.resolveDatatype(new IDatatypeURIResolver() {
+ EpochExtension<BigdataValue> ext =
+ new EpochExtension<BigdataValue>(new IDatatypeURIResolver() {
public BigdataURI resolve(URI uri) {
BigdataURI buri = vf.createURI(uri.stringValue());
buri.setIV(new TermId(VTE.URI, 1024));
@@ -743,8 +743,8 @@
final BigdataValueFactory vf = BigdataValueFactoryImpl.getInstance("test");
- ColorsEnumExtension<BigdataValue> ext = new ColorsEnumExtension<BigdataValue>();
- ext.resolveDatatype(new IDatatypeURIResolver() {
+ ColorsEnumExtension<BigdataValue> ext =
+ new ColorsEnumExtension<BigdataValue>(new IDatatypeURIResolver() {
public BigdataURI resolve(URI uri) {
BigdataURI buri = vf.createURI(uri.stringValue());
buri.setIV(new TermId(VTE.URI, 1024));
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-05 16:29:52
|
Revision: 3414
http://bigdata.svn.sourceforge.net/bigdata/?rev=3414&view=rev
Author: thompsonbry
Date: 2010-08-05 16:29:44 +0000 (Thu, 05 Aug 2010)
Log Message:
-----------
Modified WriteExecutorService, DataService, AbstractScaleOutFederation, and OverflowManager to introduce the ability to force overflow even if no shard builds are necessary. This was done in order to support benchmarking against compact shards.
Bug fix to AbstractTransactionService#notifyCommit(long). It was advancing the releaseTime without regard for the minimumReleaseAge if there were no transactions running.
Fixes for 3 unit tests in com.bigdata.resources which were failing due to (a) the bug fixed above in the AbstractTransactionService; and (b) the historical change to the OverflowManager to purge (aka delete) resources during synchronous overflow.
Fixed TestDistributedTransactionService. The tests in this suite were failing because notifyCommit(long) had at somepoint been extended to advance the release time. The mock transaction service in the unit test now explicitly overrides a method on the transaction service such
that bare commits do not advance the release time.
Bug fix in IndexManager where a NPE could be thrown if the previous journal had been deleted before calling listPartitions(long) with the timestamp of the lastCommitTime on that journal.
Added javadoc to TestAsynchronousStatementBufferFactory#test_loadAndVerify_U1() to the effect that the unit test can hang when running the entire
com.bigdata.rdf test suite with JVMs up to and including 1.6.0_17 unless you specify -XX:UseMembar as a workaround for a JVM bug. See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6822370.
com.bigdata.journal.TestTransactionService#test_newTx_readOnly() now fails occasionally.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
trunk/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
trunk/bigdata/src/java/com/bigdata/resources/IndexManager.java
trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java
trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
trunk/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
trunk/bigdata/src/java/com/bigdata/service/DataService.java
trunk/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
trunk/bigdata/src/resources/logging/log4j.properties
trunk/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
trunk/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java
trunk/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
trunk/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
trunk/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
trunk/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java
trunk/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config
trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
trunk/src/resources/config/bigdataCluster.config
Added Paths:
-----------
trunk/src/resources/scripts/dumpFed.sh
Property Changed:
----------------
trunk/
trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
trunk/src/resources/config/
Property changes on: trunk
___________________________________________________________________
Modified: svn:ignore
- ant-build
src
bin
bigdata*.jar
ant-release
standalone
test*
countersfinal.xml
events.jnl
.settings
*.jnl
TestInsertRate.out
SYSTAP-BBT-result.txt
U10load+query
*.hprof
com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
commit-log.txt
eventLog
dist
bigdata-test
com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
+ ant-build
src
bin
bigdata*.jar
ant-release
standalone
test*
countersfinal.xml
events.jnl
.settings
*.jnl
TestInsertRate.out
SYSTAP-BBT-result.txt
U10load+query
*.hprof
com.bigdata.cache.TestHardReferenceQueueWithBatchingUpdates.exp.csv
commit-log.txt
eventLog
dist
bigdata-test
com.bigdata.rdf.stress.LoadClosureAndQueryTest.*.csv
DIST.*.tgz
REL.*.tgz
Modified: trunk/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -36,6 +36,7 @@
import org.apache.log4j.Logger;
import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf;
+import com.bigdata.io.DirectBufferPool;
import com.bigdata.journal.DumpJournal;
import com.bigdata.rawstore.IRawStore;
@@ -154,6 +155,16 @@
}
+ // multi-block scan of the index segment.
+ boolean multiBlockScan = false; // @todo command line option.
+ if (multiBlockScan) {
+
+ writeBanner("dump leaves using multi-block forward scan");
+
+ dumpLeavesMultiBlockForwardScan(store);
+
+ }
+
// dump the leaves using a fast reverse scan.
boolean fastReverseScan = true;// @todo command line option
if (fastReverseScan) {
@@ -524,6 +535,36 @@
}
+ /**
+ * Dump leaves using the {@link IndexSegmentMultiBlockIterator}.
+ *
+ * @param store
+ */
+ static void dumpLeavesMultiBlockForwardScan(final IndexSegmentStore store) {
+
+ final long begin = System.currentTimeMillis();
+
+ final IndexSegment seg = store.loadIndexSegment();
+
+ final ITupleIterator<?> itr = new IndexSegmentMultiBlockIterator(seg, DirectBufferPool.INSTANCE,
+ null/* fromKey */, null/* toKey */, IRangeQuery.DEFAULT/* flags */);
+
+ int nscanned = 0;
+
+ while(itr.hasNext()) {
+
+ itr.next();
+
+ nscanned++;
+
+ }
+
+ final long elapsed = System.currentTimeMillis() - begin;
+
+ System.out.println("Visited "+nscanned+" tuples using multi-block forward scan in "+elapsed+" ms");
+
+ }
+
static void writeBanner(String s) {
System.out.println(bar);
Modified: trunk/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -1752,11 +1752,11 @@
//
// }
- /**
- * Flag may be set to force overflow processing during the next group
- * commit. The flag is cleared once an overflow has occurred.
- */
- public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
+// /**
+// * Flag may be set to force overflow processing during the next group
+// * commit. The flag is cleared once an overflow has occurred.
+// */
+// public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
/**
* Return <code>true</code> if the pre-conditions for overflow processing
@@ -1765,7 +1765,8 @@
private boolean isShouldOverflow() {
return resourceManager.isOverflowEnabled()
- && (forceOverflow.get() || resourceManager.shouldOverflow());
+// && (forceOverflow.get() || resourceManager.shouldOverflow());
+ && resourceManager.shouldOverflow();
}
@@ -1815,10 +1816,10 @@
log.error("Overflow error: "+serviceName+" : "+t, t);
- } finally {
-
- // clear force flag.
- forceOverflow.set(false);
+// } finally {
+//
+// // clear force flag.
+// forceOverflow.set(false);
}
Modified: trunk/bigdata/src/java/com/bigdata/resources/IndexManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/IndexManager.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/resources/IndexManager.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -1684,16 +1684,28 @@
final StringBuilder sb = new StringBuilder();
final AbstractJournal journal = getJournal(timestamp);
+
+ if (journal == null) {
+ /*
+ * This condition can occur if there are no shard views on the
+ * previous journal and the releaseAge is zero since the previous
+ * journal can be purged (deleted) before this method is invoked.
+ * This situation arises in a few of the unit tests which begin with
+ * an empty journal and copy everything onto the new journal such
+ * that the old journal can be immediately released.
+ */
+ return "No journal: timestamp=" + timestamp;
+ }
sb.append("timestamp="+timestamp+"\njournal="+journal.getResourceMetadata());
// historical view of Name2Addr as of that timestamp.
- final ITupleIterator itr = journal.getName2Addr(timestamp)
+ final ITupleIterator<?> itr = journal.getName2Addr(timestamp)
.rangeIterator();
while (itr.hasNext()) {
- final ITuple tuple = itr.next();
+ final ITuple<?> tuple = itr.next();
final Entry entry = EntrySerializer.INSTANCE
.deserialize(new DataInputBuffer(tuple.getValue()));
Modified: trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -287,6 +287,14 @@
*/
protected final AtomicBoolean asyncOverflowEnabled = new AtomicBoolean(true);
+ /**
+ * Flag may be set to force overflow processing during the next group
+ * commit. The flag is cleared by {@link #overflow()}.
+ *
+ * @see DataService#forceOverflow(boolean, boolean)
+ */
+ public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
+
/**
* A flag that may be set to force the next asynchronous overflow to perform
* a compacting merge for all indices that are not simply copied over to the
@@ -295,6 +303,8 @@
* made compact and SHOULD NOT be used for deployed federations</strong>).
* The state of the flag is cleared each time asynchronous overflow
* processing begins.
+ *
+ * @see DataService#forceOverflow(boolean, boolean)
*/
public final AtomicBoolean compactingMerge = new AtomicBoolean(false);
@@ -1849,6 +1859,19 @@
*/
public boolean shouldOverflow() {
+ if(forceOverflow.get()) {
+
+ /*
+ * Note: forceOverflow trumps everything else.
+ */
+
+ if (log.isInfoEnabled())
+ log.info("Forcing overflow.");
+
+ return true;
+
+ }
+
if (isTransient()) {
/*
@@ -1886,7 +1909,7 @@
return false;
}
-
+
/*
* Look for overflow condition on the "live" journal.
*/
@@ -1959,8 +1982,18 @@
*/
public Future<Object> overflow() {
- assert overflowAllowed.get();
+// assert overflowAllowed.get();
+ /*
+ * Atomically test and clear the flag. The local boolean is inspected
+ * below. When true, asynchronous overflow processing will occur unless
+ * an error occurs during synchronous overflow processing. This ensures
+ * that we can force a compacting merge on the shards of a data service
+ * even if that data service has not buffer sufficient writes to warrant
+ * a build on any of the index segments.
+ */
+ final boolean forceOverflow = this.forceOverflow.getAndSet(false/* newValue */);
+
final Event e = new Event(getFederation(), new EventResource(),
EventType.SynchronousOverflow).addDetail(
"synchronousOverflowCounter",
@@ -1982,7 +2015,12 @@
if (asyncOverflowEnabled.get()) {
- if (overflowMetadata.postProcess) {
+ /*
+ * Do overflow processing if overflow is being forced OR if we
+ * need to do a build for at least one index partition.
+ */
+
+ if (forceOverflow || overflowMetadata.postProcess) {
/*
* Post-processing SHOULD be performed.
Modified: trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -47,6 +47,7 @@
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.IndexSegment;
import com.bigdata.journal.ITransactionService;
import com.bigdata.journal.ITx;
import com.bigdata.mdi.IMetadataIndex;
@@ -492,25 +493,42 @@
}
- /**
- * Force overflow of each data service in the scale-out federation (only
- * scale-out federations support overflow processing). This method is
- * synchronous. It will not return until all {@link DataService}s have
- * initiated and completed overflow processing. Any unused resources (as
- * determined by the {@link StoreManager}) will have been purged.
- *
- * @param truncateJournal
- * When <code>true</code>, the live journal will be truncated
- * to its minimum extent (all writes will be preserved but there
- * will be no free space left in the journal). This may be used
- * to force the {@link DataService} to its minimum possible
- * footprint.
- *
- * @todo when overflow processing is enabled for the {@link MetadataService}
- * we will have to modify this to also trigger overflow for those
- * services.
- */
- public void forceOverflow(final boolean truncateJournal) {
+ /**
+ * Force overflow of each data service in the scale-out federation (only
+ * scale-out federations support overflow processing). This method is
+ * synchronous. It will not return until all {@link DataService}s have
+ * initiated and completed overflow processing. Any unused resources (as
+ * determined by the {@link StoreManager}) will have been purged.
+ * <p>
+ * This is a relatively fast operation when
+ * <code>compactingMerge := false</code>. By specifying both
+ * <code>compactingMerge := false</code> and
+ * <code>truncateJournal := false</code> you can cause the data services to
+ * close out their current journals against further writes. While this is
+ * not a global synchronous operation, it can provide a basis to obtain a
+ * "near synchronous" snapshot from the federation consisting of all writes
+ * up to the point where overflow was triggered on each data service.
+ *
+ * @param compactingMerge
+ * When <code>true</code>, each shard on each
+ * {@link IDataService} will undergo a compacting merge.
+ * Synchronous parallel compacting merge of all shards is an
+ * expensive operation. This parameter shoudl normally be
+ * <code>false</code> unless you are requesting a compacting
+ * merge for specific purposes, such as benchmarking when all
+ * data is known to exist in one {@link IndexSegment} per shard.
+ * @param truncateJournal
+ * When <code>true</code>, the live journal will be truncated to
+ * its minimum extent (all writes will be preserved but there
+ * will be no free space left in the journal). This may be used
+ * to force the {@link DataService} to its minimum possible
+ * footprint.
+ *
+ * @todo when overflow processing is enabled for the {@link MetadataService}
+ * we will have to modify this to also trigger overflow for those
+ * services.
+ */
+ public void forceOverflow(final boolean compactingMerge, final boolean truncateJournal) {
// find UUID for each data service.
final UUID[] dataServiceUUIDs = getDataServiceUUIDs(0/* maxCount */);
@@ -524,7 +542,7 @@
for (UUID serviceUUID : dataServiceUUIDs) {
tasks.add(new ForceOverflowTask(getDataService(serviceUUID),
- truncateJournal));
+ compactingMerge, truncateJournal));
}
@@ -641,16 +659,19 @@
.getLogger(ForceOverflowTask.class);
private final IDataService dataService;
+ private final boolean compactingMerge;
private final boolean truncateJournal;
- public ForceOverflowTask(final IDataService dataService,
- final boolean truncateJournal) {
+ public ForceOverflowTask(final IDataService dataService,
+ final boolean compactingMerge, final boolean truncateJournal) {
if (dataService == null)
throw new IllegalArgumentException();
this.dataService = dataService;
+ this.compactingMerge = compactingMerge;
+
this.truncateJournal = truncateJournal;
}
@@ -661,8 +682,7 @@
log.info("dataService: " + dataService.getServiceName());
// returns once synchronous overflow is complete.
- dataService
- .forceOverflow(true/* immediate */, true/* compactingMerge */);
+ dataService.forceOverflow(true/* immediate */, compactingMerge);
if (log.isInfoEnabled())
log.info("Synchronous overflow is done: "
Modified: trunk/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -43,7 +43,6 @@
import org.apache.log4j.Logger;
-import com.bigdata.concurrent.LockManager;
import com.bigdata.config.LongValidator;
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
@@ -80,9 +79,9 @@
*/
protected static final Logger log = Logger.getLogger(AbstractTransactionService.class);
- protected static final boolean INFO = log.isInfoEnabled();
+// protected static final boolean INFO = log.isInfoEnabled();
- protected static final boolean DEBUG = log.isDebugEnabled();
+// protected static final boolean DEBUG = log.isDebugEnabled();
/**
* Options understood by this service.
@@ -91,29 +90,39 @@
* @version $Id$
*/
public interface Options {
-
- /**
- * How long you want to hold onto the database history (in milliseconds)
- * or {@link Long#MAX_VALUE} for an (effectively) immortal database. The
- * {@link ITransactionService} tracks the timestamp corresponding to the
- * earliest running transaction (if any). When such a transaction
- * exists, the actual release time is:
- *
- * <pre>
- * releaseTime = min(earliestRunningTx, now - minimumReleaseAge) - 1
- * </pre>
- *
- * This ensures that history in use by running transactions is not
- * released even when the minimumReleaseAge is ZERO (0).
- *
- * @see #DEFAULT_MIN_RELEASE_AGE
- * @see #MIN_RELEASE_AGE_1H
- * @see #MIN_RELEASE_AGE_1D
- * @see #MIN_RELEASE_AGE_1W
- * @see #MIN_RELEASE_AGE_NEVER
- *
- * @see AbstractTransactionService#updateReleaseTime(long)
- */
+
+ /**
+ * How long you want to hold onto the database history (in milliseconds)
+ * or {@link Long#MAX_VALUE} for an (effectively) immortal database. The
+ * {@link ITransactionService} tracks the timestamp corresponding to the
+ * earliest running transaction (if any). When such a transaction
+ * exists, the actual release time is:
+ *
+ * <pre>
+ * releaseTime = min(lastCommitTime - 1, min(earliestRunningTx, now - minimumReleaseAge))
+ * </pre>
+ *
+ * This ensures that history in use by running transactions is not
+ * released even when the minimumReleaseAge is ZERO (0).
+ * <p>
+ * When no transactions exist the actual release time is:
+ *
+ * <pre>
+ * releaseTime = min(commitTime - 1, now - minimumReleaseAge)
+ * </pre>
+ *
+ * This ensures that the the release time advances when no transactions
+ * are in use, but that the minimum release age is still respected.
+ *
+ * @see #DEFAULT_MIN_RELEASE_AGE
+ * @see #MIN_RELEASE_AGE_1H
+ * @see #MIN_RELEASE_AGE_1D
+ * @see #MIN_RELEASE_AGE_1W
+ * @see #MIN_RELEASE_AGE_NEVER
+ *
+ * @see AbstractTransactionService#updateReleaseTime(long)
+ * @see AbstractTransactionService#notifyCommit(long)
+ */
String MIN_RELEASE_AGE = AbstractTransactionService.class.getName()
+ ".minReleaseAge";
@@ -231,7 +240,7 @@
Options.MIN_RELEASE_AGE,
Options.DEFAULT_MIN_RELEASE_AGE));
- if (INFO)
+ if (log.isInfoEnabled())
log.info(Options.MIN_RELEASE_AGE + "=" + minReleaseAge);
}
@@ -291,7 +300,7 @@
this.runState = newval;
- if (INFO) {
+ if (log.isInfoEnabled()) {
log.info("runState=" + runState);
@@ -306,7 +315,7 @@
*/
public void shutdown() {
- if(INFO)
+ if(log.isInfoEnabled())
log.info("");
lock.lock();
@@ -376,7 +385,7 @@
long elapsed = 0L;
- if(INFO)
+ if(log.isInfoEnabled())
log.info("activeCount="+getActiveCount());
while (getActiveCount() > 0) {
@@ -390,7 +399,7 @@
// update the elapsed time.
elapsed = System.nanoTime() - begin;
- if(INFO)
+ if(log.isInfoEnabled())
log.info("No transactions remaining: elapsed="+elapsed);
return;
@@ -456,7 +465,7 @@
*/
public void shutdownNow() {
- if(INFO)
+ if(log.isInfoEnabled())
log.info("");
lock.lock();
@@ -769,7 +778,7 @@
*/
public long getReleaseTime() {
- if (DEBUG)
+ if (log.isDebugEnabled())
log.debug("releaseTime=" + releaseTime + ", lastKnownCommitTime="
+ getLastCommitTime());
@@ -789,7 +798,7 @@
if(!lock.isHeldByCurrentThread())
throw new IllegalMonitorStateException();
- if (INFO)
+ if (log.isInfoEnabled())
log.info("newValue=" + newValue);
this.releaseTime = newValue;
@@ -911,7 +920,7 @@
}
- if (INFO)
+ if (log.isInfoEnabled())
log.info(state.toString());
// } finally {
@@ -981,10 +990,15 @@
synchronized (startTimeIndex) {
- isEarliestTx = startTimeIndex.findIndexOf(timestamp) == 0;
+ // Note: ZERO (0) is the first tuple in the B+Tree.
+ // Note: MINUS ONE (-1) means that the B+Tree is empty.
+ final int indexOf = startTimeIndex.findIndexOf(timestamp);
+
+ isEarliestTx = indexOf == 0;
- // remove start time from the index.
- startTimeIndex.remove(timestamp);
+ // remove start time from the index.
+ if (indexOf != -1)
+ startTimeIndex.remove(timestamp);
if (!isEarliestTx) {
@@ -1056,7 +1070,7 @@
*/
if (this.releaseTime < releaseTime) {
- if (INFO)
+ if (log.isInfoEnabled())
log.info("lastCommitTime=" + lastCommitTime
+ ", earliestTxStartTime=" + earliestTxStartTime
+ ", minReleaseAge=" + minReleaseAge + ", now="
@@ -1086,34 +1100,63 @@
try {
- synchronized (startTimeIndex) {
+ updateReleaseTimeForBareCommit(commitTime);
+
+ } finally {
- if (this.releaseTime < (commitTime - 1)
- && startTimeIndex.getEntryCount() == 0) {
+ lock.unlock();
- /*
- * If there are NO active transactions and the current
- * releaseTime is LT (commitTime-1) then advance the
- * releaseTime to (commitTime-1).
- */
+ }
- if (INFO)
- log.info("Advancing releaseTime (no active tx).");
+ }
- setReleaseTime(commitTime - 1);
+ /**
+ * If there are NO active transactions and the current releaseTime is LT
+ * (commitTime-1) then compute and set the new releaseTime.
+ * <p>
+ * Note: This method was historically part of {@link #notifyCommit(long)}.
+ * It was moved into its own method so it can be overriden for some unit
+ * tests.
+ *
+ * @throws IllegalMonitorStateException
+ * unless the caller is holding the lock.
+ */
+ protected void updateReleaseTimeForBareCommit(final long commitTime) {
- }
+ if(!lock.isHeldByCurrentThread())
+ throw new IllegalMonitorStateException();
+
+ synchronized (startTimeIndex) {
- }
+ if (this.releaseTime < (commitTime - 1)
+ && startTimeIndex.getEntryCount() == 0) {
- } finally {
+ final long lastCommitTime = commitTime;
- lock.unlock();
+ final long now = _nextTimestamp();
- }
+ final long releaseTime = Math.min(lastCommitTime - 1, now
+ - minReleaseAge);
+ if (this.releaseTime < releaseTime) {
+
+ if (log.isInfoEnabled())
+ log.info("Advancing releaseTime (no active tx)"
+ + ": lastCommitTime=" + lastCommitTime
+ + ", minReleaseAge=" + minReleaseAge + ", now="
+ + now + ", releaseTime(" + this.releaseTime
+ + "->" + releaseTime + ")");
+
+ setReleaseTime(releaseTime);
+
+ }
+
+ }
+
+ }
+
}
-
+
/**
* Return the minimum #of milliseconds of history that must be preserved.
*
@@ -1698,7 +1741,7 @@
* Note: The commits requests are placed into a partial order by sorting the
* total set of resources which the transaction declares (via this method)
* across all operations executed by the transaction and then contending for
- * locks on the named resources using a {@link LockManager}. This is
+ * locks on the named resources using a LockManager. This is
* handled by the {@link DistributedTransactionService}.
*/
public void declareResources(final long tx, final UUID dataServiceUUID,
@@ -2061,7 +2104,7 @@
// Note: sufficient to prevent deadlocks when there are shared indices.
resources.addAll(Arrays.asList(resource));
- if (INFO)
+ if (log.isInfoEnabled())
log.info("dataService=" + dataService + ", resource="
+ Arrays.toString(resource));
@@ -2201,7 +2244,7 @@
*/
public AbstractTransactionService start() {
- if(INFO)
+ if(log.isInfoEnabled())
log.info("");
lock.lock();
Modified: trunk/bigdata/src/java/com/bigdata/service/DataService.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/service/DataService.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/service/DataService.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -1845,8 +1845,8 @@
public Void call() throws Exception {
- final WriteExecutorService writeService = concurrencyManager
- .getWriteService();
+// final WriteExecutorService writeService = concurrencyManager
+// .getWriteService();
final ResourceManager resourceManager = (ResourceManager) DataService.this.resourceManager;
@@ -1859,7 +1859,8 @@
}
// trigger overflow on the next group commit.
- writeService.forceOverflow.set(true);
+// writeService.forceOverflow.set(true);
+ resourceManager.forceOverflow.set(true);
}
Modified: trunk/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -195,7 +195,7 @@
Options.SHAPSHOT_INTERVAL,
Options.DEFAULT_SHAPSHOT_INTERVAL));
- if (INFO)
+ if (log.isInfoEnabled())
log.info(Options.SHAPSHOT_INTERVAL + "=" + snapshotInterval);
isTransient = snapshotInterval == 0;
@@ -208,7 +208,7 @@
dataDir = new File(properties.getProperty(Options.DATA_DIR));
- if (INFO)
+ if (log.isInfoEnabled())
log.info(Options.DATA_DIR + "=" + dataDir);
}
@@ -218,7 +218,7 @@
setup();
- if (INFO)
+ if (log.isInfoEnabled())
log.info("lastCommitTime=" + lastCommitTime + ", #commitTimes="
+ commitTimeIndex.getEntryCount());
@@ -1891,7 +1891,7 @@
* themselves are serialized so that we do not miss any.
*/
- if (DEBUG)
+ if (log.isDebugEnabled())
log.debug("commitTime="
+ commitTime
+ ", lastKnownCommitTime="
Modified: trunk/bigdata/src/resources/logging/log4j.properties
===================================================================
--- trunk/bigdata/src/resources/logging/log4j.properties 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/resources/logging/log4j.properties 2010-08-05 16:29:44 UTC (rev 3414)
@@ -5,9 +5,9 @@
# Default log4j configuration for testing purposes.
#
# You probably want to set the default log level to ERROR.
-#
+
+#log4j.rootCategory=ALL, dest2
log4j.rootCategory=WARN, dest2
-#log4j.rootCategory=WARN, dest2
# Loggers.
# Note: logging here at INFO or DEBUG will significantly impact throughput!
@@ -66,7 +66,8 @@
#log4j.logger.com.bigdata.resources.IndexManager=INFO
#log4j.logger.com.bigdata.resources.ResourceManager=INFO
#log4j.logger.com.bigdata.resources.DefaultSplitHandler=INFO
-log4j.logger.com.bigdata.resources.OverflowManager=INFO
+#log4j.logger.com.bigdata.resources.StoreManager=INFO
+#log4j.logger.com.bigdata.resources.OverflowManager=INFO
log4j.logger.com.bigdata.resources.AsynchronousOverflowTask=INFO
log4j.logger.com.bigdata.resources.AbstractResourceManagerTask=INFO
#log4j.logger.com.bigdata.resources.AbstractResourceManagerTask=INFO
@@ -79,7 +80,6 @@
#log4j.logger.com.bigdata.service.ndx=INFO
#log4j.logger.com.bigdata.service.ndx.pipeline=DEBUG
-#log4j.logger.com.bigdata.relation.accesspath.BlockingBuffer=DEBUG
#log4j.logger.com.bigdata.concurrent.Latch=DEBUG
#log4j.logger.com.bigdata.relation=INFO
@@ -94,6 +94,7 @@
#log4j.logger.com.bigdata.relation.rule.eval.DefaultEvaluationPlan=DEBUG
#log4j.logger.com.bigdata.relation.accesspath.AbstractArrayBuffer=DEBUG
#log4j.logger.com.bigdata.relation.accesspath.BlockingBuffer=INFO
+log4j.logger.com.bigdata.relation.accesspath.BlockingBuffer=ERROR
#log4j.logger.com.bigdata.service.IBigdataFederation=DEBUG
#log4j.logger.com.bigdata.service.LoadBalancerService=INFO
Modified: trunk/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/journal/TestTransactionService.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/test/com/bigdata/journal/TestTransactionService.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -596,15 +596,17 @@
* GT the lastCommitTime since that could allow data not yet committed to
* become visible during the transaction (breaking isolation).
* <p>
- * A commitTime is identifed by looking up the callers timestamp in a log of
+ * A commitTime is identified by looking up the callers timestamp in a log of
* the historical commit times and returning the first historical commit
* time LTE the callers timestamp.
* <p>
- * The transaction start time is then choosen from the half-open interval
+ * The transaction start time is then chosen from the half-open interval
* <i>commitTime</i> (inclusive lower bound) : <i>nextCommitTime</i>
* (exclusive upper bound).
*
* @throws IOException
+ *
+ * @todo This test fails occasionally. I have not figured out why yet. BBT
*/
public void test_newTx_readOnly() throws IOException {
Modified: trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -144,39 +144,43 @@
final private UUID dataServiceUUID = UUID.randomUUID();
+ @Override
public IBigdataFederation getFederation() {
return fed;
}
+ @Override
public DataService getDataService() {
throw new UnsupportedOperationException();
}
+ @Override
public UUID getDataServiceUUID() {
return dataServiceUUID;
}
- /** Note: no failover services. */
- public UUID[] getDataServiceUUIDs() {
-
- return new UUID[] { dataServiceUUID };
-
- }
-
};
txService = new MockTransactionService(properties){
+ @Override
protected void setReleaseTime(long releaseTime) {
super.setReleaseTime(releaseTime);
+ if (log.isInfoEnabled())
+ log
+ .info("Propagating new release time to the resourceManager: releaseTime="
+ + releaseTime
+ + ", releaseAge="
+ + getMinReleaseAge());
+
// propagate the new release time to the resource manager.
resourceManager.setReleaseTime(releaseTime);
Modified: trunk/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -41,15 +41,6 @@
/**
* Test release (aka purge) of old resources.
*
- * FIXME This set of unit tests needs to be updated to reflect that
- * purgeResources() is invoked during synchronous overflow. The tests are
- * written with a different assumption, which is why they are failing.
- *
- * @todo Write a unit test for purge before, during and after the 1st overflow
- * and after a restart. Before, there should be nothing to release.
- * During, the views that are being constructed should remain safe. After,
- * we should be able to achieve a compact footprint for the data service.
- *
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
@@ -180,20 +171,19 @@
return properties;
}
-
- /**
- * Test creates an index whose view is initially defined by the initial
- * journal on the {@link ResourceManager}. An overflow of the journal
- * is then forced, which re-defines the view on the new journal. Since
- * the index is very small (it is empty), it is "copied" onto the new
- * journal rather than causing an index segment build to be scheduled.
- * Once the asynchronous overflow completes, the original journal should
- * qualify for release.
- *
- * @throws IOException
- * @throws ExecutionException
- * @throws InterruptedException
- */
+
+ /**
+ * Test creates an index whose view is initially defined by the initial
+ * journal on the {@link ResourceManager}. An overflow of the journal is
+ * then forced, which re-defines the view on the new journal. Since the
+ * index is very small (it is empty), it is "copied" onto the new
+ * journal rather than causing an index segment build to be scheduled.
+ * The original journal should not be released.
+ *
+ * @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
public void test() throws IOException,
InterruptedException, ExecutionException {
@@ -204,8 +194,9 @@
final AbstractJournal j0 = resourceManager.getLiveJournal();
// force overflow on the next commit.
- concurrencyManager.getWriteService().forceOverflow.set(true);
-
+// concurrencyManager.getWriteService().forceOverflow.set(true);
+ resourceManager.forceOverflow.set(true);
+
// disable asynchronous overflow processing to simplify the test environment.
resourceManager.asyncOverflowEnabled.set(false);
@@ -229,35 +220,33 @@
assertTrue(createTime0 < createTime1);
// verify can still open the original journal.
- // @todo unit test needs to be updated to reflect purge in sync overflow.
- assertTrue(j0 == resourceManager.openStore(j0.getResourceMetadata()
- .getUUID()));
+ assertTrue(j0 == resourceManager.openStore(j0.getResourceMetadata()
+ .getUUID()));
// verify can still open the new journal.
assertTrue(j1 == resourceManager.openStore(j1.getResourceMetadata()
.getUUID()));
- // 2 journals.
+ // 1 journals.
assertEquals(2,resourceManager.getManagedJournalCount());
// no index segments.
assertEquals(0,resourceManager.getManagedSegmentCount());
/*
- * Verify that we can the first commit record when we provide the
- * create time for a journal.
+ * Verify that the commit time index.
*/
-
- // for j0
- assertEquals(j0.getRootBlockView().getFirstCommitTime(),
- resourceManager
- .getCommitTimeStrictlyGreaterThan(createTime0));
- // for j1.
- assertEquals(j1.getRootBlockView().getFirstCommitTime(),
- resourceManager
- .getCommitTimeStrictlyGreaterThan(createTime1));
+ // for j0
+ assertEquals(j1.getRootBlockView().getFirstCommitTime(),
+ resourceManager
+ .getCommitTimeStrictlyGreaterThan(createTime1));
+ // for j1.
+ assertEquals(j1.getRootBlockView().getFirstCommitTime(),
+ resourceManager
+ .getCommitTimeStrictlyGreaterThan(createTime1));
+
/*
* Verify that the resources required for [A] are {j0, j1} when the
* probe commitTime is the timestamp when we registered [A] on [j0].
@@ -331,20 +320,20 @@
return properties;
}
-
- /**
- * Test creates an index whose view is initially defined by the initial
- * journal on the sole data service. An overflow of the journal is then
- * forced, which re-defines the view on the new journal. Since the index
- * is very small (it is empty), it is "copied" onto the new journal
- * rather than causing an index segment build to be scheduled. Once the
- * asynchronous overflow completes, the original journal should qualify
- * for release.
- *
- * @throws IOException
- * @throws ExecutionException
- * @throws InterruptedException
- */
+
+ /**
+ * Test creates an index whose view is initially defined by the initial
+ * journal on the sole data service. An overflow of the journal is then
+ * forced, which re-defines the view on the new journal. Since the index
+ * is very small (it is empty), it is "copied" onto the new journal
+ * rather than causing an index segment build to be scheduled. The
+ * original journal should be released (deleted) during synchronous
+ * overflow processing.
+ *
+ * @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
public void test() throws IOException,
InterruptedException, ExecutionException {
@@ -358,8 +347,9 @@
final UUID uuid0 = j0.getResourceMetadata().getUUID();
// force overflow on the next commit.
- concurrencyManager.getWriteService().forceOverflow.set(true);
-
+// concurrencyManager.getWriteService().forceOverflow.set(true);
+ resourceManager.forceOverflow.set(true);
+
// disable asynchronous overflow processing to simplify the test environment.
resourceManager.asyncOverflowEnabled.set(false);
@@ -371,39 +361,38 @@
// did overflow.
assertEquals(1,resourceManager.getAsynchronousOverflowCount());
- /*
- * Note: the old journal should have been closed for writes during
- * synchronous overflow processing.
- */
- // @todo unit test needs to be updated to reflect purge in sync overflow.
- assertTrue(j0.isOpen()); // still open
- assertTrue(j0.isReadOnly()); // but no longer accepts writes.
+// /*
+// * Note: the old journal should have been closed for writes during
+// * synchronous overflow processing and deleted from the file system.
+// */
+// assertTrue(j0.isOpen()); // still open
+// assertTrue(j0.isReadOnly()); // but no longer accepts writes.
+//
+// /*
+// * Purge old resources. If the index was copied to the new journal
+// * then there should be no dependency on the old journal and it
+// * should be deleted.
+// */
+// {
+//
+// final AbstractJournal liveJournal = resourceManager
+// .getLiveJournal();
+//
+// final long lastCommitTime = liveJournal.getLastCommitTime();
+//
+// final Set<UUID> actual = resourceManager
+// .getResourcesForTimestamp(lastCommitTime);
+//
+// assertSameResources(new IRawStore[] {liveJournal}, actual);
+//
+// // only retain the lastCommitTime.
+// resourceManager.setReleaseTime(lastCommitTime - 1);
+//
+// }
+//
+// resourceManager
+// .purgeOldResources(1000/* ms */, false/*truncateJournal*/);
- /*
- * Purge old resources. If the index was copied to the new journal
- * then there should be no dependency on the old journal and it
- * should be deleted.
- */
- {
-
- final AbstractJournal liveJournal = resourceManager
- .getLiveJournal();
-
- final long lastCommitTime = liveJournal.getLastCommitTime();
-
- final Set<UUID> actual = resourceManager
- .getResourcesForTimestamp(lastCommitTime);
-
- assertSameResources(new IRawStore[] {liveJournal}, actual);
-
- // only retain the lastCommitTime.
- resourceManager.setReleaseTime(lastCommitTime - 1);
-
- }
-
- resourceManager
- .purgeOldResources(1000/* ms */, false/*truncateJournal*/);
-
// verify that the old journal is no longer open.
assertFalse(j0.isOpen());
@@ -475,16 +464,17 @@
super(arg0);
}
-
- /**
- * This is the minimum release time that will be used for the test.
- * <P>
- * Note: 20000 is 20 seconds. This is what you SHOULD use for the test.
- * <p>
- * Note: 200000 is 200 seconds. This can be used for debugging, but
- * always restore the value so that the test will run in a reasonable
- * timeframe.
- */
+
+ /**
+ * This is the minimum release time that will be used for the test.
+ * <P>
+ * Note: 2000 is 2 seconds. This is what you SHOULD use for the test (it
+ * can be a little longer if you run into problems).
+ * <p>
+ * Note: 200000 is 200 seconds. This can be used for debugging, but
+ * always restore the value so that the test will run in a reasonable
+ * time frame.
+ */
final private long MIN_RELEASE_AGE = 2000;
public Properties getProperties() {
@@ -498,22 +488,22 @@
return properties;
}
-
- /**
- * Test where the index view is copied in its entirety onto the new
- * journal and the [minReleaseAge] is 2 seconds. In this case we have no
- * dependencies on the old journal, but the [minReleaseAge] is not
- * satisified immediately so no resources are released during overflow
- * processing (assuming that overflow processing is substantially faster
- * than the [minReleaseAge]). We then wait until the [minReleaseAge] has
- * passed and force overflow processing again and verify that the
- * original journal was released while the 2nd and 3rd journals are
- * retained.
- *
- * @throws IOException
- * @throws ExecutionException
- * @throws InterruptedException
- */
+
+ /**
+ * Test where the index view is copied in its entirety onto the new
+ * journal and the [minReleaseAge] is 2 seconds. In this case we have no
+ * dependencies on the old journal, but the [minReleaseAge] is not
+ * satisfied immediately so no resources are released during synchronous
+ * overflow processing (assuming that synchronous overflow processing is
+ * substantially faster than the [minReleaseAge]). We then wait until
+ * the [minReleaseAge] has passed and force overflow processing again
+ * and verify that the original journal was released while the 2nd and
+ * 3rd journals are retained.
+ *
+ * @throws IOException
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
public void test() throws IOException,
InterruptedException, ExecutionException {
@@ -527,7 +517,8 @@
final UUID uuid0 = j0.getResourceMetadata().getUUID();
// force overflow on the next commit.
- concurrencyManager.getWriteService().forceOverflow.set(true);
+// concurrencyManager.getWriteService().forceOverflow.set(true);
+ resourceManager.forceOverflow.set(true);
// disable asynchronous overflow processing to simplify the test environment.
resourceManager.asyncOverflowEnabled.set(false);
@@ -582,7 +573,8 @@
}
// force overflow on the next commit.
- concurrencyManager.getWriteService().forceOverflow.set(true);
+// concurrencyManager.getWriteService().forceOverflow.set(true);
+ resourceManager.forceOverflow.set(true);
// register another index - will force another overflow.
registerIndex("B");
Modified: trunk/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -626,20 +626,14 @@
* and when we have not since forcing overflow will trigger
* compacting merges. So you are more likely to find a problem
* if you DO NOT force overflow.
- *
- * Note: This DOES NOT guarantee that overflow is forced on a
- * given data service. it is forced if the method can gain the
- * exclusive write lock for that data service. otherwise it will
- * timeout and overflow processing will not be triggered on that
- * data service.
*/
final boolean forceOverflow = false;
if (forceOverflow) {
System.err.println("Forcing overflow: " + new Date());
- ((AbstractScaleOutFederation<?>) federation)
- .forceOverflow(true/* truncateJournal */);
+ ((AbstractScaleOutFederation<?>) federation)
+ .forceOverflow(true/* compactingMerge */, true/* truncateJournal */);
System.err.println("Forced overflow: " + new Date());
Modified: trunk/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -75,6 +75,7 @@
}
+ @Override
protected void setReleaseTime(final long releaseTime) {
lock.lock();
@@ -102,6 +103,7 @@
}
+ @Override
public MockDistributedTransactionService start() {
super.start();
@@ -110,6 +112,18 @@
}
+ /**
+ * This is overridden to be a NOP for this test suite. The unit tests in
+ * this suite depend on the ability to inject specific commit times into
+ * the transaction service without having them "release" based on the
+ * actual system clock.
+ */
+ @Override
+ protected void updateReleaseTimeForBareCommit(final long commitTime) {
+
+ return;
+ }
+
/**
* Exposed to the unit tests.
*/
@@ -129,19 +143,19 @@
*
* @return The array.
*/
- long[] toArray(CommitTimeIndex ndx) {
+ long[] toArray(final CommitTimeIndex ndx) {
synchronized(ndx) {
- long[] a = new long[ndx.getEntryCount()];
+ final long[] a = new long[ndx.getEntryCount()];
- final ITupleIterator itr = ndx.rangeIterator();
+ final ITupleIterator<?> itr = ndx.rangeIterator();
int i = 0;
while(itr.hasNext()) {
- final ITuple tuple = itr.next();
+ final ITuple<?> tuple = itr.next();
a[i] = ndx.decodeKey(tuple.getKey());
@@ -160,7 +174,6 @@
* when the release time is advanced and that it is still possible to obtain
* a read-only tx as of the timestamp immediately after the current release
* time.
- *
*/
public void test_setReleaseTime() {
@@ -169,7 +182,10 @@
properties.setProperty(DistributedTransactionService.Options.DATA_DIR,
getName());
- MockDistributedTransactionService service = new MockDistributedTransactionService(
+ properties.setProperty(DistributedTransactionService.Options.MIN_RELEASE_AGE,
+ "10");
+
+ final MockDistributedTransactionService service = new MockDistributedTransactionService(
properties).start();
try {
@@ -184,7 +200,7 @@
// verify the commit index.
{
- CommitTimeIndex ndx = service.getCommitTimeIndex();
+ final CommitTimeIndex ndx = service.getCommitTimeIndex();
synchronized (ndx) {
@@ -292,7 +308,7 @@
{
- CommitTimeIndex ndx = service.getCommitTimeIndex();
+ final CommitTimeIndex ndx = service.getCommitTimeIndex();
synchronized(ndx) {
@@ -354,7 +370,7 @@
// verify the commit time index.
{
- CommitTimeIndex ndx = service.getCommitTimeIndex();
+ final CommitTimeIndex ndx = service.getCommitTimeIndex();
synchronized(ndx) {
Modified: trunk/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -287,6 +287,7 @@
assertNotNull(subtaskStats);
+ // @todo this assert fails stochastically.
assertEquals("chunksOut", 1, subtaskStats.chunksOut.get());
assertEquals("elementsOut", 2, subtaskStats.elementsOut.get());
Modified: trunk/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -231,7 +231,7 @@
public Object getAdmin() throws RemoteException {
- if (INFO)
+ if (log.isInfoEnabled())
log.info("" + getServiceUUID());
return server.proxy;
Modified: trunk/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -113,19 +113,20 @@
*/
public interface ConfigurationOptions {
- /**
- * When <code>true</code> as an after action on the job, the
- * {@link DataService}s in the federation will be made to undergo
- * asynchronous overflow processing and the live journals will be
- * truncated so that the total size on disk of the federation is at its
- * minimum footprint for the given history retention policy (default
- * <code>false</code>). The master will block during this operation
- * so you can readily tell when it is finished. Note that this option
- * only makes sense in benchmark environments where you can contol the
- * total system otherwise asynchronous writes may continue.
- *
- * @see AbstractScaleOutFederation#forceOverflow(boolean)
- */
+ /**
+ * When <code>true</code> as an after action on the job, the
+ * {@link DataService}s in the federation will be made to undergo
+ * asynchronous overflow processing, a compacting merge will be
+ * requested for all shards, and the live journals will be truncated so
+ * that the total size on disk of the federation is at its minimum
+ * footprint for the given history retention policy (default
+ * <code>false</code>). The master will block during this operation so
+ * you can readily tell when it is finished. Note that this option only
+ * makes sense in benchmark environments where you can control the total
+ * system otherwise asynchronous writes may continue.
+ *
+ * @see AbstractScaleOutFederation#forceOverflow(boolean, boolean)
+ */
String FORCE_OVERFLOW = "forceOverflow";
/**
@@ -1763,7 +1764,7 @@
System.out.println("Forcing overflow: now=" + new Date());
- fed.forceOverflow(true/* truncateJournal */);
+ fed.forceOverflow(true/* compactingMerge */, true/* truncateJournal */);
System.out.println("Forced overflow: now=" + new Date());
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-08-05 16:29:44 UTC (rev 3414)
@@ -787,7 +787,7 @@
System.out.println("Forcing overflow: now=" + new Date());
- fed.forceOverflow(true/* truncateJournal */);
+ fed.forceOverflow(true/* compactingMerge */, true/* truncateJournal */);
System.out.println("Forced overflow: now=" + new Date());
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config 2010-08-04 20:32:16 UTC (rev 3413)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config 2010-08-05 16:29:44 UTC (rev 3414)
@@ -9,7 +9,10 @@
// Source file or directory (required).
//srcDir = new File("/data/2010_btc");
- srcDir = new File("D:/bigdata-perf-analysis/btc/btc-2009-small.nq.gz");
+ //srcDir = new File("D:/bigdata-perf-analysis/btc/btc-2009-small.nq.gz");
+ //srcDir = new File("/nas/data/bsbm/bsbm_2785/dataset.nt.gz");
+ srcDir = new File("/nas/data/bsbm/bsbm_141624/dataset.nt.gz");
+ //srcDir = new File("/nas/data/bsbm/bsbm_284826/dataset.nt.gz");
// File name filter for the srcDir (optional).
srcFilter = new com.bigdata.rdf.load.RDFFilenameFilter();
@@ -18,29 +21,34 @@
* known values are: "RDF/XML", "N-Triples", "Turtle", "N3", "TriX", "TriG",
* "nquads".
*/
- srcFormat = "nquads";
+ //srcFormat = "nquads";
+ srcFormat = "N-Triples";
// The output directory (required).
//outDir = new File("/tmp");
- outDir = new File("D:/temp");
+ //outDir = new File("D:/temp");
+ //outDir = new File("/nas/data/bsbm/bsbm_284826/split");
+ outDir = new File("/nas/data/bsbm/bsbm_141624/split");
+ //outDir = new File("/nas/data/bsbm/bsbm_2785/split");
/* The name of the default RDFFormat for the output files (optional). The
* known values are: "RDF/XML", "N-Triples", "Turtle", "N3", "TriX", "TriG",
* "nquads". The default is whatever the input file was.
*/
- outFormat = "TriG";
+ //outFormat = "TriG";
+ outFormat = "N-Triples";
// The output compression mode: None, Zip, or GZip.
...
[truncated message content] |
|
From: <tho...@us...> - 2010-08-04 20:32:22
|
Revision: 3413
http://bigdata.svn.sourceforge.net/bigdata/?rev=3413&view=rev
Author: thompsonbry
Date: 2010-08-04 20:32:16 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Changed the multi-block iterator block size default in the unit tests to 1M and removed the 10M pool for now.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
trunk/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
trunk/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
Modified: trunk/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-04 20:26:03 UTC (rev 3412)
+++ trunk/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-04 20:32:16 UTC (rev 3413)
@@ -218,12 +218,12 @@
*/
public final static DirectBufferPool INSTANCE;
- /**
- * A JVM-wide pool of direct {@link ByteBuffer}s with a default
- * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
- * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
- */
- public final static DirectBufferPool INSTANCE_10M;
+// /**
+// * A JVM-wide pool of direct {@link ByteBuffer}s with a default
+// * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
+// * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
+// */
+// public final static DirectBufferPool INSTANCE_10M;
/**
* An unbounded list of all {@link DirectBufferPool} instances.
@@ -251,11 +251,11 @@
bufferCapacity//
);
- INSTANCE_10M = new DirectBufferPool(//
- "10M",//
- Integer.MAX_VALUE, // poolCapacity
- 10 * Bytes.megabyte32 // bufferCapacity
- );
+// INSTANCE_10M = new DirectBufferPool(//
+// "10M",//
+// Integer.MAX_VALUE, // poolCapacity
+// 10 * Bytes.megabyte32 // bufferCapacity
+// );
/*
* This configuration will block if there is a concurrent demand for
Modified: trunk/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java 2010-08-04 20:26:03 UTC (rev 3412)
+++ trunk/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java 2010-08-04 20:32:16 UTC (rev 3413)
@@ -290,7 +290,7 @@
final long actualTupleCount = doEntryIteratorTest(expected
.rangeIterator(), new IndexSegmentMultiBlockIterator(actual,
- DirectBufferPool.INSTANCE_10M, null/* fromKey */,
+ DirectBufferPool.INSTANCE, null/* fromKey */,
null/* toKey */, IRangeQuery.DEFAULT));
// verifies based on what amounts to an exact range count.
Modified: trunk/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java 2010-08-04 20:26:03 UTC (rev 3412)
+++ trunk/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java 2010-08-04 20:32:16 UTC (rev 3413)
@@ -303,7 +303,7 @@
try {
final IndexSegmentMultiBlockIterator<?> itr = new IndexSegmentMultiBlockIterator(
- seg, DirectBufferPool.INSTANCE_10M, null/* fromKey */,
+ seg, DirectBufferPool.INSTANCE, null/* fromKey */,
null/* toKey */, IRangeQuery.DEFAULT);
assertFalse(itr.hasNext());
@@ -328,7 +328,8 @@
final BTree btree = BTree.createTransient(new IndexMetadata(UUID
.randomUUID()));
- final int LIMIT = 1000000;
+ final int LIMIT = 200000; // this works out to 12 1M blocks of data.
+// final int LIMIT = 1000000; // this works out to 60 1M blocks of data.
// populate the index.
for (int i = 0; i < LIMIT; i++) {
@@ -407,6 +408,8 @@
private void doRandomScanTest(final BTree groundTruth,
final IndexSegment actual, final int ntests) {
+ final DirectBufferPool pool = DirectBufferPool.INSTANCE;
+
final Random r = new Random();
final int n = groundTruth.getEntryCount();
@@ -425,7 +428,7 @@
IRangeQuery.DEFAULT, null/* filter */);
final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
- actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ actual, pool, fromKey, toKey,
IRangeQuery.DEFAULT);
assertSameEntryIterator(expectedItr, actualItr);
@@ -446,7 +449,7 @@
IRangeQuery.DEFAULT, null/* filter */);
final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
- actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ actual, pool, fromKey, toKey,
IRangeQuery.DEFAULT);
assertSameEntryIterator(expectedItr, actualItr);
@@ -468,7 +471,7 @@
IRangeQuery.DEFAULT, null/* filter */);
final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
- actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ actual, pool, fromKey, toKey,
IRangeQuery.DEFAULT);
assertSameEntryIterator(expectedItr, actualItr);
@@ -492,7 +495,7 @@
IRangeQuery.DEFAULT, null/* filter */);
final IndexSegmentMultiBlockIterator<?> actualItr = new IndexSegmentMultiBlockIterator(
- actual, DirectBufferPool.INSTANCE_10M, fromKey, toKey,
+ actual, pool, fromKey, toKey,
IRangeQuery.DEFAULT);
assertSameEntryIterator(expectedItr, actualItr);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 20:26:10
|
Revision: 3412
http://bigdata.svn.sourceforge.net/bigdata/?rev=3412&view=rev
Author: thompsonbry
Date: 2010-08-04 20:26:03 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Bug fix at com.bigdata.rdf.lexicon.LexiconRelation$ResolveTermTask.call(LexiconRelation.java:1899) an assert was using "==" when it should have been using equals() to compare IVs.
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-04 20:06:49 UTC (rev 3411)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-04 20:26:03 UTC (rev 3412)
@@ -1896,7 +1896,7 @@
* IllegalStateException if the value somehow was assigned
* the wrong term identifier (paranoia test).
*/
- assert value.getIV() == tid : "expecting tid=" + tid
+ assert value.getIV().equals(tid) : "expecting tid=" + tid
+ ", but found " + value.getIV();
assert (value).getValueFactory() == valueFactory;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 20:06:55
|
Revision: 3411
http://bigdata.svn.sourceforge.net/bigdata/?rev=3411&view=rev
Author: thompsonbry
Date: 2010-08-04 20:06:49 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Changed the multi-block iterator to use DirectBufferPool.INSTANCE rather than a custom 10M buffer pool. By using the same pool size everywhere we get better reuse out of the buffer pool.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
Modified: trunk/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-04 20:05:23 UTC (rev 3410)
+++ trunk/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-04 20:06:49 UTC (rev 3411)
@@ -2840,7 +2840,8 @@
* might also want to limit the maximum size of the reads.
*/
- final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+ final DirectBufferPool pool = DirectBufferPool.INSTANCE;
if (true
&& ((flags & REVERSE) == 0)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 20:05:29
|
Revision: 3410
http://bigdata.svn.sourceforge.net/bigdata/?rev=3410&view=rev
Author: thompsonbry
Date: 2010-08-04 20:05:23 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
added a script to start the NanoSparqlServer for a federation.
Added Paths:
-----------
trunk/src/resources/scripts/nanoSparqlServer.sh
Added: trunk/src/resources/scripts/nanoSparqlServer.sh
===================================================================
--- trunk/src/resources/scripts/nanoSparqlServer.sh (rev 0)
+++ trunk/src/resources/scripts/nanoSparqlServer.sh 2010-08-04 20:05:23 UTC (rev 3410)
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+#
+# Start a NanoSparqlServer fronting for a bigdata federation.
+#
+# usage: configFile
+
+source `dirname $0`/bigdataenv
+
+port=$1
+namespace=$2
+
+echo "port=$port namespace=$namespace config=$BIGDATA_CONFIG"
+
+java ${JAVA_OPTS} \
+ -cp ${CLASSPATH} \
+ com.bigdata.rdf.sail.bench.NanoSparqlServer \
+ $port \
+ $namespace \
+ ${BIGDATA_CONFIG} ${BIGDATA_CONFIG_OVERRIDES}
+
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 20:05:13
|
Revision: 3409
http://bigdata.svn.sourceforge.net/bigdata/?rev=3409&view=rev
Author: thompsonbry
Date: 2010-08-04 20:05:06 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Changed the log level from INFO to WARN when forcing a compacting merge of the data servivces in the federation.
Modified DumpFederation to use CSV rather than tab-delimited output.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
Modified: trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-04 18:53:35 UTC (rev 3408)
+++ trunk/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-04 20:05:06 UTC (rev 3409)
@@ -517,8 +517,7 @@
final int ndataServices = dataServiceUUIDs.length;
- if(log.isInfoEnabled())
- log.info("#dataServices=" + ndataServices + ", now=" + new Date());
+ log.warn("Forcing overflow: #dataServices=" + ndataServices + ", now=" + new Date());
final List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(ndataServices);
@@ -570,8 +569,7 @@
}
- if(log.isInfoEnabled())
- log.info("Did overflow: #ok=" + nok + ", #dataServices="
+ log.warn("Did overflow: #ok=" + nok + ", #dataServices="
+ ndataServices + ", now=" + new Date());
if (nok != tasks.size()) {
Modified: trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
===================================================================
--- trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java 2010-08-04 18:53:35 UTC (rev 3408)
+++ trunk/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java 2010-08-04 20:05:06 UTC (rev 3409)
@@ -191,7 +191,8 @@
try {
- final FormatRecord formatter = new FormatTabTable(System.out);
+ final FormatRecord formatter = new FormatCSVTable(System.out);
+// final FormatRecord formatter = new FormatTabTable(System.out);
final DumpFederation dumper = new DumpFederation(fed, tx,
formatter);
@@ -416,7 +417,7 @@
private final FormatRecord formatter;
/**
- * Interface reponsible for formatting the output.
+ * Interface responsible for formatting the output.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -632,7 +633,199 @@
}
/**
+ * Comma separated value delimited tabular output.
*
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+ public static class FormatCSVTable implements FormatRecord {
+
+ private final PrintWriter out;
+
+ public FormatCSVTable(final PrintWriter w) {
+
+ if (w == null)
+ throw new IllegalArgumentException();
+
+ this.out = w;
+
+ }
+
+ public FormatCSVTable(final Writer w) {
+
+ this( new PrintWriter( w ));
+
+ }
+
+ public FormatCSVTable(final PrintStream w) {
+
+ this(new PrintWriter(w));
+
+ }
+
+ /**
+ * @todo document the columns.
+ */
+ public void writeHeaders() {
+
+ final String s = "Timestamp"//
+ + ",IndexName" //
+ + ",IndexPartitionName"//
+ + ",PartitionId" //
+ + ",ServiceUUID" //
+ + ",ServiceName" //
+ + ",Hostname" //
+ + ",ServiceCode"//
+ /*
+ * Basic metadata about the index partition.
+ */
+ + ",SourceCount"//
+ + ",SourceJournalCount"
+ + ",SourceSegmentCount"
+ + ",SumEntryCounts" //
+ + ",SumNodeCounts" //
+ + ",SumLeafCounts" //
+ + ",SumSegmentBytes"//
+ + ",SumSegmentNodeBytes" //
+ + ",SumSegmentLeafBytes"//
+ /*
+ * Note: These values are aggregates for the data
+ * service on which the index partition resides.
+ */
+ + ",DataDirFreeSpace"//
+ + ",BytesUnderManagement"//
+ + ",JournalBytesUnderManagement"//
+ + ",IndexSegmentBytesUnderManagement"//
+ + ",ManagedJournalCount"//
+ + ",ManagedSegmentCount"//
+ + ",AsynchronousOverflowCount"//
+ /*
+ * Extended metadata about the index partition.
+ */
+ + ",LeftSeparator"//
+ + ",RightSeparator"//
+ + ",View"//
+ + ",Cause"//
+// + "\tHistory"//
+ + ",IndexMetadata"//
+ ;
+
+ out.println(s);
+
+ }
+
+ /** format row for table. */
+ public void writeRecord(final IndexPartitionRecord rec) {
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append(rec.ts);//new Date(ts));
+ sb.append("," + rec.indexName);
+ sb.append("," + DataService.getIndexPartitionName(rec.indexName,rec.locator.getPartitionId()));
+ sb.append("," + rec.locator.getPartitionId());
+ sb.append("," + rec.locator.getDataServiceUUID());
+ sb.append("," + rec.smd.getName());
+ sb.append("," + rec.smd.getHostname());
+ sb.append("," + "DS" + rec.smd.getCode());
+
+ if (rec.detailRec != null) {
+
+ // aggregate across all sources in the view.
+ final SourceDetailRecord sourceDetailRec = new SourceDetailRecord(
+ rec.detailRec.sources);
+
+ // core view stats.
+ sb.append("," + rec.detailRec.sourceCount);
+ sb.append("," + rec.detailRec.journalSourceCount);
+ sb.append("," + rec.detailRec.segmentSourceCount);
+
+ // per source stats (aggregated across sources in the view).
+ sb.append("," + sourceDetailRec.entryCount);
+ sb.append("," + sourceDetailRec.nodeCount);
+ sb.append("," + sourceDetailRec.leafCount);
+ sb.append("," + sourceDetailRec.segmentByteCount);
+ sb.append("," + sourceDetailRec.segmentNodeByteCount);
+ sb.append("," + sourceDetailRec.segmentLeafByteCount);
+
+ // stats for the entire data service
+ sb.append("," + rec.detailRec.dataDirFreeSpace);
+ sb.append("," + rec.detailRec.bytesUnderManagement);
+ sb.append("," + rec.detailRec.journalBytesUnderManagement);
+ sb.append("," + rec.detailRec.segmentBytesUnderManagement);
+ sb.append("," + rec.detailRec.managedJournalCount);
+ sb.append("," + rec.detailRec.managedSegmentCount);
+ sb.append("," + rec.detailRec.asynchronousOverflowCount);
+
+ } else {
+
+ /*
+ * Error obtaining the data of interest.
+ */
+
+ // core view stats.
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+
+ // aggregated per-source in view stats.
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+
+ // data service stats.
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+ sb.append(",N/A");
+
+ }
+
+ // extended view stats.
+ sb.append(","
+ + BytesUtil.toString(rec.locator.getLeftSeparatorKey()).replace(',', ' '));
+ sb.append(","
+ + BytesUtil.toString(rec.locator.getRightSeparatorKey()).replace(',', ' '));
+
+ if (rec.detailRec != null && rec.detailRec.pmd != null) {
+
+ // current view definition.
+ sb.append(",\""
+ + Arrays.toString(rec.detailRec.pmd.getResources()).replace(',', ';')
+ + "\"");
+
+ // cause (reason why the index partition was created).
+ sb.append(",\""
+ + rec.detailRec.pmd.getIndexPartitionCause().toString().replace(',', ';')
+ + "\"");
+
+// // history
+// sb.append(",\"" + rec.detailRec.pmd.getHistory() + "\"");
+
+ // indexMetadata
+ sb.append(",\"" + rec.detailRec.indexMetadata.toString().replace(',', ';') + "\"");
+
+ } else {
+
+ sb.append(",N/A");
+ sb.append(",N/A");
+// sb.append(",N/A");
+ sb.append(",N/A");
+
+ }
+
+ out.println(sb.toString());
+
+ }
+
+ }
+
+ /**
+ *
* @param fed
* The federation whose indices will be dump.
* @param tx
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 18:53:45
|
Revision: 3408
http://bigdata.svn.sourceforge.net/bigdata/?rev=3408&view=rev
Author: thompsonbry
Date: 2010-08-04 18:53:35 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Merged from trunk [r3368:r3407].
Rolled back changes by MikeP to IPredicate. He will roll in new versions of those changes based on updated work on handling options.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/ResourceEvents.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/TPS.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedConvertingIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIncrementalWrite.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderCacheInteraction.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithCompactingMerge.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithIncrementalBuild.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentWithBloomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestInsertLookupRemoveKeysInRootLeaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestLinearListMethods.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestMutableBTreeCursors.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestReopen.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestSplitJoinRootLeaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestSplitJoinThreeLevels.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestSplitRootLeaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestTouch.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestTransientBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestTupleFilters.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestSuccessorUtil.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/raba/codec/AbstractRabaCoderTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/raba/codec/RandomURIGenerator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestBuildTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestMergeTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestOverflow.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestResourceManagerBootstrap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestMove.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestScatterSplit.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestSplitJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTaskWithSplits.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9.txt
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/BlobOverflowHandler.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/small.rdf
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOTupleSerializer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/QUADS_QUERY_BRANCH/src/resources/analysis/queries/benchmark.txt
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config
branches/QUADS_QUERY_BRANCH/src/resources/config/log4j.properties
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DHTFilterFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9-modified.txt
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/CVSROOT/
branches/QUADS_QUERY_BRANCH/bigdata-master-pom/
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/lib/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/answers (U1)/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/scripts/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/dsi-utils/LEGAL/
branches/QUADS_QUERY_BRANCH/dsi-utils/lib/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/compression/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/io/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/util/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/io/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/util/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/osgi/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -1525,45 +1525,63 @@
}
- /**
- * Load an instance of a {@link BTree} or derived class from the store. The
- * {@link BTree} or derived class MUST declare a constructor with the
- * following signature: <code>
+ /**
+ * Load an instance of a {@link BTree} or derived class from the store. The
+ * {@link BTree} or derived class MUST declare a constructor with the
+ * following signature: <code>
*
* <i>className</i>(IRawStore store, Checkpoint checkpoint, BTreeMetadata metadata, boolean readOnly)
*
* </code>
- *
- * @param store
- * The store.
- * @param addrCheckpoint
- * The address of a {@link Checkpoint} record for the index.
- * @param readOnly
- * When <code>true</code> the {@link BTree} will be marked as
- * read-only. Marking has some advantages relating to the locking
- * scheme used by {@link Node#getChild(int)} since the root node
- * is known to be read-only at the time that it is allocated as
- * per-child locking is therefore in place for all nodes in the
- * read-only {@link BTree}. It also results in much higher
- * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
- *
- * @return The {@link BTree} or derived class loaded from that
- * {@link Checkpoint} record.
- */
+ *
+ * @param store
+ * The store.
+ * @param addrCheckpoint
+ * The address of a {@link Checkpoint} record for the index.
+ * @param readOnly
+ * When <code>true</code> the {@link BTree} will be marked as
+ * read-only. Marking has some advantages relating to the locking
+ * scheme used by {@link Node#getChild(int)} since the root node
+ * is known to be read-only at the time that it is allocated as
+ * per-child locking is therefore in place for all nodes in the
+ * read-only {@link BTree}. It also results in much higher
+ * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
+ *
+ * @return The {@link BTree} or derived class loaded from that
+ * {@link Checkpoint} record.
+ *
+ * @throws IllegalArgumentException
+ * if store is <code>null</code>.
+ */
@SuppressWarnings("unchecked")
public static BTree load(final IRawStore store, final long addrCheckpoint,
final boolean readOnly) {
+ if (store == null)
+ throw new IllegalArgumentException();
+
/*
* Read checkpoint record from store.
*/
- final Checkpoint checkpoint = Checkpoint.load(store, addrCheckpoint);
+ final Checkpoint checkpoint;
+ try {
+ checkpoint = Checkpoint.load(store, addrCheckpoint);
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not load Checkpoint: store="
+ + store + ", addrCheckpoint="
+ + store.toString(addrCheckpoint), t);
+ }
- /*
- * Read metadata record from store.
- */
- final IndexMetadata metadata = IndexMetadata.read(store, checkpoint
- .getMetadataAddr());
+ /*
+ * Read metadata record from store.
+ */
+ final IndexMetadata metadata;
+ try {
+ metadata = IndexMetadata.read(store, checkpoint.getMetadataAddr());
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not read IndexMetadata: store="
+ + store + ", checkpoint=" + checkpoint, t);
+ }
if (log.isInfoEnabled()) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -58,21 +58,21 @@
* Note: The total order of the {@link BigdataMap} is completely determined by
* {@link ITupleSerializer#serializeKey(Object)}. There is NO concept of a
* {@link Comparator}. The {@link ITupleSerializer} is responsible for coercing
- * application keys into variable length <strong>unsigned</strong> byte[]s
- * which are the keys for the underlying B+Tree. The order for the B+Tree is the
+ * application keys into variable length <strong>unsigned</strong> byte[]s which
+ * are the keys for the underlying B+Tree. The order for the B+Tree is the
* natural order for the <strong>unsigned byte[]</strong>s. {@link KeyBuilder}
* supports the generation of unsigned byte[]s from various kinds of Java
* primitives and Unicode {@link String}s and is typically used to write the
* {@link ITupleSerializer#serializeKey(Object)} method.
* <p>
- * Note: The coercion of the application keys into unsigned byte[]s is not
- * typesafe unless you either consistently use a strongly typed instance of this
+ * Note: The coercion of the application keys into unsigned byte[]s is not type
+ * safe unless you either consistently use a strongly typed instance of this
* class or specify an {@link ITupleSerializer} for the backing B+Tree that only
* allows application keys that are instances of acceptable classes. This issue
* is more critical for keys than for values since the keys define the total
* index order and the default coercion rules for keys are provided by
- * {@link KeyBuilder#asSortKey(Object)} which does not attenpt to partition the
- * key space by the application key type (keys are not safely polymorphic by
+ * {@link KeyBuilder#append(Object)} which does not attempt to partition the key
+ * space by the application key type (keys are not safely polymorphic by
* default).
* <p>
* Note: When storing Java objects in the tuple value, the value MUST be treated
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -2049,10 +2049,14 @@
// Note: default assumes NOT an index partition.
this.pmd = null;
+ /* Intern'd to reduce duplication on the heap. Will be com.bigdata.btree.BTree or
+ * com.bigdata.btree.IndexSegment and occasionally a class derived from BTree.
+ */
this.btreeClassName = getProperty(indexManager, properties, namespace,
- Options.BTREE_CLASS_NAME, BTree.class.getName().toString());
+ Options.BTREE_CLASS_NAME, BTree.class.getName()).intern();
- this.checkpointClassName = Checkpoint.class.getName();
+ // Intern'd to reduce duplication on the heap.
+ this.checkpointClassName = Checkpoint.class.getName().intern();
// this.addrSer = AddressSerializer.INSTANCE;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -207,7 +207,7 @@
if (properties != null) {
- val = properties.getProperty(key, def);
+ val = properties.getProperty(key);//, def);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -108,7 +108,7 @@
}
- ICUSortKeyGenerator(Locale locale, Object strength, DecompositionEnum mode) {
+ ICUSortKeyGenerator(final Locale locale, final Object strength, final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
@@ -132,7 +132,7 @@
} else {
- StrengthEnum str = (StrengthEnum) strength;
+ final StrengthEnum str = (StrengthEnum) strength;
if (log.isInfoEnabled())
log.info("strength=" + str);
@@ -200,9 +200,9 @@
* Buffer is reused for each {@link String} from which a sort key is
* derived.
*/
- private RawCollationKey raw = new RawCollationKey(128);
+ final private RawCollationKey raw = new RawCollationKey(128);
- public void appendSortKey(KeyBuilder keyBuilder, String s) {
+ public void appendSortKey(final KeyBuilder keyBuilder, final String s) {
// RawCollationKey raw = collator.getRawCollationKey(s, null);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -33,6 +33,7 @@
import java.util.Locale;
import java.util.Properties;
import java.util.UUID;
+
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder.Options;
@@ -88,7 +89,6 @@
* {@link #appendText(String, boolean, boolean)}.
* </p>
*
- * @see KeyBuilder#asSortKey(Object)
* @see KeyBuilder#newInstance()
* @see KeyBuilder#newUnicodeInstance()
* @see KeyBuilder#newUnicodeInstance(Properties)
@@ -97,7 +97,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface IKeyBuilder extends ISortKeyBuilder {
+public interface IKeyBuilder extends ISortKeyBuilder<Object> {
/**
* The #of bytes of data in the key.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -1065,78 +1065,8 @@
}
- /*
- * static helper methods.
- */
-
- /**
- * Used to unbox an application key (convert it to an unsigned byte[]).
- */
- static private final IKeyBuilder _keyBuilder = newUnicodeInstance();
-
- /**
- * Utility method converts an application key to a sort key (an unsigned
- * byte[] that imposes the same sort order).
- * <p>
- * Note: This method is thread-safe.
- * <p>
- * Note: Strings are Unicode safe for the default locale. See
- * {@link Locale#getDefault()}. If you require a specific local or different
- * locals at different times or for different indices then you MUST
- * provision and apply your own {@link KeyBuilder}.
- *
- * @param val
- * An application key.
- *
- * @return The unsigned byte[] equivalent of that key. This will be
- * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
- * <i>key</i> is a byte[], then the byte[] itself will be returned.
- *
- * @deprecated This method circumvents explicit configuration of the
- * {@link KeyBuilder} and is used nearly exclusively by unit
- * tests. While explicit configuration is not required for keys
- * which do not include Unicode sort key components, this method
- * also relies on a single global {@link KeyBuilder} instance
- * protected by a lock. That lock is therefore a bottleneck. The
- * correct practice is to use thread-local or per task
- * {@link IKeyBuilder}s to avoid lock contention.
- */
- @SuppressWarnings("unchecked")
- public static final byte[] asSortKey(Object val) {
+ public byte[] getSortKey(final Object val) {
- if (val == null) {
-
- return null;
-
- }
-
- if (val instanceof byte[]) {
-
- return (byte[]) val;
-
- }
-
- /*
- * Synchronize on the keyBuilder to avoid concurrent modification of its
- * state.
- */
-
- synchronized (_keyBuilder) {
-
- return _keyBuilder.getSortKey(val);
-
-// _keyBuilder.reset();
-//
-// _keyBuilder.append( key );
-//
-// return _keyBuilder.getKey();
-
- }
-
- }
-
- public byte[] getSortKey(Object val) {
-
reset();
append( val );
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -277,19 +277,19 @@
AbstractStatisticsCollector
.addGarbageCollectorMXBeanCounters(serviceRoot
.makePath(ICounterHierarchy.Memory_GarbageCollectors));
-
- /*
- * Add counters reporting on the various DirectBufferPools.
- */
- {
- // general purpose pool.
- serviceRoot.makePath(
- IProcessCounters.Memory + ICounterSet.pathSeparator
- + "DirectBufferPool").attach(
- DirectBufferPool.getCounters());
-
- }
+ // Moved since counters must be dynamically reattached to reflect pool hierarchy.
+// /*
+// * Add counters reporting on the various DirectBufferPools.
+// */
+// {
+//
+// serviceRoot.makePath(
+// IProcessCounters.Memory + ICounterSet.pathSeparator
+// + "DirectBufferPool").attach(
+// DirectBufferPool.getCounters());
+//
+// }
if (LRUNexus.INSTANCE != null) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -51,7 +51,7 @@
import com.bigdata.counters.Instrument;
import com.bigdata.journal.AbstractBufferStrategy;
import com.bigdata.journal.DiskOnlyStrategy;
-import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
+//import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.rwstore.RWStore;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -1027,33 +1027,33 @@
}
- case Disk: {
+// case Disk: {
+//
+// /*
+// * Setup the buffer strategy.
+// */
+//
+// fileMetadata = new FileMetadata(file, BufferMode.Disk,
+// useDirectBuffers, initialExtent, maximumExtent, create,
+// isEmptyFile, deleteOnExit, readOnly, forceWrites,
+// offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
+// //readOnly ? null : writeCache,
+// writeCacheEnabled,
+// validateChecksum,
+// createTime, checker, alternateRootBlock);
+//
+// _bufferStrategy = new DiskOnlyStrategy(
+// 0L/* soft limit for maximumExtent */,
+//// minimumExtension,
+// fileMetadata);
+//
+// this._rootBlock = fileMetadata.rootBlock;
+//
+// break;
+//
+// }
- /*
- * Setup the buffer strategy.
- */
-
- fileMetadata = new FileMetadata(file, BufferMode.Disk,
- useDirectBuffers, initialExtent, maximumExtent, create,
- isEmptyFile, deleteOnExit, readOnly, forceWrites,
- offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
- //readOnly ? null : writeCache,
- writeCacheEnabled,
- validateChecksum,
- createTime, checker, alternateRootBlock);
-
- _bufferStrategy = new DiskOnlyStrategy(
- 0L/* soft limit for maximumExtent */,
-// minimumExtension,
- fileMetadata);
-
- this._rootBlock = fileMetadata.rootBlock;
-
- break;
-
- }
-
-// case Disk:
+ case Disk:
case DiskWORM: {
/*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -46,6 +46,7 @@
import com.bigdata.io.DirectBufferPool;
import com.bigdata.io.FileChannelUtility;
import com.bigdata.io.IReopenChannel;
+import com.bigdata.journal.WORMStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.resources.StoreManager.ManagedJournal;
@@ -501,7 +502,7 @@
writeCache.flush();
- storeCounters.ncacheFlush++;
+// storeCounters.ncacheFlush++;
}
@@ -544,551 +545,551 @@
}
- /**
- * Counters for {@link IRawStore} access, including operations that read or
- * write through to the underlying media.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- *
- * @todo report elapsed time and average latency for force, reopen, and
- * writeRootBlock.
- *
- * @todo counters need to be atomic if we want to avoid the possibility of
- * concurrent <code>x++</code> operations failing to correctly
- * increment <code>x</code> for each request.
- */
- public static class StoreCounters {
-
- /**
- * #of read requests.
- */
- public long nreads;
-
- /**
- * #of read requests that are satisfied by our write cache (vs the
- * OS or disk level write cache).
- */
- public long ncacheRead;
-
- /**
- * #of read requests that read through to the backing file.
- */
- public long ndiskRead;
-
- /**
- * #of bytes read.
- */
- public long bytesRead;
-
- /**
- * #of bytes that have been read from the disk.
- */
- public long bytesReadFromDisk;
-
- /**
- * The size of the largest record read.
- */
- public long maxReadSize;
-
- /**
- * Total elapsed time for reads.
- */
- public long elapsedReadNanos;
-
- /**
- * Total elapsed time checking the disk write cache for records to be
- * read.
- */
- public long elapsedCacheReadNanos;
-
- /**
- * Total elapsed time for reading on the disk.
- */
- public long elapsedDiskReadNanos;
-
- /**
- * #of write requests.
- */
- public long nwrites;
-
- /**
- * #of write requests that are absorbed by our write cache (vs the OS or
- * disk level write cache).
- */
- public long ncacheWrite;
-
- /**
- * #of times the write cache was flushed to disk.
- */
- public long ncacheFlush;
-
- /**
- * #of write requests that write through to the backing file.
- */
- public long ndiskWrite;
-
- /**
- * The size of the largest record written.
- */
- public long maxWriteSize;
-
- /**
- * #of bytes written.
- */
- public long bytesWritten;
-
- /**
- * #of bytes that have been written on the disk.
- */
- public long bytesWrittenOnDisk;
-
- /**
- * Total elapsed time for writes.
- */
- public long elapsedWriteNanos;
-
- /**
- * Total elapsed time writing records into the cache (does not count
- * time to flush the cache when it is full or to write records that do
- * not fit in the cache directly to the disk).
- */
- public long elapsedCacheWriteNanos;
-
- /**
- * Total elapsed time for writing on the disk.
- */
- public long elapsedDiskWriteNanos;
-
- /**
- * #of times the data were forced to the disk.
- */
- public long nforce;
-
- /**
- * #of times the length of the file was changed (typically, extended).
- */
- public long ntruncate;
-
- /**
- * #of times the file has been reopened after it was closed by an
- * interrupt.
- */
- public long nreopen;
-
- /**
- * #of times one of the root blocks has been written.
- */
- public long nwriteRootBlock;
-
- /**
- * Initialize a new set of counters.
- */
- public StoreCounters() {
-
- }
-
- /**
- * Copy ctor.
- * @param o
- */
- public StoreCounters(final StoreCounters o) {
-
- add( o );
-
- }
-
- /**
- * Adds counters to the current counters.
- *
- * @param o
- */
- public void add(final StoreCounters o) {
-
- nreads += o.nreads;
- ncacheRead += o.ncacheRead;
- ndiskRead += o.ndiskRead;
- bytesRead += o.bytesRead;
- bytesReadFromDisk += o.bytesReadFromDisk;
- maxReadSize += o.maxReadSize;
- elapsedReadNanos += o.elapsedReadNanos;
- elapsedCacheReadNanos += o.elapsedCacheReadNanos;
- elapsedDiskReadNanos += o.elapsedDiskReadNanos;
-
- nwrites += o.nwrites;
- ncacheWrite += o.ncacheWrite;
- ncacheFlush += o.ncacheFlush;
- ndiskWrite += o.ndiskWrite;
- maxWriteSize += o.maxWriteSize;
- bytesWritten += o.bytesWritten;
- bytesWrittenOnDisk += o.bytesWrittenOnDisk;
- elapsedWriteNanos += o.elapsedWriteNanos;
- elapsedCacheWriteNanos += o.elapsedCacheWriteNanos;
- elapsedDiskWriteNanos += o.elapsedDiskWriteNanos;
-
- nforce += o.nforce;
- ntruncate += o.ntruncate;
- nreopen += o.nreopen;
- nwriteRootBlock += o.nwriteRootBlock;
-
- }
-
- /**
- * Returns a new {@link StoreCounters} containing the current counter values
- * minus the given counter values.
- *
- * @param o
- *
- * @return
- */
- public StoreCounters subtract(final StoreCounters o) {
-
- // make a copy of the current counters.
- final StoreCounters t = new StoreCounters(this);
-
- // subtract out the given counters.
- t.nreads -= o.nreads;
- t.ncacheRead -= o.ncacheRead;
- t.ndiskRead -= o.ndiskRead;
- t.bytesRead -= o.bytesRead;
- t.bytesReadFromDisk -= o.bytesReadFromDisk;
- t.maxReadSize -= o.maxReadSize;
- t.elapsedReadNanos -= o.elapsedReadNanos;
- t.elapsedCacheReadNanos -= o.elapsedCacheReadNanos;
- t.elapsedDiskReadNanos -= o.elapsedDiskReadNanos;
-
- t.nwrites -= o.nwrites;
- t.ncacheWrite -= o.ncacheWrite;
- t.ncacheFlush -= o.ncacheFlush;
- t.ndiskWrite -= o.ndiskWrite;
- t.maxWriteSize -= o.maxWriteSize;
- t.bytesWritten -= o.bytesWritten;
- t.bytesWrittenOnDisk -= o.bytesWrittenOnDisk;
- t.elapsedWriteNanos -= o.elapsedWriteNanos;
- t.elapsedCacheWriteNanos -= o.elapsedCacheWriteNanos;
- t.elapsedDiskWriteNanos -= o.elapsedDiskWriteNanos;
-
- t.nforce -= o.nforce;
- t.ntruncate -= o.ntruncate;
- t.nreopen -= o.nreopen;
- t.nwriteRootBlock -= o.nwriteRootBlock;
-
- return t;
-
- }
-
- synchronized public CounterSet getCounters() {
-
- if (root == null) {
-
- root = new CounterSet();
-
- // IRawStore API
- {
-
- /*
- * reads
- */
-
- root.addCounter("nreads", new Instrument<Long>() {
- public void sample() {
- setValue(nreads);
- }
- });
-
- root.addCounter("bytesRead", new Instrument<Long>() {
- public void sample() {
- setValue(bytesRead);
- }
- });
-
- root.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- final double elapsedReadSecs = (elapsedReadNanos / 1000000000.);
- setValue(elapsedReadSecs);
- }
- });
-
- root.addCounter("bytesReadPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double readSecs = (elapsedReadNanos / 1000000000.);
- final double bytesReadPerSec = (readSecs == 0L ? 0d
- : (bytesRead / readSecs));
- setValue(bytesReadPerSec);
- }
- });
-
- root.addCounter("maxReadSize", new Instrument<Long>() {
- public void sample() {
- setValue(maxReadSize);
- }
- });
-
- /*
- * writes
- */
-
- root.addCounter("nwrites", new Instrument<Long>() {
- public void sample() {
- setValue(nwrites);
- }
- });
-
- root.addCounter("bytesWritten", new Instrument<Long>() {
- public void sample() {
- setValue(bytesWritten);
- }
- });
-
- root.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- final double writeSecs = (elapsedWriteNanos / 1000000000.);
- setValue(writeSecs);
- }
- });
-
- root.addCounter("bytesWrittenPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double writeSecs = (elapsedWriteNanos / 1000000000.);
- final double bytesWrittenPerSec = (writeSecs == 0L ? 0d
- : (bytesWritten / writeSecs));
- setValue(bytesWrittenPerSec);
- }
- });
-
- root.addCounter("maxWriteSize", new Instrument<Long>() {
- public void sample() {
- setValue(maxWriteSize);
- }
- });
-
- }
-
- /*
- * write cache statistics
- */
- {
-
- final CounterSet writeCache = root.makePath("writeCache");
-
- /*
- * read
- */
- writeCache.addCounter("nread", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheRead);
- }
- });
-
- writeCache.addCounter("readHitRate", new Instrument<Double>() {
- public void sample() {
- setValue(nreads == 0L ? 0d : (double) ncacheRead
- / nreads);
- }
- });
-
- writeCache.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- setValue(elapsedCacheReadNanos / 1000000000.);
- }
- });
-
- /*
- * write
- */
-
- // #of writes on the write cache.
- writeCache.addCounter("nwrite", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheWrite);
- }
- });
-
- /*
- * % of writes that are buffered vs writing through to the
- * disk.
- *
- * Note: This will be 1.0 unless you are writing large
- * records. Large records are written directly to the disk
- * rather than first into the write cache. When this happens
- * the writeHitRate on the cache can be less than one.
- */
- writeCache.addCounter("writeHitRate", new Instrument<Double>() {
- public void sample() {
- setValue(nwrites == 0L ? 0d : (double) ncacheWrite
- / nwrites);
- }
- });
-
- writeCache.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- setValue(elapsedCacheWriteNanos / 1000000000.);
- }
- });
-
- // #of times the write cache was flushed to the disk.
- writeCache.addCounter("nflush", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheFlush);
- }
- });
-
- }
-
- // disk statistics
- {
- final CounterSet disk = root.makePath("disk");
-
- /*
- * read
- */
-
- disk.addCounter("nreads", new Instrument<Long>() {
- public void sample() {
- setValue(ndiskRead);
- }
- });
-
- disk.addCounter("bytesRead", new Instrument<Long>() {
- public void sample() {
- setValue(bytesReadFromDisk);
- }
- });
-
- disk.addCounter("bytesPerRead", new Instrument<Double>() {
- public void sample() {
- final double bytesPerDiskRead = (ndiskRead == 0 ? 0d
- : (bytesReadFromDisk / (double)ndiskRead));
- setValue(bytesPerDiskRead);
- }
- });
-
- disk.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- setValue(diskReadSecs);
- }
- });
-
- disk.addCounter("bytesReadPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- final double bytesReadPerSec = (diskReadSecs == 0L ? 0d
- : bytesReadFromDisk / diskReadSecs);
- setValue(bytesReadPerSec);
- }
- });
-
- disk.addCounter("secsPerRead", new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- final double readLatency = (diskReadSecs == 0 ? 0d
- : diskReadSecs / ndiskRead);
- setValue(readLatency);
- }
- });
-
- /*
- * write
- */
-
- disk.addCounter("nwrites", new Instrument<Long>() {
- public void sample() {
- setValue(ndiskWrite);
- }
- });
-
- disk.addCounter("bytesWritten", new Instrument<Long>() {
- public void sample() {
- setValue(bytesWrittenOnDisk);
- }
- });
-
- disk.addCounter("bytesPerWrite", new Instrument<Double>() {
- public void sample() {
- final double bytesPerDiskWrite = (ndiskWrite == 0 ? 0d
- : (bytesWrittenOnDisk / (double)ndiskWrite));
- setValue(bytesPerDiskWrite);
- }
- });
-
- disk.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- setValue(diskWriteSecs);
- }
- });
-
- disk.addCounter("bytesWrittenPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- final double bytesWrittenPerSec = (diskWriteSecs == 0L ? 0d
- : bytesWrittenOnDisk
- / diskWriteSecs);
- setValue(bytesWrittenPerSec);
- }
- });
-
- disk.addCounter("secsPerWrite", new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- final double writeLatency = (diskWriteSecs == 0 ? 0d
- : diskWriteSecs / ndiskWrite);
- setValue(writeLatency);
- }
- });
-
- /*
- * other
- */
-
- disk.addCounter("nforce", new Instrument<Long>() {
- public void sample() {
- setValue(nforce);
- }
- });
-
- disk.addCounter("nextend", new Instrument<Long>() {
- public void sample() {
- setValue(ntruncate);
- }
- });
-
- disk.addCounter("nreopen", new Instrument<Long>() {
- public void sample() {
- setValue(nreopen);
- }
- });
-
- disk.addCounter("rootBlockWrites", new Instrument<Long>() {
- public void sample() {
- setValue(nwriteRootBlock);
- }
- });
-
- }
-
- }
-
- return root;
-
- }
- private CounterSet root;
-
- /**
- * Human readable representation of the counters.
- */
- public String toString() {
-
- return getCounters().toString();
-
- }
-
- }
+// /**
+// * Counters for {@link IRawStore} access, including operations that read or
+// * write through to the underlying media.
+// *
+// * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+// * @version $Id$
+// *
+// * @todo report elapsed time and average latency for force, reopen, and
+// * writeRootBlock.
+// *
+// * @todo counters need to be atomic if we want to avoid the possibility of
+// * concurrent <code>x++</code> operations failing to correctly
+// * increment <code>x</code> for each request.
+// */
+// public static class StoreCounters {
+//
+// /**
+// * #of read requests.
+// */
+// public long nreads;
+//
+// /**
+// * #of read requests that are satisfied by our write cache (vs the
+// * OS or disk level write cache).
+// */
+// public long ncacheRead;
+//
+// /**
+// * #of read requests that read through to the backing file.
+// */
+// public long ndiskRead;
+//
+// /**
+// * #of bytes read.
+// */
+// public long bytesRead;
+//
+// /**
+// * #of bytes that have been read from the disk.
+// */
+// public long bytesReadFromDisk;
+//
+// /**
+// * The size of the largest record read.
+// */
+// public long maxReadSize;
+//
+// /**
+// * Total elapsed time for reads.
+// */
+// public long elapsedReadNanos;
+//
+// /**
+// * Total elapsed time checking the disk write cache for records to be
+// * read.
+// */
+// public long elapsedCacheReadNanos;
+//
+// /**
+// * Total elapsed time for reading on the disk.
+// */
+// public long elapsedDiskReadNanos;
+//
+// /**
+// * #of write requests.
+// */
+// public long nwrites;
+//
+// /**
+// * #of write requests that are absorbed by our write cache (vs the OS or
+// * disk level write cache).
+// */
+// public long ncacheWrite;
+//
+// /**
+// * #of times the write cache was flushed to disk.
+// */
+// public long ncacheFlush;
+//
+// /**
+// * #of write requests that write through to the backing file.
+// */
+// public long ndiskWrite;
+//
+// /**
+// * The size of the largest record written.
+// */
+// public long maxWriteSize;
+//
+// /**
+// * #of bytes written.
+// */
+// public long bytesWritten;
+//
+// /**
+// * #of bytes that have been written on the disk.
+// */
+// public long bytesWrittenOnDisk;
+//
+// /**
+// * Total elapsed time for writes.
+// */
+// public long elapsedWriteNanos;
+//
+// /**
+// * Total elapsed time writing records into the cache (does not count
+// * time to flush the cache when it is full or to write records that do
+// * not fit in the cache directly to the disk).
+// */
+// public long elapsedCacheWriteNanos;
+//
+// /**
+// * Total elapsed time for writing on the disk.
+// */
+// public long elapsedDiskWriteNanos;
+//
+// /**
+// * #of times the data were forced to the disk.
+// */
+// public long nforce;
+//
+// /**
+// * #of times the length of the file was changed (typically, extended).
+// */
+// public long ntruncate;
+//
+// /**
+// * #of times the file has been reopened after it was closed by an
+// * interrupt.
+// */
+// public long nreopen;
+//
+// /**
+// * #of times one of the root blocks has been written.
+// */
+// public long nwriteRootBlock;
+//
+// /**
+// * Initialize a new set of counters.
+// */
+// public StoreCounters() {
+//
+// }
+//
+// /**
+// * Copy ctor.
+// * @param o
+// */
+// public StoreCounters(final StoreCounters o) {
+//
+// add( o );
+//
+// }
+//
+// /**
+// * Adds counters to the current counters.
+// *
+// * @param o
+// */
+// public void add(final StoreCounters o) {
+//
+// nreads += o.nreads;
+// ncacheRead += o.ncacheRead;
+// ndiskRead += o.ndiskRead;
+// bytesRead += o.bytesRead;
+// bytesReadFromDisk += o.bytesReadFromDisk;
+// maxReadSize += o.maxReadSize;
+// elapsedReadNanos += o.elapsedReadNanos;
+// elapsedCacheReadNanos += o.elapsedCacheReadNanos;
+// elapsedDiskReadNanos += o.elapsedDiskReadNanos;
+//
+// nwrites += o.nwrites;
+// ncacheWrite += o.ncacheWrite;
+// ncacheFlush += o.ncacheFlush;
+// ndiskWrite += o.ndiskWrite;
+// maxWriteSize += o.maxWriteSize;
+// bytesWritten += o.bytesWritten;
+// bytesWrittenOnDisk += o.bytesWrittenOnDisk;
+// elapsedWriteNanos += o.elapsedWriteNanos;
+// elapsedCacheWriteNanos += o.elapsedCacheWriteNanos;
+// elapsedDiskWriteNanos += o.elapsedDiskWriteNanos;
+//
+// nforce += o.nforce;
+// ntruncate += o.ntruncate;
+// nreopen += o.nreopen;
+// nwriteRootBlock += o.nwriteRootBlock;
+//
+// }
+//
+// /**
+// * Returns a new {@link StoreCounters} containing the current counter values
+// * minus the given counter values.
+// *
+// * @param o
+// *
+// * @return
+// */
+// public StoreCounters subtract(final StoreCounters o) {
+//
+// // make a copy of the current counters.
+// final StoreCounters t = new StoreCounters(this);
+//
+// // subtract out the given counters.
+// t.nreads -= o.nreads;
+// t.ncacheRead -= o.ncacheRead;
+// t.ndiskRead -= o.ndiskRead;
+// t.bytesRead -= o.bytesRead;
+// t.bytesReadFromDisk -= o.bytesReadFromDisk;
+// t.maxReadSize -= o.maxReadSize;
+// t.elapsedReadNanos -= o.elapsedReadNanos;
+// t.elapsedCacheReadNanos -= o.elapsedCacheReadNanos;
+// t.elapsedDiskReadNanos -= o.elapsedDiskReadNanos;
+//
+// t.nwrites -= o.nwrites;
+// t.ncacheWrite -= o.ncacheWrite;
+// t.ncacheFlush -= o.ncacheFlush;
+// t.ndiskWrite -= o.ndiskWrite;
+// t.maxWriteSize -= o.maxWriteSize;
+// t.bytesWritten -= o.bytesWritten;
+// t.bytesWrittenOnDisk -= o.bytesWrittenOnDisk;
+// t.elapsedWriteNanos -= o.elapsedWriteNanos;
+// t.elapsedCacheWriteNanos -= o.elapsedCacheWriteNanos;
+// t.elapsedDiskWriteNanos -= o.elapsedDiskWriteNanos;
+//
+// t.nforce -= o.nforce;
+// t.ntruncate -= o.ntruncate;
+// t.nreopen -= o.nreopen;
+// t.nwriteRootBlock -= o.nwriteRootBlock;
+//
+// return t;
+//
+// }
+//
+// synchronized public CounterSet getCounters() {
+//
+// if (root == null) {
+//
+// root = new CounterSet();
+//
+// // IRawStore API
+// {
+//
+// /*
+// * reads
+// */
+//
+// root.addCounter("nreads", new Instrument<Long>() {
+// public void sample() {
+// setValue(nreads);
+// }
+// });
+//
+// root.addCounter("bytesRead", new Instrument<Long>() {
+// public void sample() {
+// setValue(bytesRead);
+// }
+// });
+//
+// root.addCounter("readSecs", new Instrument<Double>() {
+// public void sample() {
+// final double elapsedReadSecs = (elapsedReadNanos / 1000000000.);
+// setValue(elapsedReadSecs);
+// }
+// });
+//
+// root.addCounter("bytesReadPerSec",
+// new Instrument<Double>() {
+// public void sample() {
+// final double readSecs = (elapsedReadNanos / 1000000000.);
+// final double bytesReadPerSec = (readSecs == 0L ? 0d
+// : (bytesRead / readSecs));
+// setValue(bytesReadPerSec);
+// }
+// });
+//
+// root.addCounter("maxReadSize", new Instrument<Long>() {
+// public void sample() {
+// setValue(maxReadSize);
+...
[truncated message content] |
|
From: <mar...@us...> - 2010-08-04 17:08:55
|
Revision: 3407
http://bigdata.svn.sourceforge.net/bigdata/?rev=3407&view=rev
Author: martyncutcher
Date: 2010-08-04 17:08:49 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
use toPlainString to avoid exponent in stringValue
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-08-04 16:57:29 UTC (rev 3406)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-08-04 17:08:49 UTC (rev 3407)
@@ -105,9 +105,13 @@
return value.shortValue();
}
+ /**
+ * Use toPlainString to avoid expression with exponent value that
+ * would imply xsd:double rather than xsd:decimal
+ */
@Override
public String stringValue() {
- return value.toString();
+ return value.toPlainString();
}
@Override
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-08-04 16:57:35
|
Revision: 3406
http://bigdata.svn.sourceforge.net/bigdata/?rev=3406&view=rev
Author: martyncutcher
Date: 2010-08-04 16:57:29 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Add test for BigDecimal literals with large exponents
Modified Paths:
--------------
trunk/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
Modified: trunk/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2010-08-04 15:18:26 UTC (rev 3405)
+++ trunk/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2010-08-04 16:57:29 UTC (rev 3406)
@@ -2111,6 +2111,43 @@
}
/**
+ * Test with positive and negative {@link BigDecimal}s with large
+ * exponents
+ */
+ public void test_BigDecimal_largeExponents() {
+
+ final BigDecimal p1 = new BigDecimal("12000000000000000000000000");
+ final BigDecimal p2 = new BigDecimal("12000000000000000000000001");
+ final BigDecimal p3 = new BigDecimal("1.201E25");
+ final BigDecimal p4 = new BigDecimal("12020000000000000000000000");
+ final BigDecimal p5 = new BigDecimal("1.201E260");
+ final BigDecimal n1 = new BigDecimal("-12000000000000000000000000");
+ final BigDecimal n2 = new BigDecimal("-12000000000000000000000001");
+ final BigDecimal n3 = new BigDecimal("-1.2E260");
+ final BigDecimal n4 = new BigDecimal("-1.201E260");
+
+ doEncodeDecodeTest(p1);
+ doEncodeDecodeTest(p2);
+ doEncodeDecodeTest(p3);
+ doEncodeDecodeTest(p4);
+ doEncodeDecodeTest(p5);
+ doEncodeDecodeTest(n1);
+ doEncodeDecodeTest(n2);
+ doEncodeDecodeTest(n3);
+ doEncodeDecodeTest(n4);
+
+ doLTTest(p1, p2); // 1.5 LT 1.51
+ doLTTest(p1, p3); // 1.5 LT 1.51
+ doLTTest(p3, p4); // 1.5 LT 1.51
+ doLTTest(p3, p5); // 1.5 LT 1.51
+ doLTTest(n1, p1); // -1.5 LT 1.5
+ doLTTest(n2, n1); // -1.51 LT -1.5
+ doLTTest(n3, n1); // -1.51 LT -1.5
+ doLTTest(n4, n3); // -1.51 LT -1.5
+
+ }
+
+ /**
* Stress test with random byte[]s from which we then construct
* {@link BigInteger}s.
*/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-08-04 15:18:32
|
Revision: 3405
http://bigdata.svn.sourceforge.net/bigdata/?rev=3405&view=rev
Author: mrpersonick
Date: 2010-08-04 15:18:26 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
using BigDecimal.toPlainString instead of toString (which uses exponent notation)
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-08-04 12:26:44 UTC (rev 3404)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-08-04 15:18:26 UTC (rev 3405)
@@ -64,7 +64,7 @@
public V asValue(final BigdataValueFactory f,
final ILexiconConfiguration config) {
// @todo factory should cache the XSD URIs.
- final V v = (V) f.createLiteral(value.toString(),//
+ final V v = (V) f.createLiteral(value.toPlainString(),//
f.createURI(DTE.XSDDecimal.getDatatype()));
v.setIV(this);
return v;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 12:26:53
|
Revision: 3404
http://bigdata.svn.sourceforge.net/bigdata/?rev=3404&view=rev
Author: thompsonbry
Date: 2010-08-04 12:26:44 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Javadoc edits on OverflowManager and StoreManager, both of which initialize a thread pool in their ctors.
Btree.load() was modified to report more information if it is unable to load a Checkpoint or IndexMetadata record.
AbstractResourceManagerBootstrapTestCase was turning off the write cache (for dated reasons). The current WORMStrategy has a bug which prevents correct operation when the write cache is disabled (this is fixed in the HA branch). That bug was causing failures in the com.bigdata.resources.TestAll() suite.
Modified Paths:
--------------
trunk/bigdata/src/java/com/bigdata/btree/BTree.java
trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java
trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java
Modified: trunk/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-04 11:32:44 UTC (rev 3403)
+++ trunk/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-04 12:26:44 UTC (rev 3404)
@@ -1525,45 +1525,63 @@
}
- /**
- * Load an instance of a {@link BTree} or derived class from the store. The
- * {@link BTree} or derived class MUST declare a constructor with the
- * following signature: <code>
+ /**
+ * Load an instance of a {@link BTree} or derived class from the store. The
+ * {@link BTree} or derived class MUST declare a constructor with the
+ * following signature: <code>
*
* <i>className</i>(IRawStore store, Checkpoint checkpoint, BTreeMetadata metadata, boolean readOnly)
*
* </code>
- *
- * @param store
- * The store.
- * @param addrCheckpoint
- * The address of a {@link Checkpoint} record for the index.
- * @param readOnly
- * When <code>true</code> the {@link BTree} will be marked as
- * read-only. Marking has some advantages relating to the locking
- * scheme used by {@link Node#getChild(int)} since the root node
- * is known to be read-only at the time that it is allocated as
- * per-child locking is therefore in place for all nodes in the
- * read-only {@link BTree}. It also results in much higher
- * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
- *
- * @return The {@link BTree} or derived class loaded from that
- * {@link Checkpoint} record.
- */
+ *
+ * @param store
+ * The store.
+ * @param addrCheckpoint
+ * The address of a {@link Checkpoint} record for the index.
+ * @param readOnly
+ * When <code>true</code> the {@link BTree} will be marked as
+ * read-only. Marking has some advantages relating to the locking
+ * scheme used by {@link Node#getChild(int)} since the root node
+ * is known to be read-only at the time that it is allocated as
+ * per-child locking is therefore in place for all nodes in the
+ * read-only {@link BTree}. It also results in much higher
+ * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
+ *
+ * @return The {@link BTree} or derived class loaded from that
+ * {@link Checkpoint} record.
+ *
+ * @throws IllegalArgumentException
+ * if store is <code>null</code>.
+ */
@SuppressWarnings("unchecked")
public static BTree load(final IRawStore store, final long addrCheckpoint,
final boolean readOnly) {
+ if (store == null)
+ throw new IllegalArgumentException();
+
/*
* Read checkpoint record from store.
*/
- final Checkpoint checkpoint = Checkpoint.load(store, addrCheckpoint);
+ final Checkpoint checkpoint;
+ try {
+ checkpoint = Checkpoint.load(store, addrCheckpoint);
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not load Checkpoint: store="
+ + store + ", addrCheckpoint="
+ + store.toString(addrCheckpoint), t);
+ }
- /*
- * Read metadata record from store.
- */
- final IndexMetadata metadata = IndexMetadata.read(store, checkpoint
- .getMetadataAddr());
+ /*
+ * Read metadata record from store.
+ */
+ final IndexMetadata metadata;
+ try {
+ metadata = IndexMetadata.read(store, checkpoint.getMetadataAddr());
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not read IndexMetadata: store="
+ + store + ", checkpoint=" + checkpoint, t);
+ }
if (log.isInfoEnabled()) {
Modified: trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-04 11:32:44 UTC (rev 3403)
+++ trunk/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-04 12:26:44 UTC (rev 3404)
@@ -1704,7 +1704,7 @@
}
if(overflowEnabled) {
-
+ // @todo defer allocation until init() outside of ctor.
overflowService = Executors.newFixedThreadPool(1,
new DaemonThreadFactory((serviceName == null ? ""
: serviceName + "-")
Modified: trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-04 11:32:44 UTC (rev 3403)
+++ trunk/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-04 12:26:44 UTC (rev 3404)
@@ -674,7 +674,7 @@
protected final long accelerateOverflowThreshold;
/**
- * Used to run the {@link Startup}.
+ * Used to run the {@link Startup}. @todo defer to init() outside of ctor. Also, defer {@link Startup} until init() outside of ctor.
*/
private final ExecutorService startupService = Executors
.newSingleThreadExecutor(new DaemonThreadFactory
@@ -1420,7 +1420,7 @@
log.info("Waiting for concurrency manager");
for (int i = 0; i < 5; i++) {
try {
- getConcurrencyManager();
+ getConcurrencyManager(); break;
} catch (IllegalStateException ex) {
Thread.sleep(100/* ms */);
}
Modified: trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java 2010-08-04 11:32:44 UTC (rev 3403)
+++ trunk/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java 2010-08-04 12:26:44 UTC (rev 3404)
@@ -59,14 +59,15 @@
Properties properties = new Properties(super.getProperties());
- log.info("Setting " + Options.DATA_DIR + "=" + dataDir);
-
+ if (log.isInfoEnabled())
+ log.info("Setting " + Options.DATA_DIR + "=" + dataDir);
+
properties.setProperty(
com.bigdata.resources.ResourceManager.Options.DATA_DIR, dataDir
.toString());
- // disable the write cache to avoid memory leak in the test suite.
- properties.setProperty(Options.WRITE_CACHE_ENABLED, "false");
+// // disable the write cache to avoid memory leak in the test suite.
+// properties.setProperty(Options.WRITE_CACHE_ENABLED, "false");
return properties;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-04 11:32:50
|
Revision: 3403
http://bigdata.svn.sourceforge.net/bigdata/?rev=3403&view=rev
Author: thompsonbry
Date: 2010-08-04 11:32:44 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Removed the DiskOnlyStrategy test suite since it is no longer used to back the Journal. For the moment, it is still used for the temporary store, but that gets its own test suite.
Modified Paths:
--------------
trunk/bigdata/src/test/com/bigdata/journal/TestAll.java
Modified: trunk/bigdata/src/test/com/bigdata/journal/TestAll.java
===================================================================
--- trunk/bigdata/src/test/com/bigdata/journal/TestAll.java 2010-08-03 17:11:58 UTC (rev 3402)
+++ trunk/bigdata/src/test/com/bigdata/journal/TestAll.java 2010-08-04 11:32:44 UTC (rev 3403)
@@ -108,8 +108,8 @@
// }
- // @todo remove when we remove the DiskOnlyStrategy impl.
- suite.addTest( TestDiskJournal.suite() );
+ // Remove since we are no longer using the DiskOnlyStrategy for the Journal.
+ //suite.addTest( TestDiskJournal.suite() );
suite.addTest( TestWORMStrategy.suite() );
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-03 17:12:04
|
Revision: 3402
http://bigdata.svn.sourceforge.net/bigdata/?rev=3402&view=rev
Author: thompsonbry
Date: 2010-08-03 17:11:58 +0000 (Tue, 03 Aug 2010)
Log Message:
-----------
Modified NanoSparqlServer to include the query in its log messages (server side).
Modified BigdataSail to fix a problem where it was attempting to resolve a language code in a langmatch filter against the KB.
Modified Term2IdWriteTask to avoid an NPE when it returns a null IV object in the readOnly mode.
Modified BigdataEvaluationStrategy to fix a problem where a Sesame Filter object was being serialized with a rule (https://sourceforge.net/apps/trac/bigdata/ticket/135).
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java 2010-08-03 15:13:17 UTC (rev 3401)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java 2010-08-03 17:11:58 UTC (rev 3402)
@@ -258,7 +258,7 @@
* For each term whose serialized key is mapped to the current index
* partition, lookup the term in the <em>terms</em> index. If it is there
* then note its assigned termId. Otherwise, use the partition local counter
- * to assign the term identifier, note the term identifer so that it can be
+ * to assign the term identifier, note the term identifier so that it can be
* communicated back to the client, and insert the {term,termId} entry into
* the <em>terms</em> index.
*
@@ -288,8 +288,8 @@
// used to serialize term identifiers.
final DataOutputBuffer idbuf = new DataOutputBuffer();
- final TermIdEncoder encoder = new TermIdEncoder(//scaleOutTermIds,
- scaleOutTermIdBitsToReverse);
+ final TermIdEncoder encoder = readOnly ? null : new TermIdEncoder(
+ scaleOutTermIdBitsToReverse);
// #of new terms (#of writes on the index).
int nnew = 0;
@@ -532,7 +532,7 @@
}
- final public static VTE VTE(byte code) {
+ final public static VTE VTE(final byte code) {
switch(code) {
case ITermIndexCodes.TERM_CODE_URI:
@@ -574,7 +574,7 @@
}
- public Result(IV[] ivs) {
+ public Result(final IV[] ivs) {
assert ivs != null;
@@ -618,15 +618,7 @@
LongPacker.packLong(out,n);
for (int i = 0; i < n; i++) {
-
- /*
- * This is the implementation for backwards
- * compatibility. We should not see inline values here.
- */
- if (ivs[i].isInline()) {
- throw new RuntimeException();
- }
-
+
// LongPacker.packLong(out, ids[i]);
out.writeObject(ivs[i]);
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java 2010-08-03 15:13:17 UTC (rev 3401)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java 2010-08-03 17:11:58 UTC (rev 3402)
@@ -198,7 +198,7 @@
}
- AbstractKeyArrayIndexProcedureConstructor ctor =
+ final AbstractKeyArrayIndexProcedureConstructor ctor =
new Term2IdWriteProcConstructor(
readOnly, r.storeBlankNodes, r.termIdBitsToReverse);
Modified: trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
===================================================================
--- trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java 2010-08-03 15:13:17 UTC (rev 3401)
+++ trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java 2010-08-03 17:11:58 UTC (rev 3402)
@@ -1640,7 +1640,23 @@
final IJoinNexus joinNexus = joinNexusFactory.newInstance(database
.getIndexManager());
- itr1 = joinNexus.runQuery(step);
+
+// itr1 = joinNexus.runQuery(step);
+
+ if (step instanceof ProxyRuleWithSesameFilters) {
+
+ /*
+ * Note: Do not send the proxy rule down the wire. It has Sesame
+ * Filter objects which are not Serializable.
+ */
+ itr1 = joinNexus.runQuery(((ProxyRuleWithSesameFilters) step)
+ .getProxyRule());
+
+ } else {
+
+ itr1 = joinNexus.runQuery(step);
+
+ }
} catch (Exception ex) {
throw new QueryEvaluationException(ex);
Modified: trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-08-03 15:13:17 UTC (rev 3401)
+++ trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-08-03 17:11:58 UTC (rev 3402)
@@ -87,6 +87,7 @@
import org.openrdf.query.BindingSet;
import org.openrdf.query.Dataset;
import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.algebra.LangMatches;
import org.openrdf.query.algebra.QueryRoot;
import org.openrdf.query.algebra.StatementPattern;
import org.openrdf.query.algebra.TupleExpr;
@@ -3100,6 +3101,22 @@
@Override
public void meet(final ValueConstant constant) {
+ if (constant.getParentNode() instanceof LangMatches) {
+ /* Don't try to resolve for lang matches.
+ *
+ * Note: Sesame will sometimes use a Literal to represent
+ * a constant parameter to a function, such as LangMatches.
+ * For such uses, we DO NOT want to attempt to resolve the
+ * Literal against the lexicon. Instead, it should just be
+ * passed through. BigdataSailEvaluationStrategy is then
+ * responsible for recognizing cases where the lack of an
+ * IV on a constant is associated with such function calls
+ * rather than indicating that the Value is not known to
+ * the KB.
+ */
+ return;
+ }
+
final Value val = constant.getValue();
// add BigdataValue variant of the var's Value.
@@ -3186,7 +3203,14 @@
@Override
public void meet(ValueConstant constant) {
- // the Sesame Value object.
+ if (constant.getParentNode() instanceof LangMatches) {
+ /* Note: This is parallel to the meet in the visit
+ * pattern above.
+ */
+ return;
+ }
+
+ // the Sesame Value object.
final Value val = constant.getValue();
// Lookup the resolve BigdataValue object.
Modified: trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
===================================================================
--- trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java 2010-08-03 15:13:17 UTC (rev 3401)
+++ trunk/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java 2010-08-03 17:11:58 UTC (rev 3402)
@@ -1021,7 +1021,14 @@
try {
queries.put(queryId, new RunningQuery(queryId.longValue(),
queryStr, begin));
- doQuery(cxn, os);
+ try {
+ doQuery(cxn, os);
+ } catch(Throwable t) {
+ /*
+ * Log the query and the exception together.
+ */
+ log.error(t.getLocalizedMessage() + ":\n" + queryStr, t);
+ }
os.flush();
return null;
} catch (Throwable t) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-03 15:13:24
|
Revision: 3401
http://bigdata.svn.sourceforge.net/bigdata/?rev=3401&view=rev
Author: thompsonbry
Date: 2010-08-03 15:13:17 +0000 (Tue, 03 Aug 2010)
Log Message:
-----------
Bug fix for https://sourceforge.net/apps/trac/bigdata/ticket/136 (bulk loaded does not support inlining of values).
I also turned the EDS test suite back on with this change set. We currently only run the bulk loader's test suite against the EDS mode, so it was not being executed by CI. Once we get CI against a single-node cluster running, this should be changed to run against the full cluster rather than the EDS mode (EmbeddedDataService).
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/small.rdf
trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-08-03 15:11:09 UTC (rev 3400)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-08-03 15:13:17 UTC (rev 3401)
@@ -116,7 +116,6 @@
import com.bigdata.rdf.spo.SPORelation;
import com.bigdata.rdf.spo.SPOTupleSerializer;
import com.bigdata.rdf.store.AbstractTripleStore;
-import com.bigdata.rdf.store.IRawTripleStore;
import com.bigdata.rdf.store.ScaleOutTripleStore;
import com.bigdata.relation.accesspath.BlockingBuffer;
import com.bigdata.relation.accesspath.IBuffer;
@@ -2803,11 +2802,28 @@
*/
@SuppressWarnings("unchecked")
static <V extends BigdataValue> IChunkedIterator<V> newT2IdIterator(
+ final LexiconRelation r,
final Iterator<V> itr, final int chunkSize) {
- return new ChunkedWrappedIterator(new Striterator(itr), chunkSize,
- BigdataValue.class);
+ return new ChunkedWrappedIterator(new Striterator(itr)
+ .addFilter(new Filter() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected boolean isValid(final Object obj) {
+ /*
+ * Assigns the IV as a side effect iff the RDF Value can
+ * be inlined according to the governing lexicon
+ * configuration and returns true iff the value CAN NOT
+ * be inlined. Thus, inlining is doing as a side effect
+ * while the caller sees only those Values which need to
+ * be written onto the TERM2ID index.
+ */
+ return r.getInlineIV((Value) obj) == null;
+ }
+ }), chunkSize, BigdataValue.class);
+
}
/**
@@ -3064,6 +3080,13 @@
}
+ if (v.getIV().isInline()) {
+
+ // Do not write inline values on the reverse index.
+ continue;
+
+ }
+
final byte[] key = v.getIV().encode(tmp.reset())
.getKey();
@@ -3767,8 +3790,10 @@
try {
final Callable<Void> task = new AsyncTerm2IdIndexWriteTask(
- tidsLatch, lexiconRelation, newT2IdIterator(values
- .values().iterator(), producerChunkSize),
+ tidsLatch, lexiconRelation, newT2IdIterator(//
+ lexiconRelation,//
+ values.values().iterator(),//
+ producerChunkSize),
buffer_t2id);
// queue chunks onto the write buffer.
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-08-03 15:11:09 UTC (rev 3400)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-08-03 15:13:17 UTC (rev 3401)
@@ -683,7 +683,8 @@
new FileInputStream(resource))), baseURI, rdfFormat,
options);
- log.info("End of reparse: nerrors=" + nerrs + ", file="
+ if(log.isInfoEnabled())
+ log.info("End of reparse: nerrors=" + nerrs + ", file="
+ resource);
}
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/rio/small.rdf
===================================================================
(Binary files differ)
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java 2010-08-03 15:11:09 UTC (rev 3400)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java 2010-08-03 15:13:17 UTC (rev 3401)
@@ -912,8 +912,8 @@
* @param maxerrors
* The maximum #of errors before the test will abort.
*/
- public StatementVerifier(AbstractTripleStore db, int capacity,
- AtomicInteger nerrs, final int maxerrors) {
+ public StatementVerifier(final AbstractTripleStore db, final int capacity,
+ final AtomicInteger nerrs, final int maxerrors) {
super(db.getValueFactory());
@@ -927,7 +927,7 @@
Statement.class, null/* filter */) {
@Override
- protected long flush(int n, Statement[] a) {
+ protected long flush(final int n, final Statement[] a) {
verifyStatements( n , a );
@@ -945,7 +945,7 @@
* @param msg
* The error message.
*/
- private void error(String msg) {
+ private void error(final String msg) {
log.error(msg);
Modified: trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
===================================================================
--- trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java 2010-08-03 15:11:09 UTC (rev 3400)
+++ trunk/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java 2010-08-03 15:13:17 UTC (rev 3401)
@@ -83,8 +83,12 @@
// suite.addTest( TestScaleOutTripleStoreWithLocalDataServiceFederation.suite() );
-// suite.addTest(com.bigdata.rdf.store.TestScaleOutTripleStoreWithEmbeddedFederation
-// .suite());
+/*
+ * @todo We should run this test suite against a CI cluster on a single machine using
+ * the full bigdata federation rather than EDS.
+ */
+ suite.addTest(com.bigdata.rdf.store.TestScaleOutTripleStoreWithEmbeddedFederation
+ .suite());
// if (Boolean.parseBoolean(System.getProperty("maven.test.services.skip",
// "false"))
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-08-03 15:11:16
|
Revision: 3400
http://bigdata.svn.sourceforge.net/bigdata/?rev=3400&view=rev
Author: thompsonbry
Date: 2010-08-03 15:11:09 +0000 (Tue, 03 Aug 2010)
Log Message:
-----------
Made the lexiconConfiguration an eagerly initialized and final object.
Modified Paths:
--------------
trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-03 15:10:31 UTC (rev 3399)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-03 15:11:09 UTC (rev 3400)
@@ -391,6 +391,29 @@
AbstractTripleStore.Options.INLINE_BNODES,
AbstractTripleStore.Options.DEFAULT_INLINE_BNODES));
+ try {
+
+ final Class<IExtensionFactory> xfc =
+ determineExtensionFactoryClass();
+ final IExtensionFactory xFactory = xfc.newInstance();
+
+ /*
+ * Allow the extensions to resolve their datatype URIs into
+ * term identifiers.
+ */
+ xFactory.resolveDatatypes(this);
+
+ lexiconConfiguration = new LexiconConfiguration(
+ inlineLiterals, inlineBNodes, xFactory);
+
+ } catch (InstantiationException e) {
+ throw new IllegalArgumentException(
+ AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalArgumentException(
+ AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
+ }
+
}
}
@@ -1022,7 +1045,7 @@
* numTerms-1). The inline method will respect those semantics but
* then return a dense array of terms that definitely should be added
* to the lexicon indices - that is, terms in the original array at
- * index less than numTerms and not inlinable.
+ * index less than numTerms and not inlinable. MRP [BBT This is not as efficient!]
*/
final BigdataValue[] terms2 = addInlineTerms(terms, numTerms);
final int numTerms2 = terms2.length;
@@ -1252,7 +1275,7 @@
* @return
* the terms that were in consideration but could not be inlined
*/
- private BigdataValue[] addInlineTerms(BigdataValue[] terms, int numTerms) {
+ private BigdataValue[] addInlineTerms(final BigdataValue[] terms, final int numTerms) {
// these are the terms that will need to be indexed (not inline terms)
final ArrayList<BigdataValue> terms2 =
@@ -2077,7 +2100,7 @@
* The {@link ILexiconConfiguration} instance, which will determine how
* terms are encoded and decoded in the key space.
*/
- private ILexiconConfiguration lexiconConfiguration;
+ private final ILexiconConfiguration lexiconConfiguration;
/**
* Constant for the {@link LexiconRelation} namespace component.
@@ -2313,7 +2336,7 @@
* the inline internal value, or <code>null</code> if it cannot
* be converted
*/
- final public IV getInlineIV(Value value) {
+ final public IV getInlineIV(final Value value) {
return getLexiconConfiguration().createInlineIV(value);
@@ -2629,32 +2652,32 @@
*/
public ILexiconConfiguration getLexiconConfiguration() {
- if (lexiconConfiguration == null) {
-
- try {
-
- final Class<IExtensionFactory> xfc =
- determineExtensionFactoryClass();
- final IExtensionFactory xFactory = xfc.newInstance();
-
- /*
- * Allow the extensions to resolve their datatype URIs into
- * term identifiers.
- */
- xFactory.resolveDatatypes(this);
-
- lexiconConfiguration = new LexiconConfiguration(
- inlineLiterals, inlineBNodes, xFactory);
-
- } catch (InstantiationException e) {
- throw new IllegalArgumentException(
- AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException(
- AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
- }
-
- }
+// if (lexiconConfiguration == null) {
+//
+// try {
+//
+// final Class<IExtensionFactory> xfc =
+// determineExtensionFactoryClass();
+// final IExtensionFactory xFactory = xfc.newInstance();
+//
+// /*
+// * Allow the extensions to resolve their datatype URIs into
+// * term identifiers.
+// */
+// xFactory.resolveDatatypes(this);
+//
+// lexiconConfiguration = new LexiconConfiguration(
+// inlineLiterals, inlineBNodes, xFactory);
+//
+// } catch (InstantiationException e) {
+// throw new IllegalArgumentException(
+// AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
+// } catch (IllegalAccessException e) {
+// throw new IllegalArgumentException(
+// AbstractTripleStore.Options.EXTENSION_FACTORY_CLASS, e);
+// }
+//
+// }
return lexiconConfiguration;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
