Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[3896] branches/JOURNAL_HA_BRANCH/bigdata
|
From: <tho...@us...> - 2010-11-04 19:51:13
|
Revision: 3896
http://bigdata.svn.sourceforge.net/bigdata/?rev=3896&view=rev
Author: thompsonbry
Date: 2010-11-04 19:51:05 +0000 (Thu, 04 Nov 2010)
Log Message:
-----------
IndexMetadata - Raised the maximum branching factor to 4k.
AbstractLocalTripleStore - Modified log information to report the average record size for a B+Tree.
DataLoader - Added "-verbose" option to show the performance counters and details from the indices and the store.
ConcurrencyManager, Journal, RWStrategy, RWStore - fixed some issues related to performance counter reporting.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/Journal.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RWStrategy.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/ClosureStats.java
branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractLocalTripleStore.java
branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -271,7 +271,7 @@
/**
* A reasonable maximum branching factor for a {@link BTree}.
*/
- int MAX_BTREE_BRANCHING_FACTOR = 1024;
+ int MAX_BTREE_BRANCHING_FACTOR = 4196;
/**
* A reasonable maximum branching factor for an {@link IndexSegment}.
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -308,27 +308,22 @@
throw new RuntimeException(msg, cause);
}
-
- /**
- * Return interesting statistics about the transaction manager.
- */
- synchronized public CounterSet getCounters() {
-
- if (countersRoot == null) {
- countersRoot = new CounterSet();
+ /**
+ * Return interesting statistics about the transaction manager.
+ */
+ public CounterSet getCounters() {
- countersRoot.addCounter("#active", new Instrument<Integer>() {
- protected void sample() {
- setValue(activeTx.size());
- }
- });
+ final CounterSet countersRoot = new CounterSet();
- }
-
- return countersRoot;
-
- }
- private CounterSet countersRoot;
-
+ countersRoot.addCounter("#active", new Instrument<Integer>() {
+ protected void sample() {
+ setValue(activeTx.size());
+ }
+ });
+
+ return countersRoot;
+
+ }
+
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/ConcurrencyManager.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -1002,9 +1002,9 @@
*/
synchronized public CounterSet getCounters() {
- if (countersRoot == null){
+// if (countersRoot == null){
- countersRoot = new CounterSet();
+ CounterSet countersRoot = new CounterSet();
// elapsed time since the service started (milliseconds).
countersRoot.addCounter("elapsed",
@@ -1050,12 +1050,12 @@
}
- }
+// }
return countersRoot;
}
- private CounterSet countersRoot;
+// private CounterSet countersRoot;
/**
* Submit a task (asynchronous). Tasks will execute asynchronously in the
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/Journal.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/Journal.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/Journal.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -283,20 +283,15 @@
public CounterSet getCounters() {
-// if (counters == null) {
-
final CounterSet counters = super.getCounters();
counters.attach(concurrencyManager.getCounters());
counters.attach(localTransactionManager.getCounters());
-// }
-
return counters;
}
-// private CounterSet counters;
/*
* IResourceManager
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RWStrategy.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RWStrategy.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RWStrategy.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -293,7 +293,7 @@
public CounterSet getCounters() {
- return m_store.getStoreCounters().getCounters();
+ return m_store.getCounters();
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -3853,6 +3853,8 @@
/**
* Return interesting information about the write cache and file operations.
+ *
+ * @todo allocations data? user extent allocated? user extent used? etc.
*/
public CounterSet getCounters() {
Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/ClosureStats.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/ClosureStats.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/ClosureStats.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -73,10 +73,16 @@
}
+ public long triplesPerSecond() {
+
+ return ((long) (((double) mutationCount.get()) / ((double) elapsed.get()) * 1000d));
+
+ }
+
public String toString() {
return getClass().getSimpleName() + "{mutationCount=" + mutationCount.estimate_get()
- + ", elapsed=" + elapsed.estimate_get() + "ms}";
+ + ", elapsed=" + elapsed.estimate_get() + "ms, rate="+triplesPerSecond()+"}";
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractLocalTripleStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractLocalTripleStore.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractLocalTripleStore.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -90,10 +90,11 @@
final long nodesWritten = btreeCounters.getNodesWritten();
final long leavesWritten = btreeCounters.getLeavesWritten();
final long bytesWritten = btreeCounters.getBytesWritten();
+ final long bytesPerRecord = bytesWritten/(nodesWritten+leavesWritten);
- sb.append((first ? "" : ", ") + fqn + "{nodes=" + nodesWritten
- + ",leaves=" + leavesWritten + ", bytes=" + bytesWritten
- + "}");
+ sb.append((first ? "" : ", ") + fqn + "{nodes=" + nodesWritten
+ + ",leaves=" + leavesWritten + ", bytes=" + bytesWritten
+ + ", averageBytesPerRecord=" + bytesPerRecord + "}");
first = false;
@@ -113,10 +114,11 @@
final long nodesWritten = btreeCounters.getNodesWritten();
final long leavesWritten = btreeCounters.getLeavesWritten();
final long bytesWritten = btreeCounters.getBytesWritten();
+ final long bytesPerRecord = bytesWritten/(nodesWritten+leavesWritten);
- sb.append((first ? "" : ", ") + fqn + "{nodes=" + nodesWritten
- + ",leaves=" + leavesWritten + ", bytes=" + bytesWritten
- + "}");
+ sb.append((first ? "" : ", ") + fqn + "{nodes=" + nodesWritten
+ + ",leaves=" + leavesWritten + ", bytes=" + bytesWritten
+ + ", averageBytesPerRecord=" + bytesPerRecord + "}");
first = false;
Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-11-04 17:06:22 UTC (rev 3895)
+++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-11-04 19:51:05 UTC (rev 3896)
@@ -49,6 +49,7 @@
import com.bigdata.journal.ITx;
import com.bigdata.journal.Journal;
+import com.bigdata.journal.RWStrategy;
import com.bigdata.rdf.inf.ClosureStats;
import com.bigdata.rdf.inf.TruthMaintenance;
import com.bigdata.rdf.lexicon.LexiconRelation;
@@ -1190,7 +1191,7 @@
* support multiple data files within a single archive.
*
* @param args
- * [-closure][-namespace <i>namespace</i>] propertyFile (fileOrDir)+
+ * [-closure][-verbose][-namespace <i>namespace</i>] propertyFile (fileOrDir)+
*
* @throws IOException
*/
@@ -1199,6 +1200,7 @@
// default namespace.
String namespace = "kb";
boolean doClosure = false;
+ boolean verbose = false;
RDFFormat rdfFormat = null;
String baseURI = null;
@@ -1226,6 +1228,10 @@
doClosure = true;
+ } else if (arg.equals("-verbose")) {
+
+ verbose = true;
+
} else {
System.err.println("Unknown argument: " + arg);
@@ -1335,8 +1341,10 @@
jnl = new Journal(properties);
- final long firstOffset = jnl.getRootBlockView().getNextOffset();
-
+ // #of bytes on the journal before (user extent).
+// final long firstOffset = jnl.getRootBlockView().getNextOffset();
+ final long userData0 = jnl.getBufferStrategy().size();
+
System.out.println("Journal file: "+jnl.getFile());
AbstractTripleStore kb = (AbstractTripleStore) jnl
@@ -1368,9 +1376,19 @@
dataLoader.endSource();
System.out.println("Load: " + totals);
-
- if (dataLoader.closureEnum == ClosureEnum.None && doClosure) {
+
+ if (dataLoader.closureEnum == ClosureEnum.None && doClosure) {
+ if (verbose) {
+
+ System.out.println(jnl.getCounters().toString());
+
+ System.out
+ .println(((AbstractLocalTripleStore) dataLoader.database)
+ .getLocalBTreeBytesWritten(
+ new StringBuilder()).toString());
+ }
+
System.out.println("Computing closure.");
final ClosureStats stats = dataLoader.doClosure();
@@ -1378,13 +1396,38 @@
System.out.println("Closure: "+stats.toString());
}
+
+ jnl.commit();
+
+ if (verbose) {
+
+ System.out.println(jnl.getCounters().toString());
+
+ System.out
+ .println(((AbstractLocalTripleStore) dataLoader.database)
+ .getLocalBTreeBytesWritten(new StringBuilder())
+ .toString());
+
+ if (jnl.getBufferStrategy() instanceof RWStrategy) {
+
+ final StringBuilder sb = new StringBuilder();
+
+ ((RWStrategy) jnl.getBufferStrategy()).getRWStore()
+ .showAllocators(sb);
+
+ System.out.println(sb);
+
+ }
+
+ }
+
+ // #of bytes on the journal (user data only).
+ final long userData1 = jnl.getBufferStrategy().size();
- jnl.commit();
-
- final long lastOffset = jnl.getRootBlockView().getNextOffset();
+ // #of bytes written (user data only)
+ final long bytesWritten = (userData1 - userData0);
- System.out.println("Wrote: " + (lastOffset - firstOffset)
- + " bytes.");
+ System.out.println("Wrote: " + bytesWritten + " bytes.");
final long elapsedTotal = System.currentTimeMillis() - begin;
@@ -1404,7 +1447,7 @@
private static void usage() {
- System.err.println("usage: [-namespace namespace] propertyFile (fileOrDir)+");
+ System.err.println("usage: [-closure][-verbose][-namespace namespace] propertyFile (fileOrDir)+");
System.exit(1);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
