|
From: <tho...@us...> - 2014-06-09 17:42:50
|
Revision: 8454
http://sourceforge.net/p/bigdata/code/8454
Author: thompsonbry
Date: 2014-06-09 17:42:45 +0000 (Mon, 09 Jun 2014)
Log Message:
-----------
Added logic to collect the #of raw records and the IRawStore level storage bytes associated with those raw records (the backing store may have additional overhead).
We have observed some cases where a lot of raw records were appearing in the ID2TERMS index. Prior to this commit those raw records were not directly visible using DumpJournal, but only in the creation of a large number of small (64 or 128 byte) allocations.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java 2014-06-09 13:30:41 UTC (rev 8453)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java 2014-06-09 17:42:45 UTC (rev 8454)
@@ -24,6 +24,7 @@
package com.bigdata.btree;
import com.bigdata.btree.data.IAbstractNodeData;
+import com.bigdata.btree.data.ILeafData;
import com.bigdata.rawstore.IRawStore;
/**
@@ -63,6 +64,14 @@
*/
public static final int[] SLOT_SIZES = new int[] { 64, 128, 192, 320, 512,
768, 1024, 2048, 3072, 4096, 8192 };
+ /**
+ * The number of raw record allocations and the byte size of those raw
+ * record allocations.
+ *
+ * TODO We could also use a histogram over this information (raw records
+ * sizes).
+ */
+ public long nrawRecs = 0, rawRecBytes;
public PageStats() {
@@ -100,7 +109,7 @@
/** Return {@link #nodeBytes} plus {@link #leafBytes}. */
public long getTotalBytes() {
- return nodeBytes + leafBytes;
+ return nodeBytes + leafBytes + rawRecBytes;
}
/** The average bytes per node. */
@@ -113,6 +122,11 @@
return (nleaves == 0 ? 0 : leafBytes / nleaves);
}
+ /** The average bytes per raw record. */
+ public long getBytesPerRawRecord() {
+ return (nrawRecs== 0 ? 0 : rawRecBytes / nrawRecs);
+ }
+
public String toString() {
final StringBuilder sb = new StringBuilder();
sb.append(getClass().getName());
@@ -120,14 +134,17 @@
sb.append(",m=" + m);
sb.append(",nnodes=" + nnodes);
sb.append(",nleaves=" + nleaves);
+ sb.append(",nrawRecs=" + nrawRecs);
sb.append(",nodeBytes=" + nodeBytes);
sb.append(",minNodeBytes=" + minNodeBytes);
sb.append(",maxNodeBytes=" + maxNodeBytes);
sb.append(",leafBytes=" + leafBytes);
sb.append(",minLeafBytes=" + minLeafBytes);
sb.append(",maxLeafBytes=" + maxLeafBytes);
+ sb.append(",rawRecBytes=" + rawRecBytes);
sb.append(",bytesPerNode=" + getBytesPerNode());
sb.append(",bytesPerLeaf=" + getBytesPerLeaf());
+ sb.append(",bytesPerRawRec=" + getBytesPerRawRecord());
sb.append(",nerrors=" + nerrors);
final long npages = (nleaves + nnodes);
for (int i = 0; i < SLOT_SIZES.length; i++) {
@@ -177,18 +194,24 @@
sb.append('\t');
sb.append("nentries");
sb.append('\t');
+ sb.append("nrawRecs");
+ sb.append('\t');
sb.append("nerrors");
sb.append('\t');
sb.append("nodeBytes");
sb.append('\t');
sb.append("leafBytes");
sb.append('\t');
+ sb.append("rawRecBytes");
+ sb.append('\t');
sb.append("totalBytes");
sb.append('\t');
sb.append("avgNodeBytes");
sb.append('\t');
sb.append("avgLeafBytes");
sb.append('\t');
+ sb.append("avgRawRecBytes");
+ sb.append('\t');
sb.append("minNodeBytes");
sb.append('\t');
sb.append("maxNodeBytes");
@@ -246,18 +269,24 @@
sb.append('\t');
sb.append(stats.ntuples);
sb.append('\t');
+ sb.append(stats.nrawRecs);
+ sb.append('\t');
sb.append(stats.nerrors);
sb.append('\t');
sb.append(stats.nodeBytes);
sb.append('\t');
sb.append(stats.leafBytes);
sb.append('\t');
+ sb.append(stats.rawRecBytes);
+ sb.append('\t');
sb.append(stats.getTotalBytes());
sb.append('\t');
sb.append(stats.getBytesPerNode());
sb.append('\t');
sb.append(stats.getBytesPerLeaf());
sb.append('\t');
+ sb.append(stats.getBytesPerRawRecord());
+ sb.append('\t');
sb.append(stats.minNodeBytes);
sb.append('\t');
sb.append(stats.maxNodeBytes);
@@ -363,6 +392,20 @@
if (stats.maxLeafBytes < nbytes)
stats.maxLeafBytes = nbytes;
+ if (node instanceof ILeafData) {
+ final ILeafData data = (ILeafData) node;
+ if(data.hasRawRecords()) {
+ for (int i = 0; i < data.getKeys().size(); i++) {
+ final long rawAddr = data.getRawRecord(i);
+ if (rawAddr != IRawStore.NULL) {
+ stats.nrawRecs++;
+ stats.rawRecBytes += store
+ .getByteCount(rawAddr);
+ }
+ }
+ }
+ }
+
} else {
stats.nnodes++;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|