[Bigdata-commit] SF.net SVN: bigdata:[8454] branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/ com/bigdata/btree/PageStats.java

[<tho...@us...>]

Revision: 8454
          http://sourceforge.net/p/bigdata/code/8454
Author:   thompsonbry
Date:     2014-06-09 17:42:45 +0000 (Mon, 09 Jun 2014)
Log Message:
-----------
Added logic to collect the #of raw records and the IRawStore level storage bytes associated with those raw records (the backing store may have additional overhead).  

We have observed some cases where a lot of raw records were appearing in the ID2TERMS index.  Prior to this commit those raw records were not directly visible using DumpJournal, but only in the creation of a large number of small (64 or 128 byte) allocations.

Modified Paths:
--------------
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java	2014-06-09 13:30:41 UTC (rev 8453)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java	2014-06-09 17:42:45 UTC (rev 8454)
@@ -24,6 +24,7 @@
 package com.bigdata.btree;
 
 import com.bigdata.btree.data.IAbstractNodeData;
+import com.bigdata.btree.data.ILeafData;
 import com.bigdata.rawstore.IRawStore;
 
 /**
@@ -63,6 +64,14 @@
      */
     public static final int[] SLOT_SIZES = new int[] { 64, 128, 192, 320, 512,
             768, 1024, 2048, 3072, 4096, 8192 };
+    /**
+     * The number of raw record allocations and the byte size of those raw
+     * record allocations.
+     * 
+     * TODO We could also use a histogram over this information (raw records
+     * sizes).
+     */
+    public long nrawRecs = 0, rawRecBytes;
 
     public PageStats() {
 
@@ -100,7 +109,7 @@
 
     /** Return {@link #nodeBytes} plus {@link #leafBytes}. */
     public long getTotalBytes() {
-        return nodeBytes + leafBytes;
+        return nodeBytes + leafBytes + rawRecBytes;
     }
 
     /** The average bytes per node. */
@@ -113,6 +122,11 @@
         return (nleaves == 0 ? 0 : leafBytes / nleaves);
     }
 
+    /** The average bytes per raw record. */
+    public long getBytesPerRawRecord() {
+        return (nrawRecs== 0 ? 0 : rawRecBytes / nrawRecs);
+    }
+
     public String toString() {
         final StringBuilder sb = new StringBuilder();
         sb.append(getClass().getName());
@@ -120,14 +134,17 @@
         sb.append(",m=" + m);
         sb.append(",nnodes=" + nnodes);
         sb.append(",nleaves=" + nleaves);
+        sb.append(",nrawRecs=" + nrawRecs);
         sb.append(",nodeBytes=" + nodeBytes);
         sb.append(",minNodeBytes=" + minNodeBytes);
         sb.append(",maxNodeBytes=" + maxNodeBytes);
         sb.append(",leafBytes=" + leafBytes);
         sb.append(",minLeafBytes=" + minLeafBytes);
         sb.append(",maxLeafBytes=" + maxLeafBytes);
+        sb.append(",rawRecBytes=" + rawRecBytes);
         sb.append(",bytesPerNode=" + getBytesPerNode());
         sb.append(",bytesPerLeaf=" + getBytesPerLeaf());
+        sb.append(",bytesPerRawRec=" + getBytesPerRawRecord());
         sb.append(",nerrors=" + nerrors);
         final long npages = (nleaves + nnodes);
         for (int i = 0; i < SLOT_SIZES.length; i++) {
@@ -177,18 +194,24 @@
         sb.append('\t');
         sb.append("nentries");
         sb.append('\t');
+        sb.append("nrawRecs");
+        sb.append('\t');
         sb.append("nerrors");
         sb.append('\t');
         sb.append("nodeBytes");
         sb.append('\t');
         sb.append("leafBytes");
         sb.append('\t');
+        sb.append("rawRecBytes");
+        sb.append('\t');
         sb.append("totalBytes");
         sb.append('\t');
         sb.append("avgNodeBytes");
         sb.append('\t');
         sb.append("avgLeafBytes");
         sb.append('\t');
+        sb.append("avgRawRecBytes");
+        sb.append('\t');
         sb.append("minNodeBytes");
         sb.append('\t');
         sb.append("maxNodeBytes");
@@ -246,18 +269,24 @@
         sb.append('\t');
         sb.append(stats.ntuples);
         sb.append('\t');
+        sb.append(stats.nrawRecs);
+        sb.append('\t');
         sb.append(stats.nerrors);
         sb.append('\t');
         sb.append(stats.nodeBytes);
         sb.append('\t');
         sb.append(stats.leafBytes);
         sb.append('\t');
+        sb.append(stats.rawRecBytes);
+        sb.append('\t');
         sb.append(stats.getTotalBytes());
         sb.append('\t');
         sb.append(stats.getBytesPerNode());
         sb.append('\t');
         sb.append(stats.getBytesPerLeaf());
         sb.append('\t');
+        sb.append(stats.getBytesPerRawRecord());
+        sb.append('\t');
         sb.append(stats.minNodeBytes);
         sb.append('\t');
         sb.append(stats.maxNodeBytes);
@@ -363,6 +392,20 @@
                 if (stats.maxLeafBytes < nbytes)
                     stats.maxLeafBytes = nbytes;
 
+                if (node instanceof ILeafData) {
+                    final ILeafData data = (ILeafData) node;
+                    if(data.hasRawRecords()) {
+                        for (int i = 0; i < data.getKeys().size(); i++) {
+                            final long rawAddr = data.getRawRecord(i);
+                            if (rawAddr != IRawStore.NULL) {
+                                stats.nrawRecs++;
+                                stats.rawRecBytes += store
+                                        .getByteCount(rawAddr);
+                            }
+                        }
+                    }
+                }
+
             } else {
 
                 stats.nnodes++;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.