Thread: [Bigdata-commit] SF.net SVN: bigdata:[3527] branches/JOURNAL_HA_BRANCH/bigdata

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Revision: 3527
          http://bigdata.svn.sourceforge.net/bigdata/?rev=3527&view=rev
Author:   martyncutcher
Date:     2010-09-10 14:26:34 +0000 (Fri, 10 Sep 2010)

Log Message:
-----------
Add stats output for RWStore allocations

Modified Paths:
--------------
    branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java
    branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/Allocator.java
    branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java
    branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
    branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
    branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java

Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java
===================================================================

--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java	2010-09-09 17:17:21 UTC (rev 3526)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java	2010-09-10 14:26:34 UTC (rev 3527)
@@ -27,6 +27,7 @@
 import java.util.ArrayList;
 
 import com.bigdata.io.writecache.WriteCacheService;
+import com.bigdata.rwstore.RWStore.AllocationStats;
 
 /**
  * Bit maps for an allocator. The allocator is a bit map managed as int[]s.
@@ -189,10 +190,17 @@
 		return allocBits;
 	}
 
-	public String getStats() {
+	public String getStats(AllocationStats stats) {
 		final int total = m_ints * 32;
 		final int allocBits = getAllocBits();
 
+		if (stats != null) {
+			stats.m_reservedSlots += total;
+			stats.m_filledSlots += allocBits;
+			
+			return "";
+		}
+		
 		return " - start addr : " + RWStore.convertAddr(m_addr) + " [" + allocBits + "::" + total + "]";
 	}
 

Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/Allocator.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/Allocator.java	2010-09-09 17:17:21 UTC (rev 3526)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/Allocator.java	2010-09-10 14:26:34 UTC (rev 3527)
@@ -28,7 +28,9 @@
 import java.util.ArrayList;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.bigdata.rwstore.RWStore.AllocationStats;
 
+
 public interface Allocator extends Comparable {
   public int getBlockSize();
   public void setIndex(int index);
@@ -50,7 +52,7 @@
   public void addAddresses(ArrayList addrs);
   public int getRawStartAddr();
   public int getIndex();
-  public void appendShortStats(StringBuffer str);
+  public void appendShortStats(StringBuilder str, AllocationStats[] stats);
   public boolean canImmediatelyFree(int addr, int size, IAllocationContext context);
 }
 	
\ No newline at end of file

Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java	2010-09-09 17:17:21 UTC (rev 3526)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java	2010-09-10 14:26:34 UTC (rev 3527)
@@ -7,6 +7,7 @@
 import java.util.ArrayList;
 import java.util.concurrent.atomic.AtomicLong;
 
+import com.bigdata.rwstore.RWStore.AllocationStats;
 import com.bigdata.util.ChecksumUtility;
 
 /**
@@ -297,7 +298,7 @@
 		return m_hdrs[hdrIndex];
 	}
 
-	public void appendShortStats(StringBuffer str) {
+	public void appendShortStats(StringBuilder str, AllocationStats[] stats) {
 		str.append("Index: " + m_index + ", address: " + getStartAddr() + ", BLOB\n");
 	}
 

Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java	2010-09-09 17:17:21 UTC (rev 3526)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java	2010-09-10 14:26:34 UTC (rev 3527)
@@ -30,6 +30,7 @@
 
 import org.apache.log4j.Logger;
 
+import com.bigdata.rwstore.RWStore.AllocationStats;
 import com.bigdata.util.ChecksumUtility;
 
 /**
@@ -334,7 +335,7 @@
 			if (block.m_addr == 0) {
 				break;
 			}
-            sb.append(block.getStats() + "\r\n");
+            sb.append(block.getStats(null) + "\r\n");
 			counter.addAndGet(block.getAllocBits() * m_size);
 		}
 
@@ -489,14 +490,26 @@
 		return m_index;
 	}
 
-	public void appendShortStats(StringBuffer str) {
-		str.append("Index: " + m_index + ", " + m_size);
+	public void appendShortStats(StringBuilder str, AllocationStats[] stats) {
+
+		int si = -1;
+
+		if (stats == null) {
+			str.append("Index: " + m_index + ", " + m_size);
+		} else {		
+			for (int i = 0; i < stats.length; i++) {
+				if (m_size == stats[i].m_blockSize) {
+					si = i;
+					break;
+				}
+			}
+		}
 		
 		Iterator<AllocBlock> blocks = m_allocBlocks.iterator();
 		while (blocks.hasNext()) {
 			AllocBlock block = blocks.next();
 			if (block.m_addr != 0) {
-				str.append(block.getStats());
+				str.append(block.getStats(si == -1 ? null : stats[si]));
 			} else {
 				break;
 			}

Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java	2010-09-09 17:17:21 UTC (rev 3526)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java	2010-09-10 14:26:34 UTC (rev 3527)
@@ -634,7 +634,7 @@
 			// clearOutstandingDeferrels(deferredFreeListAddr, deferredFreeListEntries);
 
 			if (log.isTraceEnabled()) {
-				final StringBuffer str = new StringBuffer();
+				final StringBuilder str = new StringBuilder();
 				this.showAllocators(str);
 				log.trace(str);
 			}
@@ -778,7 +778,7 @@
 		}
 
 		if (false) {
-			StringBuffer tmp = new StringBuffer();
+			StringBuilder tmp = new StringBuilder();
 			showAllocators(tmp);
 			
 			System.out.println("Allocators: " + tmp.toString());
@@ -2076,16 +2076,50 @@
 
 	}
 	
+	public static class AllocationStats {
+		public AllocationStats(int i) {
+			m_blockSize = i;
+		}
+		long m_blockSize;
+		long m_reservedSlots;
+		long m_filledSlots;
+	}
 	/**
 	 * Utility debug outputing the allocator array, showing index, start
 	 * address and alloc type/size
+	 * 
+	 * Collected statistics are against each Allocation Block size:
+	 * total number of slots | store size
+	 * number of filled slots | store used
 	 */
-	public void showAllocators(StringBuffer str) {
+	public void showAllocators(StringBuilder str) {
+		AllocationStats[] stats = new AllocationStats[m_allocSizes.length];
+		for (int i = 0; i < stats.length; i++) {
+			stats[i] = new AllocationStats(m_allocSizes[i]*64);
+		}
 		Iterator allocs = m_allocs.iterator();
 		while (allocs.hasNext()) {
 			Allocator alloc = (Allocator) allocs.next();
-			alloc.appendShortStats(str);
+			alloc.appendShortStats(str, stats);
 		}
+		
+		// Append Summary
+		str.append("\n-------------------------\n");
+		str.append("RWStore Allocation Summary\n");
+		str.append("-------------------------\n");
+		long treserved = 0;
+		long tfilled = 0;
+		for (int i = 0; i < stats.length; i++) {
+			str.append("Allocation: " + stats[i].m_blockSize);
+			long reserved = stats[i].m_reservedSlots * stats[i].m_blockSize;
+			treserved += reserved;
+			str.append(", reserved: " + reserved);
+			long filled = stats[i].m_filledSlots * stats[i].m_blockSize;
+			tfilled += filled;
+			str.append(", filled: " + filled);
+			str.append("\n");
+		}
+		str.append("Total - file: " + convertAddr(m_fileSize) + ", reserved: " + treserved + ", filled: " + tfilled + "\n");
 	}
 
 	public ArrayList getStorageBlockAddresses() {

Modified: branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java	2010-09-09 17:17:21 UTC (rev 3526)
+++ branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java	2010-09-10 14:26:34 UTC (rev 3527)
@@ -75,11 +75,13 @@
 import com.bigdata.LRUNexus;
 import com.bigdata.btree.IndexMetadata;
 import com.bigdata.journal.AbstractJournal;
+import com.bigdata.journal.IBufferStrategy;
 import com.bigdata.journal.IIndexManager;
 import com.bigdata.journal.IJournal;
 import com.bigdata.journal.ITransactionService;
 import com.bigdata.journal.ITx;
 import com.bigdata.journal.Journal;
+import com.bigdata.journal.RWStrategy;
 import com.bigdata.journal.TimestampUtility;
 import com.bigdata.rdf.sail.BigdataSail;
 import com.bigdata.rdf.sail.BigdataSailGraphQuery;
@@ -89,6 +91,7 @@
 import com.bigdata.rdf.store.AbstractTripleStore;
 import com.bigdata.relation.AbstractResource;
 import com.bigdata.relation.RelationSchema;
+import com.bigdata.rwstore.RWStore;
 import com.bigdata.service.AbstractDistributedFederation;
 import com.bigdata.service.AbstractFederation;
 import com.bigdata.service.IBigdataFederation;
@@ -363,6 +366,17 @@
 			}
 
 			// sb.append(tripleStore.predicateUsage());
+			
+			if (tripleStore.getIndexManager() instanceof Journal) {
+				Journal journal = (Journal) tripleStore.getIndexManager();
+				IBufferStrategy strategy = journal.getBufferStrategy();
+				if (strategy instanceof RWStrategy) {
+					RWStore store = ((RWStrategy) strategy).getRWStore();
+					
+					store.showAllocators(sb);
+					
+				}
+			}
 
 		} catch (Throwable t) {
 


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Thread: [Bigdata-commit] SF.net SVN: bigdata:[3527] branches/JOURNAL_HA_BRANCH/bigdata

Fast, scalable, robust graph database platform

bigdata-commit