Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[3985] branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/ bigdata/rwstore
|
From: <mar...@us...> - 2010-11-29 09:53:36
|
Revision: 3985
http://bigdata.svn.sourceforge.net/bigdata/?rev=3985&view=rev
Author: martyncutcher
Date: 2010-11-29 09:53:27 +0000 (Mon, 29 Nov 2010)
Log Message:
-----------
1) Extend the metabits header. 2) Add more relevant data to the storage stats. 3) Enable more flexible allocations of contiguous reservations in preparation for direct buffer allocation.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/PSOutputStream.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/StorageStats.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java 2010-11-24 21:40:07 UTC (rev 3984)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/AllocBlock.java 2010-11-29 09:53:27 UTC (rev 3985)
@@ -237,4 +237,43 @@
m_commit = m_saveCommit;
m_saveCommit = null;
}
+
+ /**
+ * Must release allocations made by this allocator.
+ *
+ * The commit bits are the old transient bits, so any allocated bits
+ * set in live, but not in commit, were set within this context.
+ *
+ * The m_commit is the m_transients bits at the point of the
+ * link of the allocationContext with this allocator, bits set in m_live
+ * that are not set in m_commit, were made by this allocator for the
+ * aborted context.
+ *
+ * L 1100 0110 AC 0111 AB 0110
+ * T 1100 1110 1111 1110
+ * C 1100 1100 1110 1100
+ */
+ public void abortshadow() {
+ for (int i = 0; i < m_live.length; i++) {
+ m_live[i] &= m_commit[i];
+ m_transients[i] = m_live[i] | m_saveCommit[i];
+ }
+ m_commit = m_saveCommit;
+ }
+
+ /**
+ * When a session is active, the transient bits do not equate to an ORing
+ * of the committed bits and the live bits, but rather an ORing of the live
+ * with all the committed bits since the start of the session.
+ * When the session is released, the state is restored to an ORing of the
+ * live and the committed, thus releasing slots for re-allocation.
+ */
+ public void releaseSession() {
+ if (m_addr != 0) { // check active!
+ for (int i = 0; i < m_live.length; i++) {
+ m_transients[i] = m_live[i] | m_commit[i];
+ }
+ }
+ }
+
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2010-11-24 21:40:07 UTC (rev 3984)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/FixedAllocator.java 2010-11-29 09:53:27 UTC (rev 3985)
@@ -42,8 +42,10 @@
public class FixedAllocator implements Allocator {
private static final Logger log = Logger.getLogger(FixedAllocator.class);
+
+ private final int cModAllocation = 1 << RWStore.ALLOCATION_SCALEUP;
+ private final int cMinAllocation = cModAllocation * 1; // must be multiple of cModAllocation
-// final private RWWriteCacheService m_writeCache;
volatile private int m_freeBits;
volatile private int m_freeTransients;
@@ -111,7 +113,9 @@
final int bit = offset % allocBlockRange;
- if (RWStore.tstBit(block.m_live, bit)) {
+ if (RWStore.tstBit(block.m_live, bit)
+ || (this.m_sessionActive && RWStore.tstBit(block.m_transients, bit)))
+ {
return RWStore.convertAddr(block.m_addr) + ((long) m_size * bit);
} else {
return 0L;
@@ -145,6 +149,14 @@
}
volatile private IAllocationContext m_context;
+
+ /**
+ * Indicates whether session protection has been used to protect
+ * store from re-allocating allocations reachable from read-only
+ * requests and concurrent transactions.
+ */
+ private boolean m_sessionActive;
+
public void setAllocationContext(final IAllocationContext context) {
if (context == null && m_context != null) {
// restore commit bits in AllocBlocks
@@ -161,6 +173,21 @@
}
/**
+ * Unwinds the allocations made within the context and clears
+ */
+ public void abortAllocationContext(final IAllocationContext context) {
+ if (context != null && m_context == context) {
+ // restore commit bits in AllocBlocks
+ for (AllocBlock allocBlock : m_allocBlocks) {
+ allocBlock.abortshadow();
+ }
+ m_context = null;
+ } else {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
* write called on commit, so this is the point when "transient frees" - the
* freeing of previously committed memory can be made available since we
* are creating a new commit point - the condition being that m_freeBits
@@ -174,6 +201,8 @@
final byte[] buf = new byte[1024];
final DataOutputStream str = new DataOutputStream(new FixedOutputStream(buf));
try {
+ m_sessionActive = m_store.isSessionProtected();
+
str.writeInt(m_size);
final Iterator<AllocBlock> iter = m_allocBlocks.iterator();
@@ -185,9 +214,9 @@
str.writeInt(block.m_live[i]);
}
-// if (!m_store.isSessionPreserved()) {
+ if (!m_sessionActive) {
block.m_transients = block.m_live.clone();
-// }
+ }
/**
* If this allocator is shadowed then copy the new committed
@@ -314,29 +343,12 @@
m_size = size;
- /*
- * For smaller allocations we'll allocate a larger span, this is needed
- * to ensure the minimum allocation is large enough to guarantee
- * a unique address for a BlobAllocator.
- */
- if (m_size < 256) {
- /*
- * Note: 64 ints is 256 bytes is 2048 bits, so 2048 allocation
- * slots.
- */
- m_bitSize = 64;
- } else {
- /*
- * Note: 32 ints is 128 bytes is 1024 bits, so 1024 allocation
- * slots.
- */
- m_bitSize = 32;
- }
+ m_bitSize = calcBitSize(true, size, cMinAllocation, cModAllocation);
// m_writeCache = cache;
// number of blocks in this allocator, bitSize plus 1 for start address
- final int numBlocks = 255 / (m_bitSize + 1);
+ final int numBlocks = 254 / (m_bitSize + 1);
/*
* Create AllocBlocks for this FixedAllocator, but do not allocate
@@ -351,6 +363,82 @@
m_freeTransients = 0;
m_freeBits = 32 * m_bitSize * numBlocks;
}
+
+ /**
+ * This determines the size of the reservation required in terms of
+ * the number of ints each holding bits for 32 slots.
+ *
+ * The minimum return value will be 1, for a single int holiding 32 bits.
+ *
+ * The maximum value will be the number of ints required to fill the minimum
+ * reservation.
+ *
+ * The minimum reservation will be some multiple of the
+ * address multiplier that allows alloction blocks to address large addresses
+ * with an INT32. For example, by setting a minimum reservation at 128K, the
+ * allocation blocks INT32 start address may be multiplied by 128K to provide
+ * a physical address.
+ *
+ * The minReserve must be a power of 2, eg 1K, 2k or 4K.. etc
+ *
+ * A standard minReserve of 16K is plenty big enough, enabling 32TB of
+ * addressable store. The logical maximum used store is calculated as the
+ * maximum fixed allocation size * MAX_INT. So a store with a maximum
+ * fixed slot size of 4K could only allocated 8TB.
+ *
+ * Since the allocation size must be MOD 0 the minReserve, the lower the
+ * minReserve the smaller the allocation may be required for larger
+ * slot sizes.
+ *
+ * Another consideration is file locality. In this case the emphasis is
+ * on larger contiguous areas to improve the likely locality of allocations
+ * made by a FixedAllocator. Here the addressability implied by the reserve
+ * is not an issue, and larger reserves are chosen to improve locality. The
+ * downside is a potential for more wasted space, but this
+ * reduces as the store size grows and in large stores (> 10GB) becomes
+ * insignificant.
+ *
+ * Therefore, if a FixedAllocator is to be used in a large store and
+ * locality needs to be optimised for SATA disk access then the minReserve
+ * should be high = say 128K, while if the allocator is tuned to ByteBuffer
+ * allocation, a minallocation of 8 to 16K is more suitable.
+ *
+ * A final consideration is allocator reference efficiency in the sense
+ * to maximise the amount of allocations that can be made. By this I mean
+ * just how close we can get to MAX_INT allocations. For example, if we
+ * allow for upto 8192 allocations from a single allocator, but in
+ * practice average closer to 4096 then the maximum number of allocations
+ * comes down from MAX_INT to MAX_INT/2. This is also a consideration when
+ * considering max fixed allocator size, since if we require a large number
+ * of Blobs this reduces the amount of "virtual" allocations by at least
+ * a factro of three for each blob (at least 2 fixed allocations for
+ * content and 1 more for the header). A variation on the current Blob
+ * implementation could include the header in the first allocation, thus
+ * reducing the minimum Blob allocations from 3 to 2, but the point still
+ * holds that too small a max fixed allocation could rmatically reduce the
+ * number of allocations that could be made.
+ *
+ * @param alloc the slot size to be managed
+ * @param minReserve the minimum reservation in bytes
+ * @return the size of the int array
+ */
+ public static int calcBitSize(final boolean optDensity, final int alloc, final int minReserve, final int modAllocation) {
+ final int intAllocation = 32 * alloc; // min 32 bits
+
+ // we need to find smallest number of ints * the intAllocation
+ // such that totalAllocation % minReserve is 0
+ // example 6K intAllocation would need 8 ints for 48K for 16K min
+ // likewise a 24K intAllocation would require 2 ints
+ // if optimising for density set min ints to 8
+ int nints = optDensity ? 8 : 1;
+ while ((nints * intAllocation) < minReserve) nints++;
+
+ while ((nints * intAllocation) % modAllocation != 0) nints++;
+
+ System.out.println("calcBitSize for " + alloc + " returns " + nints);
+
+ return nints;
+ }
public String getStats(final AtomicLong counter) {
@@ -675,4 +763,14 @@
public void setBucketStats(Bucket b) {
m_statsBucket = b;
}
+
+ public void releaseSession() {
+ if (this.m_sessionActive) {
+ if (log.isTraceEnabled())
+ log.trace("Allocator: #" + m_index + " releasing session protection");
+ for (AllocBlock ab : m_allocBlocks) {
+ ab.releaseSession();
+ }
+ }
+ }
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/PSOutputStream.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/PSOutputStream.java 2010-11-24 21:40:07 UTC (rev 3984)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/PSOutputStream.java 2010-11-29 09:53:27 UTC (rev 3985)
@@ -357,11 +357,6 @@
// + ", last alloc: " + precount);
// }
- if (log.isDebugEnabled())
- log.debug("Writing BlobHdrIdx with " + m_blobHdrIdx + " allocations");
-
- // DO NOT USE BLOB ALLOCATOR
- // addr = m_store.registerBlob(addr); // returns handle
} catch (IOException e) {
// e.printStackTrace();
throw new RuntimeException(e);
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-24 21:40:07 UTC (rev 3984)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-29 09:53:27 UTC (rev 3985)
@@ -477,7 +477,7 @@
* <code>true</code> iff the backing store is open.
*/
private volatile boolean m_open = true;
-
+
class WriteCacheImpl extends WriteCache.FileChannelScatteredWriteCache {
public WriteCacheImpl(final ByteBuffer buf,
final boolean useChecksum,
@@ -852,9 +852,10 @@
final int allocBlocks = strBuf.readInt();
m_storageStatsAddr = strBuf.readLong();
- strBuf.readInt(); // reserved7
- strBuf.readInt(); // reserved8
- strBuf.readInt(); // reserved9
+ // and let's read in those reserved ints
+ for (int i = 0; i < cReservedMetaBits; i++) {
+ strBuf.readInt();
+ }
m_allocSizes = new int[allocBlocks];
for (int i = 0; i < allocBlocks; i++) {
@@ -992,7 +993,8 @@
FileChannelUtility.readAll(m_reopener, ByteBuffer.wrap(buf), addr);
- final DataInputStream strBuf = new DataInputStream(new ByteArrayInputStream(buf));
+ final ByteArrayInputStream baBuf = new ByteArrayInputStream(buf);
+ final DataInputStream strBuf = new DataInputStream(baBuf);
final int allocSize = strBuf.readInt(); // if Blob < 0
final FixedAllocator allocator;
@@ -1009,6 +1011,14 @@
freeList = m_freeFixed[index];
allocator.read(strBuf);
+ final int chk = ChecksumUtility.getCHK().checksum(buf,
+ buf.length - baBuf.available());
+
+ int tstChk = strBuf.readInt();
+ if (tstChk != chk) {
+ throw new IllegalStateException("FixedAllocator checksum error");
+ }
+
allocator.setDiskAddr(i); // store bit, not physical
// address!
allocator.setFreeList(freeList);
@@ -1420,7 +1430,7 @@
if (sze > m_maxFixedAlloc-4) {
freeBlob(addr, sze, context);
} else {
- final Allocator alloc = getBlockByAddress(addr);
+ final FixedAllocator alloc = getBlockByAddress(addr);
/*
* There are a few conditions here. If the context owns the
* allocator and the allocation was made by this context then it
@@ -1429,22 +1439,29 @@
* AllocationContexts, in this situation, the free must ALWAYS
* be deferred.
*
+ * If the MIN_RELEASE_AGE is ZERO then we can protect allocations
+ * and read-only transactions with Session protection, avoiding
+ * the need to manage deferred frees.
+ *
* FIXME We need unit tests when MIN_RELEASE_AGE is GT ZERO.
*
* FIXME We need unit test when MIN_RELEASE_AGE is ZERO AND
* there are open read-only transactions.
*/
- boolean alwaysDefer = m_minReleaseAge > 0L
- || m_activeTxCount > 0;
- if (!alwaysDefer)
- alwaysDefer = context == null && !m_contexts.isEmpty();
- if (alwaysDefer)
- if (log.isDebugEnabled())
- log.debug("Should defer " + addr + " real: " + physicalAddress(addr));
- if (alwaysDefer || !alloc.canImmediatelyFree(addr, sze, context)) {
- deferFree(addr, sze);
+ if (m_minReleaseAge == 0) {
+ immediateFree(addr, sze);
} else {
- immediateFree(addr, sze);
+ boolean alwaysDefer = m_activeTxCount > 0;
+ if (!alwaysDefer)
+ alwaysDefer = context == null && !m_contexts.isEmpty();
+ if (alwaysDefer)
+ if (log.isDebugEnabled())
+ log.debug("Should defer " + addr + " real: " + physicalAddress(addr));
+ if (alwaysDefer || !alloc.canImmediatelyFree(addr, sze, context)) {
+ deferFree(addr, sze);
+ } else {
+ immediateFree(addr, sze);
+ }
}
}
} finally {
@@ -1452,7 +1469,31 @@
}
}
+
+ long getHistoryRetention() {
+ return m_minReleaseAge;
+ }
+ boolean isSessionProtected() {
+ return m_minReleaseAge == 0 && (m_activeTxCount > 0 || !m_contexts.isEmpty());
+ }
+
+ /**
+ * Sessions will only be used to protect transactions and read-only views
+ * when the m_minReleaseAge is no zero, otherwise the deferredFree
+ * approach will be used.
+ *
+ * When called, will call through to the Allocators to re-sync the
+ * transient bits with the committed and live.
+ */
+ void releaseSessions() {
+ if (m_minReleaseAge == 0) {
+ for (FixedAllocator fa : m_allocs) {
+ fa.releaseSession();
+ }
+ }
+ }
+
private boolean freeBlob(final int hdr_addr, final int sze, final IAllocationContext context) {
if (sze < (m_maxFixedAlloc-4))
throw new IllegalArgumentException("Unexpected address size");
@@ -1498,7 +1539,7 @@
m_allocationLock.lock();
try {
- final Allocator alloc = getBlockByAddress(addr);
+ final FixedAllocator alloc = getBlockByAddress(addr);
final int addrOffset = getOffset(addr);
if (alloc == null) {
throw new IllegalArgumentException("Invalid address provided to immediateFree: " + addr + ", size: " + sze);
@@ -1879,9 +1920,10 @@
str.writeInt(m_allocSizes.length);
str.writeLong(m_storageStatsAddr);
- str.writeInt(0); // reserved7
- str.writeInt(0); // reserved8
- str.writeInt(0); // reserved9
+ // Let's reserve ourselves some space
+ for (int i = 0; i < cReservedMetaBits; i++) {
+ str.writeInt(0);
+ }
for (int i = 0; i < m_allocSizes.length; i++) {
str.writeInt(m_allocSizes[i]);
@@ -2126,18 +2168,24 @@
final private int cVersion = 0x0400;
/**
+ * cReservedMetaBits is the reserved space in the metaBits header
+ * to alloc for binary compatibility moving forward.
+ *
+ * If we need to add int values to the header we can do so and reduce the
+ * reservation by 1 each time
+ */
+ final int cReservedMetaBits = 20;
+
+ /**
* MetaBits Header
* 0 int version
* 1-2 int[2] long deferredFree
* 3 int defaultMetaBitsSize
* 4 int length of allocation sizes
- * 5 int reserved
- * 6 int reserved
- * 7 int reserved
- * 8 int reserved
- * 9 int reserved
+ * 5-6 int[2] storage stats addr
+ * + 20 reserved
*/
- final private int cMetaHdrFields = 10;
+ final private int cMetaHdrFields = 7 + cReservedMetaBits;
/**
* @see Options#META_BITS_SIZE
*/
@@ -2697,7 +2745,7 @@
if (addr > 0) {
return addr & 0xFFFFFFE0;
} else {
- final Allocator allocator = getBlock(addr);
+ final FixedAllocator allocator = getBlock(addr);
final int offset = getOffset(addr);
final long laddr = allocator.getPhysicalAddress(offset);
@@ -2709,14 +2757,14 @@
* handle dual address format, if addr is positive then it is the physical
* address, so the Allocators must be searched.
**/
- Allocator getBlockByAddress(final int addr) {
+ FixedAllocator getBlockByAddress(final int addr) {
if (addr < 0) {
return getBlock(addr);
}
final Iterator<FixedAllocator> allocs = m_allocs.iterator();
- Allocator alloc = null;
+ FixedAllocator alloc = null;
while (allocs.hasNext()) {
alloc = allocs.next();
@@ -2733,10 +2781,10 @@
// return (-addr) >>> OFFSET_BITS;
// }
- private Allocator getBlock(final int addr) {
+ private FixedAllocator getBlock(final int addr) {
final int index = (-addr) >>> OFFSET_BITS;
- return (Allocator) m_allocs.get(index);
+ return m_allocs.get(index);
}
private int getOffset(final int addr) {
@@ -3422,6 +3470,29 @@
}
}
+ /**
+ * The ContextAllocation object manages a freeList of associated allocators
+ * and an overall list of allocators. When the context is detached, all
+ * allocators must be released and any that has available capacity will be
+ * assigned to the global free lists.
+ *
+ * @param context
+ * The context to be released from all FixedAllocators.
+ */
+ public void abortContext(final IAllocationContext context) {
+ assertOpen();
+ m_allocationLock.lock();
+ try {
+ final ContextAllocation alloc = m_contexts.remove(context);
+
+ if (alloc != null) {
+ alloc.release();
+ }
+ } finally {
+ m_allocationLock.unlock();
+ }
+ }
+
/**
* The ContextAllocation class manages a set of Allocators.
*
@@ -3487,6 +3558,24 @@
// freeBlobs.addAll(m_freeBlobs);
}
+ void abort() {
+ final ArrayList<FixedAllocator> freeFixed[] = m_parent != null ? m_parent.m_freeFixed
+ : m_store.m_freeFixed;
+
+ final IAllocationContext pcontext = m_parent == null ? null
+ : m_parent.m_context;
+
+ for (FixedAllocator f : m_allFixed) {
+ f.abortAllocationContext(pcontext);
+ }
+
+ for (int i = 0; i < m_freeFixed.length; i++) {
+ freeFixed[i].addAll(m_freeFixed[i]);
+ }
+
+// freeBlobs.addAll(m_freeBlobs);
+ }
+
FixedAllocator getFreeFixed(final int i) {
final ArrayList<FixedAllocator> free = m_freeFixed[i];
if (free.size() == 0) {
@@ -4242,6 +4331,10 @@
m_activeTxCount--;
if(log.isInfoEnabled())
log.info("#activeTx="+m_activeTxCount);
+
+ if (m_activeTxCount == 0) {
+ releaseSessions();
+ }
} finally {
m_allocationLock.unlock();
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/StorageStats.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/StorageStats.java 2010-11-24 21:40:07 UTC (rev 3984)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/StorageStats.java 2010-11-29 09:53:27 UTC (rev 3985)
@@ -31,6 +31,7 @@
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.ArrayList;
+import java.util.Formatter;
/**
* Maintains stats on the RWStore allocations, useful for tuning Allocator
@@ -64,35 +65,63 @@
*
*/
public class StorageStats {
+ final int cVersion = 0x0100;
+
final int m_maxFixed;
public class BlobBucket {
final int m_size;
+ long m_allocationSize;
long m_allocations;
long m_deletes;
+ long m_deleteSize;
public BlobBucket(final int size) {
m_size = size;
}
public BlobBucket(DataInputStream instr) throws IOException {
m_size = instr.readInt();
+ m_allocationSize = instr.readLong();
m_allocations = instr.readLong();
+ m_deleteSize = instr.readLong();
m_deletes = instr.readLong();
}
public void write(DataOutputStream outstr) throws IOException {
outstr.writeInt(m_size);
+ outstr.writeLong(m_allocationSize);
outstr.writeLong(m_allocations);
+ outstr.writeLong(m_deleteSize);
outstr.writeLong(m_deletes);
}
- public void delete() {
+ public void delete(int sze) {
+ m_deleteSize += sze;
m_deletes++;
}
- public void allocate() {
+ public void allocate(int sze) {
+ m_allocationSize += sze;
m_allocations++;
}
+ public long active() {
+ return m_allocations - m_deletes;
+ }
+ public int meanAllocation() {
+ if (m_allocations == 0)
+ return 0;
+ return (int) (m_allocationSize / m_allocations);
+ }
+ public float churn() {
+ if (active() == 0)
+ return m_allocations;
+
+ BigDecimal allocs = new BigDecimal(m_allocations);
+ BigDecimal used = new BigDecimal(active());
+
+ return allocs.divide(used, 2, RoundingMode.HALF_UP).floatValue();
+ }
}
public class Bucket {
+ final int m_start;
final int m_size;
int m_allocators;
long m_totalSlots;
@@ -101,11 +130,13 @@
long m_sizeAllocations;
long m_sizeDeletes;
- public Bucket(final int size) {
+ public Bucket(final int size, final int startRange) {
m_size = size;
+ m_start = startRange;
}
public Bucket(DataInputStream instr) throws IOException {
m_size = instr.readInt();
+ m_start = instr.readInt();
m_allocators = instr.readInt();
m_slotAllocations = instr.readLong();
m_slotDeletes = instr.readLong();
@@ -115,6 +146,7 @@
}
public void write(DataOutputStream outstr) throws IOException {
outstr.writeInt(m_size);
+ outstr.writeInt(m_start);
outstr.writeInt(m_allocators);
outstr.writeLong(m_slotAllocations);
outstr.writeLong(m_slotDeletes);
@@ -126,6 +158,12 @@
if (sze < 0)
throw new IllegalArgumentException("delete requires positive size, got: " + sze);
+ if (m_size > 64 && sze < 64) {
+ // if called from deferFree then may not include size. If so then use
+ // average size of slots to date as best running estimate.
+ sze = meanAllocation();
+ }
+
if (sze > m_size) {
// sze = ((sze - 1 + m_maxFixed)/ m_maxFixed) * 4; // Blob header
@@ -151,6 +189,10 @@
return m_slotAllocations - m_slotDeletes;
}
+ public long emptySlots() {
+ return m_totalSlots - usedSlots();
+ }
+
public long usedStore() {
return m_sizeAllocations - m_sizeDeletes;
}
@@ -160,23 +202,21 @@
if (usedStore() == 0)
return 0.0f;
- BigDecimal size = new BigDecimal(m_size * usedSlots());
- BigDecimal store = new BigDecimal(100 * usedStore());
- store = store.divide(size, 2, RoundingMode.HALF_UP);
- BigDecimal total = new BigDecimal(100);
+ BigDecimal size = new BigDecimal(reservedStore());
+ BigDecimal store = new BigDecimal(100 * (reservedStore() - usedStore()));
- return total.subtract(store).floatValue();
+ return store.divide(size, 2, RoundingMode.HALF_UP).floatValue();
}
- public float totalWaste() {
+ public float totalWaste(long total) {
if (usedStore() == 0)
return 0.0f;
- BigDecimal size = new BigDecimal(m_size * m_totalSlots);
- BigDecimal store = new BigDecimal(100 * usedStore());
- store = store.divide(size, 2, RoundingMode.HALF_UP);
- BigDecimal total = new BigDecimal(100);
+ long slotWaste = reservedStore() - usedStore();
- return total.subtract(store).floatValue();
+ BigDecimal localWaste = new BigDecimal(100 * slotWaste);
+ BigDecimal totalWaste = new BigDecimal(total);
+
+ return localWaste.divide(totalWaste, 2, RoundingMode.HALF_UP).floatValue();
}
public long reservedStore() {
return m_size * m_totalSlots;
@@ -184,6 +224,40 @@
public void addAlocator() {
m_allocators++;
}
+ public float slotChurn() {
+ // Handle case where we may have deleted all allocations
+ if (usedSlots() == 0)
+ return m_slotAllocations;
+
+ BigDecimal allocs = new BigDecimal(m_slotAllocations);
+ BigDecimal used = new BigDecimal(usedSlots());
+
+ return allocs.divide(used, 2, RoundingMode.HALF_UP).floatValue();
+ }
+ public float slotsUnused() {
+ BigDecimal used = new BigDecimal(100 * (m_totalSlots-usedSlots()));
+ BigDecimal total = new BigDecimal(m_totalSlots);
+
+ return used.divide(total, 2, RoundingMode.HALF_UP).floatValue();
+ }
+ public float percentAllocations(long totalAllocations) {
+ BigDecimal used = new BigDecimal(100 * m_slotAllocations);
+ BigDecimal total = new BigDecimal(totalAllocations);
+
+ return used.divide(total, 2, RoundingMode.HALF_UP).floatValue();
+ }
+ public float percentSlotsInuse(long totalInuse) {
+ BigDecimal used = new BigDecimal(100 * usedSlots());
+ BigDecimal total = new BigDecimal(totalInuse);
+
+ return used.divide(total, 2, RoundingMode.HALF_UP).floatValue();
+ }
+ public int meanAllocation() {
+ if (m_slotAllocations == 0)
+ return 0;
+
+ return (int) (m_sizeAllocations / m_slotAllocations);
+ }
}
final ArrayList<Bucket> m_buckets;
@@ -199,8 +273,10 @@
*/
public StorageStats(final int[] buckets) {
m_buckets = new ArrayList<Bucket>();
+ int prevLimit = 0;
for (int i = 0; i < buckets.length; i++) {
- m_buckets.add(new Bucket(buckets[i]*64)); // slot sizes are 64 multiples
+ m_buckets.add(new Bucket(buckets[i]*64, prevLimit)); // slot sizes are 64 multiples
+ prevLimit = buckets[i]*64;
}
// last fixed allocator needed to compute BlobBuckets
m_maxFixed = m_buckets.get(buckets.length-1).m_size;
@@ -223,6 +299,10 @@
* @throws IOException
*/
public StorageStats(final DataInputStream instr) throws IOException {
+ int version = instr.readInt();
+ if (cVersion != version) {
+ throw new IllegalStateException("StorageStats object is wrong version");
+ }
m_buckets = new ArrayList<Bucket>();
int nbuckets = instr.readInt();
for (int i = 0; i < nbuckets; i++) {
@@ -242,6 +322,8 @@
ByteArrayOutputStream outb = new ByteArrayOutputStream();
DataOutputStream outd = new DataOutputStream(outb);
+ outd.writeInt(cVersion);
+
outd.writeInt(m_buckets.size());
for (Bucket b : m_buckets) {
@@ -266,14 +348,14 @@
m_blobAllocation += sze;
// increment blob bucket
- findBlobBucket(sze).allocate();
+ findBlobBucket(sze).allocate(sze);
}
public void deleteBlob(int sze) {
m_blobDeletion -= sze;
// decrement blob bucket
- findBlobBucket(sze).delete();
+ findBlobBucket(sze).delete(sze);
}
private BlobBucket findBlobBucket(final int sze) {
@@ -307,57 +389,88 @@
str.append("\n-------------------------\n");
str.append("RWStore Allocator Summary\n");
str.append("-------------------------\n");
- str.append(padRight("AllocatorSize", 16));
- str.append(padLeft("AllocatorCount", 16));
- str.append(padLeft("SlotsAllocated", 16));
- str.append(padLeft("SlotsRecycled", 16));
- str.append(padLeft("SlotsInUse", 16));
- str.append(padLeft("SlotsReserved", 16));
- str.append(padLeft("BytesReserved", 16));
- str.append(padLeft("BytesAppData", 16));
- str.append(padLeft("%SlotWaste", 16));
- str.append(padLeft("%StoreWaste", 16));
- str.append(padLeft("%AppData", 16));
- str.append(padLeft("%StoreFile", 16));
- str.append("\n");
+ str.append(String.format("%-16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s %16s \n",
+ "AllocatorSize",
+ "AllocatorCount",
+ "SlotsAllocated",
+ "%SlotsAllocated",
+ "SlotsRecycled",
+ "SlotChurn",
+ "SlotsInUse",
+ "%SlotsInUse",
+ "MeanAllocation",
+ "SlotsReserved",
+ "%SlotsUnused",
+ "BytesReserved",
+ "BytesAppData",
+ "%SlotWaste",
+ "%AppData",
+ "%StoreFile",
+ "%TotalWaste",
+ "%FileWaste"
+ ));
long totalAppData = 0;
long totalFileStore = 0;
+ long totalAllocations = 0;
+ long totalInuse = 0;
for (Bucket b: m_buckets) {
totalAppData += b.usedStore();
totalFileStore += b.reservedStore();
+ totalAllocations += b.m_slotAllocations;
+ totalInuse += b.usedSlots();
}
+ long totalWaste = totalFileStore - totalAppData;
+
for (Bucket b: m_buckets) {
- str.append(padRight("" + b.m_size, 16));
- str.append(padLeft("" + b.m_allocators, 16));
- str.append(padLeft("" + b.m_slotAllocations, 16));
- str.append(padLeft("" + b.m_slotDeletes, 16));
- str.append(padLeft("" + b.usedSlots(), 16));
- str.append(padLeft("" + b.m_totalSlots, 16));
- str.append(padLeft("" + b.reservedStore(), 16));
- str.append(padLeft("" + b.usedStore(), 16));
- str.append(padLeft("" + b.slotWaste() + "%", 16));
- str.append(padLeft("" + b.totalWaste() + "%", 16));
- str.append(padLeft("" + dataPercent(b.usedStore(), totalAppData) + "%", 16));
- str.append(padLeft("" + dataPercent(b.reservedStore(), totalFileStore) + "%", 16));
- str.append("\n");
+ str.append(String.format("%-16d %16d %16d %16.2f %16d %16.2f %16d %16.2f %16d %16d %16.2f %16d %16d %16.2f %16.2f %16.2f %16.2f %16.2f \n",
+ b.m_size,
+ b.m_allocators,
+ b.m_slotAllocations,
+ b.percentAllocations(totalAllocations),
+ b.m_slotDeletes,
+ b.slotChurn(),
+ b.usedSlots(),
+ b.percentSlotsInuse(totalInuse),
+ b.meanAllocation(),
+ b.m_totalSlots,
+ b.slotsUnused(),
+ b.reservedStore(),
+ b.usedStore(),
+ b.slotWaste(),
+ dataPercent(b.usedStore(), totalAppData),
+ dataPercent(b.reservedStore(), totalFileStore),
+ b.totalWaste(totalWaste),
+ b.totalWaste(totalFileStore)
+ ));
}
str.append("\n-------------------------\n");
str.append("BLOBS\n");
str.append("-------------------------\n");
- str.append(padRight("Bucket", 10));
- str.append(padLeft("Allocations", 12));
- str.append(padLeft("Deletes", 12));
- str.append(padLeft("Current", 12));
- str.append("\n");
+ str.append(String.format("%-10s %12s %12s %12s %12s %12s %12s %12s %12s\n",
+ "Bucket(K)",
+ "Allocations",
+ "Allocated",
+ "Deletes",
+ "Deleted",
+ "Current",
+ "Data",
+ "Mean",
+ "Churn"));
for (BlobBucket b: m_blobBuckets) {
- str.append(padRight("" + (b.m_size/1024) + "K", 10));
- str.append(padLeft("" + b.m_allocations, 12));
- str.append(padLeft("" + b.m_deletes, 12));
- str.append(padLeft("" + (b.m_allocations - b.m_deletes), 12));
- str.append("\n");
+ str.append(String.format("%-10d %12d %12d %12d %12d %12d %12d %12d %12.2f\n",
+ b.m_size/1024,
+ b.m_allocations,
+ b.m_allocationSize,
+ b.m_deletes,
+ b.m_deleteSize,
+ (b.m_allocations - b.m_deletes),
+ (b.m_allocationSize - b.m_deleteSize),
+ b.meanAllocation(),
+ b.churn()
+ ));
}
}
@@ -368,32 +481,4 @@
return used.divide(total, 2, RoundingMode.HALF_UP).floatValue();
}
-
- public static String padLeft(String str, int minlen) {
- if (str.length() >= minlen)
- return str;
-
- StringBuffer out = new StringBuffer();
- int pad = minlen - str.length();
- while (pad-- > 0) {
- out.append(' ');
- }
- out.append(str);
-
- return out.toString();
- }
-
- public static String padRight(String str, int minlen) {
- if (str.length() >= minlen)
- return str;
-
- StringBuffer out = new StringBuffer();
- out.append(str);
- int pad = minlen - str.length();
- while (pad-- > 0) {
- out.append(' ');
- }
-
- return out.toString();
- }
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
