Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[3942] branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/ bigdata/rwstore
|
From: <mar...@us...> - 2010-11-12 17:17:40
|
Revision: 3942
http://bigdata.svn.sourceforge.net/bigdata/?rev=3942&view=rev
Author: martyncutcher
Date: 2010-11-12 17:17:33 +0000 (Fri, 12 Nov 2010)
Log Message:
-----------
Remove BlobAllocator and handle deferFree Blobs with addressing sign convention
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/IStore.java
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Removed Paths:
-------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java
Deleted: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java 2010-11-12 16:33:28 UTC (rev 3941)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/BlobAllocator.java 2010-11-12 17:17:33 UTC (rev 3942)
@@ -1,343 +0,0 @@
-package com.bigdata.rwstore;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.log4j.Logger;
-
-import com.bigdata.rwstore.RWStore.AllocationStats;
-import com.bigdata.util.ChecksumUtility;
-
-/**
- * BlobAllocator.
- *
- * Manages Blob allocations using a list of {@link FixedAllocator}s.
- *
- * The main advantage of this is for re-allocation, since the
- * {@link FixedAllocator}s can be efficiently re-cycled where a fixed Blob
- * creates issues of best fit and fragmentation.
- *
- * Some simple patterns would cause un-reallocatable storage, consider a Blob
- * that always re-allocated to a larger size, or a pattern where several blobs
- * got larger together, in these scenarios, smaller allocations would never be
- * re-used, whilst the mechanism of component based allocation is easily
- * re-used.
- *
- * @author mgc
- */
-public class BlobAllocator implements Allocator {
-
- private static final transient Logger log = Logger.getLogger(BlobAllocator.class);
-
- final private int[] m_hdrs = new int[254];
- final private RWStore m_store;
- private int m_diskAddr;
- private int m_index;
- private int m_sortAddr;
- private ArrayList m_freeList;
- private long m_startAddr;
- /**
- * There are 256 ints in a BlobAllocator, the first is used to provide the
- * sortAddr, and the last for the checksum, leaving 254 BlobHdr addresses
- */
- private int m_freeSpots = 254;
-
- public BlobAllocator(final RWStore store, final int sortAddr) {
- m_store = store;
- m_sortAddr = sortAddr;
-
- if (log.isInfoEnabled())
- log.info("New BlobAllocator");
- }
-
- public void addAddresses(final ArrayList addrs) {
- // not relevant for BlobAllocators
- }
-
- public boolean addressInRange(final int addr) {
- // not relevant for BlobAllocators
- return false;
- }
-
- /**
- * Should not be called directly since the PSOutputStream
- * manages the blob allocations.
- */
- public int alloc(final RWStore store, final int size, final IAllocationContext context) {
- throw new UnsupportedOperationException("Blob allocators do not allocate addresses directly");
- }
-
- public boolean free(final int addr, final int sze) {
- if (sze < (m_store.m_maxFixedAlloc-4))
- throw new IllegalArgumentException("Unexpected address size");
- final int alloc = m_store.m_maxFixedAlloc-4;
- final int blcks = (alloc - 1 + sze)/alloc;
-
- int hdr_idx = (-addr) & RWStore.OFFSET_BITS_MASK;
- if (hdr_idx > m_hdrs.length)
- throw new IllegalArgumentException("free BlobAllocation problem, hdr offset: " + hdr_idx + ", avail:" + m_hdrs.length);
-
- final int hdr_addr = m_hdrs[hdr_idx];
-
- if (hdr_addr == 0) {
- return false;
- }
-
- // read in header block, then free each reference
- final byte[] hdr = new byte[(blcks+1) * 4 + 4]; // add space for checksum
- m_store.getData(hdr_addr, hdr);
-
- final DataInputStream instr = new DataInputStream(
- new ByteArrayInputStream(hdr, 0, hdr.length-4) );
- try {
- final int allocs = instr.readInt();
- for (int i = 0; i < allocs; i++) {
- final int nxt = instr.readInt();
- m_store.free(nxt, m_store.m_maxFixedAlloc);
- }
- m_store.free(hdr_addr, hdr.length);
- m_hdrs[hdr_idx] = 0;
- if (m_freeSpots++ == 0) {
- m_freeList.add(this);
- }
-
- return true;
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- }
-
- public int getFirstFixedForBlob(final int addr, final int sze) {
- if (sze < (m_store.m_maxFixedAlloc-4))
- throw new IllegalArgumentException("Unexpected address size: " + sze);
-
- final int alloc = m_store.m_maxFixedAlloc-4;
- final int blcks = (alloc - 1 + sze)/alloc;
-
- final int hdr_idx = (-addr) & RWStore.OFFSET_BITS_MASK;
- if (hdr_idx > m_hdrs.length)
- throw new IllegalArgumentException("free BlobAllocation problem, hdr offset: " + hdr_idx + ", avail:" + m_hdrs.length);
-
- final int hdr_addr = m_hdrs[hdr_idx];
-
- if (hdr_addr == 0) {
- throw new IllegalArgumentException("getFirstFixedForBlob called with unallocated address");
- }
-
- // read in header block, then free each reference
- final byte[] hdr = new byte[(blcks+1) * 4 + 4]; // add space for checksum
- m_store.getData(hdr_addr, hdr);
-
- final DataInputStream instr = new DataInputStream(
- new ByteArrayInputStream(hdr, 0, hdr.length-4) );
- try {
- final int nallocs = instr.readInt();
- final int faddr = instr.readInt();
-
- return faddr;
-
- } catch (IOException ioe) {
- throw new RuntimeException("Unable to retrieve first fixed address", ioe);
- }
- }
-
- public int getBlockSize() {
- // Not relevant for Blobs
- return 0;
- }
-
- public int getDiskAddr() {
- return m_diskAddr;
- }
-
- /**
- * returns physical address of blob header if any.
- */
- public long getPhysicalAddress(final int offset) {
- return m_store.physicalAddress(m_hdrs[offset]);
- }
-
- /**
- * Since the Blob Allocator simply manages access to FixedAllocation blocks it does not manage any
- * allocations directly.
- */
- public int getPhysicalSize(final int offset) {
- return 0;
- }
-
- /**
- * The startAddr
- */
- public long getStartAddr() {
- // not relevant for blob
- return RWStore.convertAddr(m_sortAddr);
- }
-
- public String getStats(final AtomicLong counter) {
- return "";
- }
-
- /**
- * hasFree if there are any non-zero entries in the m_hdr array;
- */
- public boolean hasFree() {
- return m_freeSpots > 0;
- }
-
- public void preserveSessionData() {
- // all data held by fixed allocators
- }
-
- /**
- * FIXME: There is a symmetry problem with read/write where one takes a Stream and the other
- * return a byte[]. This is problematical with using the checksums.
- */
- public void read(final DataInputStream str) {
- m_freeSpots = 0;
- try {
- for (int i = 0; i < 254; i++) {
- m_hdrs[i] = str.readInt();
- if (m_hdrs[i] == 0) m_freeSpots++;
- }
- final int chk = str.readInt();
- // checksum int chk = ChecksumUtility.getCHK().checksum(buf, str.size());
-
- } catch (IOException e) {
- log.error(e,e);
- throw new IllegalStateException(e);
- }
- }
-
- public void setDiskAddr(final int addr) {
- m_diskAddr = addr;
- }
-
- public void setFreeList(final ArrayList list) {
- m_freeList = list;
-
- if (hasFree()) {
- m_freeList.add(this);
- }
- }
-
- /**
- * setIndex is called in two places, firstly to set the original index and secondly on restore
- * from storage to re-establish the order.
- *
- * When called initially, the m_startAddr will be zero and so must be set by retrieving the
- * m_startAddr of the previous block (if any). Now, since a Blob must use fixed allocations we
- * are guaranteed that a BlobAllocator will not be the first allocator. To derive a startAddr that
- * can safely be used to sort a BlobAllocator against the previous (and subsequent) allocators we
- * access the previous allocators address.
- */
- public void setIndex(final int index) {
- m_index = index;
- }
-
- // @todo why is this a NOP? Javadoc.
- public boolean verify(final int addr) {
- // TODO Auto-generated method stub
- return false;
- }
-
- public byte[] write() {
- try {
- final byte[] buf = new byte[1024]; // @todo why this const?
- final DataOutputStream str = new DataOutputStream(new FixedOutputStream(buf));
-
- str.writeInt(m_sortAddr);
-
- for (int i = 0; i < 254; i++) { // @todo why this const?
- str.writeInt(m_hdrs[i]);
- }
-
- // add checksum
- final int chk = ChecksumUtility.getCHK().checksum(buf, str.size());
- str.writeInt(chk);
-
- return buf;
- } catch (IOException ioe) {
- throw new IllegalStateException(ioe);
- }
- }
-
- public int compareTo(final Object o) {
- final Allocator alloc = (Allocator) o;
-
- assert getStartAddr() != alloc.getStartAddr();
-
- return (getStartAddr() < alloc.getStartAddr()) ? -1 : 1;
- }
-
- public int register(final int addr) {
- assert m_freeSpots > 0;
-
- m_store.addToCommit(this);
-
- for (int i = 0; i < 254; i++) {
- if (m_hdrs[i] == 0) {
- m_hdrs[i] = addr;
-
- if (--m_freeSpots == 0) {
- m_freeList.remove(this);
- }
-
- final int ret = -((m_index << RWStore.OFFSET_BITS) + i);
- if (((-ret) & RWStore.OFFSET_BITS_MASK) > m_hdrs.length)
- throw new IllegalStateException("Invalid blob offset: " + ((-ret) & RWStore.OFFSET_BITS_MASK));
-
- return ret;
- }
- }
-
- throw new IllegalStateException("BlobAllocator unable to find free slot");
- }
-
- public int getRawStartAddr() {
- return m_sortAddr;
- }
-
- public int getIndex() {
- return m_index;
- }
-
- public int getBlobHdrAddress(final int hdrIndex) {
- return m_hdrs[hdrIndex];
- }
-
- public void appendShortStats(final StringBuilder str, final AllocationStats[] stats) {
- if (stats == null) {
- str.append("Index: " + m_index + ", address: " + getStartAddr() + ", BLOB\n");
- } else {
- stats[stats.length-1].m_filledSlots += 254 - m_freeSpots;
- stats[stats.length-1].m_reservedSlots += 254;
- }
- }
-
- public boolean isAllocated(final int offset) {
- return m_hdrs[offset] != 0;
- }
-
- /**
- * This is okay as a NOP. The true allocation is managed by the
- * FixedAllocators.
- */
- public void detachContext(final IAllocationContext context) {
- // NOP
- }
-
- /**
- * Since the real allocation is in the FixedAllocators, this should delegate
- * to the first address, in which case
- */
- public boolean canImmediatelyFree(final int addr, final int size, final IAllocationContext context) {
- final int faddr = this.getFirstFixedForBlob(addr, size);
-
- return m_store.getBlockByAddress(faddr).canImmediatelyFree(faddr, 0, context);
- }
-
-}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/IStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/IStore.java 2010-11-12 16:33:28 UTC (rev 3941)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/IStore.java 2010-11-12 17:17:33 UTC (rev 3942)
@@ -117,18 +117,18 @@
*/
public File getStoreFile();
- /**
- * Called by the PSOutputStream to register the header block of a blob. The
- * store must return a new address that is used to retrieve the blob header.
- * This double indirection is required to be able to manage the blobs, since
- * the blob header itself is of variable size and is handled by the standard
- * FixedAllocators in the RWStore.
- *
- * @param addr
- * The address of the header block of the blob.
- *
- * @return The
- */
- public int registerBlob(int addr);
+// /**
+// * Called by the PSOutputStream to register the header block of a blob. The
+// * store must return a new address that is used to retrieve the blob header.
+// * This double indirection is required to be able to manage the blobs, since
+// * the blob header itself is of variable size and is handled by the standard
+// * FixedAllocators in the RWStore.
+// *
+// * @param addr
+// * The address of the header block of the blob.
+// *
+// * @return The
+// */
+// public int registerBlob(int addr);
}
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-12 16:33:28 UTC (rev 3941)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-12 17:17:33 UTC (rev 3942)
@@ -320,7 +320,7 @@
private ArrayList<FixedAllocator> m_freeFixed[];
/** lists of free blob allocators. */
- private final ArrayList<BlobAllocator> m_freeBlobs;
+ // private final ArrayList<BlobAllocator> m_freeBlobs;
/** lists of blocks requiring commitment. */
private final ArrayList<Allocator> m_commitList;
@@ -520,7 +520,7 @@
m_allocs = new ArrayList<Allocator>();
- m_freeBlobs = new ArrayList<BlobAllocator>();
+ // m_freeBlobs = new ArrayList<BlobAllocator>();
try {
final RandomAccessFile m_raf = fileMetadata.getRandomAccessFile();
@@ -915,20 +915,17 @@
final int allocSize = strBuf.readInt(); // if Blob < 0
final Allocator allocator;
final ArrayList<? extends Allocator> freeList;
- if (allocSize > 0) {
- int index = 0;
- int fixedSize = m_minFixedAlloc;
- while (fixedSize < allocSize)
- fixedSize = 64 * m_allocSizes[++index];
+ assert allocSize > 0;
- allocator = new FixedAllocator(this, allocSize);//, m_writeCache);
+ int index = 0;
+ int fixedSize = m_minFixedAlloc;
+ while (fixedSize < allocSize)
+ fixedSize = 64 * m_allocSizes[++index];
- freeList = m_freeFixed[index];
- } else {
- allocator = new BlobAllocator(this, allocSize);
- freeList = m_freeBlobs;
- }
+ allocator = new FixedAllocator(this, allocSize);//, m_writeCache);
+ freeList = m_freeFixed[index];
+
allocator.read(strBuf);
allocator.setDiskAddr(i); // store bit, not physical
// address!
@@ -1757,7 +1754,7 @@
m_commitList.clear();
m_allocs.clear();
- m_freeBlobs.clear();
+ // m_freeBlobs.clear();
final int numFixed = m_allocSizes.length;
for (int i = 0; i < numFixed; i++) {
@@ -2904,34 +2901,34 @@
* blob data, the blob allocator retrieves the blob header and reads the
* data from that into the passed byte array.
*/
- public int registerBlob(final int addr) {
- m_allocationLock.lock();
- try {
- BlobAllocator ba = null;
- if (m_freeBlobs.size() > 0) {
- ba = (BlobAllocator) m_freeBlobs.get(0);
- }
- if (ba == null) {
- final Allocator lalloc = (Allocator) m_allocs.get(m_allocs.size() - 1);
- // previous block start address
- final int psa = lalloc.getRawStartAddr();
- assert (psa - 1) > m_nextAllocation;
- ba = new BlobAllocator(this, psa - 1);
- ba.setFreeList(m_freeBlobs); // will add itself to the free list
- ba.setIndex(m_allocs.size());
- m_allocs.add(ba);
- }
+// public int registerBlob(final int addr) {
+// m_allocationLock.lock();
+// try {
+// BlobAllocator ba = null;
+// if (m_freeBlobs.size() > 0) {
+// ba = (BlobAllocator) m_freeBlobs.get(0);
+// }
+// if (ba == null) {
+// final Allocator lalloc = (Allocator) m_allocs.get(m_allocs.size() - 1);
+// // previous block start address
+// final int psa = lalloc.getRawStartAddr();
+// assert (psa - 1) > m_nextAllocation;
+// ba = new BlobAllocator(this, psa - 1);
+// ba.setFreeList(m_freeBlobs); // will add itself to the free list
+// ba.setIndex(m_allocs.size());
+// m_allocs.add(ba);
+// }
+//
+// if (!m_commitList.contains(ba)) {
+// m_commitList.add(ba);
+// }
+//
+// return ba.register(addr);
+// } finally {
+// m_allocationLock.unlock();
+// }
+// }
- if (!m_commitList.contains(ba)) {
- m_commitList.add(ba);
- }
-
- return ba.register(addr);
- } finally {
- m_allocationLock.unlock();
- }
- }
-
public void addToCommit(final Allocator allocator) {
if (!m_commitList.contains(allocator)) {
m_commitList.add(allocator);
@@ -3108,11 +3105,11 @@
public void deferFree(final int rwaddr, final int sze) {
m_allocationLock.lock();
try {
- m_deferredFreeOut.writeInt(rwaddr);
-
- final Allocator alloc = getBlockByAddress(rwaddr);
- if (alloc instanceof BlobAllocator) {
+ if (sze > this.m_maxFixedAlloc) {
+ m_deferredFreeOut.writeInt(-rwaddr);
m_deferredFreeOut.writeInt(sze);
+ } else {
+ m_deferredFreeOut.writeInt(rwaddr);
}
} catch (IOException e) {
throw new RuntimeException("Could not free: rwaddr=" + rwaddr
@@ -3207,12 +3204,11 @@
while (nxtAddr != 0) { // while (false && addrs-- > 0) {
- final Allocator alloc = getBlock(nxtAddr);
- if (alloc instanceof BlobAllocator) {
+ if (nxtAddr > 0) { // Blob
final int bloblen = strBuf.readInt();
assert bloblen > 0; // a Blob address MUST have a size
- immediateFree(nxtAddr, bloblen);
+ immediateFree(-nxtAddr, bloblen);
} else {
immediateFree(nxtAddr, 0); // size ignored for FreeAllocators
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
