Menu
▾
▴
bigdata-commit
|
From: <mar...@us...> - 2010-09-01 18:27:15
|
Revision: 3488
http://bigdata.svn.sourceforge.net/bigdata/?rev=3488&view=rev
Author: martyncutcher
Date: 2010-09-01 18:27:08 +0000 (Wed, 01 Sep 2010)
Log Message:
-----------
Use aged release time for checkfreeable
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-01 17:46:11 UTC (rev 3487)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-01 18:27:08 UTC (rev 3488)
@@ -2615,10 +2615,11 @@
Long freeTime = m_transactionService.tryCallWithLock(new Callable<Long>() {
public Long call() throws Exception {
+ long now = System.currentTimeMillis();
if (m_transactionService.getActiveCount() == 0) {
- return System.currentTimeMillis();
+ return now;
} else {
- return m_transactionService.getEarliestTxStartTime();
+ return now - m_transactionService.getMinReleaseAge(); // getEarliestTxStartTime();
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-09-01 18:44:05
|
Revision: 3490
http://bigdata.svn.sourceforge.net/bigdata/?rev=3490&view=rev
Author: martyncutcher
Date: 2010-09-01 18:43:58 +0000 (Wed, 01 Sep 2010)
Log Message:
-----------
refine deferred delete release time
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-01 18:27:35 UTC (rev 3489)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-01 18:43:58 UTC (rev 3490)
@@ -2616,10 +2616,13 @@
public Long call() throws Exception {
long now = System.currentTimeMillis();
+ long earliest = m_transactionService.getEarliestTxStartTime();
+ long aged = now - m_transactionService.getMinReleaseAge();
+
if (m_transactionService.getActiveCount() == 0) {
- return now;
- } else {
- return now - m_transactionService.getMinReleaseAge(); // getEarliestTxStartTime();
+ return aged;
+ } else {
+ return aged < earliest ? aged : earliest;
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-09-03 11:16:55
|
Revision: 3502
http://bigdata.svn.sourceforge.net/bigdata/?rev=3502&view=rev
Author: martyncutcher
Date: 2010-09-03 11:16:49 +0000 (Fri, 03 Sep 2010)
Log Message:
-----------
freeCurrentDeferrals should not update lastReleaseTime
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-03 11:14:17 UTC (rev 3501)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-03 11:16:49 UTC (rev 3502)
@@ -60,6 +60,7 @@
import com.bigdata.journal.Options;
import com.bigdata.journal.RWStrategy.FileMetadataView;
import com.bigdata.quorum.Quorum;
+import com.bigdata.util.ChecksumError;
import com.bigdata.util.ChecksumUtility;
/**
@@ -1119,7 +1120,9 @@
}
+ private long immediateFreeCount = 0;
private void immediateFree(final int addr, final int sze) {
+
switch (addr) {
case 0:
case -1:
@@ -2629,7 +2632,7 @@
}, 5L, TimeUnit.MILLISECONDS);
- if (false) freeAllDeferrals(freeTime);
+ freeCurrentDeferrals(freeTime);
} catch (RuntimeException e) {
// fine, will try again later
} catch (Exception e) {
@@ -2644,12 +2647,12 @@
*
* @param txnRelease - the max release time
*/
- protected void freeAllDeferrals(long txnRelease) {
- System.out.println("freeAllDeferrals");
+ protected void freeCurrentDeferrals(long txnRelease) {
m_deferFreeLock.lock();
try {
- if (m_lastDeferredReleaseTime <= txnRelease) {
+ if (m_rb.getLastCommitTime() <= txnRelease) {
+// System.out.println("freeCurrentDeferrals");
final Iterator<Integer> curdefers = m_currentTxnFreeList.iterator();
while (curdefers.hasNext()) {
final int rwaddr = curdefers.next();
@@ -2664,8 +2667,6 @@
}
}
m_currentTxnFreeList.clear();
-
- m_lastDeferredReleaseTime = txnRelease;
}
} finally {
m_deferFreeLock.unlock();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-09-10 17:12:40
|
Revision: 3529
http://bigdata.svn.sourceforge.net/bigdata/?rev=3529&view=rev
Author: martyncutcher
Date: 2010-09-10 17:12:33 +0000 (Fri, 10 Sep 2010)
Log Message:
-----------
Refined RWStore allocation stats display
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-10 16:02:26 UTC (rev 3528)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-09-10 17:12:33 UTC (rev 3529)
@@ -194,9 +194,9 @@
* com.bigdata.rwstore.RWStore.allocSizes=1,2,3,5...
*
*/
- // static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 12, 16, 32, 48, 64, 128, 192, 320, 512, 832, 1344, 2176, 3520 };
- private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 };
- // static final int[] ALLOC_SIZES = { 1, 2, 4, 8, 16, 32, 64, 128 };
+ private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 12, 16, 32, 48, 64, 128, 192, 320, 512, 832, 1344, 2176, 3520 };
+ // private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 };
+ // private static final int[] ALLOC_SIZES = { 1, 2, 4, 8, 16, 32, 64, 128 };
final int m_maxFixedAlloc;
final int m_minFixedAlloc;
@@ -2108,18 +2108,23 @@
str.append("RWStore Allocation Summary\n");
str.append("-------------------------\n");
long treserved = 0;
+ long treservedSlots = 0;
long tfilled = 0;
+ long tfilledSlots = 0;
for (int i = 0; i < stats.length; i++) {
- str.append("Allocation: " + stats[i].m_blockSize);
long reserved = stats[i].m_reservedSlots * stats[i].m_blockSize;
treserved += reserved;
- str.append(", reserved: " + reserved);
+ treservedSlots += stats[i].m_reservedSlots;
long filled = stats[i].m_filledSlots * stats[i].m_blockSize;
tfilled += filled;
- str.append(", filled: " + filled);
+ tfilledSlots += stats[i].m_filledSlots;
+
+ str.append("Allocation: " + stats[i].m_blockSize);
+ str.append(", slots: " + stats[i].m_filledSlots + "/" + stats[i].m_reservedSlots);
+ str.append(", storage: " + filled + "/" + reserved);
str.append("\n");
}
- str.append("Total - file: " + convertAddr(m_fileSize) + ", reserved: " + treserved + ", filled: " + tfilled + "\n");
+ str.append("Total - file: " + convertAddr(m_fileSize) + ", slots: " + tfilledSlots + "/" + treservedSlots + ", storage: " + tfilled + "/" + treserved + "\n");
}
public ArrayList getStorageBlockAddresses() {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-10-28 18:39:56
|
Revision: 3849
http://bigdata.svn.sourceforge.net/bigdata/?rev=3849&view=rev
Author: thompsonbry
Date: 2010-10-28 18:39:50 +0000 (Thu, 28 Oct 2010)
Log Message:
-----------
Fixed Divided by Zero error in showAllocators
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-10-27 16:25:14 UTC (rev 3848)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-10-28 18:39:50 UTC (rev 3849)
@@ -2139,7 +2139,7 @@
str.append("Allocation: " + stats[i].m_blockSize);
str.append(", slots: " + stats[i].m_filledSlots + "/" + stats[i].m_reservedSlots);
str.append(", storage: " + filled + "/" + reserved);
- str.append(", usage: " + (filled * 100 / reserved) + "%");
+ str.append(", usage: " + (reserved==0?0:(filled * 100 / reserved)) + "%");
str.append("\n");
}
str.append("Total - file: " + convertAddr(m_fileSize) + ", slots: " + tfilledSlots + "/" + treservedSlots + ", storage: " + tfilled + "/" + treserved + "\n");
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-03 13:23:16
|
Revision: 3875
http://bigdata.svn.sourceforge.net/bigdata/?rev=3875&view=rev
Author: thompsonbry
Date: 2010-11-03 13:14:44 +0000 (Wed, 03 Nov 2010)
Log Message:
-----------
Fixed a few findbugs complaints.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 11:52:07 UTC (rev 3874)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 13:14:44 UTC (rev 3875)
@@ -298,15 +298,6 @@
* significant contention may be avoided.
*/
final private ReentrantLock m_allocationLock = new ReentrantLock();
-
- /**
- * This lock controls access to the deferredFree structures used
- * in deferFree.
- *
- * The deferral of freeing storage supports processing of read-only
- * transactions concurrent with modifying/mutation tasks
- */
- final private ReentrantLock m_deferFreeLock = new ReentrantLock();
/**
* The deferredFreeList is simply an array of releaseTime,freeListAddrs
@@ -516,21 +507,18 @@
}
}
- public void close() {
- try {
- if (m_bufferedWrite != null) {
- m_bufferedWrite.release();
- m_bufferedWrite = null;
- }
- m_writeCache.close();
- m_raf.close();
- } catch (IOException e) {
- // ..oooh err... only trying to help
- } catch (InterruptedException e) {
- // thrown from writeCache?
- e.printStackTrace();
- }
- }
+ synchronized public void close() {
+ try {
+ if (m_bufferedWrite != null) {
+ m_bufferedWrite.release();
+ m_bufferedWrite = null;
+ }
+ m_writeCache.close();
+ m_raf.close();
+ } catch (Throwable t) {
+ throw new RuntimeException(t);
+ }
+ }
/**
* Basic check on key root block validity
@@ -1049,8 +1037,8 @@
m_diskReads++;
}
- } catch (Exception e) {
- e.printStackTrace();
+ } catch (Throwable e) {
+ log.error(e,e);
throw new IllegalArgumentException("Unable to read data", e);
}
@@ -1504,20 +1492,23 @@
final int len = 4 * (2 + 1 + m_allocSizes.length + m_metaBits.length);
final byte buf[] = new byte[len];
- final FixedOutputStream str = new FixedOutputStream(buf);
-
- str.writeLong(m_lastDeferredReleaseTime);
-
- str.writeInt(m_allocSizes.length);
- for (int i = 0; i < m_allocSizes.length; i++) {
- str.writeInt(m_allocSizes[i]);
- }
- for (int i = 0; i < m_metaBits.length; i++) {
- str.writeInt(m_metaBits[i]);
- }
+ final FixedOutputStream str = new FixedOutputStream(buf);
+ try {
+ str.writeLong(m_lastDeferredReleaseTime);
- str.flush();
+ str.writeInt(m_allocSizes.length);
+ for (int i = 0; i < m_allocSizes.length; i++) {
+ str.writeInt(m_allocSizes[i]);
+ }
+ for (int i = 0; i < m_metaBits.length; i++) {
+ str.writeInt(m_metaBits[i]);
+ }
+ str.flush();
+ } finally {
+ str.close();
+ }
+
final long addr = physicalAddress(m_metaBitsAddr);
if (addr == 0) {
throw new IllegalStateException("Invalid metabits address: " + m_metaBitsAddr);
@@ -1648,7 +1639,9 @@
+ ", old addr: " + old + ", new addr: " + naddr);
try {
- m_writeCache.write(metaBit2Addr(naddr), ByteBuffer.wrap(allocator.write()), 0, false); // do not use checksum
+ // do not use checksum
+ m_writeCache.write(metaBit2Addr(naddr), ByteBuffer
+ .wrap(allocator.write()), 0, false);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -1678,9 +1671,12 @@
} catch (IOException e) {
throw new StorageTerminalError("Unable to commit transaction", e);
} finally {
-// m_committing = false;
- m_recentAlloc = false;
- m_allocationLock.unlock();
+ try {
+ // m_committing = false;
+ m_recentAlloc = false;
+ } finally {
+ m_allocationLock.unlock();
+ }
}
checkCoreAllocations();
@@ -2069,9 +2065,12 @@
} catch (Throwable t) {
throw new RuntimeException("Force Reopen", t);
} finally {
- m_extendingFile = false;
- m_readsAtExtend = this.m_diskReads;
- writeLock.unlock();
+ try {
+ m_extendingFile = false;
+ m_readsAtExtend = this.m_diskReads;
+ } finally {
+ writeLock.unlock();
+ }
}
}
@@ -2726,7 +2725,7 @@
* DeferredFrees are written to the deferred PSOutputStream
*/
public void deferFree(final int rwaddr, final int sze) {
- m_deferFreeLock.lock();
+ m_allocationLock.lock();
try {
m_deferredFreeOut.writeInt(rwaddr);
@@ -2738,7 +2737,7 @@
throw new RuntimeException("Could not free: rwaddr=" + rwaddr
+ ", size=" + sze, e);
} finally {
- m_deferFreeLock.unlock();
+ m_allocationLock.unlock();
}
}
@@ -2779,7 +2778,7 @@
* @return the address of the deferred addresses saved on the store
*/
public long saveDeferrals() {
- m_deferFreeLock.lock();
+ m_allocationLock.lock();
try {
if (m_deferredFreeOut.getBytesWritten() == 0) {
return 0;
@@ -2798,7 +2797,7 @@
} catch (IOException e) {
throw new RuntimeException("Cannot write to deferred free", e);
} finally {
- m_deferFreeLock.unlock();
+ m_allocationLock.unlock();
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-03 14:48:38
|
Revision: 3878
http://bigdata.svn.sourceforge.net/bigdata/?rev=3878&view=rev
Author: thompsonbry
Date: 2010-11-03 14:48:32 +0000 (Wed, 03 Nov 2010)
Log Message:
-----------
passing over to martyn.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 14:33:52 UTC (rev 3877)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 14:48:32 UTC (rev 3878)
@@ -198,7 +198,6 @@
* @todo This array should be configurable and must be written into the
* store so changing values does not break older stores.
* com.bigdata.rwstore.RWStore.allocSizes=1,2,3,5...
- *
*/
private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 12, 16, 32, 48, 64, 128, 192, 320, 512, 832, 1344, 2176, 3520 };
// private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 };
@@ -206,6 +205,7 @@
final int m_maxFixedAlloc;
final int m_minFixedAlloc;
+
/**
* The fixed size of any allocator on the disk in bytes. The #of allocations
* managed by an allocator is this value times 8 because each slot uses one
@@ -214,15 +214,15 @@
* However, the {@link FixedAllocator} only incrementally allocates the
* {@link AllocBlock}s.
*/
- static final int ALLOC_BLOCK_SIZE = 1024;
+ static private final int ALLOC_BLOCK_SIZE = 1024;
- // from 32 bits, need 13 to hold max offset of 8 * 1024, leaving 19 for number of blocks: 256K
- static final int BLOCK_INDEX_BITS = 19;
+// // from 32 bits, need 13 to hold max offset of 8 * 1024, leaving 19 for number of blocks: 256K
+// static final int BLOCK_INDEX_BITS = 19;
static final int OFFSET_BITS = 13;
static final int OFFSET_BITS_MASK = 0x1FFF; // was 0xFFFF
static final int ALLOCATION_SCALEUP = 16; // multiplier to convert allocations based on minimum allocation of 32k
- static final int META_ALLOCATION = 8; // 8 * 32K is size of meta Allocation
+ static private final int META_ALLOCATION = 8; // 8 * 32K is size of meta Allocation
static final int BLOB_FIXED_ALLOCS = 1024;
// private ICommitCallback m_commitCallback;
@@ -253,17 +253,19 @@
*/
private final ArrayList<Allocator> m_allocs;
- // lists of free alloc blocks
+ /** lists of free alloc blocks. */
private ArrayList<FixedAllocator> m_freeFixed[];
- // lists of free blob allocators
+ /** lists of free blob allocators. */
private final ArrayList<BlobAllocator> m_freeBlobs;
- // lists of blocks requiring commitment
+ /** lists of blocks requiring commitment. */
private final ArrayList<Allocator> m_commitList;
- private WriteBlock m_writes;
+// private WriteBlock m_writes;
+
private final Quorum<?,?> m_quorum;
+
private final RWWriteCacheService m_writeCache;
private int[] m_allocSizes;
@@ -321,9 +323,13 @@
// private final ArrayList<Integer> m_currentTxnFreeList = new ArrayList<Integer>();
private final PSOutputStream m_deferredFreeOut;
- private ReopenFileChannel m_reopener = null;
+ /**
+ * Used to transparently re-open the backing channel if it has been closed
+ * by an interrupt during an IO.
+ */
+ private final ReopenFileChannel m_reopener;
- private BufferedWrite m_bufferedWrite;
+ private volatile BufferedWrite m_bufferedWrite;
class WriteCacheImpl extends WriteCache.FileChannelScatteredWriteCache {
public WriteCacheImpl(final ByteBuffer buf,
@@ -332,8 +338,9 @@
final IReopenChannel<FileChannel> opener)
throws InterruptedException {
- super(buf, useChecksum, m_quorum!=null&&m_quorum
- .isHighlyAvailable(), bufferHasData, opener, m_bufferedWrite);
+ super(buf, useChecksum, m_quorum != null
+ && m_quorum.isHighlyAvailable(), bufferHasData, opener,
+ m_bufferedWrite);
}
@@ -1523,14 +1530,14 @@
/**
* This must now update the root block which is managed by FileMetadata in
- * almost guaranteed secure manner
+ * almost guaranteed secure manner.
*
* It is not the responsibility of the store to write this out, this is
* handled by whatever is managing the FileMetadata that this RWStore was
- * initialised from and should be forced by newRootBlockView
+ * initialised from and should be forced by newRootBlockView.
*
* It should now only be called by extend file to ensure that the metaBits
- * are set correctly
+ * are set correctly.
*
* In order to ensure that the new block is the one that would be chosen, we need to
* duplicate the rootBlock. This does mean that we lose the ability to roll
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-03 14:58:51
|
Revision: 3879
http://bigdata.svn.sourceforge.net/bigdata/?rev=3879&view=rev
Author: thompsonbry
Date: 2010-11-03 14:58:45 +0000 (Wed, 03 Nov 2010)
Log Message:
-----------
Javadoc edits.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 14:48:32 UTC (rev 3878)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 14:58:45 UTC (rev 3879)
@@ -137,7 +137,7 @@
* with a minimum block allocation of 64K, and a minimum bit number per block of
* 32.
* <p>
- * Where possible lists and roving pointers will be used to minimise searching
+ * Where possible lists and roving pointers will be used to minimize searching
* of the potentially large structures.
* <p>
* Since the memory is allocated on (at least) a 128 byte boundary, there is
@@ -187,18 +187,18 @@
private static final transient Logger log = Logger.getLogger(RWStore.class);
- /**
- * The sizes of the slots managed by a {@link FixedAllocator} are 64 times
- * the values in this array.
- *
- * @todo good to have 4k and 8k boundaries for better efficiency on SSD.
- * NB A 1K boundary is % 16, so 4K % 64
- * - can still use fibonacci base, but from 4K start
- *
- * @todo This array should be configurable and must be written into the
- * store so changing values does not break older stores.
- * com.bigdata.rwstore.RWStore.allocSizes=1,2,3,5...
- */
+ /**
+ * The sizes of the slots managed by a {@link FixedAllocator} are 64 times
+ * the values in this array. This array is written into the store so
+ * changing the values does not break older stores. This array is
+ * configurable using {@link com.bigdata.journal.Options#RW_ALLOCATIONS}.
+ * <p>
+ * Note: It is good to have 4k and 8k boundaries for better efficiency on
+ * SSD. A 1K boundary is expressed as <code>16</code> in the allocation
+ * sizes, so a 4K boundary is expressed as <code>64</code>. The default
+ * series of allocation sizes is based on the Fibonacci sequence, but is
+ * pegged to the closest 4K boundary for values larger than 4k.
+ */
private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 12, 16, 32, 48, 64, 128, 192, 320, 512, 832, 1344, 2176, 3520 };
// private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 };
// private static final int[] ALLOC_SIZES = { 1, 2, 4, 8, 16, 32, 64, 128 };
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-03 23:05:07
|
Revision: 3885
http://bigdata.svn.sourceforge.net/bigdata/?rev=3885&view=rev
Author: thompsonbry
Date: 2010-11-03 23:05:00 +0000 (Wed, 03 Nov 2010)
Log Message:
-----------
Found something else which can be final.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 18:53:49 UTC (rev 3884)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-03 23:05:00 UTC (rev 3885)
@@ -1726,7 +1726,7 @@
volatile private int m_fileSize;
volatile private int m_nextAllocation;
- volatile private int m_maxFileSize;
+ final private int m_maxFileSize;
// private int m_headerSize = 2048;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-04 12:46:53
|
Revision: 3888
http://bigdata.svn.sourceforge.net/bigdata/?rev=3888&view=rev
Author: thompsonbry
Date: 2010-11-04 12:46:45 +0000 (Thu, 04 Nov 2010)
Log Message:
-----------
javadoc.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-04 12:45:18 UTC (rev 3887)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-04 12:46:45 UTC (rev 3888)
@@ -2618,14 +2618,21 @@
}
- public long getTotalAllocations() {
+ /** The # of allocation requests made. */
+ public long getTotalAllocations() {
return m_allocations;
}
+ /**
+ * The # of free requests made
+ */
public long getTotalFrees() {
return m_frees;
}
+ /**
+ * The # of bytes requested - as opposed to the size of the slots allocated.
+ */
public long getTotalAllocationsSize() {
return m_nativeAllocBytes;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-11-04 15:01:08
|
Revision: 3891
http://bigdata.svn.sourceforge.net/bigdata/?rev=3891&view=rev
Author: martyncutcher
Date: 2010-11-04 15:01:01 +0000 (Thu, 04 Nov 2010)
Log Message:
-----------
Fix abort problem by handling uninitialised rootblock with default initialisation
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-04 13:13:49 UTC (rev 3890)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-04 15:01:01 UTC (rev 3891)
@@ -255,7 +255,7 @@
* FIXME This is initially true and is never set to false. Should this all
* go away?
*/
- private boolean m_preserveSession = true;
+ private boolean m_preserveSession = false;
// private boolean m_readOnly;
/**
@@ -493,49 +493,14 @@
try {
if (m_rb.getNextOffset() == 0) { // if zero then new file
- final String buckets = fileMetadata.getProperty(
- Options.RW_ALLOCATIONS, null/* default */);
- if (buckets == null) {
- m_allocSizes = DEFAULT_ALLOC_SIZES;
- } else {
- final String[] specs = buckets.split(",");
- m_allocSizes = new int[specs.length];
- int prevSize = 0;
- for (int i = 0; i < specs.length; i++) {
- final int nxtSize = Integer.parseInt(specs[i]);
- if (nxtSize <= prevSize)
- throw new IllegalArgumentException("Invalid AllocSizes property");
- m_allocSizes[i] = nxtSize;
- prevSize = nxtSize;
- }
- }
+ setAllocations(fileMetadata);
+
+ defaultInit();
+
+ m_maxFixedAlloc = m_allocSizes[m_allocSizes.length-1]*64;
+ m_minFixedAlloc = m_allocSizes[0]*64;
- final int numFixed = m_allocSizes.length;
-
- m_freeFixed = new ArrayList[numFixed];
-
- for (int i = 0; i < numFixed; i++) {
- m_freeFixed[i] = new ArrayList<FixedAllocator>();
- }
-
- m_fileSize = convertFromAddr(m_fd.length());
-
- // make space for meta-allocators
- m_metaBits[0] = -1;
- m_metaTransientBits[0] = -1;
- m_nextAllocation = -(1 + META_ALLOCATION); // keep on a minimum 8K boundary
-
- if (m_fileSize > m_nextAllocation) {
- m_fileSize = m_nextAllocation;
- }
-
- m_reopener.raf.setLength(convertAddr(m_fileSize));
-
- m_maxFixedAlloc = m_allocSizes[m_allocSizes.length-1]*64;
- m_minFixedAlloc = m_allocSizes[0]*64;
-
- commitChanges(null);
-
+ // commitChanges(null);
} else {
initfromRootBlock(m_rb);
@@ -551,7 +516,50 @@
throw new StorageTerminalError("Unable to initialize store", e);
}
}
+
+ private void setAllocations(final FileMetadata fileMetadata) throws IOException {
+ final String buckets = fileMetadata.getProperty(
+ Options.RW_ALLOCATIONS, null/* default */);
+ if (buckets == null) {
+ m_allocSizes = DEFAULT_ALLOC_SIZES;
+ } else {
+ final String[] specs = buckets.split(",");
+ m_allocSizes = new int[specs.length];
+ int prevSize = 0;
+ for (int i = 0; i < specs.length; i++) {
+ final int nxtSize = Integer.parseInt(specs[i]);
+ if (nxtSize <= prevSize)
+ throw new IllegalArgumentException("Invalid AllocSizes property");
+ m_allocSizes[i] = nxtSize;
+ prevSize = nxtSize;
+ }
+ }
+ }
+
+ private void defaultInit() throws IOException {
+ final int numFixed = m_allocSizes.length;
+ m_freeFixed = new ArrayList[numFixed];
+
+ for (int i = 0; i < numFixed; i++) {
+ m_freeFixed[i] = new ArrayList<FixedAllocator>();
+ }
+
+ m_fileSize = convertFromAddr(m_fd.length());
+
+ // make space for meta-allocators
+ m_metaBits[0] = -1;
+ m_metaTransientBits[0] = -1;
+ m_nextAllocation = -(1 + META_ALLOCATION); // keep on a minimum 8K boundary
+
+ if (m_fileSize > m_nextAllocation) {
+ m_fileSize = m_nextAllocation;
+ }
+
+ m_reopener.raf.setLength(convertAddr(m_fileSize));
+
+ }
+
public boolean isOpen() {
return m_open;
}
@@ -643,83 +651,89 @@
// m_rb = m_fmv.getRootBlock();
assert(m_rb != null);
-// m_commitCounter = m_rb.getCommitCounter();
-
- final long nxtOffset = m_rb.getNextOffset();
- m_nextAllocation = -(int) (nxtOffset >> 32);
-
- m_metaBitsAddr = -(int) nxtOffset;
+ if (m_rb.getNextOffset() == 0) {
+ defaultInit();
+ } else {
+ final long nxtOffset = m_rb.getNextOffset();
+ m_nextAllocation = -(int) (nxtOffset >> 32);
+
+ if (m_nextAllocation == 0) {
+ m_nextAllocation = -(1 + META_ALLOCATION);
+ }
+
+ m_metaBitsAddr = -(int) nxtOffset;
+
+ if (log.isInfoEnabled()) {
+ log.info("MetaBitsAddr: " + m_metaBitsAddr);
+ }
+
+ final long metaAddr = m_rb.getMetaStartAddr();
+ m_fileSize = (int) -(metaAddr & 0xFFFFFFFF);
+
+ long rawmbaddr = m_rb.getMetaBitsAddr();
+
+ /*
+ * Take bottom 16 bits (even 1K of metabits is more than sufficient)
+ */
+ final int metaBitsStore = (int) (rawmbaddr & 0xFFFF);
+
+ if (metaBitsStore > 0) {
+ rawmbaddr >>= 16;
- if (log.isInfoEnabled()) {
- log.info("MetaBitsAddr: " + m_metaBitsAddr);
- }
-
- final long metaAddr = m_rb.getMetaStartAddr();
- m_fileSize = (int) -(metaAddr & 0xFFFFFFFF);
-
- long rawmbaddr = m_rb.getMetaBitsAddr();
+ // RWStore now restore metabits
+ final byte[] buf = new byte[metaBitsStore * 4];
+
+ FileChannelUtility.readAll(m_reopener, ByteBuffer.wrap(buf), rawmbaddr);
- /*
- * Take bottom 16 bits (even 1K of metabits is more than sufficient)
- */
- final int metaBitsStore = (int) (rawmbaddr & 0xFFFF);
+ final DataInputStream strBuf = new DataInputStream(new ByteArrayInputStream(buf));
+
+ m_lastDeferredReleaseTime = strBuf.readLong();
+
+ final int allocBlocks = strBuf.readInt();
+ m_allocSizes = new int[allocBlocks];
+ for (int i = 0; i < allocBlocks; i++) {
+ m_allocSizes[i] = strBuf.readInt();
+ }
+ m_metaBitsSize = metaBitsStore - allocBlocks - 3; // allow for deferred free
+ m_metaBits = new int[m_metaBitsSize];
+ if (log.isInfoEnabled()) {
+ log.info("Raw MetaBitsAddr: " + rawmbaddr);
+ }
+ for (int i = 0; i < m_metaBitsSize; i++) {
+ m_metaBits[i] = strBuf.readInt();
+ }
+ m_metaTransientBits = (int[]) m_metaBits.clone();
- if (metaBitsStore > 0) {
- rawmbaddr >>= 16;
+ final int numFixed = m_allocSizes.length;
- // RWStore now restore metabits
- final byte[] buf = new byte[metaBitsStore * 4];
-
- FileChannelUtility.readAll(m_reopener, ByteBuffer.wrap(buf), rawmbaddr);
+ m_freeFixed = new ArrayList[numFixed];
- final DataInputStream strBuf = new DataInputStream(new ByteArrayInputStream(buf));
-
- m_lastDeferredReleaseTime = strBuf.readLong();
-
- final int allocBlocks = strBuf.readInt();
- m_allocSizes = new int[allocBlocks];
- for (int i = 0; i < allocBlocks; i++) {
- m_allocSizes[i] = strBuf.readInt();
- }
- m_metaBitsSize = metaBitsStore - allocBlocks - 3; // allow for deferred free
- m_metaBits = new int[m_metaBitsSize];
- if (log.isInfoEnabled()) {
- log.info("Raw MetaBitsAddr: " + rawmbaddr);
- }
- for (int i = 0; i < m_metaBitsSize; i++) {
- m_metaBits[i] = strBuf.readInt();
- }
- m_metaTransientBits = (int[]) m_metaBits.clone();
+ for (int i = 0; i < numFixed; i++) {
+ m_freeFixed[i] = new ArrayList<FixedAllocator>();
+ }
- final int numFixed = m_allocSizes.length;
-
- m_freeFixed = new ArrayList[numFixed];
-
- for (int i = 0; i < numFixed; i++) {
- m_freeFixed[i] = new ArrayList<FixedAllocator>();
- }
-
- checkCoreAllocations();
+ checkCoreAllocations();
+
+ readAllocationBlocks();
+
+ // clearOutstandingDeferrels(deferredFreeListAddr, deferredFreeListEntries);
- readAllocationBlocks();
-
- // clearOutstandingDeferrels(deferredFreeListAddr, deferredFreeListEntries);
-
- if (log.isTraceEnabled()) {
- final StringBuilder str = new StringBuilder();
- this.showAllocators(str);
- log.trace(str);
+ if (log.isTraceEnabled()) {
+ final StringBuilder str = new StringBuilder();
+ this.showAllocators(str);
+ log.trace(str);
+ }
+
+ if (physicalAddress(m_metaBitsAddr) == 0) {
+ throw new IllegalStateException("Free/Invalid metaBitsAddr on load");
+ }
+
}
- if (physicalAddress(m_metaBitsAddr) == 0) {
- throw new IllegalStateException("Free/Invalid metaBitsAddr on load");
- }
-
+ if (log.isInfoEnabled())
+ log.info("restored from RootBlock: " + m_nextAllocation
+ + ", " + m_metaBitsAddr);
}
-
- if (log.isInfoEnabled())
- log.info("restored from RootBlock: " + m_nextAllocation
- + ", " + m_metaBitsAddr);
}
// /*
@@ -3282,6 +3296,7 @@
}
+
/**
* Striped performance counters for {@link IRawStore} access, including
* operations that read or write through to the underlying media.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-05 19:32:16
|
Revision: 3904
http://bigdata.svn.sourceforge.net/bigdata/?rev=3904&view=rev
Author: thompsonbry
Date: 2010-11-05 19:32:10 +0000 (Fri, 05 Nov 2010)
Log Message:
-----------
javadoc
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-05 17:14:05 UTC (rev 3903)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-05 19:32:10 UTC (rev 3904)
@@ -202,42 +202,33 @@
*/
public interface Options {
- /**
- * Option defines the Allocation block sizes for the RWStore. The values
- * defined are multiplied by 64 to provide the actual allocations. The
- * list of allocations should be ',' delimited and in increasing order.
- * For example,
- *
- * <pre>
- * "1,2,4,8,116,32,64"
- * </pre>
- *
- * defines allocations from 64 to 4K in size. The default allocations
- * are:
- *
- * <pre>
- * "1,2,3,5,8,12,16,32,48,64,128,192,320,512,832,1344,2176,3520"
- * </pre>
- *
- * providing blocks up to 220K aligned on 4K boundaries as soon as
- * possible to optimize IO - particularly relevant for SSDs.
- *
- * @see #DEFAULT_ALLOCATION_SIZES
- */
+ /**
+ * Option defines the Allocation block sizes for the RWStore. The values
+ * defined are multiplied by 64 to provide the actual allocations. The
+ * list of allocations should be ',' delimited and in increasing order.
+ * This array is written into the store so changing the values does not
+ * break older stores. For example,
+ *
+ * <pre>
+ * "1,2,4,8,116,32,64"
+ * </pre>
+ *
+ * defines allocations from 64 to 4K in size. It is a good to define
+ * block sizes on 4K boundaries as soon as possible to optimize IO. This
+ * is particularly relevant for SSDs. A 1K boundary is expressed as
+ * <code>16</code> in the allocation sizes, so a 4K boundary is
+ * expressed as <code>64</code> and an 8k boundary as <code>128</code>.
+ * <p>
+ * The default allocations are {@value #DEFAULT_ALLOCATION_SIZES}.
+ *
+ * @see #DEFAULT_ALLOCATION_SIZES
+ */
String ALLOCATION_SIZES = RWStore.class.getName() + ".allocationSizes";
/**
- * The sizes of the slots managed by a {@link FixedAllocator} are 64 times
- * the values in this array. This array is written into the store so
- * changing the values does not break older stores. This array is
- * configurable using {@link com.bigdata.journal.Options#ALLOCATION_SIZES}.
- * <p>
- * Note: It is good to have 4k and 8k boundaries for better efficiency on
- * SSD. A 1K boundary is expressed as <code>16</code> in the allocation
- * sizes, so a 4K boundary is expressed as <code>64</code>. The default
- * series of allocation sizes is based on the Fibonacci sequence, but is
- * pegged to the closest 4K boundary for values larger than 4k.
+ * @see #ALLOCATION_SIZES
*/
+ //String DEFAULT_ALLOCATION_SIZES = "1, 2, 3, 5, 8, 12, 16, 32, 48, 64, 128";
String DEFAULT_ALLOCATION_SIZES = "1, 2, 3, 5, 8, 12, 16, 32, 48, 64, 128, 192, 320, 512, 832, 1344, 2176, 3520";
// private static final int[] DEFAULT_ALLOC_SIZES = { 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610, 987, 1597, 2584, 4181 };
// private static final int[] ALLOC_SIZES = { 1, 2, 4, 8, 16, 32, 64, 128 };
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-11-08 13:10:20
|
Revision: 3910
http://bigdata.svn.sourceforge.net/bigdata/?rev=3910&view=rev
Author: martyncutcher
Date: 2010-11-08 13:10:14 +0000 (Mon, 08 Nov 2010)
Log Message:
-----------
add formatting to showAllocators
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-08 11:09:09 UTC (rev 3909)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-08 13:10:14 UTC (rev 3910)
@@ -2392,6 +2392,13 @@
str.append("\n-------------------------\n");
str.append("RWStore Allocation Summary\n");
str.append("-------------------------\n");
+ str.append(padRight("Allocator", 10));
+ str.append(padLeft("Slots used", 12));
+ str.append(padLeft("available", 12));
+ str.append(padLeft("Store used", 14));
+ str.append(padLeft("available", 14));
+ str.append(padLeft("Usage", 8));
+ str.append("\n");
long treserved = 0;
long treservedSlots = 0;
long tfilled = 0;
@@ -2404,17 +2411,47 @@
tfilled += filled;
tfilledSlots += stats[i].m_filledSlots;
- str.append("Allocation: " + stats[i].m_blockSize);
- str.append(", slots: " + stats[i].m_filledSlots + "/" + stats[i].m_reservedSlots);
- str.append(", storage: " + filled + "/" + reserved);
- str.append(", usage: " + (reserved==0?0:(filled * 100 / reserved)) + "%");
+ str.append(padRight("" + stats[i].m_blockSize, 10));
+ str.append(padLeft("" + stats[i].m_filledSlots, 12) + padLeft("" + stats[i].m_reservedSlots, 12));
+ str.append(padLeft("" + filled, 14) + padLeft("" + reserved, 14));
+ str.append(padLeft("" + (reserved==0?0:(filled * 100 / reserved)) + "%", 8));
str.append("\n");
}
- str.append("Total - file: " + convertAddr(m_fileSize) + //
- ", slots: " + tfilledSlots + "/" + treservedSlots + //
- ", storage: " + tfilled + "/" + treserved + //
- "\n");
+ str.append(padRight("Totals", 10));
+ str.append(padLeft("" + tfilledSlots, 12));
+ str.append(padLeft("" + treservedSlots, 12));
+ str.append(padLeft("" + tfilled, 14));
+ str.append(padLeft("" + treserved, 14));
+ str.append(padLeft("" + (treserved==0?0:(tfilled * 100 / treserved)) + "%", 8));
+ str.append("\nFile size: " + convertAddr(m_fileSize) + "bytes\n");
}
+
+ private String padLeft(String str, int minlen) {
+ if (str.length() >= minlen)
+ return str;
+
+ StringBuffer out = new StringBuffer();
+ int pad = minlen - str.length();
+ while (pad-- > 0) {
+ out.append(' ');
+ }
+ out.append(str);
+
+ return out.toString();
+ }
+ private String padRight(String str, int minlen) {
+ if (str.length() >= minlen)
+ return str;
+
+ StringBuffer out = new StringBuffer();
+ out.append(str);
+ int pad = minlen - str.length();
+ while (pad-- > 0) {
+ out.append(' ');
+ }
+
+ return out.toString();
+ }
// public ArrayList<Allocator> getStorageBlockAddresses() {
// final ArrayList<Allocator> addrs = new ArrayList<Allocator>(m_allocs.size());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-11-11 15:02:10
|
Revision: 3936
http://bigdata.svn.sourceforge.net/bigdata/?rev=3936&view=rev
Author: martyncutcher
Date: 2010-11-11 15:02:03 +0000 (Thu, 11 Nov 2010)
Log Message:
-----------
add reserved bytes to header and check for extent in establishExtent
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-11 14:39:43 UTC (rev 3935)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-11 15:02:03 UTC (rev 3936)
@@ -762,6 +762,12 @@
cDefaultMetaBitsSize = strBuf.readInt();
final int allocBlocks = strBuf.readInt();
+ strBuf.readInt(); // reserved5
+ strBuf.readInt(); // reserved6
+ strBuf.readInt(); // reserved7
+ strBuf.readInt(); // reserved8
+ strBuf.readInt(); // reserved9
+
m_allocSizes = new int[allocBlocks];
for (int i = 0; i < allocBlocks; i++) {
m_allocSizes[i] = strBuf.readInt();
@@ -1763,9 +1769,15 @@
try {
str.writeInt(cVersion);
str.writeLong(m_lastDeferredReleaseTime);
- str.writeInt(cDefaultMetaBitsSize);
+ str.writeInt(cDefaultMetaBitsSize);
+ str.writeInt(m_allocSizes.length);
- str.writeInt(m_allocSizes.length);
+ str.writeInt(0); // reserved5
+ str.writeInt(0); // reserved6
+ str.writeInt(0); // reserved7
+ str.writeInt(0); // reserved8
+ str.writeInt(0); // reserved9
+
for (int i = 0; i < m_allocSizes.length; i++) {
str.writeInt(m_allocSizes[i]);
}
@@ -1973,16 +1985,21 @@
* 0x0200 == 2.00
* 0x0320 == 3.20
*/
- final private int cVersion = 0x0200;
+ final private int cVersion = 0x0300;
/**
* MetaBits Header
- * int version
- * long deferredFree
- * int defaultMetaBitsSize
- * int length of allocation sizes
+ * 0 int version
+ * 1-2 int[2] long deferredFree
+ * 3 int defaultMetaBitsSize
+ * 4 int length of allocation sizes
+ * 5 int reserved
+ * 6 int reserved
+ * 7 int reserved
+ * 8 int reserved
+ * 9 int reserved
*/
- final private int cMetaHdrFields = 5;
+ final private int cMetaHdrFields = 10;
/**
* @see Options#META_BITS_SIZE
*/
@@ -2965,10 +2982,12 @@
final long currentExtent = convertAddr(m_fileSize);
- if (extent != currentExtent) {
+ if (extent > currentExtent) {
extendFile(convertFromAddr(extent - currentExtent));
+ } else if (extent < currentExtent) {
+ throw new IllegalArgumentException("Cannot shrink RWStore extent");
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-18 15:29:20
|
Revision: 3958
http://bigdata.svn.sourceforge.net/bigdata/?rev=3958&view=rev
Author: thompsonbry
Date: 2010-11-18 15:29:14 +0000 (Thu, 18 Nov 2010)
Log Message:
-----------
Updated javadoc for showAllocators.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-18 14:32:43 UTC (rev 3957)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-18 15:29:14 UTC (rev 3958)
@@ -2521,16 +2521,18 @@
* <dl>
* <dt>AllocatorSize</dt><dd>The #of bytes in the allocated slots issued by this allocator.</dd>
* <dt>AllocatorCount</dt><dd>The #of fixed allocators for that slot size.</dd>
+ * <dt>SlotsInUse</dt><dd>The difference between the two previous columns (net slots in use for this slot size).</dd>
+ * <dt>SlotsReserved</dt><dd>The #of slots in this slot size which have had storage reserved for them.</dd>
* <dt>SlotsAllocated</dt><dd>Cumulative allocation of slots to date in this slot size (regardless of the transaction outcome).</dd>
* <dt>SlotsRecycled</dt><dd>Cumulative recycled slots to date in this slot size (regardless of the transaction outcome).</dd>
- * <dt>SlotsInUse</dt><dd>The difference between the two previous columns (net slots in use for this slot size).</dd>
- * <dt>SlotsReserved</dt><dd>The #of slots in this slot size which have had storage reserved for them.</dd>
+ * <dt>SlotsChurn</dt><dd>How frequently slots of this size are re-allocated (SlotsInUse/SlotsAllocated).</dd>
+ * <dt>%SlotsUnused</dt><dd>The percentage of slots of this size which are not in use (1-(SlotsInUse/SlotsReserved)).</dd>
* <dt>BytesReserved</dt><dd>The space reserved on the backing file for those allocation slots</dd>
* <dt>BytesAppData</dt><dd>The #of bytes in the allocated slots which are used by application data (including the record checksum).</dd>
- * <dt>%SlotWaste</dt><dd>BytesAppData/(SlotsInUse*AllocatorSize) : How well the application data fits in the slots.</dd>
- * <dt>%StoreWaste</dt><dd>BytesAppData/BytesReserved : How much of the reserved space is in use by application data.</dd>
- * <dt>%AppData</dt><dd>BytesAppData/Sum(BytesAppData). This is how much of your data is stored by each allocator.</dd>
- * <dt>%BackingFile</dt><dd>BytesReserved/Sum(BytesReserved). This is how much of the backing file is reserved for each allocator.</dd>
+ * <dt>%SlotWaste</dt><dd>How well the application data fits in the slots (BytesAppData/(SlotsInUse*AllocatorSize)).</dd>
+ * <dt>%AppData</dt><dd>How much of your data is stored by each allocator (BytesAppData/Sum(BytesAppData)).</dd>
+ * <dt>%StoreFile</dt><dd>How much of the backing file is reserved for each allocator (BytesReserved/Sum(BytesReserved)).</dd>
+ * <dt>%StoreWaste</dt><dd>How much of the total waste on the store is waste for this allocator size ((BytesReserved-BytesAppData)/(Sum(BytesReserved)-Sum(BytesAppData))).</dd>
* </dl>
*/
public void showAllocators(final StringBuilder str) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-18 16:50:29
|
Revision: 3959
http://bigdata.svn.sourceforge.net/bigdata/?rev=3959&view=rev
Author: thompsonbry
Date: 2010-11-18 16:50:22 +0000 (Thu, 18 Nov 2010)
Log Message:
-----------
Added logic to notice when MIN_RELEASE_AGE is non-zero and always defer frees.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-18 15:29:14 UTC (rev 3958)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-18 16:50:22 UTC (rev 3959)
@@ -50,6 +50,7 @@
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.BTree.Counter;
+import com.bigdata.config.LongValidator;
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
import com.bigdata.counters.striped.StripedCounters;
@@ -73,6 +74,7 @@
import com.bigdata.journal.ha.HAWriteMessage;
import com.bigdata.quorum.Quorum;
import com.bigdata.rawstore.IRawStore;
+import com.bigdata.service.AbstractTransactionService;
import com.bigdata.util.ChecksumUtility;
/**
@@ -441,6 +443,7 @@
// * serves as a BLOB header when there is more than a single entry with
// * the same txReleaseTime.
// private static final int MAX_DEFERRED_FREE = 4094; // fits in 16k block
+ private final long m_minReleaseAge;
private volatile long m_lastDeferredReleaseTime = 0L;
// private final ArrayList<Integer> m_currentTxnFreeList = new ArrayList<Integer>();
private final PSOutputStream m_deferredFreeOut;
@@ -525,6 +528,14 @@
if (fileMetadata == null)
throw new IllegalArgumentException();
+ this.m_minReleaseAge = LongValidator.GTE_ZERO.parse(
+ AbstractTransactionService.Options.MIN_RELEASE_AGE,
+ AbstractTransactionService.Options.DEFAULT_MIN_RELEASE_AGE);
+
+ if (log.isInfoEnabled())
+ log.info(AbstractTransactionService.Options.MIN_RELEASE_AGE + "="
+ + m_minReleaseAge);
+
cDefaultMetaBitsSize = Integer.valueOf(fileMetadata.getProperty(
Options.META_BITS_SIZE,
Options.DEFAULT_META_BITS_SIZE));
@@ -1404,19 +1415,26 @@
freeBlob(addr, sze, context);
} else {
final Allocator alloc = getBlockByAddress(addr);
- /*
- * There are a few conditions here. If the context owns the
- * allocator and the allocation was made by this context then
- * it can be freed immediately.
- * The problem comes when the context is null and the allocator
- * is NOT owned, BUT there are active AllocationContexts, in this
- * situation, the free must ALWAYS be deferred.
- */
- final boolean alwaysDefer = context == null && m_contexts.size() > 0;
- if (alwaysDefer)
+ /*
+ * There are a few conditions here. If the context owns the
+ * allocator and the allocation was made by this context then it
+ * can be freed immediately. The problem comes when the context
+ * is null and the allocator is NOT owned, BUT there are active
+ * AllocationContexts, in this situation, the free must ALWAYS
+ * be deferred.
+ *
+ * FIXME We need unit tests when MIN_RELEASE_AGE is GT ZERO.
+ *
+ * FIXME We need unit test when MIN_RELEASE_AGE is ZERO AND
+ * there are open read-only transactions.
+ */
+ boolean alwaysDefer = m_minReleaseAge > 0L;
+ if (!alwaysDefer)
+ alwaysDefer = context == null && !m_contexts.isEmpty();
+ if (alwaysDefer)
if (log.isDebugEnabled())
log.debug("Should defer " + physicalAddress(addr));
- if (/*alwaysDefer ||*/ !alloc.canImmediatelyFree(addr, sze, context)) {
+ if (alwaysDefer || !alloc.canImmediatelyFree(addr, sze, context)) {
deferFree(addr, sze);
} else {
immediateFree(addr, sze);
@@ -3487,6 +3505,12 @@
private ContextAllocation establishContextAllocation(
final IAllocationContext context) {
+
+ /*
+ * The allocation lock MUST be held to make changes in the membership of
+ * m_contexts atomic with respect to free().
+ */
+ assert m_allocationLock.isHeldByCurrentThread();
ContextAllocation ret = m_contexts.get(context);
@@ -3500,8 +3524,9 @@
}
- log.warn("Context: ncontexts=" + m_contexts.size() + ", context="
- + context);
+ if (log.isInfoEnabled())
+ log.info("Context: ncontexts=" + m_contexts.size()
+ + ", context=" + context);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-11-18 17:12:30
|
Revision: 3961
http://bigdata.svn.sourceforge.net/bigdata/?rev=3961&view=rev
Author: thompsonbry
Date: 2010-11-18 17:12:23 +0000 (Thu, 18 Nov 2010)
Log Message:
-----------
deactivateTx() was incrementing rather than decrementing.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-18 17:10:17 UTC (rev 3960)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-11-18 17:12:23 UTC (rev 3961)
@@ -4227,7 +4227,7 @@
public void deactivateTx() {
m_allocationLock.lock();
try {
- m_activeTxCount++;
+ m_activeTxCount--;
} finally {
m_allocationLock.unlock();
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-12-14 17:15:17
|
Revision: 4009
http://bigdata.svn.sourceforge.net/bigdata/?rev=4009&view=rev
Author: thompsonbry
Date: 2010-12-14 17:15:11 +0000 (Tue, 14 Dec 2010)
Log Message:
-----------
Added Option to RWStore to disable the double buffering of writes.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-12-14 17:06:17 UTC (rev 4008)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-12-14 17:15:11 UTC (rev 4009)
@@ -285,6 +285,16 @@
String DEFAULT_FREE_BITS_THRESHOLD = "300";
+ /**
+ * When <code>true</code>, scattered writes which are strictly ascending
+ * will be coalesced within a buffer and written out as a single IO
+ * (default {@value #DEFAULT_DOUBLE_BUFFER_WRITES}). This improves write
+ * performance for SATA, SAS, and even SSD.
+ */
+ String DOUBLE_BUFFER_WRITES = RWStore.class.getName() + ".doubleBuffer";
+
+ String DEFAULT_DOUBLE_BUFFER_WRITES = "true";
+
}
/*
@@ -594,10 +604,16 @@
} catch (IOException e1) {
throw new RuntimeException(e1);
}
-
- try {
- m_bufferedWrite = new BufferedWrite(this);
- } catch (InterruptedException e1) {
+
+ if (Boolean.valueOf(fileMetadata.getProperty(
+ Options.DOUBLE_BUFFER_WRITES,
+ Options.DEFAULT_DOUBLE_BUFFER_WRITES))) {
+ try {
+ m_bufferedWrite = new BufferedWrite(this);
+ } catch (InterruptedException e1) {
+ m_bufferedWrite = null;
+ }
+ } else {
m_bufferedWrite = null;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X