Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[8508] branches/BIGDATA_RELEASE_1_3_0/bigdata/src
|
From: <mar...@us...> - 2014-06-30 11:32:35
|
Revision: 8508
http://sourceforge.net/p/bigdata/code/8508
Author: martyncutcher
Date: 2014-06-30 11:32:23 +0000 (Mon, 30 Jun 2014)
Log Message:
-----------
For ticket #936, modified metabits demispace use to default to "false" and support an explicit, programmatic, request to toggle between fixed allocation and demispace.
Provide a utility MetabitsUtil class with "main" to toggle from a command line invocation.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java
Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/MetabitsUtil.java 2014-06-30 11:32:23 UTC (rev 8508)
@@ -0,0 +1,105 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2014. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+package com.bigdata.rwstore;
+
+import java.io.File;
+import java.util.Properties;
+
+import com.bigdata.journal.BufferMode;
+import com.bigdata.journal.RWStrategy;
+import com.bigdata.journal.Journal;
+import com.bigdata.journal.Journal.Options;
+import com.bigdata.rawstore.IRawStore;
+
+/**
+ * A utility class to explicitly change the metabits storage to allow for
+ * compatibility with previous versions.
+ * <p>
+ * There is an option to use a demispace rather than standard allocations to
+ * support stores with large numbers of allocations. If such a store needs to be
+ * opened by an earlier code-base, then the store must be amended to store the
+ * metabits in a standard allocation.
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class MetabitsUtil {
+
+ static String getArg(final String[] args, final String arg, final String def) {
+ for (int p = 0; p < args.length; p += 2) {
+ if (arg.equals(args[p]))
+ return args[p + 1];
+ }
+
+ return def;
+ }
+
+ static Journal getStore(String storeFile) {
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Options.FILE, storeFile);
+
+ properties.setProperty(Options.BUFFER_MODE,
+ BufferMode.DiskRW.toString());
+
+ return new Journal(properties);// .getBufferStrategy();
+
+ }
+
+ /**
+ * Example usage:
+ * <p>
+ * MatabitsUtil -store "/path/store.jnl" -usedemispace true
+ */
+ static public void main(final String[] args) {
+ final String store = getArg(args, "-store", null);
+ if (store == null) {
+ System.err.println("file must be specificed with -store");
+ return;
+ }
+ final File file = new File(store);
+ if (!file.exists()) {
+ System.err.println("Specified file '" + store + "' not found");
+ return;
+ }
+
+ final boolean usedemi = "true".equals(getArg(args, "-usedemispace",
+ "true"));
+
+ final Journal jnl = getStore(store);
+
+ try {
+ final RWStore rws = ((RWStrategy) jnl.getBufferStrategy())
+ .getStore();
+
+ if (rws.ensureMetabitsDemispace(usedemi)) { // changed
+ jnl.commit();
+ }
+ } finally {
+ jnl.close();
+ }
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-06-28 01:45:04 UTC (rev 8507)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2014-06-30 11:32:23 UTC (rev 8508)
@@ -342,7 +342,7 @@
*/
String META_BITS_DEMI_SPACE = RWStore.class.getName() + ".metabitsDemispace";
- String DEFAULT_META_BITS_DEMI_SPACE = "true";
+ String DEFAULT_META_BITS_DEMI_SPACE = "false";
/**
* Defines the number of bits that must be free in a FixedAllocator for
@@ -1478,11 +1478,17 @@
// Can handle minor store version incompatibility
final int storeVersion = strBuf.readInt();
- if ((storeVersion & 0xFF00) != (cVersion & 0xFF00)) {
+
+ switch ((storeVersion & 0xFF00)) {
+ case (cVersion & 0xFF00):
+ case (cVersionDemispace & 0xFF00):
+ break;
+ default:
throw new IllegalStateException(
"Incompatible RWStore header version: storeVersion="
+ storeVersion + ", cVersion=" + cVersion);
}
+
m_lastDeferredReleaseTime = strBuf.readLong();
if (strBuf.readInt() != cDefaultMetaBitsSize) {
throw new IllegalStateException("Store opened with unsupported metabits size");
@@ -3031,7 +3037,7 @@
final FixedOutputStream str = new FixedOutputStream(buf);
try {
- str.writeInt(cVersion);
+ str.writeInt(m_metaBitsAddr > 0 ? cVersionDemispace : cVersion);
str.writeLong(m_lastDeferredReleaseTime);
str.writeInt(cDefaultMetaBitsSize);
str.writeInt(m_allocSizes.length);
@@ -3068,7 +3074,7 @@
* writeMetaBits().
*/
//final long addr = physicalAddress(m_metaBitsAddr);
- final long addr = ((long) m_metaBitsAddr) << ALLOCATION_SCALEUP;
+ final long addr = m_metaBitsAddr < 0 ? physicalAddress(m_metaBitsAddr) : ((long) m_metaBitsAddr) << ALLOCATION_SCALEUP;
if (addr == 0) {
throw new IllegalStateException("Invalid metabits address: " + m_metaBitsAddr);
}
@@ -3081,7 +3087,7 @@
// Similar to writeMetaBits, we are no longer writing to a FixedAllocator managed region,
// so no latched address is provided
- m_writeCacheService.write(addr, ByteBuffer.wrap(buf), 0/*chk*/, false/*useChecksum*/, 0 /*latchedAddr*/);
+ m_writeCacheService.write(addr, ByteBuffer.wrap(buf), 0/*chk*/, false/*useChecksum*/, m_metaBitsAddr < 0 ? m_metaBitsAddr : 0 /*latchedAddr*/);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
@@ -3134,42 +3140,69 @@
* that we do not need to reallocate the metabits region when we are
* writing out the updated versions of the FixedAllocators).
*/
-// final long oldMetaBits = m_metaBitsAddr;
-// final int oldMetaBitsSize = (m_metaBits.length + m_allocSizes.length + 1) * 4;
-// m_metaBitsAddr = alloc(getRequiredMetaBitsStorage(), null);
+ if (m_metaBitsAddr > 0) {
+ // already using demi-space, remove from WCS
+ m_writeCacheService.removeWriteToAddr(convertAddr(-m_metaBitsAddr), 0);
+ } else {
+ final int reqmbc = getRequiredMetaBitsStorage();
+ int nmbaddr = 0;
+ // if > max alloc or explicitly use the demi-space, then drop through for demi-space
+ if ((!m_useMetabitsDemispace) && reqmbc < m_maxFixedAlloc) {
+ nmbaddr = alloc(reqmbc, null);
+ }
- /*
- * If m_metaBitsAddr < 0 then was allocated from FixedAllocators (for existing-store compatibility)
- */
- if (m_metaBitsAddr < 0) {
- if (physicalAddress(m_metaBitsAddr) == 0) {
- throw new IllegalStateException("Returned MetaBits Address not valid!");
- }
-
- final int oldMetaBitsSize = (m_metaBits.length + m_allocSizes.length + 1) * 4;
- // Call immediateFree - no need to defer freeof metaBits, this
- // has to stop somewhere!
- // No more allocations must be made
- immediateFree((int) m_metaBitsAddr, oldMetaBitsSize);
-
- m_metaBitsAddr = 0;
- }
-
+ // If existing allocation, then free it
+ if (m_metaBitsAddr < 0) {
+
+ final int oldMetaBitsSize = (m_metaBits.length
+ + m_allocSizes.length + 1) * 4;
+
+ // Call immediateFree - no need to defer freeof metaBits, this
+ // has to stop somewhere!
+ // No more allocations must be made
+ immediateFree((int) m_metaBitsAddr, oldMetaBitsSize);
+
+ }
+
+ m_metaBitsAddr = nmbaddr;
+ }
+
if (m_metaBitsAddr == 0) {
// Allocate special region to be able to store maximum metabits (128k of 2 64K demi-space
// Must be aligned on 128K boundary and allocations are made in units of 64K.
+ //
+ // May need to extend the file for teh demi-space!
while (m_nextAllocation % 2 != 0) {
m_nextAllocation--;
}
m_metaBitsAddr = -m_nextAllocation; // must be positive to differentiate from FixedAllocator address
m_nextAllocation -= 2; // allocate 2 * 64K
- } else { // remove previous write from WCS
- m_writeCacheService.removeWriteToAddr(convertAddr(-m_metaBitsAddr), 0);
+
+ // Check for file extension
+ while (m_nextAllocation <= m_fileSize) {
+ extendFile();
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Using Demi-space metabits");
}
- // Now "toggle" m_metaBitsAddr - 64K boundary
- m_metaBitsAddr ^= 0x01; // toggle zero or 64K offset
+ if (m_metaBitsAddr > 0) { // Demi-Space
+ // Now "toggle" m_metaBitsAddr - 64K boundary
+ m_metaBitsAddr ^= 0x01; // toggle zero or 64K offset
+ }
+ if (log.isDebugEnabled()) {
+ final long mbaddr;
+ if (m_metaBitsAddr < 0) {
+ mbaddr = physicalAddress((int) m_metaBitsAddr);
+ } else {
+ mbaddr = convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
+ }
+
+ log.debug("Writing metabits at " + mbaddr);
+ }
+
// There must be no buffered deferred frees
// assert m_deferredFreeOut.getBytesWritten() == 0;
@@ -3451,8 +3484,10 @@
* Versions
* 0x0300 - extended header to include reserved ints
* 0x0400 - removed explicit BlobAllocators
+ * 0x0500 - using metaBits demi-space
*/
final private int cVersion = 0x0400;
+ final private int cVersionDemispace = 0x0500;
/**
* cReservedMetaBits is the reserved space in the metaBits header
@@ -4233,31 +4268,41 @@
*
* @return long representation of metaBitsAddr PLUS the size
*/
- public long getMetaBitsAddr() {
- assert m_metaBitsAddr > 0;
-
- // long ret = physicalAddress((int) m_metaBitsAddr);
- long ret = convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
- ret <<= 16;
-
- // include space for version, allocSizes and deferred free info AND cDefaultMetaBitsSize
- final int metaBitsSize = cMetaHdrFields + m_metaBits.length + m_allocSizes.length;
- ret += metaBitsSize;
-
- if (log.isTraceEnabled())
- log.trace("Returning metabitsAddr: " + ret + ", for "
- + m_metaBitsAddr + " - " + m_metaBits.length + ", "
- + metaBitsSize);
+ public long getMetaBitsAddr() {
+ long ret = 0;
- return ret;
- }
+ if (m_metaBitsAddr < 0) {
+ ret = physicalAddress((int) m_metaBitsAddr);
+ } else {
+ // long ret = physicalAddress((int) m_metaBitsAddr);
+ ret = convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
+ }
+ ret <<= 16;
+ // include space for version, allocSizes and deferred free info AND
+ // cDefaultMetaBitsSize
+ final int metaBitsSize = cMetaHdrFields + m_metaBits.length
+ + m_allocSizes.length;
+ ret += metaBitsSize;
+
+ if (log.isTraceEnabled())
+ log.trace("Returning metabitsAddr: " + ret + ", for "
+ + m_metaBitsAddr + " - " + m_metaBits.length + ", "
+ + metaBitsSize);
+
+ return ret;
+ }
+
/**
*
- * @return the address of the metaBits demi-space
+ * @return the address of the metaBits
*/
- public long getMetaBitsDemiSpace() {
- return convertAddr(-m_metaBitsAddr);
+ public long getMetaBitsStoreAddress() {
+ if (m_metaBitsAddr < 0) {
+ return physicalAddress((int) m_metaBitsAddr);
+ } else {
+ return convertAddr(-m_metaBitsAddr); // maximum 48 bit address range
+ }
}
/**
@@ -7188,6 +7233,31 @@
return ret;
}
+
+ /**
+ * Forces a reset of the metabits allocation on the next commit.
+ * <p>
+ * Note that a side-effect of this is that there will be a memory leak
+ * of either a FixedAllocation slot or an existing demi-space.
+ * <p>
+ * @param useDemispace
+ * @return whether the storage has been modified.
+ */
+ public boolean ensureMetabitsDemispace(final boolean useDemispace) {
+ final boolean isDemispace = m_metaBitsAddr > 0;
+
+ if (isDemispace != useDemispace || m_useMetabitsDemispace != useDemispace) {
+ m_useMetabitsDemispace = useDemispace;
+
+ m_metaBitsAddr = 0;
+
+ m_recentAlloc = true; // force commit
+
+ return true;
+ } else {
+ return false;
+ }
+ }
// public void prepareForRebuild(final HARebuildRequest req) {
// assert m_rebuildRequest == null;
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-06-28 01:45:04 UTC (rev 8507)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2014-06-30 11:32:23 UTC (rev 8508)
@@ -1540,23 +1540,116 @@
public void test_metaAlloc() {
Journal store = (Journal) getStore();
- try {
+ try {
- final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
+ final RWStrategy bs = (RWStrategy) store.getBufferStrategy();
- final RWStore rw = bs.getStore();
- long realAddr = 0;
- for (int i = 0; i < 100000; i++) {
- int allocAddr = rw.metaAlloc();
+ final RWStore rw = bs.getStore();
+ long realAddr = 0;
+ for (int r = 0; r < 100; r++) {
+ for (int i = 0; i < 1000; i++) {
+ int allocAddr = rw.metaAlloc();
+
+ realAddr = rw.metaBit2Addr(allocAddr);
+ }
+ rw.commit();
+ }
- realAddr = rw.metaBit2Addr(allocAddr);
- }
- if(log.isInfoEnabled())log.info("metaAlloc lastAddr: " + realAddr);
+ if (log.isInfoEnabled())
+ log.info("metaAlloc lastAddr: " + realAddr);
} finally {
store.destroy();
}
}
+
+ /**
+ * Tests the MetabitsUtil to switch the demispace.
+ *
+ * If the address is a demispace then addr % 64K == 0.
+ *
+ * If the address is NOT a demispace then it should be less than first
+ * demispace
+ */
+ public void test_metabitsDemispace() {
+ Journal store = (Journal) getStore();
+ try {
+ RWStrategy bs = (RWStrategy) store.getBufferStrategy();
+ RWStore rw = bs.getStore();
+ final String fname = rw.getStoreFile().getAbsolutePath();
+
+ store.commit();
+
+ final long fa1 = rw.getMetaBitsStoreAddress();
+
+ rw.ensureMetabitsDemispace(true);
+ store.commit();
+
+ final long ds1 = rw.getMetaBitsStoreAddress();
+
+ assertTrue((ds1 & 0xFFFF) == 0); // MOD 64K
+ assertTrue(ds1 > fa1);
+
+ rw.ensureMetabitsDemispace(false);
+ store.commit();
+
+ final long fa2 = rw.getMetaBitsStoreAddress();
+
+ assertTrue(ds1 > fa2);
+
+ rw.ensureMetabitsDemispace(true);
+ store.commit();
+
+ final long ds2 = rw.getMetaBitsStoreAddress();
+
+ assertTrue((ds2 & 0xFFFF) == 0);
+ assertTrue(ds2 > ds1);
+
+ // Now use MetaBitsUtil
+
+ store.close();
+
+ MetabitsUtil.main(new String[] { "-store", fname, "-usedemispace", "false"});
+
+ store = getExplicitStore(fname);
+
+ bs = (RWStrategy) store.getBufferStrategy();
+ rw = bs.getStore();
+ final long fa3 = rw.getMetaBitsStoreAddress();
+
+ assertTrue(fa3 < ds1);
+
+ store.close();
+
+ MetabitsUtil.main(new String[] { "-store", fname, "-usedemispace", "true"});
+
+ store = getExplicitStore(fname);
+
+ bs = (RWStrategy) store.getBufferStrategy();
+ rw = bs.getStore();
+
+ final long ds3 = rw.getMetaBitsStoreAddress();
+ assertTrue((ds3 & 0xFFFF) == 0);
+ assertTrue(ds3 > ds2);
+
+ } finally {
+ store.destroy();
+ }
+ }
+
+ Journal getExplicitStore(String storeFile) {
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Options.FILE, storeFile);
+
+ properties.setProperty(Options.BUFFER_MODE,
+ BufferMode.DiskRW.toString());
+
+ return new Journal(properties);// .getBufferStrategy();
+
+ }
+
static class DummyAllocationContext implements IAllocationContext {
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
