Menu
▾
▴
bigdata-commit
|
From: <tho...@us...> - 2014-03-16 10:55:04
|
Revision: 7983
http://sourceforge.net/p/bigdata/code/7983
Author: thompsonbry
Date: 2014-03-16 10:55:01 +0000 (Sun, 16 Mar 2014)
Log Message:
-----------
Modified DumpJournal to track the #of errors when dumping the pages of an index and continue (unless interrupted or cancelled).
See #855 (AssertionError: Child does not have persistent identity)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-03-15 14:00:37 UTC (rev 7982)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-03-16 10:55:01 UTC (rev 7983)
@@ -86,6 +86,7 @@
import com.bigdata.resources.OverflowManager;
import com.bigdata.service.DataService;
import com.bigdata.service.Split;
+import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
@@ -1537,11 +1538,29 @@
for (int i = 0; i <= nkeys; i++) {
- // normal read following the node hierarchy, using cache, etc.
- final AbstractNode<?> child = ((Node) node).getChild(i);
+ try {
+
+ // normal read following the node hierarchy, using cache, etc.
+ final AbstractNode<?> child = ((Node) node).getChild(i);
- // recursive dump
- dumpPages(ndx, child, stats);
+ // recursive dump
+ dumpPages(ndx, child, stats);
+
+ } catch (Throwable t) {
+
+ if (InnerCause.isInnerCause(t, InterruptedException.class)
+ || InnerCause.isInnerCause(t,
+ InterruptedException.class)) {
+ throw new RuntimeException(t);
+ }
+ /*
+ * Log the error and track the #of errors, but keep scanning
+ * the index.
+ */
+ stats.nerrors++;
+ log.error("Error reading child[i=" + i + "]: " + t, t);
+ continue;
+ }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java 2014-03-15 14:00:37 UTC (rev 7982)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/PageStats.java 2014-03-16 10:55:01 UTC (rev 7983)
@@ -54,6 +54,8 @@
* {@link #SLOT_SIZES}.
*/
public long blobs;
+ /** The #of errors encountered during traversal. */
+ public long nerrors;
/**
* This map is used to report the histogram of pages based on the actual
* byte count of the user data in the allocation when the backing slot size
@@ -126,6 +128,7 @@
sb.append(",maxLeafBytes=" + maxLeafBytes);
sb.append(",bytesPerNode=" + getBytesPerNode());
sb.append(",bytesPerLeaf=" + getBytesPerLeaf());
+ sb.append(",nerros=" + nerrors);
final long npages = (nleaves + nnodes);
for (int i = 0; i < SLOT_SIZES.length; i++) {
final long slotsThisSize = histogram[i];
@@ -174,6 +177,8 @@
sb.append('\t');
sb.append("nentries");
sb.append('\t');
+ sb.append("nerrors");
+ sb.append('\t');
sb.append("nodeBytes");
sb.append('\t');
sb.append("leafBytes");
@@ -241,6 +246,8 @@
sb.append('\t');
sb.append(stats.ntuples);
sb.append('\t');
+ sb.append(stats.nerrors);
+ sb.append('\t');
sb.append(stats.nodeBytes);
sb.append('\t');
sb.append(stats.leafBytes);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-09-02 13:35:02
|
Revision: 8632
http://sourceforge.net/p/bigdata/code/8632
Author: martyncutcher
Date: 2014-09-02 13:34:48 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Provide protection against eviction of Mutable BTree nodes during read-only operation #855. Achieved by checking for read-lock held on current thread and avoiding a touch that could cause eviction.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -36,9 +36,14 @@
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.FutureTask;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -87,6 +92,7 @@
import com.bigdata.service.DataService;
import com.bigdata.service.Split;
import com.bigdata.util.InnerCause;
+import com.bigdata.util.StackInfoReport;
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
@@ -249,6 +255,8 @@
* mutation.
*/
final protected boolean readOnly;
+
+ final ReentrantReadWriteLock lock;
/**
* Optional cache for {@link INodeData} and {@link ILeafData} instances and
@@ -959,6 +967,8 @@
this.store = store;
this.readOnly = readOnly;
+
+ this.lock = UnisolatedReadWriteIndex.getReadWriteLock(this);
// /*
// * The Memoizer is not used by the mutable B+Tree since it is not safe
@@ -1976,6 +1986,8 @@
}
};
+
+ volatile Throwable error;
final public Object insert(Object key, Object value) {
@@ -3368,7 +3380,14 @@
}
- doSyncTouch(node);
+ // If a mutable tree and a read-only operation then ensure there are no evictions from this thread
+ final Integer rcount = threadLockMap.get(Thread.currentThread().getId());
+ if (!isReadOnly() && rcount != null /* && false this should stochastically fail!*/) {
+ // NOP
+ assert rcount.intValue() > 0;
+ } else {
+ doSyncTouch(node);
+ }
}
@@ -3432,7 +3451,7 @@
*/
// assert isReadOnly() || ndistinctOnWriteRetentionQueue > 0;
-
+
node.referenceCount++;
if (!writeRetentionQueue.add(node)) {
@@ -3597,6 +3616,15 @@
}
+ private void badNode(final AbstractNode<?> node) {
+// try {
+// Thread.sleep(50);
+// } catch (InterruptedException e) {
+// // ignore;
+// }
+ throw new AssertionError("ReadOnly and identity: " + node.identity);
+ }
+
/**
* Codes the node and writes the coded record on the store (non-recursive).
* The node MUST be dirty. If the node has a parent, then the parent is
@@ -3616,7 +3644,31 @@
* @return The persistent identity assigned by the store.
*/
protected long writeNodeOrLeaf(final AbstractNode<?> node) {
-
+
+ /*
+ * The check for the writeLock held on an update is currently disabled since
+ * a number of Unit tests failed with it active. The exception information
+ * indicates that the writeLock is not held in any of the test failure scenarios, so
+ * the thread check does not fail due to submitted tasks.
+ * TODO: we need to clarify the requirements for holding the writeLock across any update
+ */
+ if (false && !lock.isWriteLockedByCurrentThread()) {
+ throw new AssertionError("WriteLock not held, readOnly: " + this.isReadOnly() + ", locked by other: " + lock.isWriteLocked());
+ }
+
+ if (error != null) {
+ throw new IllegalStateException("BTree is in an error state", error);
+ }
+
+// if (node.writing != null) {
+// final Throwable remote = node.writing;
+// node.writing = null;
+//
+// throw new AssertionError("Duplicate Call - ", remote);
+// }
+//
+ // node.writing = new StackInfoReport("Write Guard");
+
assert root != null; // i.e., isOpen().
assert node != null;
assert node.btree == this;
@@ -3640,7 +3692,10 @@
* TestMROWTransactions might also demonstrate an issue
* occasionally. If so, then check for the same root cause.
*/
- assert !node.isReadOnly();
+ if (node.isReadOnly()) {
+ badNode(node); // supports debugging
+ }
+ assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. Also StressTestUnisolatedReadWriteIndex. See http://trac.bigdata.com/ticket/343
assertNotReadOnly();
/*
@@ -3740,6 +3795,14 @@
// No longer dirty (prevents re-coding on re-eviction).
node.setDirty(false);
+// if (node.writing == null) {
+// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
+//
+// throw new AssertionError("Concurrent modification of thread guard");
+// }
+
+// node.writing = null;
+
return 0L;
}
@@ -3829,6 +3892,14 @@
}
+// if (node.writing == null) {
+// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
+//
+// throw new AssertionError("Concurrent modification of thread guard");
+// }
+//
+// node.writing = null;
+
return addr;
}
@@ -3855,40 +3926,6 @@
if (addr == IRawStore.NULL)
throw new IllegalArgumentException();
-// final Long addr2 = Long.valueOf(addr);
-//
-// if (storeCache != null) {
-//
-// // test cache : will touch global LRU iff found.
-// final IAbstractNodeData data = (IAbstractNodeData) storeCache
-// .get(addr);
-//
-// if (data != null) {
-//
-// // Node and Leaf MUST NOT make it into the global LRU or store
-// // cache!
-// assert !(data instanceof AbstractNode<?>);
-//
-// final AbstractNode<?> node;
-//
-// if (data.isLeaf()) {
-//
-// node = nodeSer.nodeFactory.allocLeaf(this, addr,
-// (ILeafData) data);
-//
-// } else {
-//
-// node = nodeSer.nodeFactory.allocNode(this, addr,
-// (INodeData) data);
-//
-// }
-//
-// // cache hit.
-// return node;
-//
-// }
-//
-// }
final ByteBuffer tmp;
{
@@ -3945,21 +3982,6 @@
}
-// if (storeCache != null) {
-//
-// // update cache : will touch global LRU iff cache is modified.
-// final IAbstractNodeData data2 = (IAbstractNodeData) storeCache
-// .putIfAbsent(addr2, data);
-//
-// if (data2 != null) {
-//
-// // concurrent insert, use winner's value.
-// data = data2;
-//
-// }
-//
-// }
-
// wrap as Node or Leaf.
final AbstractNode<?> node = nodeSer.wrap(this, addr, data);
@@ -4303,4 +4325,35 @@
}
+
+ /**
+ * Maintain count of readLocks on this Thread. Use ThreadLocal for now, but
+ * could switch to HashMap if necessary
+ */
+// ThreadLocal<AtomicInteger> threadReadLockCount = new ThreadLocal<AtomicInteger>();
+ ConcurrentHashMap<Long, Integer> threadLockMap = new ConcurrentHashMap<Long, Integer>();
+
+ public void readLockedThread() {
+// final AtomicInteger ai = threadReadLockCount.get();
+// if (ai == null) {
+// threadReadLockCount.set(new AtomicInteger(1));
+// } else {
+// ai.incrementAndGet();
+// }
+ final long thisThreadId = Thread.currentThread().getId();
+ final Integer entry = threadLockMap.get(thisThreadId);
+ final Integer newVal = entry == null ? 1 : 1 + entry.intValue();
+ threadLockMap.put(thisThreadId, newVal);
+ }
+
+ public void readUnlockedThread() {
+ final long thisThreadId = Thread.currentThread().getId();
+ final Integer entry = threadLockMap.get(thisThreadId);
+ assert entry != null;
+ if (entry.intValue() == 1) {
+ threadLockMap.remove(thisThreadId);
+ } else {
+ threadLockMap.put(thisThreadId, entry.intValue() - 1);
+ }
+ }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -92,6 +92,12 @@
* on hand so that we can set this field.
*/
final transient protected AbstractBTree btree;
+
+ /**
+ * Can be used in debugging, testing or high integrity live scenarios to check
+ * for concurrent writes triggered by evictions. This is currently disabled.
+ */
+ // volatile transient Throwable writing = null;
/**
* The parent of this node. This is null for the root node. The parent is
@@ -537,8 +543,11 @@
*/
parent = (Node) parent.copyOnWrite(oldId);
+ assert !parent.isPersistent();
+ } else {
+ assert !parent.isPersistent();
}
-
+
/*
* Replace the reference to this child with the reference to the
* new child. This makes the old child inaccessible via
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -29,6 +29,7 @@
import java.lang.ref.WeakReference;
import java.lang.reflect.Constructor;
import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
@@ -899,23 +900,24 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/343
* @see https://sourceforge.net/apps/trac/bigdata/ticket/440
*/
-// final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
- final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
- lock.lock();
- try {
+ final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
+ //final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
+ //lock.lock();
+ try {
+ //synchronized(this) {
+ if (/* autoCommit && */needsCheckpoint()) {
+
+ /*
+ * Flush the btree, write a checkpoint record, and return the
+ * address of that checkpoint record. The [checkpoint] reference
+ * is also updated.
+ */
+
+ return _writeCheckpoint2();
+
+ }
+ //}
- if (/* autoCommit && */needsCheckpoint()) {
-
- /*
- * Flush the btree, write a checkpoint record, and return the
- * address of that checkpoint record. The [checkpoint] reference
- * is also updated.
- */
-
- return _writeCheckpoint2();
-
- }
-
/*
* There have not been any writes on this btree or auto-commit is
* disabled.
@@ -1109,14 +1111,14 @@
@Override
final public long getRecordVersion() {
- return recordVersion;
+ return recordVersion;
}
@Override
final public long getMetadataAddr() {
- return metadata.getMetadataAddr();
+ return metadata.getMetadataAddr();
}
@@ -1312,7 +1314,7 @@
@Override
public long handleCommit(final long commitTime) {
- return writeCheckpoint2().getCheckpointAddr();
+ return writeCheckpoint2().getCheckpointAddr();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -60,88 +60,78 @@
}
final AbstractBTree btree = node.btree;
+
+ if (btree.error != null) {
+ throw new IllegalStateException("BTree is in an error state", btree.error);
+ }
+
+ try {
+ // Note: This assert can be violated for a read-only B+Tree since
+ // there is less synchronization.
+ assert btree.isReadOnly()
+ || btree.ndistinctOnWriteRetentionQueue > 0;
- // Note: This assert can be violated for a read-only B+Tree since there
- // is less synchronization.
- assert btree.isReadOnly() || btree.ndistinctOnWriteRetentionQueue > 0;
+ btree.ndistinctOnWriteRetentionQueue--;
- btree.ndistinctOnWriteRetentionQueue--;
-
- if (node.deleted) {
+ if (node.deleted) {
- /*
- * Deleted nodes are ignored as they are evicted from the queue.
- */
+ /*
+ * Deleted nodes are ignored as they are evicted from the queue.
+ */
- return;
+ return;
- }
+ }
- // this does not permit transient nodes to be coded.
- if (node.dirty && btree.store != null) {
-// // this causes transient nodes to be coded on eviction.
-// if (node.dirty) {
-
- if (node.isLeaf()) {
+ // this does not permit transient nodes to be coded.
+ if (node.dirty && btree.store != null) {
+ // // this causes transient nodes to be coded on eviction.
+ // if (node.dirty) {
- /*
- * A leaf is written out directly.
- */
-
- btree.writeNodeOrLeaf(node);
+ if (node.isLeaf()) {
- } else {
+ /*
+ * A leaf is written out directly.
+ */
- /*
- * A non-leaf node must be written out using a post-order
- * traversal so that all dirty children are written through
- * before the dirty parent. This is required in order to
- * assign persistent identifiers to the dirty children.
- */
+ btree.writeNodeOrLeaf(node);
- btree.writeNodeRecursive(node);
+ } else {
- }
+ /*
+ * A non-leaf node must be written out using a post-order
+ * traversal so that all dirty children are written through
+ * before the dirty parent. This is required in order to
+ * assign persistent identifiers to the dirty children.
+ */
- // is a coded data record.
- assert node.isCoded();
-
- // no longer dirty.
- assert !node.dirty;
-
- if (btree.store != null) {
-
- // object is persistent (has assigned addr).
- assert ref.identity != PO.NULL;
-
- }
-
- } // isDirty
+ btree.writeNodeRecursive(node);
- // This does not insert into the cache. That is handled by writeNodeOrLeaf.
-// if (btree.globalLRU != null) {
-//
-// /*
-// * Add the INodeData or ILeafData object to the global LRU, NOT the
-// * Node or Leaf.
-// *
-// * Note: The global LRU touch only occurs on eviction from the write
-// * retention queue. This is nice because it limits the touches on
-// * the global LRU, which could otherwise be a hot spot. We do a
-// * touch whether or not the node was persisted since we are likely
-// * to return to the node in either case.
-// */
-//
-// final IAbstractNodeData delegate = node.getDelegate();
-//
-// assert delegate != null : node.toString();
-//
-// assert delegate.isCoded() : node.toString();
-//
-// btree.globalLRU.add(delegate);
-//
-// }
+ }
+ // is a coded data record.
+ assert node.isCoded();
+
+ // no longer dirty.
+ assert !node.dirty;
+
+ if (btree.store != null) {
+
+ // object is persistent (has assigned addr).
+ assert ref.identity != PO.NULL;
+
+ }
+
+ } // isDirty
+ } catch (Throwable e) {
+ // If the btree is mutable that we are trying to evict a node then
+ // mark it in error
+ if (!btree.readOnly)
+ btree.error = e;
+
+ throw new Error(e);
+ }
+
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 12:48:16 UTC (rev 8631)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 13:34:48 UTC (rev 8632)
@@ -30,6 +30,7 @@
import java.util.Iterator;
import java.util.WeakHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -159,7 +160,7 @@
}
-// writeLock.lock();
+ // writeLock.lock();
if(!writeLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -189,8 +190,6 @@
*/
public Lock readLock() {
- final Lock readLock = readWriteLock.readLock();
-
try {
if(log.isDebugEnabled()) {
@@ -201,7 +200,7 @@
}
-// readLock.lock();
+ // readLock.lock();
if(!readLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -214,12 +213,73 @@
throw new RuntimeException(ex);
}
+
+ // We have attained the read lock
return readLock;
}
/**
+ * WrappedLock is used to intercept lock/unlock calls to the readLock to
+ * trigger calls to the AbstractBTree thread lock management that can be
+ * used to identify whether the readlock is held by the current thread.
+ * <p>
+ * This is tested in AbstractBTree.touch() to determine whether the touch
+ * should be ignored or trigger potential evictions.
+ */
+ class WrappedLock implements Lock {
+
+ final Lock delegate;
+
+ WrappedLock(Lock delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void lock() {
+ delegate.lock();
+
+ ndx.readLockedThread();
+ }
+
+ @Override
+ public void lockInterruptibly() throws InterruptedException {
+ delegate.lockInterruptibly();
+ }
+
+ @Override
+ public boolean tryLock() {
+ final boolean ret = delegate.tryLock();
+ if (ret) {
+ ndx.readLockedThread();
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean tryLock(long time, TimeUnit unit)
+ throws InterruptedException {
+ final boolean ret = delegate.tryLock(time, unit);
+ if (ret) {
+ ndx.readLockedThread();
+ }
+ return ret;
+ }
+
+ @Override
+ public void unlock() {
+ delegate.unlock();
+ ndx.readUnlockedThread();
+ }
+
+ @Override
+ public Condition newCondition() {
+ return delegate.newCondition();
+ }
+ }
+
+ /**
* Acquire an appropriate lock depending on whether or not the procedure
* asserts that it is read-only.
*
@@ -268,7 +328,9 @@
*/
final private ReadWriteLock readWriteLock;
- /**
+ final private Lock readLock;
+
+ /**
* Canonicalizing mapping for the locks used to control access to the
* unisolated index.
*/
@@ -349,6 +411,8 @@
this.readWriteLock = getReadWriteLock(ndx);
+ this.readLock = new WrappedLock(readWriteLock.readLock());
+
}
/**
@@ -382,8 +446,8 @@
return readWriteLock;
}
- }
-
+ }
+
@Override
public String toString() {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2014-09-02 14:36:45
|
Revision: 8634
http://sourceforge.net/p/bigdata/code/8634
Author: martyncutcher
Date: 2014-09-02 14:36:42 +0000 (Tue, 02 Sep 2014)
Log Message:
-----------
Undo previous commit r8032, prior to re-syncing post-review
Revision Links:
--------------
http://sourceforge.net/p/bigdata/code/8032
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Dec 19, 2006
+ *
*/
package com.bigdata.btree;
@@ -36,14 +37,9 @@
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.FutureTask;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -92,7 +88,6 @@
import com.bigdata.service.DataService;
import com.bigdata.service.Split;
import com.bigdata.util.InnerCause;
-import com.bigdata.util.StackInfoReport;
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
@@ -255,8 +250,6 @@
* mutation.
*/
final protected boolean readOnly;
-
- final ReentrantReadWriteLock lock;
/**
* Optional cache for {@link INodeData} and {@link ILeafData} instances and
@@ -967,8 +960,6 @@
this.store = store;
this.readOnly = readOnly;
-
- this.lock = UnisolatedReadWriteIndex.getReadWriteLock(this);
// /*
// * The Memoizer is not used by the mutable B+Tree since it is not safe
@@ -1986,8 +1977,6 @@
}
};
-
- volatile Throwable error;
final public Object insert(Object key, Object value) {
@@ -3380,14 +3369,7 @@
}
- // If a mutable tree and a read-only operation then ensure there are no evictions from this thread
- final Integer rcount = threadLockMap.get(Thread.currentThread().getId());
- if (!isReadOnly() && rcount != null /* && false this should stochastically fail!*/) {
- // NOP
- assert rcount.intValue() > 0;
- } else {
- doSyncTouch(node);
- }
+ doSyncTouch(node);
}
@@ -3451,7 +3433,7 @@
*/
// assert isReadOnly() || ndistinctOnWriteRetentionQueue > 0;
-
+
node.referenceCount++;
if (!writeRetentionQueue.add(node)) {
@@ -3616,15 +3598,6 @@
}
- private void badNode(final AbstractNode<?> node) {
-// try {
-// Thread.sleep(50);
-// } catch (InterruptedException e) {
-// // ignore;
-// }
- throw new AssertionError("ReadOnly and identity: " + node.identity);
- }
-
/**
* Codes the node and writes the coded record on the store (non-recursive).
* The node MUST be dirty. If the node has a parent, then the parent is
@@ -3644,31 +3617,7 @@
* @return The persistent identity assigned by the store.
*/
protected long writeNodeOrLeaf(final AbstractNode<?> node) {
-
- /*
- * The check for the writeLock held on an update is currently disabled since
- * a number of Unit tests failed with it active. The exception information
- * indicates that the writeLock is not held in any of the test failure scenarios, so
- * the thread check does not fail due to submitted tasks.
- * TODO: we need to clarify the requirements for holding the writeLock across any update
- */
- if (false && !lock.isWriteLockedByCurrentThread()) {
- throw new AssertionError("WriteLock not held, readOnly: " + this.isReadOnly() + ", locked by other: " + lock.isWriteLocked());
- }
-
- if (error != null) {
- throw new IllegalStateException("BTree is in an error state", error);
- }
-
-// if (node.writing != null) {
-// final Throwable remote = node.writing;
-// node.writing = null;
-//
-// throw new AssertionError("Duplicate Call - ", remote);
-// }
-//
- // node.writing = new StackInfoReport("Write Guard");
-
+
assert root != null; // i.e., isOpen().
assert node != null;
assert node.btree == this;
@@ -3692,10 +3641,7 @@
* TestMROWTransactions might also demonstrate an issue
* occasionally. If so, then check for the same root cause.
*/
- if (node.isReadOnly()) {
- badNode(node); // supports debugging
- }
- assert !node.isReadOnly(); // FIXME Occasional CI errors on this assert for TestMROWTransactions. Also StressTestUnisolatedReadWriteIndex. See http://trac.bigdata.com/ticket/343
+ assert !node.isReadOnly();
assertNotReadOnly();
/*
@@ -3795,14 +3741,6 @@
// No longer dirty (prevents re-coding on re-eviction).
node.setDirty(false);
-// if (node.writing == null) {
-// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
-//
-// throw new AssertionError("Concurrent modification of thread guard");
-// }
-
-// node.writing = null;
-
return 0L;
}
@@ -3892,14 +3830,6 @@
}
-// if (node.writing == null) {
-// log.warn("Concurrent modification of thread guard", new RuntimeException("WTF2: " + node.hashCode()));
-//
-// throw new AssertionError("Concurrent modification of thread guard");
-// }
-//
-// node.writing = null;
-
return addr;
}
@@ -3926,6 +3856,40 @@
if (addr == IRawStore.NULL)
throw new IllegalArgumentException();
+// final Long addr2 = Long.valueOf(addr);
+//
+// if (storeCache != null) {
+//
+// // test cache : will touch global LRU iff found.
+// final IAbstractNodeData data = (IAbstractNodeData) storeCache
+// .get(addr);
+//
+// if (data != null) {
+//
+// // Node and Leaf MUST NOT make it into the global LRU or store
+// // cache!
+// assert !(data instanceof AbstractNode<?>);
+//
+// final AbstractNode<?> node;
+//
+// if (data.isLeaf()) {
+//
+// node = nodeSer.nodeFactory.allocLeaf(this, addr,
+// (ILeafData) data);
+//
+// } else {
+//
+// node = nodeSer.nodeFactory.allocNode(this, addr,
+// (INodeData) data);
+//
+// }
+//
+// // cache hit.
+// return node;
+//
+// }
+//
+// }
final ByteBuffer tmp;
{
@@ -3982,6 +3946,21 @@
}
+// if (storeCache != null) {
+//
+// // update cache : will touch global LRU iff cache is modified.
+// final IAbstractNodeData data2 = (IAbstractNodeData) storeCache
+// .putIfAbsent(addr2, data);
+//
+// if (data2 != null) {
+//
+// // concurrent insert, use winner's value.
+// data = data2;
+//
+// }
+//
+// }
+
// wrap as Node or Leaf.
final AbstractNode<?> node = nodeSer.wrap(this, addr, data);
@@ -4325,35 +4304,4 @@
}
-
- /**
- * Maintain count of readLocks on this Thread. Use ThreadLocal for now, but
- * could switch to HashMap if necessary
- */
-// ThreadLocal<AtomicInteger> threadReadLockCount = new ThreadLocal<AtomicInteger>();
- ConcurrentHashMap<Long, Integer> threadLockMap = new ConcurrentHashMap<Long, Integer>();
-
- public void readLockedThread() {
-// final AtomicInteger ai = threadReadLockCount.get();
-// if (ai == null) {
-// threadReadLockCount.set(new AtomicInteger(1));
-// } else {
-// ai.incrementAndGet();
-// }
- final long thisThreadId = Thread.currentThread().getId();
- final Integer entry = threadLockMap.get(thisThreadId);
- final Integer newVal = entry == null ? 1 : 1 + entry.intValue();
- threadLockMap.put(thisThreadId, newVal);
- }
-
- public void readUnlockedThread() {
- final long thisThreadId = Thread.currentThread().getId();
- final Integer entry = threadLockMap.get(thisThreadId);
- assert entry != null;
- if (entry.intValue() == 1) {
- threadLockMap.remove(thisThreadId);
- } else {
- threadLockMap.put(thisThreadId, entry.intValue() - 1);
- }
- }
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Nov 15, 2006
+ *
*/
package com.bigdata.btree;
@@ -92,12 +93,6 @@
* on hand so that we can set this field.
*/
final transient protected AbstractBTree btree;
-
- /**
- * Can be used in debugging, testing or high integrity live scenarios to check
- * for concurrent writes triggered by evictions. This is currently disabled.
- */
- // volatile transient Throwable writing = null;
/**
* The parent of this node. This is null for the root node. The parent is
@@ -543,11 +538,8 @@
*/
parent = (Node) parent.copyOnWrite(oldId);
- assert !parent.isPersistent();
- } else {
- assert !parent.isPersistent();
}
-
+
/*
* Replace the reference to this child with the reference to the
* new child. This makes the old child inaccessible via
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/BTree.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,13 +23,13 @@
*/
/*
* Created on Nov 15, 2006
+ *
*/
package com.bigdata.btree;
import java.lang.ref.WeakReference;
import java.lang.reflect.Constructor;
import java.util.Iterator;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
@@ -900,24 +900,23 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/343
* @see https://sourceforge.net/apps/trac/bigdata/ticket/440
*/
- final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
- //final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
- //lock.lock();
- try {
- //synchronized(this) {
- if (/* autoCommit && */needsCheckpoint()) {
-
- /*
- * Flush the btree, write a checkpoint record, and return the
- * address of that checkpoint record. The [checkpoint] reference
- * is also updated.
- */
-
- return _writeCheckpoint2();
-
- }
- //}
+// final Lock lock = new UnisolatedReadWriteIndex(this).writeLock();
+ final Lock lock = UnisolatedReadWriteIndex.getReadWriteLock(this).writeLock();
+ lock.lock();
+ try {
+ if (/* autoCommit && */needsCheckpoint()) {
+
+ /*
+ * Flush the btree, write a checkpoint record, and return the
+ * address of that checkpoint record. The [checkpoint] reference
+ * is also updated.
+ */
+
+ return _writeCheckpoint2();
+
+ }
+
/*
* There have not been any writes on this btree or auto-commit is
* disabled.
@@ -1111,14 +1110,14 @@
@Override
final public long getRecordVersion() {
- return recordVersion;
+ return recordVersion;
}
@Override
final public long getMetadataAddr() {
- return metadata.getMetadataAddr();
+ return metadata.getMetadataAddr();
}
@@ -1314,7 +1313,7 @@
@Override
public long handleCommit(final long commitTime) {
- return writeCheckpoint2().getCheckpointAddr();
+ return writeCheckpoint2().getCheckpointAddr();
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/DefaultEvictionListener.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Nov 17, 2006
+ *
*/
package com.bigdata.btree;
@@ -60,78 +61,88 @@
}
final AbstractBTree btree = node.btree;
-
- if (btree.error != null) {
- throw new IllegalStateException("BTree is in an error state", btree.error);
- }
-
- try {
- // Note: This assert can be violated for a read-only B+Tree since
- // there is less synchronization.
- assert btree.isReadOnly()
- || btree.ndistinctOnWriteRetentionQueue > 0;
- btree.ndistinctOnWriteRetentionQueue--;
+ // Note: This assert can be violated for a read-only B+Tree since there
+ // is less synchronization.
+ assert btree.isReadOnly() || btree.ndistinctOnWriteRetentionQueue > 0;
- if (node.deleted) {
+ btree.ndistinctOnWriteRetentionQueue--;
+
+ if (node.deleted) {
- /*
- * Deleted nodes are ignored as they are evicted from the queue.
- */
+ /*
+ * Deleted nodes are ignored as they are evicted from the queue.
+ */
- return;
+ return;
- }
+ }
- // this does not permit transient nodes to be coded.
- if (node.dirty && btree.store != null) {
- // // this causes transient nodes to be coded on eviction.
- // if (node.dirty) {
+ // this does not permit transient nodes to be coded.
+ if (node.dirty && btree.store != null) {
+// // this causes transient nodes to be coded on eviction.
+// if (node.dirty) {
+
+ if (node.isLeaf()) {
- if (node.isLeaf()) {
+ /*
+ * A leaf is written out directly.
+ */
+
+ btree.writeNodeOrLeaf(node);
- /*
- * A leaf is written out directly.
- */
+ } else {
- btree.writeNodeOrLeaf(node);
+ /*
+ * A non-leaf node must be written out using a post-order
+ * traversal so that all dirty children are written through
+ * before the dirty parent. This is required in order to
+ * assign persistent identifiers to the dirty children.
+ */
- } else {
+ btree.writeNodeRecursive(node);
- /*
- * A non-leaf node must be written out using a post-order
- * traversal so that all dirty children are written through
- * before the dirty parent. This is required in order to
- * assign persistent identifiers to the dirty children.
- */
+ }
- btree.writeNodeRecursive(node);
+ // is a coded data record.
+ assert node.isCoded();
+
+ // no longer dirty.
+ assert !node.dirty;
+
+ if (btree.store != null) {
+
+ // object is persistent (has assigned addr).
+ assert ref.identity != PO.NULL;
+
+ }
+
+ } // isDirty
- }
+ // This does not insert into the cache. That is handled by writeNodeOrLeaf.
+// if (btree.globalLRU != null) {
+//
+// /*
+// * Add the INodeData or ILeafData object to the global LRU, NOT the
+// * Node or Leaf.
+// *
+// * Note: The global LRU touch only occurs on eviction from the write
+// * retention queue. This is nice because it limits the touches on
+// * the global LRU, which could otherwise be a hot spot. We do a
+// * touch whether or not the node was persisted since we are likely
+// * to return to the node in either case.
+// */
+//
+// final IAbstractNodeData delegate = node.getDelegate();
+//
+// assert delegate != null : node.toString();
+//
+// assert delegate.isCoded() : node.toString();
+//
+// btree.globalLRU.add(delegate);
+//
+// }
- // is a coded data record.
- assert node.isCoded();
-
- // no longer dirty.
- assert !node.dirty;
-
- if (btree.store != null) {
-
- // object is persistent (has assigned addr).
- assert ref.identity != PO.NULL;
-
- }
-
- } // isDirty
- } catch (Throwable e) {
- // If the btree is mutable that we are trying to evict a node then
- // mark it in error
- if (!btree.readOnly)
- btree.error = e;
-
- throw new Error(e);
- }
-
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 13:45:09 UTC (rev 8633)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java 2014-09-02 14:36:42 UTC (rev 8634)
@@ -23,6 +23,7 @@
*/
/*
* Created on Jan 10, 2008
+ *
*/
package com.bigdata.btree;
@@ -30,7 +31,6 @@
import java.util.Iterator;
import java.util.WeakHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -160,7 +160,7 @@
}
- // writeLock.lock();
+// writeLock.lock();
if(!writeLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -190,6 +190,8 @@
*/
public Lock readLock() {
+ final Lock readLock = readWriteLock.readLock();
+
try {
if(log.isDebugEnabled()) {
@@ -200,7 +202,7 @@
}
- // readLock.lock();
+// readLock.lock();
if(!readLock.tryLock( LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
@@ -213,73 +215,12 @@
throw new RuntimeException(ex);
}
-
- // We have attained the read lock
return readLock;
}
/**
- * WrappedLock is used to intercept lock/unlock calls to the readLock to
- * trigger calls to the AbstractBTree thread lock management that can be
- * used to identify whether the readlock is held by the current thread.
- * <p>
- * This is tested in AbstractBTree.touch() to determine whether the touch
- * should be ignored or trigger potential evictions.
- */
- class WrappedLock implements Lock {
-
- final Lock delegate;
-
- WrappedLock(Lock delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public void lock() {
- delegate.lock();
-
- ndx.readLockedThread();
- }
-
- @Override
- public void lockInterruptibly() throws InterruptedException {
- delegate.lockInterruptibly();
- }
-
- @Override
- public boolean tryLock() {
- final boolean ret = delegate.tryLock();
- if (ret) {
- ndx.readLockedThread();
- }
- return ret;
- }
-
- @Override
- public boolean tryLock(long time, TimeUnit unit)
- throws InterruptedException {
- final boolean ret = delegate.tryLock(time, unit);
- if (ret) {
- ndx.readLockedThread();
- }
- return ret;
- }
-
- @Override
- public void unlock() {
- delegate.unlock();
- ndx.readUnlockedThread();
- }
-
- @Override
- public Condition newCondition() {
- return delegate.newCondition();
- }
- }
-
- /**
* Acquire an appropriate lock depending on whether or not the procedure
* asserts that it is read-only.
*
@@ -328,9 +269,7 @@
*/
final private ReadWriteLock readWriteLock;
- final private Lock readLock;
-
- /**
+ /**
* Canonicalizing mapping for the locks used to control access to the
* unisolated index.
*/
@@ -411,8 +350,6 @@
this.readWriteLock = getReadWriteLock(ndx);
- this.readLock = new WrappedLock(readWriteLock.readLock());
-
}
/**
@@ -446,8 +383,8 @@
return readWriteLock;
}
- }
-
+ }
+
@Override
public String toString() {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X