Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4523] branches/QUADS_QUERY_BRANCH
|
From: <tho...@us...> - 2011-05-18 18:14:55
|
Revision: 4523
http://bigdata.svn.sourceforge.net/bigdata/?rev=4523&view=rev
Author: thompsonbry
Date: 2011-05-18 18:14:45 +0000 (Wed, 18 May 2011)
Log Message:
-----------
Merge INT64_BRANCH to QUADS_QUERY_BRANCH [r4485:r4522]. Conflicts: 1 (htree.xls). Added: 2 Updated: 85.
The htree.xls conflict was resolved by accepting the incoming version. I am working on that file and I will handle the conflict when I merge this update back into my working copy.
The branch is closed.
See https://sourceforge.net/apps/trac/bigdata/ticket/294
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/segment math.xls
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/AtomicBlockAppendProc.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractNode.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeStatistics.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeUtilizationReport.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BloomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/Checkpoint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IBTreeStatistics.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ILinearList.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentCheckpoint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentPlan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/Leaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/MutableLeafData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/MutableNodeData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/Node.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ResultSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/AbstractReadOnlyNodeData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/DefaultLeafCoder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/DefaultNodeCoder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/IAbstractNodeData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/ILeafData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/INodeData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/data/ISpannedTupleCountData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/htree/HTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/htree/HashBucket.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/htree/MutableBucketData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/CommitRecordIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/mdi/MetadataIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BuildViewMetadata.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/JournalIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/PurgeResult.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/ResourceManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/SplitUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/CommitTimeIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/MetadataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestFindChild.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentCheckpoint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentPlan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentWithBloomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestLinearListMethods.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/data/AbstractNodeDataRecordTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/data/AbstractNodeOrLeafDataRecordTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/data/MockLeafData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/data/MockNodeData.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestFixedLengthPrefixShardSplits.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestSparseRowStoreSplitHandler.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestSnapshotHelper.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTaskWithSplits.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/XXXCShardSplitHandler.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestXXXCShardSplitHandler.java
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/
branches/QUADS_QUERY_BRANCH/bigdata/lib/jetty/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/aggregate/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/fast/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/rto/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/util/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/htree/raba/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/jsr166/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/joinGraph/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/joinGraph/fast/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/joinGraph/rto/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/util/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/jsr166/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/util/httpd/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/lib/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/answers (U1)/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/scripts/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/error/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/relation/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/samples/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/aggregate/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/relation/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/
branches/QUADS_QUERY_BRANCH/dsi-utils/
branches/QUADS_QUERY_BRANCH/dsi-utils/LEGAL/
branches/QUADS_QUERY_BRANCH/dsi-utils/lib/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/osgi/
branches/QUADS_QUERY_BRANCH/src/resources/bin/config/
Property changes on: branches/QUADS_QUERY_BRANCH
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/JOURNAL_HA_BRANCH:2596-4066
/branches/LARGE_LITERALS_REFACTOR:4175-4387
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/dev-btm:2574-2730
/branches/fko:3150-3194
/trunk:3392-3437,3656-4061
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/INT64_BRANCH:4486-4522
/branches/JOURNAL_HA_BRANCH:2596-4066
/branches/LARGE_LITERALS_REFACTOR:4175-4387
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/dev-btm:2574-2730
/branches/fko:3150-3194
/trunk:3392-3437,3656-4061
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/lib/jetty
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/INT64_BRANCH/bigdata/lib/jetty:4486-4522
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/segment math.xls
===================================================================
(Binary files differ)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/AtomicBlockAppendProc.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/AtomicBlockAppendProc.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/AtomicBlockAppendProc.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -274,7 +274,7 @@
* key. Otherwise this will be the first block written for that
* file.
*/
- int toIndex = tmp.indexOf(toKey);
+ long toIndex = tmp.indexOf(toKey);
assert toIndex < 0 : "Expecting insertion point: id=" + id
+ ", version=" + version + ", toIndex=" + toIndex;
@@ -285,7 +285,7 @@
toIndex = -(toIndex + 1); // convert to an index.
// #of entries in the index.
- final int entryCount = ((AbstractBTree) ndx).getEntryCount();
+ final long entryCount = ((AbstractBTree) ndx).getEntryCount();
if (log.isDebugEnabled())
log.debug("toIndex=" + toIndex + ", entryCount=" + entryCount);
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/aggregate
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/INT64_BRANCH/bigdata/src/java/com/bigdata/bop/aggregate:4486-4522
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleIndex.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleIndex.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleIndex.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -368,13 +368,13 @@
*/
/** The #of tuples to be skipped after every tuple visited. */
- private transient int skipCount;
+ private transient long skipCount;
/** The #of tuples accepted so far. */
private transient int nread = 0;
/** The inclusive lower bound of the first tuple actually visited. */
- private transient int fromIndex;
+ private transient long fromIndex;
/** The exclusive upper bound of the last tuple which could be visited. */
- private transient int toIndex;
+ private transient long toIndex;
/**
*
@@ -393,7 +393,7 @@
final AbstractBTree ndx = (AbstractBTree) src.getIndex();
- final int currentIndex = ndx.indexOf(tuple.getKey());
+ final long currentIndex = ndx.indexOf(tuple.getKey());
if (nread == 0) {
@@ -409,9 +409,9 @@
toIndex = -toIndex + 1;
}
- final int rangeCount = (toIndex - fromIndex);
+ final long rangeCount = (toIndex - fromIndex);
- skipCount = Math.max(1, rangeCount / limit);
+ skipCount = Math.max(1L, rangeCount / limit);
// minus one since src.next() already consumed one tuple.
skipCount -= 1;
@@ -429,7 +429,7 @@
* If the skip count is positive, then skip over N tuples.
*/
- final int nextIndex = Math.min(ndx.getEntryCount() - 1,
+ final long nextIndex = Math.min(ndx.getEntryCount() - 1,
currentIndex + skipCount);
src.seek(ndx.keyAt(nextIndex));
@@ -474,13 +474,13 @@
*/
/** The offset of each tuple to be sampled. */
- private transient int[] offsets;
+ private transient long[] offsets;
/** The #of tuples accepted so far. */
private transient int nread = 0;
/** The inclusive lower bound of the first tuple actually visited. */
- private transient int fromIndex;
+ private transient long fromIndex;
/** The exclusive upper bound of the last tuple which could be visited. */
- private transient int toIndex;
+ private transient long toIndex;
/**
*
@@ -539,7 +539,7 @@
* Skip to the next tuple.
*/
- final int nextIndex = offsets[nread];
+ final long nextIndex = offsets[nread];
// System.err.println("limit=" + limit + ", rangeCount="
// + (toIndex - fromIndex) + ", fromIndex=" + fromIndex
@@ -688,8 +688,8 @@
* @throws IllegalArgumentException
* unless <i>toIndex</i> is GT <i>fromIndex</i>.
*/
- int[] getOffsets(final long seed, int limit, final int fromIndex,
- final int toIndex);
+ long[] getOffsets(long seed, int limit, long fromIndex, long toIndex);
+
}
/**
@@ -703,8 +703,8 @@
/**
* {@inheritDoc}
*/
- public int[] getOffsets(final long seed, int limit,
- final int fromIndex, final int toIndex) {
+ public long[] getOffsets(final long seed, int limit,
+ final long fromIndex, final long toIndex) {
if (limit < 1)
throw new IllegalArgumentException();
@@ -715,10 +715,15 @@
if (toIndex <= fromIndex)
throw new IllegalArgumentException();
- final int rangeCount = (toIndex - fromIndex);
+ final long rangeCount = (toIndex - fromIndex);
- if (limit > rangeCount)
- limit = rangeCount;
+ if (limit > rangeCount) {
+ /*
+ * Note: cast valid since limit is int32 and limit LT rangeCount
+ * so rangeCount may be cast to int32.
+ */
+ limit = (int) rangeCount;
+ }
if (limit == rangeCount) {
@@ -782,8 +787,8 @@
* if <i>limit!=rangeCount</i> (after adjusting for limits
* greater than the rangeCount).
*/
- public int[] getOffsets(final long seed, int limit,
- final int fromIndex, final int toIndex) {
+ public long[] getOffsets(final long seed, int limit,
+ final long fromIndex, final long toIndex) {
if (limit < 1)
throw new IllegalArgumentException();
@@ -794,16 +799,21 @@
if (toIndex <= fromIndex)
throw new IllegalArgumentException();
- final int rangeCount = (toIndex - fromIndex);
+ final long rangeCount = (toIndex - fromIndex);
- if (limit > rangeCount)
- limit = rangeCount;
+ if (limit > rangeCount) {
+ /*
+ * Note: cast valid since limit is int32 and limit LT rangeCount
+ * so rangeCount may be cast to int32.
+ */
+ limit = (int) rangeCount;
+ }
if (limit != rangeCount)
throw new UnsupportedOperationException();
// offsets of tuples to visit.
- final int[] offsets = new int[limit];
+ final long[] offsets = new long[limit];
for (int i = 0; i < limit; i++) {
@@ -832,8 +842,18 @@
*/
static public class BitVectorOffsetSampler implements IOffsetSampler {
- public int[] getOffsets(final long seed, int limit,
- final int fromIndex, final int toIndex) {
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Note: The utility of this class is limited to smaller range counts
+ * (32k is fine, 2x or 4k that is also Ok) so it will reject anything
+ * with a very large range count.
+ *
+ * @throws UnsupportedOperationException
+ * if the rangeCount is GT {@link Integer#MAX_VALUE}
+ */
+ public long[] getOffsets(final long seed, int limit,
+ final long fromIndex, final long toIndex) {
if (limit < 1)
throw new IllegalArgumentException();
@@ -844,13 +864,25 @@
if (toIndex <= fromIndex)
throw new IllegalArgumentException();
- final int rangeCount = (toIndex - fromIndex);
+ final long rangeCount2 = (toIndex - fromIndex);
- if (limit > rangeCount)
+ if (rangeCount2 > Integer.MAX_VALUE) {
+ /*
+ * The utility of this class is limited to smaller range counts
+ * so it will reject anything with a very large range count.
+ */
+ throw new UnsupportedOperationException();
+ }
+
+ // known to be an int32 value.
+ final int rangeCount = (int) rangeCount2;
+
+ if (limit > rangeCount) {
limit = rangeCount;
+ }
// offsets of tuples to visit.
- final int[] offsets = new int[limit];
+ final long [] offsets = new long [limit];
// create a cleared bit vector of the stated capacity.
final BitVector v = LongArrayBitVector.ofLength(//
@@ -921,8 +953,17 @@
*/
static public class AcceptanceSetOffsetSampler implements IOffsetSampler {
- public int[] getOffsets(final long seed, int limit,
- final int fromIndex, final int toIndex) {
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Note: The utility of this class is limited to moderate range counts
+ * (~100k) so it will reject anything with a very large range count.
+ *
+ * @throws UnsupportedOperationException
+ * if the rangeCount is GT {@link Integer#MAX_VALUE}
+ */
+ public long[] getOffsets(final long seed, int limit,
+ final long fromIndex, final long toIndex) {
if (limit < 1)
throw new IllegalArgumentException();
@@ -933,13 +974,19 @@
if (toIndex <= fromIndex)
throw new IllegalArgumentException();
- final int rangeCount = (toIndex - fromIndex);
+ final long rangeCount2 = (toIndex - fromIndex);
- if (limit > rangeCount)
+ if (rangeCount2 > Integer.MAX_VALUE)
+ throw new UnsupportedOperationException();
+
+ final int rangeCount = (int) rangeCount2;
+
+ if (limit > rangeCount) {
limit = rangeCount;
+ }
// offsets of tuples to visit.
- final int[] offsets = new int[limit];
+ final long [] offsets = new long[limit];
// hash set of accepted offsets.
final IntOpenHashSet v = new IntOpenHashSet(
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -31,9 +31,6 @@
import com.bigdata.btree.AbstractBTree;
import com.bigdata.btree.BTree;
-import com.bigdata.btree.BTreeUtilizationReport;
-import com.bigdata.btree.IBTreeStatistics;
-import com.bigdata.btree.IBTreeUtilizationReport;
import com.bigdata.journal.Journal;
/**
@@ -235,54 +232,55 @@
}
- private static class MockBTreeStatistics implements IBTreeStatistics {
+// private static class MockBTreeStatistics implements IBTreeStatistics {
+//
+// private final int m;
+//
+// private final int height;
+//
+// private final long leafCount;
+//
+// private final long nodeCount;
+//
+// private final long entryCount;
+//
+// private final IBTreeUtilizationReport utilReport;
+//
+// public MockBTreeStatistics(final int m, final int height,
+// final long nodeCount, final long leafCount,
+// final long entryCount) {
+// this.m = m;
+// this.height = height;
+// this.nodeCount = nodeCount;
+// this.leafCount = leafCount;
+// this.entryCount = entryCount;
+// this.utilReport = new BTreeUtilizationReport(this);
+// }
+//
+// public int getBranchingFactor() {
+// return m;
+// }
+//
+// public int getHeight() {
+// return height;
+// }
+//
+// public long getNodeCount() {
+// return nodeCount;
+// }
+//
+// public long getLeafCount() {
+// return leafCount;
+// }
+//
+// public long getEntryCount() {
+// return entryCount;
+// }
+//
+// public IBTreeUtilizationReport getUtilization() {
+// return utilReport;
+// }
+//
+// } // MockBTreeStatistics
- private final int m;
-
- private final int entryCount;
-
- private final int height;
-
- private final int leafCount;
-
- private final int nodeCount;
-
- private final IBTreeUtilizationReport utilReport;
-
- public MockBTreeStatistics(final int m, final int entryCount,
- final int height, final int leafCount, final int nodeCount) {
- this.m = m;
- this.entryCount = entryCount;
- this.height = height;
- this.leafCount = leafCount;
- this.nodeCount = nodeCount;
- this.utilReport = new BTreeUtilizationReport(this);
- }
-
- public int getBranchingFactor() {
- return m;
- }
-
- public int getEntryCount() {
- return entryCount;
- }
-
- public int getHeight() {
- return height;
- }
-
- public int getLeafCount() {
- return leafCount;
- }
-
- public int getNodeCount() {
- return nodeCount;
- }
-
- public IBTreeUtilizationReport getUtilization() {
- return utilReport;
- }
-
- }
-
}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/INT64_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph:4486-4522
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/fast
___________________________________________________________________
Deleted: svn:mergeinfo
-
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/rto
___________________________________________________________________
Deleted: svn:mergeinfo
-
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/util
___________________________________________________________________
Modified: svn:mergeinfo
-
+ /branches/INT64_BRANCH/bigdata/src/java/com/bigdata/bop/util:4486-4522
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -802,13 +802,13 @@
tmp.addCounter("height",
new OneShotInstrument<Integer>(getHeight()));
- tmp.addCounter("nodeCount", new OneShotInstrument<Integer>(
+ tmp.addCounter("nodeCount", new OneShotInstrument<Long>(
getNodeCount()));
- tmp.addCounter("leafCount", new OneShotInstrument<Integer>(
+ tmp.addCounter("leafCount", new OneShotInstrument<Long>(
getLeafCount()));
- tmp.addCounter("tupleCount", new OneShotInstrument<Integer>(
+ tmp.addCounter("tupleCount", new OneShotInstrument<Long>(
getEntryCount()));
/*
@@ -842,15 +842,15 @@
* to avoid dragging in the B+Tree reference.
*/
- final int entryCount = getEntryCount();
+ final long entryCount = getEntryCount();
final long bytes = btreeCounters.bytesOnStore_nodesAndLeaves.get()
+ btreeCounters.bytesOnStore_rawRecords.get();
- final int bytesPerTuple = (int) (entryCount == 0 ? 0d
+ final long bytesPerTuple = (long) (entryCount == 0 ? 0d
: (bytes / entryCount));
- tmp.addCounter("bytesPerTuple", new OneShotInstrument<Integer>(
+ tmp.addCounter("bytesPerTuple", new OneShotInstrument<Long>(
bytesPerTuple));
}
@@ -1606,24 +1606,11 @@
abstract public int getHeight();
- abstract public int getNodeCount();
+ abstract public long getNodeCount();
- abstract public int getLeafCount();
+ abstract public long getLeafCount();
- /**
- * {@inheritDoc}
- *
- * @todo this could be re-defined as the exact entry count if we tracked the
- * #of deleted index entries and subtracted that from the total #of
- * index entries before returning the result. The #of deleted index
- * entries would be stored in the index {@link Checkpoint} record.
- * <p>
- * Since {@link #getEntryCount()} is also used to give the total #of
- * index enties (and we need that feature) so we need to either add
- * another method with the appropriate semantics, add a boolean flag,
- * or add a method returning the #of deleted entries, etc.
- */
- abstract public int getEntryCount();
+ abstract public long getEntryCount();
/**
* Return a statistics snapshot of the B+Tree.
@@ -2343,7 +2330,7 @@
}
- public int indexOf(final byte[] key) {
+ public long indexOf(final byte[] key) {
if (key == null)
throw new IllegalArgumentException();
@@ -2357,7 +2344,7 @@
}
- public byte[] keyAt(final int index) {
+ public byte[] keyAt(final long index) {
if (index < 0)
throw new IndexOutOfBoundsException(ERROR_LESS_THAN_ZERO);
@@ -2371,7 +2358,7 @@
}
- public byte[] valueAt(final int index) {
+ public byte[] valueAt(final long index) {
final Tuple tuple = getLookupTuple();
@@ -2381,7 +2368,7 @@
}
- final public Tuple valueAt(final int index, final Tuple tuple) {
+ final public Tuple valueAt(final long index, final Tuple tuple) {
if (index < 0)
throw new IndexOutOfBoundsException(ERROR_LESS_THAN_ZERO);
@@ -2516,9 +2503,9 @@
if (toKey != null)
assert rangeCheck(toKey,true);
- int fromIndex = (fromKey == null ? 0 : root.indexOf(fromKey));
+ long fromIndex = (fromKey == null ? 0 : root.indexOf(fromKey));
- int toIndex = (toKey == null ? getEntryCount() : root.indexOf(toKey));
+ long toIndex = (toKey == null ? getEntryCount() : root.indexOf(toKey));
// Handle case when fromKey is not found.
if (fromIndex < 0)
@@ -3183,22 +3170,22 @@
if (info) {
+ final int branchingFactor = getBranchingFactor();
+
final int height = getHeight();
- final int nnodes = getNodeCount();
+ final long nnodes = getNodeCount();
- final int nleaves = getLeafCount();
+ final long nleaves = getLeafCount();
- final int nentries = getEntryCount();
+ final long nentries = getEntryCount();
- final int branchingFactor = getBranchingFactor();
-
- out.println("height=" + height + ", branchingFactor="
- + branchingFactor + ", #nodes=" + nnodes + ", #leaves="
- + nleaves + ", #entries=" + nentries + ", nodeUtil="
- + utils.getNodeUtilization() + "%, leafUtil="
- + utils.getLeafUtilization() + "%, utilization="
- + utils.getTotalUtilization() + "%");
+ out.println("branchingFactor=" + branchingFactor + ", height="
+ + height + ", #nodes=" + nnodes + ", #leaves=" + nleaves
+ + ", #entries=" + nentries + ", nodeUtil="
+ + utils.getNodeUtilization() + "%, leafUtil="
+ + utils.getLeafUtilization() + "%, utilization="
+ + utils.getTotalUtilization() + "%");
}
if (root != null) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractNode.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractNode.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -57,7 +57,7 @@
* DO-NOT-USE-GENERIC-HERE. The compiler will fail under Linux (JDK 1.6.0_14,
* _16).
*/
-> extends PO implements IAbstractNode, IAbstractNodeData, IKeysData, ISpannedTupleCountData {
+> extends PO implements IAbstractNode, IAbstractNodeData, IKeysData {
/**
* Log for node and leaf operations.
@@ -1214,7 +1214,7 @@
* this guarantees that the return value will be >= 0 if and only if
* the key is found.
*/
- abstract public int indexOf(byte[] searchKey);
+ abstract public long indexOf(byte[] searchKey);
/**
* Recursive search locates the entry at the specified index position in the
@@ -1231,7 +1231,7 @@
* @exception IndexOutOfBoundsException
* if index is greater than the #of entries.
*/
- abstract public byte[] keyAt(int index);
+ abstract public byte[] keyAt(long index);
/**
* Recursive search locates the entry at the specified index position in the
@@ -1249,7 +1249,7 @@
* @exception IndexOutOfBoundsException
* if index is greater than the #of entries.
*/
- abstract public void valueAt(int index, Tuple tuple);
+ abstract public void valueAt(long index, Tuple tuple);
/**
* Dump the data onto the {@link PrintStream} (non-recursive).
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -168,19 +168,19 @@
}
- final public int getNodeCount() {
+ final public long getNodeCount() {
return nnodes;
}
- final public int getLeafCount() {
+ final public long getLeafCount() {
return nleaves;
}
- final public int getEntryCount() {
+ final public long getEntryCount() {
return nentries;
@@ -252,18 +252,27 @@
/**
* The #of non-leaf nodes in the btree. The is zero (0) for a new btree.
*/
- protected int nnodes;
+ protected long nnodes;
/**
* The #of leaf nodes in the btree. This is one (1) for a new btree.
*/
- protected int nleaves;
+ protected long nleaves;
/**
* The #of entries in the btree. This is zero (0) for a new btree.
*/
- protected int nentries;
+ protected long nentries;
+ /**
+ * The value of the record version number that will be assigned to the next
+ * node or leaf written onto the backing store. This number is incremented
+ * each time a node or leaf is written onto the backing store. The initial
+ * value is ZERO (0). The first value assigned to a node or leaf will be
+ * ZERO (0).
+ */
+ protected long recordVersion;
+
/**
* The mutable counter exposed by #getCounter()}.
* <p>
@@ -417,6 +426,8 @@
this.counter = new AtomicLong( checkpoint.getCounter() );
+ this.recordVersion = checkpoint.getRecordVersion();
+
}
/**
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeStatistics.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeStatistics.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeStatistics.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -44,22 +44,22 @@
private final int m;
- private final int entryCount;
-
private final int height;
- private final int leafCount;
+ private final long nodeCount;
- private final int nodeCount;
+ private final long leafCount;
+ private final long entryCount;
+
private final IBTreeUtilizationReport utilReport;
public BTreeStatistics(final AbstractBTree btree) {
this.m = btree.getBranchingFactor();
- this.entryCount = btree.getEntryCount();
this.height = btree.getHeight();
+ this.nodeCount = btree.getNodeCount();
this.leafCount = btree.getLeafCount();
- this.nodeCount = btree.getNodeCount();
+ this.entryCount = btree.getEntryCount();
this.utilReport = btree.getUtilization();
}
@@ -67,20 +67,20 @@
return m;
}
- public int getEntryCount() {
- return entryCount;
- }
-
public int getHeight() {
return height;
}
- public int getLeafCount() {
+ public long getNodeCount() {
+ return nodeCount;
+ }
+
+ public long getLeafCount() {
return leafCount;
}
- public int getNodeCount() {
- return nodeCount;
+ public long getEntryCount() {
+ return entryCount;
}
public IBTreeUtilizationReport getUtilization() {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeUtilizationReport.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeUtilizationReport.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeUtilizationReport.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -51,13 +51,13 @@
public BTreeUtilizationReport(final IBTreeStatistics stats) {
- final int nnodes = stats.getNodeCount();
+ final long nnodes = stats.getNodeCount();
- final int nleaves = stats.getLeafCount();
+ final long nleaves = stats.getLeafCount();
- final int nentries = stats.getEntryCount();
+ final long nentries = stats.getEntryCount();
- final int numNonRootNodes = nnodes + nleaves - 1;
+ final long numNonRootNodes = nnodes + nleaves - 1;
final int branchingFactor = stats.getBranchingFactor();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BloomFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BloomFilter.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BloomFilter.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -55,11 +55,7 @@
*/
public class BloomFilter implements IBloomFilter, Externalizable {
- protected static final transient Logger log = Logger.getLogger(BloomFilter.class);
-
-// protected static final transient boolean INFO = log.isInfoEnabled();
-//
-// protected static final transient boolean DEBUG = log.isDebugEnabled();
+ private static final transient Logger log = Logger.getLogger(BloomFilter.class);
/**
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/Checkpoint.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/Checkpoint.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/Checkpoint.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -36,13 +36,15 @@
private long addrMetadata;
private long addrRoot; // of root node/leaf for BTree; rootDir for HTree.
private int height; // height for BTree; globalDepth for HTree.
- private int nnodes; // #of directories for HTree
- private int nleaves; // #of buckets for HTree.
- private int nentries; // #of tuples in the index.
- private long counter;
+ private long nnodes; // #of directories for HTree
+ private long nleaves; // #of buckets for HTree.
+ private long nentries; // #of tuples in the index.
+ private long counter; // B+Tree local counter.
private long addrBloomFilter;
+ private long recordVersion; // #of node or leaf records written to date.
+
/**
* Added in {@link #VERSION1}. This is a short field allowing for 65536
* different possible index types.
@@ -181,7 +183,7 @@
/**
* The #of non-leaf nodes (B+Tree) or directories (HTree).
*/
- public final int getNodeCount() {
+ public final long getNodeCount() {
return nnodes;
@@ -190,7 +192,7 @@
/**
* The #of leaves (B+Tree) or hash buckets (HTree).
*/
- public final int getLeafCount() {
+ public final long getLeafCount() {
return nleaves;
@@ -199,22 +201,34 @@
/**
* The #of index entries (aka tuple count).
*/
- public final int getEntryCount() {
+ public final long getEntryCount() {
return nentries;
}
- /**
- * Return the value of the counter stored in the {@link Checkpoint}
- * record.
- */
+ /**
+ * Return the value of the B+Tree local counter stored in the
+ * {@link Checkpoint} record.
+ */
public final long getCounter() {
return counter;
}
+ /**
+ * Return the value of the next record version number to be assigned that is
+ * stored in the {@link Checkpoint} record. This number is incremented each
+ * time a node or leaf is written onto the backing store. The initial value
+ * is ZERO (0). The first value assigned to a node or leaf will be ZERO (0).
+ */
+ public final long getRecordVersion() {
+
+ return counter;
+
+ }
+
/**
* A human readable representation of the state of the {@link Checkpoint}
* record.
@@ -262,10 +276,11 @@
0L,// No root yet.
0L,// No bloom filter yet.
0, // height
- 0, // nnodes
- 0, // nleaves
- 0, // nentries
+ 0L, // nnodes
+ 0L, // nleaves
+ 0L, // nentries
0L, // counter
+ 0L, // recordVersion
IndexTypeEnum.BTree // indexType
);
@@ -292,10 +307,11 @@
0L,// No root yet.
0L,// No bloom filter yet.
0, // height
- 0, // nnodes
- 0, // nleaves
- 0, // nentries
+ 0L, // nnodes
+ 0L, // nleaves
+ 0L, // nentries
oldCheckpoint.counter,//
+ 0L, // recordVersion
IndexTypeEnum.BTree//
);
@@ -350,14 +366,16 @@
btree.nleaves,//
btree.nentries,//
btree.counter.get(),//
+ btree.recordVersion,//
IndexTypeEnum.BTree//
);
}
private Checkpoint(final long addrMetadata, final long addrRoot,
- final long addrBloomFilter, final int height, final int nnodes,
- final int nleaves, final int nentries, final long counter,
+ final long addrBloomFilter, final int height, final long nnodes,
+ final long nleaves, final long nentries, final long counter,
+ final long recordVersion,
final IndexTypeEnum indexType) {
assert indexType != null;
@@ -389,7 +407,9 @@
this.nentries = nentries;
this.counter = counter;
-
+
+ this.recordVersion = recordVersion;
+
this.indexType = indexType;
}
@@ -412,11 +432,36 @@
* which is present only for {@link IndexTypeEnum#HTree}.
*/
private static transient final int VERSION1 = 0x1;
+
+ /**
+ * Adds and/or modifies the following fields.
+ * <dl>
+ * <dt>nodeCount</dt>
+ * <dd>Changed from int32 to int64.</dd>
+ * <dt>leafCount</dt>
+ * <dd>Changed from int32 to int64.</dd>
+ * <dt>entryCount</dt>
+ * <dd>Changed from int32 to int64.</dd>
+ * <dt>recordVersion</dt>
+ * <dd>Added a new field which record the <em>next</em> record version
+ * identifier to be used when the next node or leaf data record is written
+ * onto the backing store. The field provides sequential numbering of those
+ * data record which can facilitate certain kinds of forensics. For example,
+ * all updated records falling between two checkpoints may be identified by
+ * a file scan filtering for the index UUID and a record version number GT
+ * the last record version written for the first checkpoint and LTE the last
+ * record version number written for the second checkpoint. The initial
+ * value is ZERO (0).</dd>
+ * </dl>
+ * In addition, the <strong>size</strong> of the checkpoint record has been
+ * increased in order to provide room for future expansions.
+ */
+ private static transient final int VERSION2 = 0x2;
/**
* The current version.
*/
- private static transient final int VERSION = VERSION1;
+ private static transient final int currentVersion = VERSION2;
/**
* Write the {@link Checkpoint} record on the store, setting
@@ -474,25 +519,42 @@
switch (version) {
case VERSION0:
case VERSION1:
+ case VERSION2:
break;
default:
throw new IOException("Unknown version: " + version);
}
- this.addrMetadata = in.readLong();
+ this.addrMetadata = in.readLong();
- this.addrRoot = in.readLong();
+ this.addrRoot = in.readLong();
- this.addrBloomFilter = in.readLong();
-
- this.height = in.readInt();
+ this.addrBloomFilter = in.readLong();
- this.nnodes = in.readInt();
+ this.height = in.readInt();
- this.nleaves = in.readInt();
+ if (version <= VERSION1) {
- this.nentries = in.readInt();
+ this.nnodes = in.readInt();
+ this.nleaves = in.readInt();
+
+ this.nentries = in.readInt();
+
+ this.recordVersion = 0L;
+
+ } else {
+
+ this.nnodes = in.readLong();
+
+ this.nleaves = in.readLong();
+
+ this.nentries = in.readLong();
+
+ this.recordVersion = in.readLong();
+
+ }
+
this.counter = in.readLong();
switch (version) {
@@ -501,6 +563,7 @@
indexType = IndexTypeEnum.BTree;
break;
case VERSION1:
+ case VERSION2:
this.indexType = IndexTypeEnum.valueOf(in.readShort());
in.readShort();// ignored.
in.readInt();// ignored.
@@ -510,12 +573,23 @@
}
in.readLong(); // unused.
+
+ if (version >= VERSION2) {
+
+ // Read some additional padding added to the record in VERSION2.
+ for (int i = 0; i < 10; i++) {
+
+ in.readLong();
+
+ }
+
+ }
}
public void writeExternal(final ObjectOutput out) throws IOException {
- out.writeInt(VERSION);
+ out.writeInt(currentVersion);
out.writeLong(addrMetadata);
@@ -525,12 +599,33 @@
out.writeInt(height);
- out.writeInt(nnodes);
+ if (currentVersion <= VERSION1) {
- out.writeInt(nleaves);
+ if (nnodes > Integer.MAX_VALUE)
+ throw new RuntimeException();
+ if (nleaves > Integer.MAX_VALUE)
+ throw new RuntimeException();
+ if (nentries > Integer.MAX_VALUE)
+ throw new RuntimeException();
+
+ out.writeInt((int)nnodes);
- out.writeInt(nentries);
+ out.writeInt((int)nleaves);
+ out.writeInt((int)nentries);
+
+ } else {
+
+ out.writeLong(nnodes);
+
+ out.writeLong(nleaves);
+
+ out.writeLong(nentries);
+
+ out.writeLong(recordVersion);
+
+ }
+
out.writeLong(counter);
/*
@@ -541,12 +636,21 @@
out.writeShort(0/* unused */);
out.writeInt(0/* unused */);
- /*
- * 8 bytes follow.
- */
+ /*
+ * 8 bytes follow.
+ */
out.writeLong(0L/* unused */);
+ /*
+ * Additional space added in VERSION2.
+ */
+ for (int i = 0; i < 10; i++) {
+
+ out.writeLong(0L/* unused */);
+
+ }
+
}
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IBTreeStatistics.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IBTreeStatistics.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IBTreeStatistics.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -59,20 +59,20 @@
* The #of non-leaf nodes in the {@link AbstractBTree}. This is zero (0)
* for a new btree.
*/
- int getNodeCount();
+ long getNodeCount();
/**
* The #of leaf nodes in the {@link AbstractBTree}. This is one (1) for a
* new btree.
*/
- int getLeafCount();
+ long getLeafCount();
/**
* The #of entries (aka tuples) in the {@link AbstractBTree}. This is zero
* (0) for a new B+Tree. When the B+Tree supports delete markers, this value
* also includes tuples which have been marked as deleted.
*/
- int getEntryCount();
+ long getEntryCount();
/**
* Computes and returns the utilization of the tree. The utilization figures
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ILinearList.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ILinearList.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ILinearList.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -29,7 +29,18 @@
/**
* Interface for methods that return or accept an ordinal index into the entries
- * in the B+-TRee.
+ * in the B+-Tree. The semantics of this interface are build over the #of
+ * spanned tuples for each child as recorded within each node of the B+Tree.
+ * This provides a fast means to compute the linear index into the B+Tree of any
+ * given tuple. However, this interface is only available for a local B+Tree
+ * object (versus scale-out) since the spanned tuple count metadata is not exact
+ * across shards. Further, when delete markers are used, the deleted tuples
+ * remain in the B+Tree and the {@link ILinearList} interface will continue to
+ * count them until they have been purged. Thus deleting a tuple does not change
+ * the {@link #indexOf(byte[])} keys after that tuple, {@link #keyAt(long)} can
+ * return the key for a deleted tuple, and {@link #valueAt(long)} will return
+ * <code>null</code> if the tuple at that index is marked as deleted within the
+ * B+Tree.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -59,10 +70,10 @@
* in the index.
*
*
- * @see #keyAt(int)
- * @see #valueAt(int)
+ * @see #keyAt(long)
+ * @see #valueAt(long)
*/
- public int indexOf(byte[] key);
+ public long indexOf(byte[] key);
/**
* Return the key for the identified entry. This performs an efficient
@@ -72,17 +83,17 @@
* @param index
* The index position of the entry (origin zero).
*
- * @return The key at that index position (not a copy).
+ * @return The key at that index position.
*
* @exception IndexOutOfBoundsException
* if index is less than zero.
* @exception IndexOutOfBoundsException
* if index is greater than or equal to the #of entries.
*
- * @see #indexOf(Object)
- * @see #valueAt(int)
+ * @see #indexOf(byte[])
+ * @see #valueAt(long)
*/
- public byte[] keyAt(int index);
+ public byte[] keyAt(long index);
/**
* Return the value for the identified entry. This performs an efficient
@@ -100,9 +111,9 @@
* @exception IndexOutOfBoundsException
* if index is greater than or equal to the #of entries.
*
- * @see #indexOf(Object)
- * @see #keyAt(int)
+ * @see #indexOf(byte[])
+ * @see #keyAt(long)
*/
- public byte[] valueAt(int index);
+ public byte[] valueAt(long index);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegment.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegment.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -129,23 +129,23 @@
}
- final public int getLeafCount() {
+ final public long getNodeCount() {
reopen();
- return fileStore.getCheckpoint().nleaves;
+ return fileStore.getCheckpoint().nnodes;
}
- final public int getNodeCount() {
+ final public long getLeafCount() {
reopen();
- return fileStore.getCheckpoint().nnodes;
+ return fileStore.getCheckpoint().nleaves;
}
- final public int getEntryCount() {
+ final public long getEntryCount() {
reopen();
@@ -718,10 +718,7 @@
/**
* @param btree
* @param addr
- * @param branchingFactor
- * @param nentries
- * @param keys
- * @param childKeys
+ * @param data
*/
protected ImmutableNode(final AbstractBTree btree, final long addr,
final INodeData data) {
@@ -843,9 +840,9 @@
return Long.MAX_VALUE;
}
- final public int getSpannedTupleCount() {
- return 0;
- }
+// final public int getSpannedTupleCount() {
+// return 0;
+// }
final public boolean isLeaf() {
return true;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentBuilder.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -48,7 +48,6 @@
import com.bigdata.btree.data.IAbstractNodeData;
import com.bigdata.btree.data.ILeafData;
import com.bigdata.btree.data.INodeData;
-import com.bigdata.btree.data.ISpannedTupleCountData;
import com.bigdata.btree.raba.IRaba;
import com.bigdata.btree.raba.MutableKeyBuffer;
import com.bigdata.btree.raba.MutableValueBuffer;
@@ -261,7 +260,7 @@
/**
* The value specified to the ctor.
*/
- final public int entryCount;
+ final public long entryCount;
/**
* The iterator specified to the ctor. This is the source for the keys and
@@ -557,7 +556,7 @@
/**
* The #of tuples written for the output tree.
*/
- int ntuplesWritten;
+ long ntuplesWritten;
/**
* The #of nodes written for the output tree. This will be zero if all
@@ -1106,7 +1105,7 @@
public static IndexSegmentBuilder newInstance(//
final File outFile,//
final File tmpDir,//
- final int entryCount,//
+ final long entryCount,//
final ITupleIterator<?> entryIterator, //
final int m,//
final IndexMetadata metadata,//
@@ -1196,7 +1195,7 @@
protected IndexSegmentBuilder(//
final File outFile,//
final File tmpDir,//
- final int entryCount,//
+ final long entryCount,//
final ITupleIterator<?> entryIterator, //
final int m,//
IndexMetadata metadata,//
@@ -1415,21 +1414,26 @@
stack[plan.height] = leaf;
- /*
- * Setup optional bloom filter.
- *
- * Note: For read-only {@link IndexSegment} we always know the #of
- * keys exactly at the time that we provision the bloom filter. This
- * makes it easy for us to tune the filter for a desired false
- * positive rate.
- */
- if (metadata.getBloomFilterFactory() != null && plan.nentries > 0) {
+ /*
+ * Setup optional bloom filter.
+ *
+ * Note: For read-only {@link IndexSegment} we always know the #of
+ * keys exactly at the time that we provision the bloom filter. This
+ * makes it easy for us to tune the filter for a desired false
+ * positive rate.
+ *
+ * Note: The bloom filter can not be used with very large indices
+ * due to the space requirements of the filter. However, very large
+ * in this case is MAX_INT tuples!
+ */
+ if (metadata.getBloomFilterFactory() != null && plan.nentries > 0
+ && plan.nentries < Integer.MAX_VALUE) {
// the desired error rate for the bloom filter.
final double p = metadata.getBloomFilterFactory().p;
- // create the bloom filter.
- bloomFilter = new BloomFilter(plan.nentries, p);
+ // create the bloom filter.
+ bloomFilter = new BloomFilter((int) plan.nentries, p);
} else {
@@ -2065,7 +2069,8 @@
final AbstractSimpleNodeData child) {
// #of entries spanned by this node.
- final int nentries = child.getSpannedTupleCount();
+ final long nentries = (child.isLeaf() ? child.getKeyCount()
+ : ((INodeData) child).getSpannedTupleCount());
if (parent.nchildren == parent.max) {
@@ -3238,11 +3243,21 @@
outChannel.position(0);
+ /*
+ * Note: The build plan is restricted to MAX_INT leaves and there
+ * are always more leaves than nodes in a B+Tree both nnodes and
+ * nleaves are int32 values.
+ */
+ if(nnodesWritten>Integer.MAX_VALUE)
+ throw new AssertionError();
+ if(nleavesWritten>Integer.MAX_VALUE)
+ throw new AssertionError();
+
final IndexSegmentCheckpoint md = new IndexSegmentCheckpoint(
addressManager.getOffsetBits(), //
plan.height, // will always be correct.
- nleavesWritten, // actual #of leaves written.
- nnodesWritten, // actual #of nodes written.
+ (int)nleavesWritten, // actual #of leaves written.
+ (int)nnodesWritten, // actual #of nodes written.
ntuplesWritten, // actual #of tuples written.
maxNodeOrLeafLength,//
offsetLeaves, extentLeaves, offsetNodes, extentNodes,
@@ -3292,7 +3307,7 @@
* Thompson</a>
*/
abstract protected static class AbstractSimpleNodeData implements
- IAbstractNodeData, ISpannedTupleCountData {
+ IAbstractNodeData {
/**
* The level in the output tree for this node or leaf (origin zero). The
@@ -3478,11 +3493,11 @@
}
- final public int getSpannedTupleCount() {
-
- return keys.size();
-
- }
+// final public int getSpannedTupleCount() {
+//
+// return keys.size();
+//
+// }
final public int getValueCount() {
@@ -3632,12 +3647,12 @@
/**
* The #of entries spanned by this node.
*/
- int nentries;
+ long nentries;
/**
* The #of entries spanned by each child of this node.
*/
- final int[] childEntryCount;
+ final long [] childEntryCount;
/**
* <code>true</code> iff the node is tracking the min/max tuple revision
@@ -3645,7 +3660,7 @@
*/
final boolean hasVersionTimestamps;
- final public int getSpannedTupleCount() {
+ final public long getSpannedTupleCount() {
return nentries;
@@ -3660,7 +3675,7 @@
}
- final public int getChildEntryCount(final int index) {
+ final public long getChildEntryCount(final int index) {
if (index < 0 || index > keys.size() + 1)
throw new IllegalArgumentException();
@@ -3676,7 +3691,7 @@
this.childAddr = new long[m];
- this.childEntryCount = new int[m];
+ this.childEntryCount = new long[m];
this.hasVersionTimestamps = hasVersionTimestamps;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentCheckpoint.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentCheckpoint.java 2011-05-18 17:58:13 UTC (rev 4522)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegmentCheckpoint.java 2011-05-18 18:14:45 UTC (rev 4523)
@@ -86,15 +86,20 @@
static final int SIZEOF_TIMESTAMP = Bytes.SIZEOF_LONG;
static final int SI...
[truncated message content] |
