Menu
▾
▴
[cweb-CVS] bigdata/src/java/com/bigdata/objndx IReadOnlyBatchOp.java, NONE, 1.1 IndexSegmentBuilder.java, 1.27, 1.28 BTree.java, 1.38, 1.39 IndexSegmentFileStore.java, 1.10, 1.11 AbstractBTree.java, 1.19, 1.20 BatchContains.java, 1.2, 1.3 NodeSerializer.java, 1.33, 1.34 BTreeMetadata.java, 1.13, 1.14 ISimpleBTree.java, 1.2, 1.3 BatchLookup.java, 1.3, 1.4 IndexSegment.java, 1.17, 1.18
|
From: Bryan T. <tho...@us...> - 2007-03-15 16:11:47
|
Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv10595/src/java/com/bigdata/objndx Modified Files: IndexSegmentBuilder.java BTree.java IndexSegmentFileStore.java AbstractBTree.java BatchContains.java NodeSerializer.java BTreeMetadata.java ISimpleBTree.java BatchLookup.java IndexSegment.java Added Files: IReadOnlyBatchOp.java Log Message: Refactoring to define service apis (data service, transaction manager service) and some approximate implementations of those services (not supporting service discovery, network protocol, or service robustness). Copied in the UML model so that it will actually get committed to CVS.... Index: IndexSegmentFileStore.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/IndexSegmentFileStore.java,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -d -r1.10 -r1.11 *** IndexSegmentFileStore.java 8 Mar 2007 18:14:05 -0000 1.10 --- IndexSegmentFileStore.java 15 Mar 2007 16:11:08 -0000 1.11 *************** *** 41,45 **** * of the queue does not require any IOs. */ ! protected final ByteBuffer buf_nodes; /** --- 41,45 ---- * of the queue does not require any IOs. */ ! private ByteBuffer buf_nodes; /** *************** *** 51,65 **** * The random access file used to read the index segment. */ ! protected final RandomAccessFile raf; /** * A read-only view of the metadata record for the index segment. */ ! protected final IndexSegmentMetadata metadata; /** * A read-only view of the extension metadata record for the index segment. */ ! protected final IndexSegmentExtensionMetadata extensionMetadata; /** --- 51,65 ---- * The random access file used to read the index segment. */ ! private RandomAccessFile raf; /** * A read-only view of the metadata record for the index segment. */ ! protected IndexSegmentMetadata metadata; /** * A read-only view of the extension metadata record for the index segment. */ ! protected IndexSegmentExtensionMetadata extensionMetadata; /** *************** *** 87,90 **** --- 87,108 ---- this.file = file; + reopen(); + + } + + /** + * Re-open a closed store. This operation should succeed if the backing file + * is still accessible. + * + * @exception IllegalStateException + * if the store is not closed. + * + * @see #close() + */ + public void reopen() { + + if (open) + throw new IllegalStateException("Already open."); + if (!file.exists()) { *************** *** 132,136 **** } ! /** * Load the {@link IndexSegment} or derived class from the store. The --- 150,154 ---- } ! /** * Load the {@link IndexSegment} or derived class from the store. The *************** *** 196,199 **** --- 214,222 ---- } + /** + * Closes the file and releases the internal buffers and metadata records. + * This operation may be reversed by {@link #reopen()} as long as the + * backing file remains available. + */ public void close() { *************** *** 204,207 **** --- 227,240 ---- raf.close(); + + raf = null; + + buf_nodes = null; + + metadata = null; + + extensionMetadata = null; + + open = false; } catch (IOException ex) { *************** *** 211,216 **** } - open = false; - } --- 244,247 ---- --- NEW FILE: IReadOnlyBatchOp.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Mar 14, 2007 */ package com.bigdata.objndx; /** * A batch operation that does not allow mutation operations. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public interface IReadOnlyBatchOp extends IBatchOp { } Index: IndexSegmentBuilder.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/IndexSegmentBuilder.java,v retrieving revision 1.27 retrieving revision 1.28 diff -C2 -d -r1.27 -r1.28 *** IndexSegmentBuilder.java 8 Mar 2007 18:14:05 -0000 1.27 --- IndexSegmentBuilder.java 15 Mar 2007 16:11:08 -0000 1.28 *************** *** 67,70 **** --- 67,71 ---- import com.bigdata.isolation.Value; import com.bigdata.journal.Journal; + import com.bigdata.journal.ResourceManager; import com.bigdata.journal.TemporaryRawStore; import com.bigdata.objndx.IndexSegment.CustomAddressSerializer; *************** *** 789,792 **** --- 790,796 ---- + "MB"+", rate="+fpf.format(mbPerSec)+"MB/sec"); + // report event + ResourceManager.buildIndexSegment(null/* name */, + outFile.toString(), plan.nentries, elapsed, md.length); } catch (Throwable ex) { Index: ISimpleBTree.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/ISimpleBTree.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** ISimpleBTree.java 15 Feb 2007 01:34:22 -0000 1.2 --- ISimpleBTree.java 15 Mar 2007 16:11:08 -0000 1.3 *************** *** 57,65 **** * @version $Id$ * ! * FIXME re-define this interface for byte[] keys ! * ! * @todo implement a strongly typed subclass of BTree using a default ! * {@link KeyBuilder} and use that to run the various test suites that ! * have a dependency on int keys. * * @see KeyBuilder, which may be used to encode one or more primitive data type --- 57,61 ---- * @version $Id$ * ! * @todo re-define this interface for byte[] keys * * @see KeyBuilder, which may be used to encode one or more primitive data type Index: BTreeMetadata.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/BTreeMetadata.java,v retrieving revision 1.13 retrieving revision 1.14 diff -C2 -d -r1.13 -r1.14 *** BTreeMetadata.java 11 Mar 2007 11:41:44 -0000 1.13 --- BTreeMetadata.java 15 Mar 2007 16:11:08 -0000 1.14 *************** *** 88,91 **** --- 88,93 ---- protected BTreeMetadata(BTree btree) { + assert btree.isOpen(); + this.addrRoot = btree.root.getIdentity(); Index: BatchContains.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/BatchContains.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** BatchContains.java 15 Feb 2007 01:34:22 -0000 1.2 --- BatchContains.java 15 Mar 2007 16:11:08 -0000 1.3 *************** *** 57,61 **** * @version $Id$ */ ! public class BatchContains implements IBatchOp { /** --- 57,61 ---- * @version $Id$ */ ! public class BatchContains implements IReadOnlyBatchOp { /** Index: BatchLookup.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/BatchLookup.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** BatchLookup.java 15 Feb 2007 01:34:22 -0000 1.3 --- BatchLookup.java 15 Mar 2007 16:11:08 -0000 1.4 *************** *** 54,58 **** * @version $Id$ */ ! public class BatchLookup implements IBatchOp { /** --- 54,58 ---- * @version $Id$ */ ! public class BatchLookup implements IReadOnlyBatchOp { /** Index: IndexSegment.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/IndexSegment.java,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** IndexSegment.java 8 Mar 2007 18:14:05 -0000 1.17 --- IndexSegment.java 15 Mar 2007 16:11:08 -0000 1.18 *************** *** 8,11 **** --- 8,12 ---- import com.bigdata.cache.HardReferenceQueue; + import com.bigdata.journal.ResourceManager; import com.bigdata.rawstore.Addr; import com.bigdata.rawstore.Bytes; *************** *** 36,41 **** * Type safe reference to the backing store. */ ! final protected IndexSegmentFileStore fileStore; ! /** * An optional bloom filter that will be used to filter point tests. Since --- 37,42 ---- * Type safe reference to the backing store. */ ! protected final IndexSegmentFileStore fileStore; ! /** * An optional bloom filter that will be used to filter point tests. Since *************** *** 44,94 **** * structures. */ ! final it.unimi.dsi.mg4j.util.BloomFilter bloomFilter; ! /** ! * Text of a message used in exceptions for mutation operations on the ! * index segment. */ ! final protected static String MSG_READ_ONLY = "Read-only index"; public int getBranchingFactor() { ! return fileStore.metadata.branchingFactor; ! } public int getHeight() { ! return fileStore.metadata.height; ! } public int getLeafCount() { ! return fileStore.metadata.nleaves; ! } public int getNodeCount() { ! return fileStore.metadata.nnodes; ! } public int getEntryCount() { return fileStore.metadata.nentries; ! } ! public IndexSegment(IndexSegmentFileStore fileStore ) { ! this(fileStore, new HardReferenceQueue<PO>( new DefaultEvictionListener(), BTree.DEFAULT_HARD_REF_QUEUE_CAPACITY, BTree.DEFAULT_HARD_REF_QUEUE_SCAN)); ! } ! /** * Open a read-only index segment. --- 45,109 ---- * structures. */ ! it.unimi.dsi.mg4j.util.BloomFilter bloomFilter; ! /** ! * Text of a message used in exceptions for mutation operations on the index ! * segment. */ ! final protected transient static String MSG_READ_ONLY = "Read-only index"; public int getBranchingFactor() { ! ! reopen(); ! return fileStore.metadata.branchingFactor; ! } public int getHeight() { ! ! reopen(); ! return fileStore.metadata.height; ! } public int getLeafCount() { ! ! reopen(); ! return fileStore.metadata.nleaves; ! } public int getNodeCount() { ! ! reopen(); ! return fileStore.metadata.nnodes; ! } public int getEntryCount() { + reopen(); + return fileStore.metadata.nentries; ! } ! public IndexSegment(IndexSegmentFileStore fileStore) { ! this(fileStore, new HardReferenceQueue<PO>( new DefaultEvictionListener(), BTree.DEFAULT_HARD_REF_QUEUE_CAPACITY, BTree.DEFAULT_HARD_REF_QUEUE_SCAN)); ! ! // report on the event. ! ResourceManager.openIndexSegment(null/* name */, fileStore.getFile() ! .toString(), fileStore.size()); ! } ! /** * Open a read-only index segment. *************** *** 124,139 **** // Type-safe reference to the backing store. this.fileStore = (IndexSegmentFileStore) fileStore; ! // Read the root node. this.root = readNodeOrLeaf(fileStore.metadata.addrRoot); ! if( fileStore.metadata.addrBloom == 0L ) { ! /* * No bloom filter. */ ! this.bloomFilter = null; ! } else { --- 139,201 ---- // Type-safe reference to the backing store. this.fileStore = (IndexSegmentFileStore) fileStore; ! ! _open(); ! ! } ! ! /** ! * Extended to also close the backing file. ! */ ! public void close() { ! ! if (root == null) { ! ! throw new IllegalStateException("Already closed."); ! ! } ! ! // close the backing file. ! fileStore.close(); ! ! // release the optional bloom filter. ! bloomFilter = null; ! ! // release buffers and hard reference to the root node. ! super.close(); ! ! // report event. ! ResourceManager.closeIndexSegment(fileStore.getFile().toString()); ! ! } ! ! /** ! * Re-opens the backing file. ! */ ! protected void reopen() { ! ! if (root == null) { ! ! // reopen the file. ! fileStore.reopen(); ! ! _open(); ! ! } ! ! } ! ! private void _open() { ! // Read the root node. this.root = readNodeOrLeaf(fileStore.metadata.addrRoot); ! if (fileStore.metadata.addrBloom == 0L) { ! /* * No bloom filter. */ ! this.bloomFilter = null; ! } else { *************** *** 151,270 **** } - - } - - } ! public void close() { ! ! fileStore.close(); ! ! } ! ! // /** ! // * Overrides the base class to use the optional bloom filter when present. ! // * ! // * @todo Verify that the bloom filter is safe for concurrent readers ! // * ! // * FIXME restore use of the bloom filter once I update the api to byte[]s. ! // * ! // * FIXME use the bloom filter for the batch lookup api as well. ! // */ ! // public Object lookup(Object key) { ! // ! // if (key == null) { ! // ! // throw new IllegalArgumentException(); ! // ! // } ! // ! // Object key2; ! // if(stride > 1) { ! // /* ! // * When the stride is greater than one the application needs to ! // * provide an array parameter anyway so you do not need to copy ! // * anything. ! // */ ! // key2 = key; ! // } else { ! // /* ! // * unautobox the key. When unboxing a key, we need to allocate a new ! // * buffer each time in order to support concurrent readers. ! // */ ! // key2 = ArrayType.alloc(keyType, 1, stride); ! // unbox(key,key2); ! // } ! // ! // if( bloomFilter != null && ! containsKey(key2)) { ! // ! // /* ! // * If the bloom filter reports that the key does not exist then we ! // * always believe it. ! // */ ! // ! // counters.nbloomRejects++; ! // ! // return null; ! // ! // } ! // ! // /* ! // * Either there is no bloom filter or the bloom filter believes that the ! // * key exists. Either way we now lookup the entry in the btree. Again, ! // * we allocate temporary arrays in order to support concurrent readers. ! // */ ! // ! // final Object[] values = new Object[1]; ! // ! // /* ! // * use the super class implementation since we already tested the bloom ! // * filter. ! // */ ! // super.lookup(1,key2,values); ! // ! // return values[0]; ! // ! // } ! ! /** ! * Operation is not supported. ! */ ! public void insert(int ntuples, Object keys, Object[] values) { - throw new UnsupportedOperationException(); - } ! /** ! * Operation is not supported. */ - public void remove(int ntuples, Object keys, Object[] values) { - - throw new UnsupportedOperationException(); - - } - - // /** - // * Used to unbox an application key into a supplied buffer. - // * - // * @param src - // * The application key (Integer, Long, etc). - // * @param dst - // * A polymorphic array with room for a single key. - // */ - // private void unbox(Object src,Object dst) { - // assert stride == 1; - // switch(keyType) { - // case BYTE: ((byte[])dst)[0] = ((Byte)src).byteValue(); break; - // case SHORT: ((short[])dst)[0] = ((Short)src).shortValue(); break; - // case CHAR: ((char[])dst)[0] = ((Character)src).charValue(); break; - // case INT: ((int[])dst)[0] = ((Integer)src).intValue(); break; - // case LONG: ((long[])dst)[0] = ((Long)src).longValue(); break; - // case FLOAT: ((float[])dst)[0] = ((Float)src).floatValue(); break; - // case DOUBLE: ((double[])dst)[0] = ((Double)src).doubleValue(); break; - // case OBJECT: ((Object[])dst)[0] = src; break; - // default: throw new UnsupportedOperationException(); - // } - // } /** --- 213,224 ---- } ! } } ! /* ! * bloom filter support. */ /** *************** *** 275,280 **** * * @return True if the bloom filter believes that the key is present in the ! * index. When true, you must still test the key to verify that it ! * is, in fact, present in the index. When false, you do NOT need to * test the index. * --- 229,234 ---- * * @return True if the bloom filter believes that the key is present in the ! * index. When true, you MUST still test the key to verify that it ! * is, in fact, present in the index. When false, you SHOULD NOT * test the index. * *************** *** 284,293 **** final protected boolean containsKey(byte[] key) { assert bloomFilter != null; ! return bloomFilter.contains(key); ! } /** * Operation is disallowed. --- 238,254 ---- final protected boolean containsKey(byte[] key) { + reopen(); + assert bloomFilter != null; ! return bloomFilter.contains(key); ! } + /* + * ISimpleBTree (disallows mutation operations, applies the optional bloom + * filter when present). + */ + /** * Operation is disallowed. *************** *** 307,311 **** } ! /** * Factory for immutable nodes and leaves used by the {@link NodeSerializer}. --- 268,401 ---- } ! ! /** ! * Applies the optional bloom filter if it exists. If the bloom filter ! * reports true, then verifies that the key does in fact exist in the index. ! */ ! public boolean contains(byte[] key) { ! ! if (bloomFilter != null) { ! ! if (!containsKey(key)) { ! ! // rejected by the bloom filter. ! return false; ! ! } ! ! // test the index. ! return super.contains(key); ! ! } ! ! // test the index. ! return super.contains(key); ! ! } ! ! /** ! * Applies the optional bloom filter if it exists. If the bloom filter ! * exists and reports true, then looks up the value for the key in the index ! * (note that the key might not exist in the index since a bloom filter ! * allows false positives). ! */ ! public Object lookup(Object key) { ! ! if (bloomFilter != null) { ! ! byte[] _key; ! ! if (key instanceof byte[]) { ! ! _key = (byte[]) key; ! ! } else { ! _key = unbox(key); ! ! } ! ! if (!containsKey(_key)) { ! ! // rejected by the bloom filter. ! return null; ! ! } ! ! /* ! * Test the index (may be a false positive and we need the value ! * paired to the key in any case). ! */ ! return super.lookup(_key); ! ! } ! ! // test the index. ! return super.lookup(key); ! ! } ! ! /* ! * IBatchBTree (disallows mutation operations, applies optional bloom filter ! * for batch operations). ! */ ! ! /** ! * Disallowed. ! */ ! public void insert(BatchInsert op) { ! ! throw new UnsupportedOperationException(MSG_READ_ONLY); ! ! } ! ! /** ! * Disallowed. ! */ ! public void remove(BatchRemove op) { ! ! throw new UnsupportedOperationException(MSG_READ_ONLY); ! ! } ! ! /** ! * Apply a batch lookup operation. The bloom filter is used iff it is ! * defined. ! */ ! public void lookup(BatchLookup op) { ! ! if( bloomFilter != null ) { ! ! op.apply(this); ! ! } else { ! ! super.lookup(op); ! ! } ! ! } ! ! /** ! * Apply a batch existence test operation. The bloom filter is used iff it ! * is defined. ! */ ! public void contains(BatchContains op) { ! ! if( bloomFilter != null ) { ! ! op.apply(this); ! ! } else { ! ! super.contains(op); ! ! } ! ! } ! ! /* ! * INodeFactory ! */ ! /** * Factory for immutable nodes and leaves used by the {@link NodeSerializer}. *************** *** 314,320 **** public static final INodeFactory INSTANCE = new ImmutableNodeFactory(); ! ! private ImmutableNodeFactory() {} ! public ILeafData allocLeaf(IIndex btree, long addr, int branchingFactor, IKeyBuffer keys, Object[] values) { --- 404,411 ---- public static final INodeFactory INSTANCE = new ImmutableNodeFactory(); ! ! private ImmutableNodeFactory() { ! } ! public ILeafData allocLeaf(IIndex btree, long addr, int branchingFactor, IKeyBuffer keys, Object[] values) { *************** *** 357,362 **** long[] childKeys, int[] childEntryCount) { ! super(btree, addr, branchingFactor, nentries, keys, ! childKeys, childEntryCount); } --- 448,453 ---- long[] childKeys, int[] childEntryCount) { ! super(btree, addr, branchingFactor, nentries, keys, childKeys, ! childEntryCount); } *************** *** 365,385 **** throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object insert(Object key,Object val) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object remove(Object key) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! } ! /** * Immutable leaf throws {@link UnsupportedOperationException} for the --- 456,476 ---- throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object insert(Object key, Object val) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object remove(Object key) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! } ! /** * Immutable leaf throws {@link UnsupportedOperationException} for the *************** *** 402,430 **** protected ImmutableLeaf(AbstractBTree btree, long addr, int branchingFactor, IKeyBuffer keys, Object[] values) { ! super(btree, addr, branchingFactor, keys, values); ! } ! public void delete() { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object insert(Object key,Object val) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object remove(Object key) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } } ! } --- 493,521 ---- protected ImmutableLeaf(AbstractBTree btree, long addr, int branchingFactor, IKeyBuffer keys, Object[] values) { ! super(btree, addr, branchingFactor, keys, values); ! } ! public void delete() { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object insert(Object key, Object val) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } ! public Object remove(Object key) { throw new UnsupportedOperationException(MSG_READ_ONLY); ! } } ! } *************** *** 471,477 **** */ public CustomAddressSerializer() { ! this.offsetNodes = 0; ! } --- 562,568 ---- */ public CustomAddressSerializer() { ! this.offsetNodes = 0; ! } *************** *** 491,511 **** */ public CustomAddressSerializer(long offsetNodes) { ! /* * Note: trim to int (we restrict the maximum size of the segment). */ this.offsetNodes = (int) offsetNodes; ! ! // System.err.println("offsetNodes="+offsetNodes); ! } ! /** ! * This over-estimates the space requirements. */ public int getSize(int n) { ! return Bytes.SIZEOF_LONG * n; ! } --- 582,602 ---- */ public CustomAddressSerializer(long offsetNodes) { ! /* * Note: trim to int (we restrict the maximum size of the segment). */ this.offsetNodes = (int) offsetNodes; ! ! // System.err.println("offsetNodes="+offsetNodes); ! } ! /** ! * This over-estimates the space requirements. */ public int getSize(int n) { ! return Bytes.SIZEOF_LONG * n; ! } *************** *** 514,519 **** * to the conventions of this class. */ ! public void putChildAddresses(DataOutputStream os, long[] childAddr, int nchildren) throws IOException { ! for (int i = 0; i < nchildren; i++) { --- 605,611 ---- * to the conventions of this class. */ ! public void putChildAddresses(DataOutputStream os, long[] childAddr, ! int nchildren) throws IOException { ! for (int i = 0; i < nchildren; i++) { *************** *** 525,549 **** if (addr == 0L) { ! throw new RuntimeException("Child is not persistent: index=" ! + i); } ! // test the low bit. when set this is a node; otherwise a leaf. final boolean isLeaf = (addr & 1) == 0; ! // strip off the low bit. addr >>= 1; ! final int offset = Addr.getOffset(addr); ! final int nbytes = Addr.getByteCount(addr); ! final int adjustedOffset = (isLeaf ? (offset << 1) : ((offset << 1) | 1)); ! // write the adjusted offset (requires decoding). LongPacker.packLong(os, adjustedOffset); ! // write the #of bytes (does not require decoding). LongPacker.packLong(os, nbytes); --- 617,641 ---- if (addr == 0L) { ! throw new RuntimeException( ! "Child is not persistent: index=" + i); } ! // test the low bit. when set this is a node; otherwise a leaf. final boolean isLeaf = (addr & 1) == 0; ! // strip off the low bit. addr >>= 1; ! final int offset = Addr.getOffset(addr); ! final int nbytes = Addr.getByteCount(addr); ! final int adjustedOffset = (isLeaf ? (offset << 1) : ((offset << 1) | 1)); ! // write the adjusted offset (requires decoding). LongPacker.packLong(os, adjustedOffset); ! // write the #of bytes (does not require decoding). LongPacker.packLong(os, nbytes); *************** *** 561,565 **** // check that we know the offset for deserialization. assert offsetNodes > 0; ! for (int i = 0; i < nchildren; i++) { --- 653,657 ---- // check that we know the offset for deserialization. assert offsetNodes > 0; ! for (int i = 0; i < nchildren; i++) { *************** *** 570,582 **** * whether the referent is a node (1) or a leaf (0). */ ! /* * offset (this field must be decoded). */ long v = LongPacker.unpackLong(is); ! assert v <= Integer.MAX_VALUE; ! ! // test the low bit. when set this is a node; otherwise a leaf. final boolean isLeaf = (v & 1) == 0; --- 662,674 ---- * whether the referent is a node (1) or a leaf (0). */ ! /* * offset (this field must be decoded). */ long v = LongPacker.unpackLong(is); ! assert v <= Integer.MAX_VALUE; ! ! // test the low bit. when set this is a node; otherwise a leaf. final boolean isLeaf = (v & 1) == 0; *************** *** 585,598 **** // compute the real offset into the file. ! final int offset = isLeaf? (int)v : (int)v + offsetNodes; ! /* * nbytes (this field does not need any further interpretation). */ ! v = LongPacker.unpackLong(is); ! assert v <= Integer.MAX_VALUE; ! final int nbytes = (int) v; --- 677,690 ---- // compute the real offset into the file. ! final int offset = isLeaf ? (int) v : (int) v + offsetNodes; ! /* * nbytes (this field does not need any further interpretation). */ ! v = LongPacker.unpackLong(is); ! assert v <= Integer.MAX_VALUE; ! final int nbytes = (int) v; *************** *** 601,609 **** */ final long addr = Addr.toLong(nbytes, offset); ! if (addr == 0L) { throw new RuntimeException( ! "Child does not have persistent address: index=" + i); } --- 693,702 ---- */ final long addr = Addr.toLong(nbytes, offset); ! if (addr == 0L) { throw new RuntimeException( ! "Child does not have persistent address: index=" ! + i); } *************** *** 630,650 **** * @return The encoded address. */ ! static public long encode(int nbytes,int offset,boolean isLeaf) { ! long addr = Addr.toLong(nbytes, (int) offset); ! addr <<= 1; // (addr << 1) ! if (!isLeaf) { ! addr |= 1; // addr++; ! } ! return addr; ! } } ! } --- 723,743 ---- * @return The encoded address. */ ! static public long encode(int nbytes, int offset, boolean isLeaf) { ! long addr = Addr.toLong(nbytes, (int) offset); ! addr <<= 1; // (addr << 1) ! if (!isLeaf) { ! addr |= 1; // addr++; ! } ! return addr; ! } } ! } Index: BTree.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/BTree.java,v retrieving revision 1.38 retrieving revision 1.39 diff -C2 -d -r1.38 -r1.39 *** BTree.java 12 Mar 2007 18:06:12 -0000 1.38 --- BTree.java 15 Mar 2007 16:11:08 -0000 1.39 *************** *** 482,486 **** * Read the root node of the btree. */ ! this.root = readNodeOrLeaf( metadata.addrRoot ); } --- 482,487 ---- * Read the root node of the btree. */ ! // this.root = readNodeOrLeaf( metadata.addrRoot ); ! reopen(); } *************** *** 503,506 **** --- 504,550 ---- } + + /** + * Uses {@link #handleCommit()} to flush any dirty nodes to the store and + * update the metadata so we can clear the hard reference queue and release + * the hard reference to the root node. {@link #reopen()} is responsible for + * reloading the root node. + */ + public void close() { + + /* + * flush any dirty records, noting the address of the metadata record + * so that we can reload the store. + */ + handleCommit(); + + /* + * this will clear the hard reference cache, release the node serializer + * buffers, and release the hard reference to the root node. + */ + super.close(); + + } + + /** + * Reloads the root node iff it is <code>null</code> (indicating a closed + * index). + * + * @see #close() + */ + protected void reopen() { + + if (root == null) { + + /* + * reload the root node. + */ + + root = readNodeOrLeaf(metadata.addrRoot); + + } + + } + /** * Writes dirty nodes using a post-order traversal that first writes any *************** *** 519,522 **** --- 563,568 ---- */ public long write() { + + assert root != null; // i.e., isOpen(). if (root.dirty) { *************** *** 563,569 **** /** ! * Method returns the metadata record persisted by {@link #write()}. You ! * MUST override this method to return a subclass of {@link BTreeMetadata} ! * in order to persist additional metadata with the btree. * * @return A new metadata object that can be used to restore the btree. --- 609,616 ---- /** ! * Method returns the metadata record persisted by {@link #write()}. ! * <p> ! * Note: In order to persist additional metadata with the btree you MUST ! * override this method to return a subclass of {@link BTreeMetadata}. * * @return A new metadata object that can be used to restore the btree. *************** *** 571,574 **** --- 618,623 ---- protected BTreeMetadata newMetadata() { + assert root != null; // i.e., isOpen(). + return new BTreeMetadata(this); *************** *** 578,591 **** * Handle request for a commit by {@link #write()}ing dirty nodes and * leaves onto the store, writing a new metadata record, and returning the ! * address of that metadata record.< * <p> * Note: In order to avoid needless writes the existing metadata record is ! * always returned iff all of the folowing are true: * <ol> ! * <li> it metadata record is defined (it is not defined when a btree is ! * first created).</li> ! * <li> the root of the btree is NOT dirty </li> ! * <li> the persistent address of the root of the btree is the same as the ! * address record in the metadata record.</li> * </ol> * --- 627,644 ---- * Handle request for a commit by {@link #write()}ing dirty nodes and * leaves onto the store, writing a new metadata record, and returning the ! * address of that metadata record. * <p> * Note: In order to avoid needless writes the existing metadata record is ! * always returned if: * <ol> ! * <li> the metadata record is defined (it is not defined when a btree is ! * first created) -AND- </li> ! * <li> the root of the btree is NOT dirty and the persistent address of the ! * root of the btree is the same as the address record in the metadata ! * record -OR- </li> ! * <li> the root is <code>null</code>, indicating that the index is ! * closed (flushing the index to disk and updating the metadata record is ! * part of the close protocol so we know that the metadata address is ! * current in this case).</li> * </ol> * *************** *** 595,600 **** public long handleCommit() { ! if (metadata != null && !root.isDirty() ! && metadata.addrRoot == root.getIdentity()) { /* --- 648,654 ---- public long handleCommit() { ! if (metadata != null ! && (root == null || !root.dirty ! && metadata.addrRoot == root.getIdentity())) { /* Index: AbstractBTree.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/AbstractBTree.java,v retrieving revision 1.19 retrieving revision 1.20 diff -C2 -d -r1.19 -r1.20 *** AbstractBTree.java 12 Mar 2007 18:06:12 -0000 1.19 --- AbstractBTree.java 15 Mar 2007 16:11:08 -0000 1.20 *************** *** 152,158 **** * at which point it is replaced by a node. The root is also replaced each * time copy-on-write triggers a cascade of updates. */ protected AbstractNode root; ! /** * Used to serialize and de-serialize the nodes and leaves of the tree. --- 152,168 ---- * at which point it is replaced by a node. The root is also replaced each * time copy-on-write triggers a cascade of updates. + * <p> + * This hard reference is cleared to <code>null</code> if an index is + * {@link #close() closed}. {@link #getRoot()} automatically uses + * {@link #reopen()} to reload the root so that closed indices may be + * transparently made ready for further use (indices are closed to reduce + * their resource burden, not to make their references invalid). The + * {@link AbstractNode} and derived classes <em>assume</em> that the root + * is non-null. This assumption is valid if {@link #close()} is invoked by + * the application in a manner consistent with the single-threaded contract + * for the {@link AbstractBTree}. */ protected AbstractNode root; ! /** * Used to serialize and de-serialize the nodes and leaves of the tree. *************** *** 161,164 **** --- 171,187 ---- /** + * Count of the #of times that a reference to this {@link AbstractBTree} + * occurs on a {@link HardReferenceQueue}. This field will remain zero(0) + * unless the {@link AbstractBTree} is placed onto a + * {@link HardReferenceQueue} maintained by the application. + * <p> + * Note: <em>DO NOT MODIFY THIS FIELD DIRECTLY</em> -- The journal is + * responsible for setting up the {@link HardReferenceQueue}, updating this + * field, and {@link #close() closing} {@link AbstractBTree}s that are not + * in use. + */ + public int referenceCount = 0; + + /** * Leaves (and nodes) are added to a hard reference queue when they are * created or read from the store. On eviction from the queue a dirty leaf *************** *** 314,318 **** /** ! * The persistence store. */ public IRawStore getStore() { --- 337,431 ---- /** ! * The contract for close is to reduce the resource burden of the index (by ! * discarding buffers) while not rendering the index inoperative. An index ! * that has been {@link #close() closed} MAY be {@link #reopen() reopened} ! * at any time (conditional on the continued availability of the backing ! * store). The index reference remains valid after a {@link #close()}. A ! * closed index is transparently restored by either {@link #getRoot()} or ! * {@link #reopen()}. ! * <p> ! * This implementation clears the hard reference queue (releasing all node ! * references), releases the hard reference to the root node, and releases ! * the buffers on the {@link NodeSerializer} (they will be naturally ! * reallocated on reuse). ! * <p> ! * Note: {@link AbstractBTree} is NOT thread-safe and {@link #close()} MUST ! * be invoked in a context in which there will not be concurrent threads -- ! * the natural choice being the single-threaded commit service on the ! * journal. ! * ! * @exception IllegalStateException ! * if the root is <code>null</code>, indicating that the ! * index is already closed. ! * ! * @exception IllegalStateException ! * if the root is dirty (this implies that this is a mutable ! * btree and there are mutations that have not been written ! * through to the store) ! */ ! public void close() { ! ! if(root==null) { ! ! throw new IllegalStateException("Already closed"); ! ! } ! ! if (root.dirty) { ! ! throw new IllegalStateException("Root node is dirty"); ! ! } ! ! /* ! * Release buffers. ! */ ! nodeSer.close(); ! ! /* ! * Clear the hard reference queue (this will not trigger any writes ! * since we know as a pre-condition that the root node is clean). ! */ ! leafQueue.evictAll(true); ! ! /* ! * Clear the reference to the root node (permits GC). ! */ ! root = null; ! ! } ! ! /** ! * This is part of a {@link #close()}/{@link #reopen()} protocol that may ! * be used to reduce the resource burden of an {@link AbstractBTree}. The ! * implementation must reload the root node of the tree iff {@link #root} is ! * <code>null</code> (indicating that the index has been closed). This ! * method is automatically invoked by a variety of methods that need to ! * ensure that the index is available for use. ! * ! * @see #close() ! * @see #isOpen() ! * @see #getRoot() ! */ ! abstract protected void reopen(); ! ! /** ! * An "open" index has its buffers and root node in place rather than having ! * to reallocate buffers or reload the root node from the store. ! * ! * @return If the index is "open". ! * ! * @see #close() ! * @see #reopen() ! * @see #getRoot() ! */ ! final public boolean isOpen() { ! ! return root != null; ! ! } ! ! /** ! * The backing store. */ public IRawStore getStore() { *************** *** 374,380 **** --- 487,500 ---- * at which point it is replaced by a node. The root is also replaced each * time copy-on-write triggers a cascade of updates. + * <p> + * The hard reference to the root node is cleared if the index is + * {@link #close() closed}. This method automatically {@link #reopen()}s + * the index if it is closed, making it available for use. */ final public AbstractNode getRoot() { + // make sure that the root is defined. + if(root == null) reopen(); + return root; *************** *** 421,425 **** * Each call MAY process more than one tuple. */ ! int nused = root.batchInsert(op); assert nused > 0; --- 541,545 ---- * Each call MAY process more than one tuple. */ ! int nused = getRoot().batchInsert(op); assert nused > 0; *************** *** 469,473 **** * Each call MAY process more than one tuple. */ ! int nused = root.batchLookup(op); assert nused > 0; --- 589,593 ---- * Each call MAY process more than one tuple. */ ! int nused = getRoot().batchLookup(op); assert nused > 0; *************** *** 499,503 **** * Each call MAY process more than one tuple. */ ! int nused = root.batchContains(op); assert nused > 0; --- 619,623 ---- * Each call MAY process more than one tuple. */ ! int nused = getRoot().batchContains(op); assert nused > 0; *************** *** 520,524 **** * Each call MAY process more than one tuple. */ ! int nused = root.batchRemove(op); assert nused > 0; --- 640,644 ---- * Each call MAY process more than one tuple. */ ! int nused = getRoot().batchRemove(op); assert nused > 0; *************** *** 552,556 **** * as soon as I update the test suites. */ ! final private byte[] unbox(Object key) { return keyBuilder.reset().append(((Integer) key).intValue()).getKey(); --- 672,676 ---- * as soon as I update the test suites. */ ! final protected byte[] unbox(Object key) { return keyBuilder.reset().append(((Integer) key).intValue()).getKey(); *************** *** 567,575 **** if (key instanceof byte[]) { ! return root.insert((byte[]) key,value); } else { ! return root.insert( unbox(key), value ); } --- 687,695 ---- if (key instanceof byte[]) { ! return getRoot().insert((byte[]) key,value); } else { ! return getRoot().insert( unbox(key), value ); } *************** *** 586,594 **** if (key instanceof byte[]) { ! return root.lookup((byte[])key); } else { ! return root.lookup(unbox(key)); } --- 706,714 ---- if (key instanceof byte[]) { ! return getRoot().lookup((byte[])key); } else { ! return getRoot().lookup(unbox(key)); } *************** *** 603,607 **** counters.nfinds++; ! return root.contains((byte[])key); } --- 723,727 ---- counters.nfinds++; ! return getRoot().contains((byte[])key); } *************** *** 616,624 **** if (key instanceof byte[]) { ! return root.remove((byte[])key); } else { ! return root.remove(unbox(key)); } --- 736,744 ---- if (key instanceof byte[]) { ! return getRoot().remove((byte[])key); } else { ! return getRoot().remove(unbox(key)); } *************** *** 633,637 **** counters.nindexOf++; ! int index = root.indexOf(key); return index; --- 753,757 ---- counters.nindexOf++; ! int index = getRoot().indexOf(key); return index; *************** *** 649,653 **** counters.ngetKey++; ! return root.keyAt(index); } --- 769,773 ---- counters.ngetKey++; ! return getRoot().keyAt(index); } *************** *** 663,667 **** counters.ngetKey++; ! return root.valueAt(index); } --- 783,787 ---- counters.ngetKey++; ! return getRoot().valueAt(index); } *************** *** 675,679 **** * are non-null) before calling rangeIterator on the root node. */ ! return root.rangeIterator(fromKey, toKey); } --- 795,799 ---- * are non-null) before calling rangeIterator on the root node. */ ! return getRoot().rangeIterator(fromKey, toKey); } *************** *** 681,684 **** --- 801,806 ---- public int rangeCount(byte[] fromKey, byte[] toKey) { + AbstractNode root = getRoot(); + int fromIndex = (fromKey == null ? 0 : root.indexOf(fromKey)); *************** *** 712,716 **** public IEntryIterator entryIterator() { ! return root.entryIterator(); } --- 834,838 ---- public IEntryIterator entryIterator() { ! return getRoot().entryIterator(); } *************** *** 726,730 **** protected Iterator leafIterator() { ! return new Striterator(root.postOrderIterator()) .addFilter(new Filter() { --- 848,852 ---- protected Iterator leafIterator() { ! return new Striterator(getRoot().postOrderIterator()) .addFilter(new Filter() { *************** *** 823,827 **** } ! return root.dump(level, out, 0, true); } --- 945,953 ---- } ! if (root != null) { ! ! return root.dump(level, out, 0, true); ! ! } else return true; } *************** *** 923,926 **** --- 1049,1053 ---- protected void writeNodeRecursive(AbstractNode node) { + assert root != null; // i.e., isOpen(). assert node != null; assert node.dirty; *************** *** 998,1001 **** --- 1125,1129 ---- protected long writeNodeOrLeaf(AbstractNode node) { + assert root != null; // i.e., isOpen(). assert node != null; assert node.btree == this; Index: NodeSerializer.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/NodeSerializer.java,v retrieving revision 1.33 retrieving revision 1.34 diff -C2 -d -r1.33 -r1.34 *** NodeSerializer.java 6 Mar 2007 20:38:05 -0000 1.33 --- NodeSerializer.java 15 Mar 2007 16:11:08 -0000 1.34 *************** *** 315,318 **** --- 315,320 ---- */ private final ChecksumUtility chk; + + private final int initialBufferCapacity; public IValueSerializer getValueSerializer() { *************** *** 416,419 **** --- 418,423 ---- } + this.initialBufferCapacity = initialBufferCapacity; + this._buf = alloc(initialBufferCapacity); *************** *** 440,443 **** --- 444,465 ---- /** + * Releases any buffers. They will be automatically reallocated if the + * {@link NodeSerializer} is used again. + * + * @todo write tests of this feature, including random closes during the + * {@link NodeSerializer} stress test and with and without record + * compression (the {@link #cbuf} field is not being automatically + * (re-)allocated right now so that will break if we clear the + * buffer). + */ + public void close() { + + _buf = null; + + cbuf = null; + + } + + /** * Allocate a buffer of the stated capacity. * *************** *** 600,604 **** */ public ByteBuffer putNodeOrLeaf(IAbstractNodeData node) { ! if(node instanceof INodeData) { --- 622,626 ---- */ public ByteBuffer putNodeOrLeaf(IAbstractNodeData node) { ! if(node instanceof INodeData) { *************** *** 626,629 **** --- 648,658 ---- public ByteBuffer putNode(INodeData node) { + if( _buf == null ) { + + // the buffer was released so we reallocate it. + _buf = alloc(initialBufferCapacity); + + } + while (true) { *************** *** 643,647 **** private ByteBuffer putNode(ByteBuffer buf, INodeData node) { ! assert buf != null; assert node != null; --- 672,676 ---- private ByteBuffer putNode(ByteBuffer buf, INodeData node) { ! assert buf != null; assert node != null; *************** *** 935,938 **** --- 964,974 ---- public ByteBuffer putLeaf(ILeafData leaf) { + if( _buf == null ) { + + // the buffer was released so we reallocate it. + _buf = alloc(initialBufferCapacity); + + } + while (true) { *************** *** 1362,1366 **** * Buffer for compressed records. */ ! final private ByteBuffer cbuf; /** --- 1398,1402 ---- * Buffer for compressed records. */ ! private ByteBuffer cbuf; /** |
