cweb-cvs

[Bryan T. <tho...@us...>]

Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx
In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv5461/src/java/com/bigdata/objndx

Modified Files:
	IndexSegmentBuilder.java BTree.java IndexSegmentFileStore.java 
	AbstractBTree.java IndexSegmentMerger.java NodeSerializer.java 
	BTreeMetadata.java 
Log Message:
Further work supporting transactional isolation.

Index: IndexSegmentFileStore.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/IndexSegmentFileStore.java,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** IndexSegmentFileStore.java	12 Feb 2007 21:51:01 -0000	1.6
--- IndexSegmentFileStore.java	21 Feb 2007 20:17:21 -0000	1.7
***************
*** 117,120 ****
--- 117,126 ----
      }
      
+     public boolean isFullyBuffered() {
+         
+         return false;
+         
+     }
+     
      public void close() {
  
***************
*** 153,157 ****
       * buffer. Otherwise this reads through to the backing file.
       */
!     public ByteBuffer read(long addr, ByteBuffer dst) {
  
          if (!open)
--- 159,163 ----
       * buffer. Otherwise this reads through to the backing file.
       */
!     public ByteBuffer read(long addr) {
  
          if (!open)
***************
*** 164,167 ****
--- 170,175 ----
          final int offsetNodes = Addr.getOffset(metadata.addrNodes);
  
+         ByteBuffer dst;
+ 
          if (offset >= offsetNodes && buf_nodes != null) {
  
***************
*** 189,200 ****
          } else {
  
!             /*
!              *  Allocate if not provided by the caller.
!              */
!             if (dst == null) {
! 
!                 dst = ByteBuffer.allocate(length);
! 
!             }
  
              /*
--- 197,202 ----
          } else {
  
!             // Allocate buffer.
!             dst = ByteBuffer.allocate(length);
  
              /*

Index: IndexSegmentBuilder.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/IndexSegmentBuilder.java,v
retrieving revision 1.24
retrieving revision 1.25
diff -C2 -d -r1.24 -r1.25
*** IndexSegmentBuilder.java	15 Feb 2007 22:01:18 -0000	1.24
--- IndexSegmentBuilder.java	21 Feb 2007 20:17:21 -0000	1.25
***************
*** 64,68 ****
  
  import com.bigdata.journal.Journal;
! import com.bigdata.journal.TemporaryStore;
  import com.bigdata.objndx.IndexSegment.CustomAddressSerializer;
  import com.bigdata.rawstore.Addr;
--- 64,68 ----
  
  import com.bigdata.journal.Journal;
! import com.bigdata.journal.TemporaryRawStore;
  import com.bigdata.objndx.IndexSegment.CustomAddressSerializer;
  import com.bigdata.rawstore.Addr;
***************
*** 164,168 ****
       * a region of the {@link #outFile}.
       */
!     protected TemporaryStore leafBuffer;
      
      /**
--- 164,168 ----
       * a region of the {@link #outFile}.
       */
!     protected TemporaryRawStore leafBuffer;
      
      /**
***************
*** 170,174 ****
       * a region of the {@link #outFile}.
       */
!     protected TemporaryStore nodeBuffer;
      
      /**
--- 170,174 ----
       * a region of the {@link #outFile}.
       */
!     protected TemporaryRawStore nodeBuffer;
      
      /**
***************
*** 348,352 ****
       * 
       * @todo make checksum, and record compression parameters in this
!      *       constructor variant
       * 
       * FIXME test with and without each of these options { useChecksum,
--- 348,352 ----
       * 
       * @todo make checksum, and record compression parameters in this
!      *       constructor variant.
       * 
       * FIXME test with and without each of these options { useChecksum,
***************
*** 358,362 ****
      
          this(outFile, tmpDir, btree.getEntryCount(), btree.entryIterator(), m,
!                 btree.nodeSer.valueSerializer, false/* useChecksum */,
                  null/* new RecordCompressor() */, errorRate);
          
--- 358,362 ----
      
          this(outFile, tmpDir, btree.getEntryCount(), btree.entryIterator(), m,
!                 btree.nodeSer.valueSerializer, true/* useChecksum */,
                  null/* new RecordCompressor() */, errorRate);
          
***************
*** 387,391 ****
       *            Used to serialize values in the new {@link IndexSegment}.
       * @param useChecksum
!      *            whether or not checksums are computed for nodes and leaves.
       * @param recordCompressor
       *            An object to compress leaves and nodes or <code>null</code>
--- 387,394 ----
       *            Used to serialize values in the new {@link IndexSegment}.
       * @param useChecksum
!      *            Whether or not checksums are computed for nodes and leaves.
!      *            The use of checksums on the read-only indices provides a check
!      *            for corrupt media and definately makes the database more
!      *            robust.
       * @param recordCompressor
       *            An object to compress leaves and nodes or <code>null</code>
***************
*** 549,553 ****
               * index build operation.
               */
!             leafBuffer = new TemporaryStore();
              
              /*
--- 552,556 ----
               * index build operation.
               */
!             leafBuffer = new TemporaryRawStore();
              
              /*
***************
*** 559,563 ****
               * abstraction for a disk file.
               */
!             nodeBuffer = plan.nnodes > 0 ? new TemporaryStore() : null;
  
              /*
--- 562,566 ----
               * abstraction for a disk file.
               */
!             nodeBuffer = plan.nnodes > 0 ? new TemporaryRawStore() : null;
  
              /*

Index: BTreeMetadata.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/BTreeMetadata.java,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** BTreeMetadata.java	17 Feb 2007 21:34:18 -0000	1.10
--- BTreeMetadata.java	21 Feb 2007 20:17:21 -0000	1.11
***************
*** 149,154 ****
      public static BTreeMetadata read(IRawStore store, long addr) {
          
!         return (BTreeMetadata) SerializerUtil.deserialize(store
!                 .read(addr, null));
          
      }
--- 149,153 ----
      public static BTreeMetadata read(IRawStore store, long addr) {
          
!         return (BTreeMetadata) SerializerUtil.deserialize(store.read(addr));
          
      }

Index: BTree.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/BTree.java,v
retrieving revision 1.34
retrieving revision 1.35
diff -C2 -d -r1.34 -r1.35
*** BTree.java	17 Feb 2007 21:34:21 -0000	1.34
--- BTree.java	21 Feb 2007 20:17:21 -0000	1.35
***************
*** 409,413 ****
                  hardReferenceQueue,
                  PackedAddressSerializer.INSTANCE, valueSer,
!                 NodeFactory.INSTANCE, recordCompressor, true /* useChecksum */);
  
          /*
--- 409,417 ----
                  hardReferenceQueue,
                  PackedAddressSerializer.INSTANCE, valueSer,
!                 NodeFactory.INSTANCE, //
!                 recordCompressor, //
!                 // FIXME only use checksum for stores that are not fully buffered.
!                 true || !store.isFullyBuffered()/* useChecksum */
!                 );
  
          /*
***************
*** 453,457 ****
                  PackedAddressSerializer.INSTANCE, 
                  metadata.valueSer, NodeFactory.INSTANCE,
!                 metadata.recordCompressor, metadata.useChecksum);
          
          // save a reference to the immutable metadata record.
--- 457,463 ----
                  PackedAddressSerializer.INSTANCE, 
                  metadata.valueSer, NodeFactory.INSTANCE,
!                 metadata.recordCompressor,//
!                 metadata.useChecksum // use checksum iff used on create.
!                 );
          
          // save a reference to the immutable metadata record.

Index: AbstractBTree.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/AbstractBTree.java,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** AbstractBTree.java	15 Feb 2007 14:23:49 -0000	1.16
--- AbstractBTree.java	21 Feb 2007 20:17:21 -0000	1.17
***************
*** 133,137 ****
       * flag turns on some more expensive assertions.
       */
!     final protected boolean debug = false;
  
      /**
--- 133,137 ----
       * flag turns on some more expensive assertions.
       */
!     final protected boolean debug = DEBUG||true;
  
      /**
***************
*** 163,179 ****
  
      /**
!      * Leaves are added to a hard reference queue when they are created or read
!      * from the store. On eviction from the queue the leaf is serialized by a
!      * listener against the {@link IRawStore}. Once the leaf is no longer
!      * strongly reachable its weak references may be cleared by the VM. Note
!      * that leaves are evicted as new leaves are added to the hard reference
!      * queue. This occurs in two situations: (1) when a new leaf is created
!      * during a split of an existing leaf; and (2) when a leaf is read in from
!      * the store. The minimum capacity for the hard reference queue is two (2)
!      * so that a split may occur without forcing eviction of either leaf in the
       * split. Incremental writes basically make it impossible for the commit IO
       * to get "too large" where too large is defined by the size of the hard
!      * reference cache.
!      * 
       * Note: The code in {@link Node#postOrderIterator(boolean)} and
       * {@link DirtyChildIterator} MUST NOT touch the hard reference queue since
--- 163,183 ----
  
      /**
!      * Leaves (and nodes) are added to a hard reference queue when they are
!      * created or read from the store. On eviction from the queue a dirty leaf
!      * (or node) is serialized by a listener against the {@link IRawStore}.
!      * Once the leaf is no longer strongly reachable its weak references may be
!      * cleared by the VM.
!      * <p>
!      * Note that leaves (and nodes) are evicted as new leaves (or nodes) are
!      * added to the hard reference queue. This occurs in two situations: (1)
!      * when a new leaf (or node) is created during a split of an existing leaf
!      * (or node); and (2) when a leaf (or node) is read in from the store.
!      * <p>
!      * The minimum capacity for the hard reference queue is two (2) so that a
!      * split may occur without forcing eviction of either leaf (or node) in the
       * split. Incremental writes basically make it impossible for the commit IO
       * to get "too large" where too large is defined by the size of the hard
!      * reference cache and help to ensure fast commit operations on the store.
!      * <p>
       * Note: The code in {@link Node#postOrderIterator(boolean)} and
       * {@link DirtyChildIterator} MUST NOT touch the hard reference queue since
***************
*** 198,202 ****
       *       a fixed memory burden for the index? As it stands the #of nodes and
       *       the #of leaves in memory can vary and leaves require much more
!      *       memory than nodes (for most trees).
       */
      final protected HardReferenceQueue<PO> leafQueue;
--- 202,217 ----
       *       a fixed memory burden for the index? As it stands the #of nodes and
       *       the #of leaves in memory can vary and leaves require much more
!      *       memory than nodes (for most trees). (As an alternative, allow a
!      *       btree to retain some #of levels of the nodes in memory using a
!      *       separate node cache.)
!      * 
!      * FIXME Verify that memory allocated for leaves or nodes on the queue is
!      * reclaimed when copy-on-write is triggered since those data are no longer
!      * reachable by this instance of the btree. This is essentially a memory
!      * leak. Note that we can not just clear the hard reference on the queue,
!      * but we can release the keys and values for the node, which constitute
!      * most of its state. The node will already be marked as "!dirty" since copy
!      * on write was triggered, so it will NOT be serialized when it is evicted
!      * from the hard reference queue.
       */
      final protected HardReferenceQueue<PO> leafQueue;
***************
*** 978,982 ****
       * identity assigned to the node by the store. This method is NOT recursive
       * and dirty children of a node will NOT be visited.
!      * 
       * Note: This will throw an exception if the backing store is read-only.
       * 
--- 993,997 ----
       * identity assigned to the node by the store. This method is NOT recursive
       * and dirty children of a node will NOT be visited.
!      * <p>
       * Note: This will throw an exception if the backing store is read-only.
       * 
***************
*** 1026,1039 ****
          }
  
-         // /*
-         // * Convert the keys buffer to an immutable keys buffer. The immutable
-         // * keys buffer is potentially much more compact.
-         // */
-         // if( node.keys instanceof MutableKeyBuffer ) {
-         //
-         // node.keys = new ImmutableKeyBuffer((MutableKeyBuffer)node.keys);
-         //                
-         // }
- 
          /*
           * Serialize the node or leaf onto a shared buffer.
--- 1041,1044 ----
***************
*** 1099,1109 ****
      protected AbstractNode readNodeOrLeaf(long addr) {
  
!         /*
!          * offer the node serializer's buffer to the IRawStore. it will be used
!          * iff it is large enough and the store does not prefer to return a
!          * read-only slice.
!          */
!         // nodeSer.buf.clear();
!         ByteBuffer tmp = store.read(addr, nodeSer._buf);
          assert tmp.position() == 0;
          assert tmp.limit() == Addr.getByteCount(addr);
--- 1104,1114 ----
      protected AbstractNode readNodeOrLeaf(long addr) {
  
! //        /*
! //         * offer the node serializer's buffer to the IRawStore. it will be used
! //         * iff it is large enough and the store does not prefer to return a
! //         * read-only slice.
! //         */
! //        ByteBuffer tmp = store.read(addr, nodeSer._buf);
!         ByteBuffer tmp = store.read(addr);
          assert tmp.position() == 0;
          assert tmp.limit() == Addr.getByteCount(addr);

Index: NodeSerializer.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/NodeSerializer.java,v
retrieving revision 1.31
retrieving revision 1.32
diff -C2 -d -r1.31 -r1.32
*** NodeSerializer.java	8 Feb 2007 21:32:13 -0000	1.31
--- NodeSerializer.java	21 Feb 2007 20:17:21 -0000	1.32
***************
*** 310,317 ****
      
      /**
!      * The object index is used in a single threaded context. Therefore a
!      * single private instance is used to compute checksums.
       */
!     private static final ChecksumUtility chk = new ChecksumUtility();
      
      public IValueSerializer getValueSerializer() {
--- 310,318 ----
      
      /**
!      * A private instance is used to compute checksums for each
!      * {@link AbstractBTree}. This makes is possible to have concurrent reads
!      * or writes on multiple btrees that are backed by different stores.
       */
!     private final ChecksumUtility chk;
      
      public IValueSerializer getValueSerializer() {
***************
*** 405,408 ****
--- 406,411 ----
  
          this.useChecksum = useChecksum;
+ 
+         this.chk = useChecksum ? new ChecksumUtility() : null;
          
          if (initialBufferCapacity == 0) {
***************
*** 420,431 ****
           * FIXME The capacity of this buffer is a SWAG. If it is too small then
           * an EOFException will be thrown. This needs to be modified start with
!          * a smaller buffer and grow as required.  An alternative would be to
           * re-allocate this whenever _buf is resize since the compressed data
           * should never be larger than the original data.
           */
      
          cbuf = recordCompressor != null //
! //                ? ByteBuffer.allocate(Bytes.megabyte32) //
!                 ? ByteBuffer.allocateDirect(Bytes.megabyte32*2) //
                  : null;
          
--- 423,438 ----
           * FIXME The capacity of this buffer is a SWAG. If it is too small then
           * an EOFException will be thrown. This needs to be modified start with
!          * a smaller buffer and grow as required. An alternative would be to
           * re-allocate this whenever _buf is resize since the compressed data
           * should never be larger than the original data.
+          * 
+          * @todo consider discarding [buf] and [cbuf] if the node serializer
+          * becomes inactive in order to minimize memory use. they can be
+          * reallocated as necesssary.
           */
      
          cbuf = recordCompressor != null //
!                 ? ByteBuffer.allocate(Bytes.megabyte32) //
! //                ? ByteBuffer.allocateDirect(Bytes.megabyte32*2) //
                  : null;
          
***************
*** 542,546 ****
              /*
               * @todo it is extremely weird, but this assertion (and the parallel
!              * one below) trips during the TestBTreeWithJournal stress tests.
               * this is odd because the code explicitly resets the position of
               * the buffer that it is manipulating as its last step in getNode()
--- 549,553 ----
              /*
               * @todo it is extremely weird, but this assertion (and the parallel
!              * one below) trips during the AbstractBTreeWithJournalTestCase stress tests.
               * this is odd because the code explicitly resets the position of
               * the buffer that it is manipulating as its last step in getNode()

Index: IndexSegmentMerger.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/objndx/IndexSegmentMerger.java,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** IndexSegmentMerger.java	9 Feb 2007 18:56:58 -0000	1.8
--- IndexSegmentMerger.java	21 Feb 2007 20:17:21 -0000	1.9
***************
*** 195,199 ****
                  in1.nodeSer.keySerializer,
                  in1.nodeSer.valueSerializer,
!                 new RecordCompressor(),
                  useChecksum
                  );
--- 195,199 ----
                  in1.nodeSer.keySerializer,
                  in1.nodeSer.valueSerializer,
!                 null, //new RecordCompressor(),
                  useChecksum
                  );