[cweb-CVS] bigdata/src/java/com/bigdata/journal TemporaryStore.java, NONE, 1.1 OverflowException.ja

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/journal
In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv1690/src/java/com/bigdata/journal

Modified Files:
	AbstractBufferStrategy.java FileMetadata.java Tx.java 
	BasicBufferStrategy.java DiskOnlyStrategy.java 
Added Files:
	TemporaryStore.java OverflowException.java 
Log Message:
Added a TemporaryStore that starts in memory and then automatically overflows to disk.

Index: DiskOnlyStrategy.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java,v
retrieving revision 1.15
retrieving revision 1.16
diff -C2 -d -r1.15 -r1.16
*** DiskOnlyStrategy.java	9 Feb 2007 16:13:18 -0000	1.15
--- DiskOnlyStrategy.java	15 Feb 2007 20:59:21 -0000	1.16
***************
*** 61,64 ****
--- 61,76 ----
      private boolean open;

+     public File getFile() {
+         
+         return file;
+         
+     }
+     
+ //    public FileChannel getFileChannel() {
+ //        
+ //        return channel;
+ //        
+ //    }
+     
      DiskOnlyStrategy(long maximumExtent, FileMetadata fileMetadata) {

***************
*** 255,259 ****
                  if (!overflow((int) needed)) {

!                     throw new RuntimeException("overflow");

                  }
--- 267,271 ----
                  if (!overflow((int) needed)) {

!                     throw new OverflowException();

                  }

Index: AbstractBufferStrategy.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** AbstractBufferStrategy.java	9 Feb 2007 16:13:18 -0000	1.8
--- AbstractBufferStrategy.java	15 Feb 2007 20:59:21 -0000	1.9
***************
*** 101,106 ****
       */
      public boolean overflow(int needed) {

!         if( getUserExtent() +needed > Integer.MAX_VALUE) {

              // Would overflow int32 bytes.
--- 101,110 ----
       */
      public boolean overflow(int needed) {
+ 
+         final long userExtent = getUserExtent();

!         final long required = userExtent + needed;
!         
!         if ( required > Integer.MAX_VALUE) {

              // Would overflow int32 bytes.
***************
*** 110,117 ****
          }

          /*
           * Increase by the initial extent or by 32M, whichever is greater.
           */
!         long newExtent = getUserExtent()
                  + Math.max(initialExtent, Bytes.megabyte * 32);

--- 114,129 ----
          }

+         if( required > maximumExtent ) {
+             
+             // Would exceed the maximum extent.
+             
+             return false;
+             
+         }
+         
          /*
           * Increase by the initial extent or by 32M, whichever is greater.
           */
!         long newExtent = userExtent
                  + Math.max(initialExtent, Bytes.megabyte * 32);

***************
*** 124,128 ****

          /*
!          * Extent the capacity.
           */
          truncate( newExtent );
--- 136,140 ----

          /*
!          * Extend the capacity.
           */
          truncate( newExtent );

Index: FileMetadata.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/FileMetadata.java,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** FileMetadata.java	8 Feb 2007 21:32:10 -0000	1.10
--- FileMetadata.java	15 Feb 2007 20:59:21 -0000	1.11
***************
*** 91,95 ****
       * journal.
       */
!     final int headerSize0  = SIZE_MAGIC + SIZE_VERSION + (SIZEOF_ROOT_BLOCK * 2);

      /**
--- 91,95 ----
       * journal.
       */
!     static final int headerSize0 = SIZE_MAGIC + SIZE_VERSION + (SIZEOF_ROOT_BLOCK * 2);

      /**
***************
*** 116,120 ****
       *            The unique segment identifier.
       * @param file
!      *            The name of the file to be opened.
       * @param bufferMode
       *            The {@link BufferMode}.
--- 116,122 ----
       *            The unique segment identifier.
       * @param file
!      *            The name of the file to be opened - when null, a file with a
!      *            unique name will be created in the default temporary
!      *            directory.
       * @param bufferMode
       *            The {@link BufferMode}.
***************
*** 140,149 ****
       *             journal.
       */
!     FileMetadata(int segmentId, File file, BufferMode bufferMode, boolean useDirectBuffers,
!             long initialExtent, boolean create, boolean readOnly,
!             ForceEnum forceWrites) throws RuntimeException {

!         if (file == null)
!             throw new IllegalArgumentException();

          if (bufferMode == null)
--- 142,151 ----
       *             journal.
       */
!     FileMetadata(int segmentId, File file, BufferMode bufferMode,
!             boolean useDirectBuffers, long initialExtent, boolean create,
!             boolean readOnly, ForceEnum forceWrites) throws RuntimeException {

! //        if (file == null)
! //            throw new IllegalArgumentException();

          if (bufferMode == null)
***************
*** 175,180 ****
          this.segment = segmentId;

-         this.file = file;
-         
          this.bufferMode = bufferMode;

--- 177,180 ----
***************
*** 183,187 ****
          this.readOnly = readOnly;

!         exists = file.exists();

          if (exists) {
--- 183,187 ----
          this.readOnly = readOnly;

!         this.exists = file != null && file.exists();

          if (exists) {
***************
*** 208,215 ****
              }

              System.err.println("Will create file: " + file.getAbsoluteFile());

          }
! 
          try {

--- 208,231 ----
              }

+             if (file == null) {
+                 
+                 try {
+ 
+                     file = File.createTempFile("bigdata", ".store");
+                     
+                 } catch (IOException ex) {
+ 
+                     throw new RuntimeException(ex);
+ 
+                 }
+                 
+             }
+ 
              System.err.println("Will create file: " + file.getAbsoluteFile());

          }
!         
!         this.file = file;
!         
          try {

Index: Tx.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/Tx.java,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** Tx.java	15 Feb 2007 14:23:49 -0000	1.26
--- Tx.java	15 Feb 2007 20:59:21 -0000	1.27
***************
*** 58,62 ****
  import com.bigdata.objndx.IndexSegment;
  import com.bigdata.rawstore.IRawStore;
- import com.bigdata.rawstore.SimpleMemoryRawStore;
  import com.bigdata.scaleup.MetadataIndex;
  import com.bigdata.scaleup.PartitionedIndex;
--- 58,61 ----
***************
*** 147,163 ****

      /**
!      * A store used to hold write sets for the transaction.  The same store can
!      * be used to buffer resolved write-write conflicts.
!      * 
!      * @todo This uses a memory-based store to avoid issues with maintaining
!      *       transactions across journal boundaries. This could be improved on
!      *       trivially by transparently promoting the store from memory-based to
!      *       disk-based if it overflows some set maximum capacity. In such a
!      *       scenario, the file backing the on disk store would be flagged for
!      *       deletion on exit of the JVM and allocated in a temporary directory.<br>
!      *       A further improvement would allow transactions to use partitioned
!      *       indices.
       */
!     final private IRawStore tmpStore = new SimpleMemoryRawStore();

      /**
--- 146,161 ----

      /**
!      * A store used to hold write sets for the transaction. The same store can
!      * be used to buffer resolved write-write conflicts. This uses a
!      * {@link TemporaryStore} to avoid issues with maintaining transactions
!      * across journal boundaries. This places a limit on transactions of 2G in
!      * their serialized write set. Since the indices use a copy-on-write model,
!      * the amount of user data can be significantly less due to multiple
!      * versions of the same btree nodes. Using smaller branching factors in the
!      * isolated index helps significantly to increase the effective utilization
!      * of the store since copy-on-write causes fewer bytes to be copied each
!      * time it is invoked.
       */
!     final private IRawStore tmpStore = new TemporaryStore();

      /**

--- NEW FILE: TemporaryStore.java ---
/**

The Notice below must appear in each file of the Source Code of any
copy you distribute of the Licensed Product.  Contributors to any
Modifications may add their own copyright notices to identify their
own contributions.

License:

The contents of this file are subject to the CognitiveWeb Open Source
License Version 1.1 (the License).  You may not copy or use this file,
in either source code or executable form, except in compliance with
the License.  You may obtain a copy of the License from

  http://www.CognitiveWeb.org/legal/license/

Software distributed under the License is distributed on an AS IS
basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.  See
the License for the specific language governing rights and limitations
under the License.

Copyrights:

Portions created by or assigned to CognitiveWeb are Copyright
(c) 2003-2003 CognitiveWeb.  All Rights Reserved.  Contact
information for CognitiveWeb is available at

  http://www.CognitiveWeb.org

Portions Copyright (c) 2002-2003 Bryan Thompson.

Acknowledgements:

Special thanks to the developers of the Jabber Open Source License 1.0
(JOSL), from which this License was derived.  This License contains
terms that differ from JOSL.

Special thanks to the CognitiveWeb Open Source Contributors for their
suggestions and support of the Cognitive Web.

Modifications:

*/
/*
 * Created on Feb 15, 2007
 */

package com.bigdata.journal;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;

import com.bigdata.rawstore.Addr;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;

/**
 * A non-restart-safe store for temporary data that buffers data in memory until
 * a maximum capacity has been reached and then converts to a disk-based store
 * with a maximum capacity of 2G. The maximum capacity constraint is imposed by
 * {@link Addr}. On conversion to a disk-backed store, the disk file is created
 * using the temporary file mechansism and is marked for eventual deletion no
 * later than when the JVM exits and as soon as the store is {@link #close()}.
 * 
 * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
 * @version $Id$
 * 
 * @todo The {@link TemporaryStore} would benefit from any caching or AIO
 *       solutions developed for the {@link DiskOnlyStrategy}.
 */
public class TemporaryStore implements IRawStore {

    /**
     * The initial size of the in-memory buffer. This buffer will grow as
     * necessary until {@link #maximumInMemoryExtent} at which point the store
     * will convert over to a disk-based mechanism.
     */
    public final long initialInMemoryExtent;

    /**
     * The maximum capacity of the in-memory buffer.
     */
    public final long maximumInMemoryExtent;

    /**
     * Whether or not a direct buffer was used for the in-memory store (not
     * recommended).
     */
    public final boolean useDirectBuffers;

    private boolean open = true;
    private IBufferStrategy buf;

    public IBufferStrategy getBufferStrategy() {

        return buf;

    }

    /**
     * Create a {@link TemporaryStore} with an initial in-memory capacity of 10M
     * that will grow up to 100M before converting into a disk-based store
     * backed by a temporary file.
     */
    public TemporaryStore() {

        this( Bytes.megabyte*10, Bytes.megabyte*100, false );

    }

    /**
     * Create a {@link TemporaryStore} with the specified configuration.
     * 
     * @param initialInMemoryExtent
     *            The initial size of the in-memory buffer. This buffer will
     *            grow as necessary until <i>maximumInMemoryExtent</i> at which
     *            point the store will convert over to a disk-based mechanism.
     * @param maximumInMemoryExtent
     *            The maximum capacity of the in-memory buffer. The actual
     *            maximum may differ slightly based on the buffer growth policy.
     * @param useDirectBuffers
     *            Whether or not the in-memory buffer will be direct. The use of
     *            a direct buffer here is NOT recommended.
     */
    public TemporaryStore(long initialInMemoryExtent,
            long maximumInMemoryExtent, boolean useDirectBuffers) {

        buf = new TransientBufferStrategy(initialInMemoryExtent, maximumInMemoryExtent,
                useDirectBuffers);

        this.initialInMemoryExtent = initialInMemoryExtent;

        this.maximumInMemoryExtent = maximumInMemoryExtent;

        this.useDirectBuffers = useDirectBuffers;

    }

    /**
     * Close the store and delete the associated file, if any.
     */
    public void close() {

        if(!open) throw new IllegalStateException();

        open = false;

        buf.close();

        buf.deleteFile();

    }

    public void force(boolean metadata) {

        if(!open) throw new IllegalStateException();

        buf.force(metadata);

    }

    public boolean isOpen() {

        return open;

    }

    /**
     * Always returns <code>false</code> since the store will be deleted as soon
     * as it is closed.
     */
    public boolean isStable() {
        return false;
    }

    public ByteBuffer read(long addr, ByteBuffer dst) {

        if(!open) throw new IllegalStateException();

        return buf.read(addr, dst);

    }

    public long write(ByteBuffer data) {

        if(!open) throw new IllegalStateException();

        try {

            return buf.write(data);

        } catch(OverflowException ex) {

            if(buf instanceof TransientBufferStrategy) {

                overflowToDisk();

                return buf.write(data);

            } else {

                throw ex;

            }

        }

    }

    protected void overflowToDisk() {

        TransientBufferStrategy tmp = (TransientBufferStrategy)buf;

        int segmentId = 0;

        File file = null; // request a unique filename.

        /*
         * Set the initial extent to be large enough for the root blocks plus
         * twice the data in the in-memory buffer.
         */
        long initialExtent = FileMetadata.headerSize0 + tmp.getUserExtent() * 2;

        final boolean create = true;

        final boolean readOnly = false;

        // Do not force writes since the store is not restart safe.
        final ForceEnum forceWrites = ForceEnum.No;

        /* Create a unique store file and setup the root blocks.  The file
         * will be pre-extended to the requested initialExtent.
         */
        FileMetadata fileMetadata = new FileMetadata(segmentId, file,
                BufferMode.Disk, useDirectBuffers, initialExtent,
                create, readOnly, forceWrites);

        // Mark the file for deletion on exit.
        fileMetadata.file.deleteOnExit();

        // Open the disk-based store file.
        DiskOnlyStrategy diskBuf = new DiskOnlyStrategy(Bytes.gigabyte * 2,
                fileMetadata);

        try {

            /*
             * Transfer the data from the in-memory buffer to the disk.  The
             * write begins immediately after the file header.
             */

            // setup the transfer source.
            ByteBuffer b = tmp.directBuffer;
            b.limit(tmp.nextOffset);
            b.position(0);

            // write the data on the channel.
            diskBuf.channel.write(b,diskBuf.headerSize);

            // increment the offset.
            diskBuf.nextOffset += tmp.nextOffset;

        } catch(IOException ex) {

            try {

                diskBuf.close();

            } catch (Throwable ex2) {

            }

            throw new RuntimeException();

        }

        this.buf = diskBuf;

    }

    /**
     * Return a temporary file for use by the store. The file will be
     * automatically deleted if the JVM exits.
     * 
     * @return A temporary file for use by the store.
     */
    protected File getTempFile() {

        try {

            File file = File.createTempFile("transientOverflow", ".store");

            file.deleteOnExit();

            return file;

        } catch(IOException ex) {

            throw new RuntimeException(ex);

        }

    }

}

Index: BasicBufferStrategy.java
===================================================================
RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/BasicBufferStrategy.java,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** BasicBufferStrategy.java	9 Feb 2007 16:13:18 -0000	1.13
--- BasicBufferStrategy.java	15 Feb 2007 20:59:21 -0000	1.14
***************
*** 91,97 ****
          if (needed > 0) {

!             if(!overflow((int)needed)) {

!                 throw new RuntimeException("overflow");

              }
--- 91,97 ----
          if (needed > 0) {

!             if (!overflow((int) needed)) {

!                 throw new OverflowException();

              }
***************
*** 164,170 ****
      }

-     /**
-      * FIXME write tests of this method.
-      */
      public void truncate(long newExtent) {

--- 164,167 ----

--- NEW FILE: OverflowException.java ---
/**