[cweb-CVS] bigdata/src/test/com/bigdata/rawstore SimpleMemoryRawStore.java, NONE, 1.1 SimpleFileRawStore.java, NONE, 1.1

[Bryan T. <tho...@us...>]

Update of /cvsroot/cweb/bigdata/src/test/com/bigdata/rawstore
In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv15000/src/test/com/bigdata/rawstore

Added Files:
	SimpleMemoryRawStore.java SimpleFileRawStore.java 
Log Message:
Updated the UML model and added a ZIP containing an HTML presentation of the model.
Working on partitioned index support.

--- NEW FILE: SimpleFileRawStore.java ---
/**

The Notice below must appear in each file of the Source Code of any
copy you distribute of the Licensed Product.  Contributors to any
Modifications may add their own copyright notices to identify their
own contributions.

License:

The contents of this file are subject to the CognitiveWeb Open Source
License Version 1.1 (the License).  You may not copy or use this file,
in either source code or executable form, except in compliance with
the License.  You may obtain a copy of the License from

  http://www.CognitiveWeb.org/legal/license/

Software distributed under the License is distributed on an AS IS
basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.  See
the License for the specific language governing rights and limitations
under the License.

Copyrights:

Portions created by or assigned to CognitiveWeb are Copyright
(c) 2003-2003 CognitiveWeb.  All Rights Reserved.  Contact
information for CognitiveWeb is available at

  http://www.CognitiveWeb.org

Portions Copyright (c) 2002-2003 Bryan Thompson.

Acknowledgements:

Special thanks to the developers of the Jabber Open Source License 1.0
(JOSL), from which this License was derived.  This License contains
terms that differ from JOSL.

Special thanks to the CognitiveWeb Open Source Contributors for their
suggestions and support of the Cognitive Web.

Modifications:

*/
/*
 * Created on Jan 31, 2007
 */

package com.bigdata.rawstore;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.HashSet;
import java.util.Set;

import com.bigdata.journal.TemporaryRawStore;


/**
 * A simple persistent unbuffered implementation backed by a file. The maximum
 * size of the file is restricted to {@link Integer#MAX_VALUE} bytes since we
 * must code the offset into the file using {@link Addr#toLong(int, int)}.
 * 
 * @see {@link TemporaryRawStore}, which provides a more solution for temporary
 *      data that begins with the benefits of a memory-resident buffer and then
 *      converts to a disk-based store on overflow.
 * 
 * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
 * @version $Id$
 */
public class SimpleFileRawStore implements IRawStore {

    private boolean open = true;
    
    public final File file;
    protected final RandomAccessFile raf;
    
    /**
     * This provides a purely transient means to identify deleted records.  This
     * data does NOT survive restart of the store. 
     */
    private final Set<Long> deleted = new HashSet<Long>();

    /**
     * Open a store. The file will be created if it does not exist and it is
     * opened for writing. If the file is opened for writing, then an exception
     * will be thrown unless an exclusive lock can be obtained on the file.
     * 
     * @param file
     *            The name of the file to use as the backing store.
     * @param mode
     *            The file open mode for
     *            {@link RandomAccessFile#RandomAccessFile(File, String)()}.
     */
    public SimpleFileRawStore(File file, String mode) throws IOException {
        
        if (file == null)
            throw new IllegalArgumentException("file is null");
        
        this.file = file;
        
        raf = new RandomAccessFile(file,mode);

        if( mode.indexOf("w") != -1 ) {

            if (raf.getChannel().tryLock() == null) {

                throw new IOException("Could not lock file: "
                        + file.getAbsoluteFile());

            }
            
        }
        
    }
    
    public boolean isOpen() {

        return open;
        
    }
    
    public boolean isStable() {
        
        return true;
        
    }

    public boolean isFullyBuffered() {
        
        return false;
        
    }
    
    public File getFile() {
        
        return file;
        
    }
    
    /**
     * This also releases the lock if any obtained by the constructor.
     */
    public void close() {
        
        if( !open ) throw new IllegalStateException();

        open = false;

        try {

            raf.close();
            
        } catch(IOException ex) { 
            
            throw new RuntimeException(ex);
            
        }
        
    }

    public void closeAndDelete() {
        
        close();
        
        if(!file.delete()) {
            
            System.err.println("WARN: Could not delete: "+file.getAbsolutePath());
            
        }
        
    }

    public ByteBuffer read(long addr) {

        if (addr == 0L)
            throw new IllegalArgumentException("Address is 0L");

        final int offset = Addr.getOffset(addr);

        final int nbytes = Addr.getByteCount(addr);

        if (nbytes == 0) {

            throw new IllegalArgumentException(
                    "Address encodes record length of zero");

        }

        if (deleted.contains(addr)) {

            throw new IllegalArgumentException(
                    "Address was deleted in this session");

        }

        try {

            if (offset + nbytes > raf.length()) {

                throw new IllegalArgumentException("Address never written.");

            }

            // allocate a new buffer of the exact capacity.

            ByteBuffer dst = ByteBuffer.allocate(nbytes);

            // copy the data into the buffer.

            raf.getChannel().read(dst, (long) offset);

            // flip for reading.

            dst.flip();

            // return the buffer.

            return dst;

        } catch (IOException ex) {

            throw new RuntimeException(ex);

        }
        
    }

    public long write(ByteBuffer data) {

        if (data == null)
            throw new IllegalArgumentException("Buffer is null");

        // #of bytes to store.
        final int nbytes = data.remaining();

        if (nbytes == 0)
            throw new IllegalArgumentException("No bytes remaining in buffer");

        try {

            // the new record will be appended to the end of the file.
            long pos = raf.length();

            if (pos + nbytes > Integer.MAX_VALUE) {

                throw new IOException("Would exceed int32 bytes in file.");

            }

            // the offset into the file at which the record will be written.
            final int offset = (int) pos;

            // // extend the file to have sufficient space for this record.
            // raf.setLength(pos + nbytes);

            // write the data onto the end of the file.
            raf.getChannel().write(data, pos);

            // formulate the address that can be used to recover that record.
            return Addr.toLong(nbytes, offset);

        } catch (IOException ex) {

            throw new RuntimeException(ex);

        }

    }

//    /**
//     * Note: the delete implementation checks its arguments and makes a
//     * <em>transient</em> note that the record has been deleted but that
//     * information does NOT survive restart of the store.
//     */
//    public void delete(long addr) {
//
//        if(addr==0L) throw new IllegalArgumentException("Address is 0L");
//        
//        final int offset = Addr.getOffset(addr);
//        
//        final int nbytes = Addr.getByteCount(addr);
//
//        if(nbytes==0) {
//            
//            throw new IllegalArgumentException(
//                    "Address encodes record length of zero");
//            
//        }
//        
//        try {
//
//            if (offset + nbytes > raf.length()) {
//
//                throw new IllegalArgumentException("Address never written.");
//
//            }
//
//        } catch (IOException ex) {
//
//            throw new RuntimeException(ex);
//
//        }
//
//        Long l = Long.valueOf(addr);
//        
//        if(deleted.contains(l)) {
//        
//            throw new IllegalArgumentException("Address was deleted in this session");
//            
//        }
//        
//        deleted.add(l);
//        
//    }

    public void force(boolean metadata) {

        try {
            
            raf.getChannel().force(metadata);
            
        } catch( IOException ex) {
            
            throw new RuntimeException(ex);
            
        }
                
    }

    public long size() {
        
        try {
        
            return raf.length();
            
        } catch(IOException ex) {
            
            throw new RuntimeException(ex);
            
        }
        
    }
    
}

--- NEW FILE: SimpleMemoryRawStore.java ---
/**

The Notice below must appear in each file of the Source Code of any
copy you distribute of the Licensed Product.  Contributors to any
Modifications may add their own copyright notices to identify their
own contributions.

License:

The contents of this file are subject to the CognitiveWeb Open Source
License Version 1.1 (the License).  You may not copy or use this file,
in either source code or executable form, except in compliance with
the License.  You may obtain a copy of the License from

  http://www.CognitiveWeb.org/legal/license/

Software distributed under the License is distributed on an AS IS
basis, WITHOUT WARRANTY OF ANY KIND, either express or implied.  See
the License for the specific language governing rights and limitations
under the License.

Copyrights:

Portions created by or assigned to CognitiveWeb are Copyright
(c) 2003-2003 CognitiveWeb.  All Rights Reserved.  Contact
information for CognitiveWeb is available at

  http://www.CognitiveWeb.org

Portions Copyright (c) 2002-2003 Bryan Thompson.

Acknowledgements:

Special thanks to the developers of the Jabber Open Source License 1.0
(JOSL), from which this License was derived.  This License contains
terms that differ from JOSL.

Special thanks to the CognitiveWeb Open Source Contributors for their
suggestions and support of the Cognitive Web.

Modifications:

*/
/*
 * Created on Jan 31, 2007
 */

package com.bigdata.rawstore;

import java.io.File;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import com.bigdata.journal.TemporaryRawStore;

/**
 * A purely transient append-only implementation useful when data need to be
 * buffered in memory. The writes are stored in an {@link ArrayList}.
 * 
 * @see {@link TemporaryRawStore}, which provides a more scalable solution for temporary
 *      data.
 * 
 * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
 * @version $Id$
 */
public class SimpleMemoryRawStore implements IRawStore {

    private boolean open = true;
    
    /**
     * The #of bytes written so far. This is used to generate the
     * {@link Addr address} values returned by {@link #write(ByteBuffer)}. This
     * is necessary in order for this implementation to assign addresses in the
     * same manner as they would be assigned by an implementation using an
     * append only byte[] or file.
     */
    protected int nextOffset = 0;
    
    /**
     * Maps an {@link Addr} onto the index in {@link #records} at which the
     * data for that address was written.
     */
    private final Map<Long,Integer> addrs;
    
    /**
     * The buffered records in the order written. If a record is deleted then
     * that element in the list will be a [null] value.
     */
    protected final ArrayList<byte[]> records;
    
    /**
     * Uses an initial capacity of 1000 records.
     */
    public SimpleMemoryRawStore() {

        this(1000);
        
    }
    
    /**
     * 
     * @param capacity
     *            The #of records that you expect to store (non-negative). If
     *            the capacity is exceeded then the internal {@link ArrayList}
     *            will be grown as necessary.
     */
    public SimpleMemoryRawStore(int capacity) {
        
        if (capacity < 0)
            throw new IllegalArgumentException("capacity is negative");
        
        records = new ArrayList<byte[]>(capacity);
        
        // estimate hash table capacity to avoid resizing.
        addrs = new HashMap<Long,Integer>((int)(capacity*1.25));

    }
    
    public boolean isOpen() {

        return open;
        
    }

    public boolean isStable() {
        
        return false;
        
    }

    public boolean isFullyBuffered() {
        
        return true;
        
    }
    
    /**
     * This always returns <code>null</code>.
     */
    public File getFile() {
        
        return null;
        
    }
    
    public void close() {
        
        if( !open ) throw new IllegalStateException();

        open = false;
        
        // discard all the records.
        records.clear();
        
    }

    public void closeAndDelete() {
        
        close();
        
    }

    public ByteBuffer read(long addr) {

        if (addr == 0L)
            throw new IllegalArgumentException("Address is 0L");
        
//        final int offset = Addr.getOffset(addr);
        
        final int nbytes = Addr.getByteCount(addr);

        if(nbytes==0) {
            
            throw new IllegalArgumentException(
                    "Address encodes record length of zero");
            
        }
        
        Integer index = addrs.get(addr);
        
        if(index==null) {
            
            throw new IllegalArgumentException("Address never written.");

        }

        final byte[] b = records.get(index);

        if(b == null) {
            
            throw new IllegalArgumentException("Record was deleted");
            
        }
        
        if(b.length != nbytes) {
            
            throw new RuntimeException("Bad address / data");
            
        }
        
        // return a read-only view onto the data in the store.
            
        return ByteBuffer.wrap(b).asReadOnlyBuffer();
        
    }

    public long write(ByteBuffer data) {

        if (data == null)
            throw new IllegalArgumentException("Buffer is null");

        // #of bytes to store.
        final int nbytes = data.remaining();

        if (nbytes == 0)
            throw new IllegalArgumentException("No bytes remaining in buffer");

        // allocate a new array that is an exact fit for the data.
        final byte[] b = new byte[nbytes];

        // copy the data into the array.
        data.get(b);
        
        // the next offset.
        final int offset = nextOffset;
        
        // increment by the #of bytes written.
        nextOffset += nbytes;

        // the position in the records[] where this record is stored.
        final int index = records.size();
        
        // add the record to the records array.
        records.add(b);
        
        // formulate the address that can be used to recover that record.
        long addr = Addr.toLong(nbytes, offset);
        
        // save the mapping from the addr to the records[].
        addrs.put(addr,index);
        
        return addr;
        
    }

//    public void delete(long addr) {
//
//        if(addr==0L) throw new IllegalArgumentException("Address is 0L");
//        
////        final int offset = Addr.getOffset(addr);
//        
//        final int nbytes = Addr.getByteCount(addr);
//
//        if(nbytes==0) {
//            
//            throw new IllegalArgumentException(
//                    "Address encodes record length of zero");
//            
//        }
//        
//        Integer index = addrs.get(addr);
//        
//        if(index==null) {
//         
//            throw new IllegalArgumentException("Address never written.");
//
//        }
//        
//        byte[] b = records.get(index);
//
//        if(b == null) {
//            
//            throw new IllegalArgumentException("Record was deleted");
//            
//        }
//
//        if(b.length != nbytes) {
//            
//            throw new RuntimeException("Bad address / data");
//            
//        }
//        
//        // release that record.
//        records.set(index, null);
//        
//    }

    public void force(boolean metadata) {
        
        // NOP.
        
    }
    
    public long size() {
        
        return nextOffset;
        
    }
    
}