Menu
▾
▴
[cweb-CVS] bigdata/src/java/com/bigdata/journal IIndexStore.java, NONE, 1.1 Tx.java, 1.34, 1.35 ITx.java, 1.5, 1.6 IRootBlockView.java, 1.9, 1.10 AbstractTx.java, 1.2, 1.3 ITransactionManager.java, 1.3, 1.4 IJournal.java, 1.8, 1.9 Name2Addr.java, 1.4, 1.5 IBufferStrategy.java, 1.15, 1.16 ICommitter.java, 1.1, 1.2 Journal.java, 1.59, 1.60 IIndexManager.java, 1.2, 1.3 IStore.java, 1.8, NONE
|
From: Bryan T. <tho...@us...> - 2007-03-11 11:43:29
|
Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/journal In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv5433/src/java/com/bigdata/journal Modified Files: Tx.java ITx.java IRootBlockView.java AbstractTx.java ITransactionManager.java IJournal.java Name2Addr.java IBufferStrategy.java ICommitter.java Journal.java IIndexManager.java Added Files: IIndexStore.java Removed Files: IStore.java Log Message: Continued minor refactoring in line with model updates. Index: IJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/IJournal.java,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** IJournal.java 8 Mar 2007 18:14:06 -0000 1.8 --- IJournal.java 11 Mar 2007 11:42:45 -0000 1.9 *************** *** 94,101 **** /** ! * Return the named index isolated by the transaction having the ! * specified start time. * ! * @param startTime The transaction start time. */ public IIndex getIndex(String name, long startTime); --- 94,113 ---- /** ! * Return the named index as isolated by the transaction having the ! * specified transaction start time. * ! * @param name ! * The index name. ! * @param startTime ! * The transaction start time, which serves as the unique ! * identifier for the transaction. ! * ! * @return The isolated index. ! * ! * @exception IllegalArgumentException ! * if <i>name</i> is <code>null</code> ! * ! * @exception IllegalStateException ! * if there is no active transaction with that timestamp. */ public IIndex getIndex(String name, long startTime); Index: AbstractTx.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/AbstractTx.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** AbstractTx.java 8 Mar 2007 18:14:07 -0000 1.2 --- AbstractTx.java 11 Mar 2007 11:42:45 -0000 1.3 *************** *** 74,81 **** * The start startTime assigned to this transaction. * <p> ! * Note: Transaction {@link #startTime} and {@link #commitTime}s ! * are assigned by a global time service. The time service must provide ! * unique times for transaction start and commit timestamps and for commit ! * times for unisolated {@link Journal#commit()}s. */ final protected long startTime; --- 74,81 ---- * The start startTime assigned to this transaction. * <p> ! * Note: Transaction {@link #startTime} and {@link #commitTime}s are ! * assigned by a global time service. The time service must provide unique ! * times for transaction start and commit timestamps and commit times for ! * unisolated {@link Journal#commit()}s. */ final protected long startTime; Index: Name2Addr.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/Name2Addr.java,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** Name2Addr.java 21 Feb 2007 20:17:21 -0000 1.4 --- Name2Addr.java 11 Mar 2007 11:42:45 -0000 1.5 *************** *** 44,49 **** * expensive operation of encoding unicode names to byte[] keys. * ! * @todo use a weak value cache so that unused indices may be swept by the ! * GC. */ private Map<String,IIndex> name2BTree = new HashMap<String,IIndex>(); --- 44,56 ---- * expensive operation of encoding unicode names to byte[] keys. * ! * FIXME This is the place to solve the resource (RAM) burden for indices is ! * Name2Addr. Currently, indices are never closed once opened which is a ! * resource leak. We need to close them out eventually based on LRU plus ! * timeout plus NOT IN USE. The way to approach this is a weak reference ! * cache combined with an LRU or hard reference queue that tracks reference ! * counters (just like the BTree hard reference cache for leaves). Eviction ! * events lead to closing an index iff the reference counter is zero. ! * Touches keep recently used indices from closing even though they may have ! * a zero reference count. */ private Map<String,IIndex> name2BTree = new HashMap<String,IIndex>(); *************** *** 148,152 **** // re-load btree from the store. ! btree = BTreeMetadata.load(this.store, entry.addr); // save name -> btree mapping in transient cache. --- 155,159 ---- // re-load btree from the store. ! btree = BTree.load(this.store, entry.addr); // save name -> btree mapping in transient cache. Index: IIndexManager.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/IIndexManager.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** IIndexManager.java 8 Mar 2007 18:14:07 -0000 1.2 --- IIndexManager.java 11 Mar 2007 11:42:46 -0000 1.3 *************** *** 57,61 **** * @version $Id$ */ ! public interface IIndexManager extends IStore { /** --- 57,61 ---- * @version $Id$ */ ! public interface IIndexManager extends IIndexStore { /** Index: ITransactionManager.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/ITransactionManager.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** ITransactionManager.java 28 Feb 2007 13:59:10 -0000 1.3 --- ITransactionManager.java 11 Mar 2007 11:42:45 -0000 1.4 *************** *** 131,154 **** // */ // public long newReadCommittedTx(); ! ! /** ! * Return the named index as isolated by the transaction. ! * ! * @param name ! * The index name. ! * @param startTime ! * The transaction start time, which serves as the unique ! * identifier for the transaction. ! * ! * @return The isolated index. ! * ! * @exception IllegalArgumentException ! * if <i>name</i> is <code>null</code> ! * ! * @exception IllegalStateException ! * if there is no active transaction with that timestamp. ! */ ! public IIndex getIndex(String name, long startTime); ! /** * Abort the transaction. --- 131,135 ---- // */ // public long newReadCommittedTx(); ! /** * Abort the transaction. Index: ICommitter.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/ICommitter.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** ICommitter.java 5 Feb 2007 18:17:40 -0000 1.1 --- ICommitter.java 11 Mar 2007 11:42:46 -0000 1.2 *************** *** 1,32 **** package com.bigdata.journal; import com.bigdata.rawstore.Addr; - import com.bigdata.rawstore.IRawStore; - /** ! * An interface implemented by a persistence capable data structure such as ! * a btree so that it can participate in the commit protocol for the store. * <p> ! * This interface is invoked by {@link #commit()} for each registered ! * committer. The {@link Addr} returned by {@link #commit()} will be saved ! * on the root block in the slot identified to the committer when it ! * registered itself. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ * ! * @see IRawStore#registerCommitter(int, ! * com.bigdata.objndx.IRawStore.ICommitter) */ public interface ICommitter { /** ! * Flush all dirty records to disk in preparation for an atomic commit. * ! * @return The {@link Addr address} of the record from which the ! * persistence capable data structure can load itself. If no ! * changes have been made then the previous address should be ! * returned as it is still valid. */ public long handleCommit(); --- 1,74 ---- + /** + + The Notice below must appear in each file of the Source Code of any + copy you distribute of the Licensed Product. Contributors to any + Modifications may add their own copyright notices to identify their + own contributions. + + License: + + The contents of this file are subject to the CognitiveWeb Open Source + License Version 1.1 (the License). You may not copy or use this file, + in either source code or executable form, except in compliance with + the License. You may obtain a copy of the License from + + http://www.CognitiveWeb.org/legal/license/ + + Software distributed under the License is distributed on an AS IS + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + Copyrights: + + Portions created by or assigned to CognitiveWeb are Copyright + (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact + information for CognitiveWeb is available at + + http://www.CognitiveWeb.org + + Portions Copyright (c) 2002-2003 Bryan Thompson. + + Acknowledgements: + + Special thanks to the developers of the Jabber Open Source License 1.0 + (JOSL), from which this License was derived. This License contains + terms that differ from JOSL. + + Special thanks to the CognitiveWeb Open Source Contributors for their + suggestions and support of the Cognitive Web. + + Modifications: + + */ package com.bigdata.journal; import com.bigdata.rawstore.Addr; /** ! * An interface implemented by a persistence capable data structure such as a ! * btree so that it can participate in the commit protocol for the store. * <p> ! * This interface is invoked by {@link Journal#commit()} for each registered ! * {@link ICommitter}. The {@link Addr} returned by {@link #handleCommit()} ! * will be saved in the {@link ICommitRecord} under the index identified by the ! * {@link ICommitter} when it was registered. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ * ! * @see IAtomicStore#setCommitter(int, ICommitter) */ public interface ICommitter { /** ! * Flush dirty state to the store in preparation for an atomic commit and ! * return the {@link Addr address} from which the persistence capable data ! * structure may be reloaded. * ! * @return The {@link Addr address} of the record from which the persistence ! * capable data structure may be reloaded. If no changes have been ! * made then the previous address should be returned as it is still ! * valid. */ public long handleCommit(); Index: Tx.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/Tx.java,v retrieving revision 1.34 retrieving revision 1.35 diff -C2 -d -r1.34 -r1.35 *** Tx.java 28 Feb 2007 13:59:09 -0000 1.34 --- Tx.java 11 Mar 2007 11:42:44 -0000 1.35 *************** *** 61,65 **** import com.bigdata.rawstore.Bytes; import com.bigdata.scaleup.MetadataIndex; ! import com.bigdata.scaleup.PartitionedIndex; /** --- 61,65 ---- import com.bigdata.rawstore.Bytes; import com.bigdata.scaleup.MetadataIndex; ! import com.bigdata.scaleup.PartitionedIndexView; /** *************** *** 101,122 **** * * @todo Support transactions where the indices isolated by the transactions are ! * {@link PartitionedIndex}es. ! * ! * @todo Track which {@link IndexSegment}s and {@link Journal}s are required ! * to support the {@link IsolatedBTree}s in use by a {@link Tx}. Deletes ! * of old journals and index segments MUST be deferred until no ! * transaction remains which can read those data. This metadata must be ! * restart-safe so that resources are eventually deleted. On restart, ! * active transactions will have been discarded abort and their resources ! * released. (Do we need a restart-safe means to indicate the set of ! * running transactions?)<br> ! * There is also a requirement for quickly locating the specific journal ! * and index segments required to support isolation of an index. This ! * probably means an index into the history of the {@link MetadataIndex} ! * (so we don't throw it away until no transactions can reach back that ! * far) as well as an index into the named indices index -- perhaps simply ! * an index by startTime into the root addresses (or whole root block ! * views, or moving the root addresses out of the root block and into the ! * store with only the address of the root addresses in the root block). * * @todo The various public methods on this API that have {@link RunState} --- 101,105 ---- * * @todo Support transactions where the indices isolated by the transactions are ! * {@link PartitionedIndexView}es. * * @todo The various public methods on this API that have {@link RunState} *************** *** 129,133 **** * server failed to test the pre-conditions and they were not met */ ! public class Tx extends AbstractTx implements IStore, ITx { /** --- 112,116 ---- * server failed to test the pre-conditions and they were not met */ ! public class Tx extends AbstractTx implements IIndexStore, ITx { /** Index: Journal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/Journal.java,v retrieving revision 1.59 retrieving revision 1.60 diff -C2 -d -r1.59 -r1.60 *** Journal.java 8 Mar 2007 18:14:07 -0000 1.59 --- Journal.java 11 Mar 2007 11:42:46 -0000 1.60 *************** *** 64,73 **** import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.objndx.BTree; - import com.bigdata.objndx.BTreeMetadata; import com.bigdata.objndx.IIndex; import com.bigdata.objndx.IndexSegment; import com.bigdata.rawstore.Addr; import com.bigdata.rawstore.Bytes; ! import com.bigdata.scaleup.PartitionedJournal.Options; import com.bigdata.util.concurrent.DaemonThreadFactory; --- 64,72 ---- import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.objndx.BTree; import com.bigdata.objndx.IIndex; import com.bigdata.objndx.IndexSegment; import com.bigdata.rawstore.Addr; import com.bigdata.rawstore.Bytes; ! import com.bigdata.scaleup.MasterJournal.Options; import com.bigdata.util.concurrent.DaemonThreadFactory; *************** *** 108,135 **** * FIXME Priority items are: * <ol> - * <li> Transaction isolation (correctness tests, isolatedbtree tests, fused - * view tests).</li> * <li> Concurrent load for RDFS w/o rollback.</li> * <li> Group commit for higher transaction throughput.<br> ! * Note: TPS is basically constant for a given combination of the buffer mode ! * and whether or not commits are forced to disk. This means that the #of ! * clients is not a strong influence on performance. The big wins are Transient ! * and Force := No since neither conditions synchs to disk. This suggests that ! * the big win for TPS throughput is going to be group commit. </li> ! * <li> Scale out database (automatic re-partitioning of indices and processing * of deletion markers).</li> * <li> AIO for the Direct and Disk modes.</li> ! * <li> Distributed database protocols.</li> ! * <li> Segment server (mixture of journal server and read-optimized database * server).</li> ! * <li> Testing of an "embedded database" using both a journal only and a ! * journal + read-optimized database design. This can be tested up to the GPO ! * layer.</li> ! * <li>Support primary key (clustered) indices in GPO/PO layer.</li> ! * <li>Implement backward validation and state-based conflict resolution with ! * custom merge rules for RDFS, persistent objects, generic objects, and primary ! * key indices, and secondary indexes.</li> ! * <li> Architecture using queues from GOM to journal/database segment server ! * supporting both embedded and remote scenarios.</li> * </ol> * --- 107,135 ---- * FIXME Priority items are: * <ol> * <li> Concurrent load for RDFS w/o rollback.</li> * <li> Group commit for higher transaction throughput.<br> ! * Note: For short transactions, TPS is basically constant for a given ! * combination of the buffer mode and whether or not commits are forced to disk. ! * This means that the #of clients is not a strong influence on performance. The ! * big wins are Transient and Force := No since neither conditions synchs to ! * disk. This suggests that the big win for TPS throughput is going to be group ! * commit. </li> ! * <li> Scale-up database (automatic re-partitioning of indices and processing * of deletion markers).</li> * <li> AIO for the Direct and Disk modes.</li> ! * <li> GOM integration, including: support for primary key (clustered) indices; ! * using queues from GOM to journal/database segment server supporting both ! * embedded and remote scenarios; and using state-based conflict resolution to ! * obtain high concurrency for generic objects, link set metadata, and indices.</li> ! * <li> Scale-out database, including: ! * <ul> ! * <li> Data server (mixture of journal server and read-optimized database * server).</li> ! * <li> Transaction service (low-latency with failover).</li> ! * <li> Metadata index services (one per named index with failover).</li> ! * <li> Resource reclaimation. </li> ! * <li> Job scheduler to map functional programs across the data (possible ! * Hadoop integration point).</li> ! * </ul> * </ol> * *************** *** 549,557 **** val = File.createTempFile("bigdata-" + bufferMode + "-", ".jnl", tmpDir).toString(); - - // // the file that gets opened. - // properties.setProperty(Options.FILE, val); - // // turn off this property to facilitate re-open of the same file. - // properties.setProperty(Options.CREATE_TEMP_FILE,"false"); } catch(IOException ex) { --- 549,552 ---- *************** *** 935,942 **** * that method is the address from which the {@link ICommitter} may be * reloaded (and its previous address if its state has not changed). That ! * address is saved in the slot of the root block under which that committer ! * was {@link #registerCommitter(int, ICommitter) registered}. We then ! * force the data to stable store, update the root block, and force the root ! * block and the file metadata to stable store. */ public long commit() { --- 930,937 ---- * that method is the address from which the {@link ICommitter} may be * reloaded (and its previous address if its state has not changed). That ! * address is saved in the {@link ICommitRecord} under the index for which ! * that committer was {@link #registerCommitter(int, ICommitter) registered}. ! * We then force the data to stable store, update the root block, and force ! * the root block and the file metadata to stable store. */ public long commit() { *************** *** 947,953 **** /** ! * Handle the {@link #commit()} and integrations with transaction support so ! * that we can update the first and last transaction identifiers on the root ! * block as necessary. * * @param tx --- 942,948 ---- /** ! * Handles the {@link #commit()} and integrations with transaction support ! * so that we can update the first and last transaction identifiers on the ! * root block as necessary. * * @param tx *************** *** 1233,1237 **** */ ! name2Addr = (Name2Addr) BTreeMetadata.load(this, addr); } --- 1228,1232 ---- */ ! name2Addr = (Name2Addr) BTree.load(this, addr); } *************** *** 1276,1280 **** */ ! ndx = (CommitRecordIndex) BTreeMetadata.load(this, addr); } --- 1271,1275 ---- */ ! ndx = (CommitRecordIndex) BTree.load(this, addr); } *************** *** 1367,1371 **** */ ! return ((Name2Addr) BTreeMetadata.load(this, commitRecord .getRootAddr(ROOT_NAME2ADDR))).get(name); --- 1362,1366 ---- */ ! return ((Name2Addr) BTree.load(this, commitRecord .getRootAddr(ROOT_NAME2ADDR))).get(name); Index: ITx.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/ITx.java,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** ITx.java 28 Feb 2007 13:59:10 -0000 1.5 --- ITx.java 11 Mar 2007 11:42:44 -0000 1.6 *************** *** 57,61 **** * @version $Id$ */ ! public interface ITx extends IStore { /** --- 57,61 ---- * @version $Id$ */ ! public interface ITx extends IIndexStore { /** Index: IRootBlockView.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/IRootBlockView.java,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** IRootBlockView.java 28 Feb 2007 13:59:10 -0000 1.9 --- IRootBlockView.java 11 Mar 2007 11:42:45 -0000 1.10 *************** *** 51,55 **** import com.bigdata.rawstore.Addr; ! import com.bigdata.scaleup.PartitionedJournal; /** --- 51,55 ---- import com.bigdata.rawstore.Addr; ! import com.bigdata.scaleup.MasterJournal; /** *************** *** 171,175 **** * a featured used to support transactional isolation. * <p> ! * Note: When using a {@link PartitionedJournal} the {@link Addr address} of * the {@link ICommitRecord} MAY refer to a historical {@link SlaveJournal} * and care MUST be exercised to resolve the address against the appropriate --- 171,175 ---- * a featured used to support transactional isolation. * <p> ! * Note: When using a {@link MasterJournal} the {@link Addr address} of * the {@link ICommitRecord} MAY refer to a historical {@link SlaveJournal} * and care MUST be exercised to resolve the address against the appropriate --- IStore.java DELETED --- Index: IBufferStrategy.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/journal/IBufferStrategy.java,v retrieving revision 1.15 retrieving revision 1.16 diff -C2 -d -r1.15 -r1.16 *** IBufferStrategy.java 21 Feb 2007 20:17:21 -0000 1.15 --- IBufferStrategy.java 11 Mar 2007 11:42:45 -0000 1.16 *************** *** 87,94 **** * and whether or not the file metadata for the journal is forced * to stable storage. See {@link Options#FORCE_ON_COMMIT}. - * - * @todo this is basically an atomic commit. Reconcile it with - * {@link IAtomicStore#commit()}. Note that the latter works with - * registered committers, while this does not. */ public void writeRootBlock(IRootBlockView rootBlock, --- 87,90 ---- --- NEW FILE: IIndexStore.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Oct 25, 2006 */ package com.bigdata.journal; import com.bigdata.objndx.IIndex; /** * Interface for reading and writing persistent data using one or more named * indices. Persistent data are stored as ordered key-value tuples in indices. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public interface IIndexStore { /** * Return a named index. * * @param name * The index name. * * @return The named index or <code>null</code> if no index was registered * under that name. * * @see IJournal#registerIndex(String, IIndex) */ public IIndex getIndex(String name); } |
