Menu
▾
▴
[cweb-CVS] bigdata/src/java/com/bigdata/scaleup ResourceState.java, NONE, 1.1 IPartitionTask.java, NONE, 1.1 JournalMetadata.java, NONE, 1.1 AbstractPartitionTask.java, NONE, 1.1 IResourceMetadata.java, NONE, 1.1 PartitionedIndex.java, 1.7, 1.8 SegmentMetadata.java, 1.2, 1.3 SlaveJournal.java, 1.2, 1.3 PartitionMetadata.java, 1.2, 1.3 PartitionedJournal.java, 1.12, 1.13 IndexSegmentLifeCycleEnum.java, 1.1, NONE
|
From: Bryan T. <tho...@us...> - 2007-03-08 18:14:58
|
Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv15000/src/java/com/bigdata/scaleup Modified Files: PartitionedIndex.java SegmentMetadata.java SlaveJournal.java PartitionMetadata.java PartitionedJournal.java Added Files: ResourceState.java IPartitionTask.java JournalMetadata.java AbstractPartitionTask.java IResourceMetadata.java Removed Files: IndexSegmentLifeCycleEnum.java Log Message: Updated the UML model and added a ZIP containing an HTML presentation of the model. Working on partitioned index support. --- NEW FILE: IPartitionTask.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ package com.bigdata.scaleup; import com.bigdata.objndx.IndexSegment; /** * Interface for a scheduleable task that produces one or more * {@link IndexSegment}s, updates the {@link MetadataIndex} to reflect the * existence of the new {@link IndexSegment}s and notifies existing views * with a depedency on the source(s) that they must switch over to the new * {@link IndexSegment}s. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public interface IPartitionTask extends java.util.concurrent.Callable<Object> { /** * Run the task. * * @return No return semantics are defined. * * @throws Exception * The exception thrown by the task. */ public Object call() throws Exception; } --- IndexSegmentLifeCycleEnum.java DELETED --- --- NEW FILE: JournalMetadata.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ package com.bigdata.scaleup; import java.io.File; import com.bigdata.journal.Journal; /** * Metadata required to locate a {@link Journal} resource. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ * * @todo make this persistence capable by modifying the value serializer to * use the {@link IResourceMetadata} interface. */ public class JournalMetadata implements IResourceMetadata { protected final String filename; protected final ResourceState state; public File getFile() { return new File(filename); } /** * Always returns ZERO (0L) since we can not accurately estimate the #of * bytes on the journal dedicated to a given partition of a named index. */ public long size() { return 0L; } public ResourceState state() { return state; } public JournalMetadata(Journal journal, ResourceState state) { if(journal.getFile()==null) { throw new IllegalArgumentException("Journal is not persistent."); } this.filename = journal.getFile().toString(); this.state = state; } } --- NEW FILE: IResourceMetadata.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ package com.bigdata.scaleup; import java.io.File; import com.bigdata.journal.Journal; import com.bigdata.objndx.IndexSegment; /** * Interface for metadata about a {@link Journal} or {@link IndexSegment}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public interface IResourceMetadata { /** * The store file. */ public File getFile(); /** * The #of bytes in the store file. */ public long size(); /** * The life cycle state of that store file. */ public ResourceState state(); // public int hashCode(); // // public boolean equals(IResourceMetadata o); } Index: SlaveJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/SlaveJournal.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** SlaveJournal.java 17 Feb 2007 21:34:21 -0000 1.2 --- SlaveJournal.java 8 Mar 2007 18:14:06 -0000 1.3 *************** *** 47,50 **** --- 47,51 ---- import java.util.Properties; + import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.journal.IJournal; import com.bigdata.journal.Journal; *************** *** 83,86 **** --- 84,93 ---- private final PartitionedJournal master; + protected PartitionedJournal getMaster() { + + return master; + + } + public SlaveJournal(PartitionedJournal master,Properties properties) { *************** *** 157,179 **** } /** ! * Registers and returns {@link PartitionedIndex} under the given name ! * and assigns the supplied {@link BTree} to absorb writes for that * {@link PartitionedIndex}. * <p> ! * A {@link MetadataIndex} is also registered under the given name and ! * an initial partition for that index is created using the separator ! * key <code>new byte[]{}</code>. The partition will initially ! * consist of zero {@link IndexSegment}s. * <p> ! * Note: The returned object is invalid once the ! * {@link PartitionedJournal} {@link PartitionedJournal#overflow()}s. * * @todo use a prototype model so that the registered btree type is ! * preserved? (Only the metadata extensions are preserved right ! * now). One way to do this is by putting the constructor on the ! * metadata object. Another is to make the btree Serializable and ! * then just declare everything else as transient. */ public IIndex registerIndex(String name, IIndex btree) { --- 164,212 ---- } + + /** + * Registers and returns a {@link PartitionedIndex} under the given name and + * assigns an {@link UnisolatedBTree} to absorb writes for that + * {@link PartitionedIndex}. The resulting index will support transactional + * isolation. + * <p> + * A {@link MetadataIndex} is also registered under the given name and an + * initial partition for that index is created using the separator key + * <code>new byte[]{}</code>. The partition will initially consist of + * zero {@link IndexSegment}s. + * <p> + * Note: The returned object is invalid once the {@link PartitionedJournal} + * {@link PartitionedJournal#overflow()}s. + * <p> + * Note: You MUST {@link #commit()} before the registered index will be + * either restart-safe or visible to new transactions. + */ + public IIndex registerIndex(String name) { + + return registerIndex(name, new UnisolatedBTree(this)); + + } /** ! * Registers and returns a {@link PartitionedIndex} under the given name and ! * assigns the supplied {@link IIndex} to absorb writes for that * {@link PartitionedIndex}. * <p> ! * A {@link MetadataIndex} is also registered under the given name and an ! * initial partition for that index is created using the separator key ! * <code>new byte[]{}</code>. The partition will initially consist of ! * zero {@link IndexSegment}s. * <p> ! * Note: The returned object is invalid once the {@link PartitionedJournal} ! * {@link PartitionedJournal#overflow()}s. ! * <p> ! * Note: You MUST {@link #commit()} before the registered index will be ! * either restart-safe or visible to new transactions. * * @todo use a prototype model so that the registered btree type is ! * preserved? (Only the metadata extensions are preserved right now). ! * One way to do this is by putting the constructor on the metadata ! * object. Another is to make the btree Serializable and then just ! * declare everything else as transient. */ public IIndex registerIndex(String name, IIndex btree) { Index: SegmentMetadata.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/SegmentMetadata.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** SegmentMetadata.java 6 Mar 2007 20:38:06 -0000 1.2 --- SegmentMetadata.java 8 Mar 2007 18:14:05 -0000 1.3 *************** *** 1,45 **** /** ! The Notice below must appear in each file of the Source Code of any ! copy you distribute of the Licensed Product. Contributors to any ! Modifications may add their own copyright notices to identify their ! own contributions. ! License: ! The contents of this file are subject to the CognitiveWeb Open Source ! License Version 1.1 (the License). You may not copy or use this file, ! in either source code or executable form, except in compliance with ! the License. You may obtain a copy of the License from ! http://www.CognitiveWeb.org/legal/license/ ! Software distributed under the License is distributed on an AS IS ! basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See ! the License for the specific language governing rights and limitations ! under the License. ! Copyrights: ! Portions created by or assigned to CognitiveWeb are Copyright ! (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact ! information for CognitiveWeb is available at ! http://www.CognitiveWeb.org ! Portions Copyright (c) 2002-2003 Bryan Thompson. ! Acknowledgements: ! Special thanks to the developers of the Jabber Open Source License 1.0 ! (JOSL), from which this License was derived. This License contains ! terms that differ from JOSL. ! Special thanks to the CognitiveWeb Open Source Contributors for their ! suggestions and support of the Cognitive Web. ! Modifications: ! */ package com.bigdata.scaleup; --- 1,45 ---- /** ! The Notice below must appear in each file of the Source Code of any ! copy you distribute of the Licensed Product. Contributors to any ! Modifications may add their own copyright notices to identify their ! own contributions. ! License: ! The contents of this file are subject to the CognitiveWeb Open Source ! License Version 1.1 (the License). You may not copy or use this file, ! in either source code or executable form, except in compliance with ! the License. You may obtain a copy of the License from ! http://www.CognitiveWeb.org/legal/license/ ! Software distributed under the License is distributed on an AS IS ! basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See ! the License for the specific language governing rights and limitations ! under the License. ! Copyrights: ! Portions created by or assigned to CognitiveWeb are Copyright ! (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact ! information for CognitiveWeb is available at ! http://www.CognitiveWeb.org ! Portions Copyright (c) 2002-2003 Bryan Thompson. ! Acknowledgements: ! Special thanks to the developers of the Jabber Open Source License 1.0 ! (JOSL), from which this License was derived. This License contains ! terms that differ from JOSL. ! Special thanks to the CognitiveWeb Open Source Contributors for their ! suggestions and support of the Cognitive Web. ! Modifications: ! */ package com.bigdata.scaleup; *************** *** 47,51 **** import com.bigdata.objndx.IndexSegment; - import com.bigdata.scaleup.PartitionedJournal.IViewMetadata; /** --- 47,50 ---- *************** *** 55,59 **** * @version $Id$ */ ! public class SegmentMetadata implements IViewMetadata { /** --- 54,58 ---- * @version $Id$ */ ! public class SegmentMetadata implements IResourceMetadata { /** *************** *** 70,76 **** * The life-cycle state for that {@link IndexSegment}. */ ! final public IndexSegmentLifeCycleEnum state; ! public SegmentMetadata(String filename,long nbytes,IndexSegmentLifeCycleEnum state) { this.filename = filename; --- 69,75 ---- * The life-cycle state for that {@link IndexSegment}. */ ! final public ResourceState state; ! public SegmentMetadata(String filename,long nbytes,ResourceState state) { this.filename = filename; *************** *** 103,107 **** } ! public IndexSegmentLifeCycleEnum status() { return state; } --- 102,106 ---- } ! public ResourceState state() { return state; } Index: PartitionedJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/PartitionedJournal.java,v retrieving revision 1.12 retrieving revision 1.13 diff -C2 -d -r1.12 -r1.13 *** PartitionedJournal.java 6 Mar 2007 20:38:06 -0000 1.12 --- PartitionedJournal.java 8 Mar 2007 18:14:06 -0000 1.13 *************** *** 53,59 **** import java.util.Iterator; import java.util.Properties; - import java.util.concurrent.Executors; ! import com.bigdata.isolation.IIsolatableIndex; import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.isolation.Value; --- 53,59 ---- import java.util.Iterator; import java.util.Properties; [...967 lines suppressed...] + + } + + return seg; + } + + /** + * The maximum #of index segments that will be held open without a hard + * reference existing for that index segment in the application. + */ + final int INDEX_SEGMENT_LRU_CAPACITY = 5; + + /** + * A cache for recently used index segments designed to prevent their being + * swept by the VM between uses. + */ + private final WeakValueCache<String/*filename*/, IndexSegment> resourceCache; } Index: PartitionMetadata.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/PartitionMetadata.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** PartitionMetadata.java 9 Feb 2007 18:56:59 -0000 1.2 --- PartitionMetadata.java 8 Mar 2007 18:14:06 -0000 1.3 *************** *** 116,120 **** for (int i = 0; i < segs.length; i++) { ! if (segs[i].state == IndexSegmentLifeCycleEnum.LIVE) count++; --- 116,120 ---- for (int i = 0; i < segs.length; i++) { ! if (segs[i].state == ResourceState.Live) count++; *************** *** 138,142 **** for (int i = 0; i < segs.length; i++) { ! if (segs[i].state == IndexSegmentLifeCycleEnum.LIVE) { files[k++] = segs[i].filename; --- 138,142 ---- for (int i = 0; i < segs.length; i++) { ! if (segs[i].state == ResourceState.Live) { files[k++] = segs[i].filename; *************** *** 333,337 **** long nbytes = is.readLong(); ! IndexSegmentLifeCycleEnum state = IndexSegmentLifeCycleEnum .valueOf(is.readInt()); --- 333,337 ---- long nbytes = is.readLong(); ! ResourceState state = ResourceState .valueOf(is.readInt()); --- NEW FILE: AbstractPartitionTask.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ package com.bigdata.scaleup; import java.io.File; import java.util.concurrent.Executors; import com.bigdata.isolation.IIsolatableIndex; import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.isolation.Value; import com.bigdata.objndx.AbstractBTree; import com.bigdata.objndx.IValueSerializer; import com.bigdata.objndx.IndexSegment; import com.bigdata.objndx.IndexSegmentBuilder; import com.bigdata.objndx.IndexSegmentMerger; import com.bigdata.objndx.IndexSegmentMetadata; import com.bigdata.objndx.RecordCompressor; import com.bigdata.objndx.IndexSegmentMerger.MergedEntryIterator; import com.bigdata.objndx.IndexSegmentMerger.MergedLeafIterator; import com.bigdata.rawstore.Bytes; /** * Abstract base class for tasks that build {@link IndexSegment}(s). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ * * @todo write test suite for executing partition task schedules. * * @todo add a * {@link Executors#newSingleThreadExecutor(java.util.concurrent.ThreadFactory)} * that will be used to run these tasks and modify the shutdown logic * for the master to also shutdown that thread. * * @todo add result to persistent schedule outcome so that this is restart * safe. * * @todo once the {@link IndexSegment} is ready the metadata index needs to * be updated to reflect that the indexsegment is live and the views * that rely on the partition on the old journal need to be * invalidated so new views utilize the new indexsegment rather than * the data on the old journal. * * @todo the old journal is not available for release until all partitions * for all indices have been evicted. we need to track that in a * restart safe manner. * * @todo parameterize useChecksum, recordCompressor. * * @todo assiging sequential segment identifiers may impose an unnecessary * message overhead since we can just use the temporary file mechanism * and the inspect the {@link IndexSegmentMetadata} to learn more * about a given store file. * * @todo try performance with and without checksums and with and without * record compression. */ abstract public class AbstractPartitionTask implements IPartitionTask { protected final PartitionedJournal master; /** * Branching factor used for generated {@link IndexSegment}(s). */ protected final int branchingFactor; protected final double errorRate; protected final String name; protected final int partId; protected final byte[] fromKey; protected final byte[] toKey; /** * Branching factor used for temporary file(s). */ protected final int tmpFileBranchingFactor = Bytes.kilobyte32*4; /** * When true, pre-record checksum are generated for the output * {@link IndexSegment}. */ protected final boolean useChecksum = false; /** * When non-null, a {@link RecordCompressor} will be applied to the * output {@link IndexSegment}. */ protected final RecordCompressor recordCompressor = null; /** * The serializer used by all {@link IIsolatableIndex}s. */ static protected final IValueSerializer valSer = Value.Serializer.INSTANCE; /** * * @param master * The master. * @param name * The index name. * @param branchingFactor * The branching factor to be used on the new * {@link IndexSegment}(s). * @param errorRate * The error rate for the bloom filter for the new * {@link IndexSegment}(s) -or- zero(0d) if no bloom filter * is desired. * @param partId * The unique partition identifier for the partition. * @param fromKey * The first key that would be accepted into that partition * (aka the separator key for that partition).<br> * Note: Failure to use the actual separatorKeys for the * partition will result in changing the separator key in a * manner that is incoherent! The change arises since the * updates are stored in the metadata index based on the * supplied <i>fromKey</i>. * @param toKey * The first key that would NOT be accepted into that * partition (aka the separator key for the right sibling * partition and <code>null</code> iff there is no right * sibling). * * @todo It is an error to supply a <i>fromKey</i> that is not also the * separatorKey for the partition, however the code does not * detect this error. * * @todo It is possible for the partition to be redefined (joined with a * sibling or split into two partitions). In this case any already * scheduled operations MUST abort and new operations with the * correct separator keys must be scheduled. */ public AbstractPartitionTask(PartitionedJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey) { this.master = master; this.branchingFactor = branchingFactor; this.errorRate = errorRate; this.name = name; this.partId = partId; this.fromKey = fromKey; this.toKey = toKey; } /** * * @todo update the metadata index (synchronized) and invalidate * existing partitioned index views that depend on the source * index. * * @param resources */ protected void updatePartition(IResourceMetadata[] resources) { throw new UnsupportedOperationException(); } /** * Task builds an {@link IndexSegment} for a partition from data on a * historical read-only {@link SlaveJournal}. When the {@link IndexSegment} * is ready the metadata index is updated to make the segment "live" and * existing views are notified that the source data on the partition must be * invalidated in favor of the new {@link IndexSegment}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ * * @todo this must use a merge rule that knows about deletion markers and is * only usable when the input is an {@link UnisolatedBTree}. The * output {@link IndexSegment} will contain timestamps and deletion * markers and support isolation. */ public static class BuildTask extends AbstractPartitionTask { private final IResourceMetadata src; private final int segId; /** * * @param src * The source for the build operation. Only those entries in * the described key range will be used. * @param segId * The output segment identifier. */ public BuildTask(PartitionedJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, IResourceMetadata src, int segId) { super(master,name,branchingFactor,errorRate,partId,fromKey,toKey); this.src = src; this.segId = segId; } /** * Build an {@link IndexSegment} from a key range corresponding to an * index partition. * * @todo this needs to use the rangeIterator on the root of the btree so * that the {@link Value}s will be visible, including both * version counters and deletion markers. */ public Object call() throws Exception { AbstractBTree src = master.getIndex(name,this.src); File outFile = master.getSegmentFile(name, partId, segId); new IndexSegmentBuilder(outFile, master.tmpDir, src.rangeCount( fromKey, toKey), src.rangeIterator(fromKey, toKey), branchingFactor, valSer, useChecksum, recordCompressor, errorRate); IResourceMetadata[] resources = new SegmentMetadata[] { new SegmentMetadata( "" + outFile, outFile.length(), ResourceState.New) }; updatePartition(resources); return null; } } /** * Abstract base class for compacting merge tasks. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ abstract static class AbstractMergeTask extends AbstractPartitionTask { protected final int segId; protected final boolean fullCompactingMerge; /** * A compacting merge of two or more resources. * * @param segId * The output segment identifier. * * @param fullCompactingMerge * True iff this will be a full compacting merge. */ protected AbstractMergeTask(PartitionedJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, int segId, boolean fullCompactingMerge) { super(master, name, branchingFactor, errorRate, partId, fromKey, toKey); this.segId = segId; this.fullCompactingMerge = fullCompactingMerge; } /** * Compacting merge of two or more resources - deletion markers are * preserved iff {@link #fullCompactingMerge} is <code>false</code>. * * @todo make sure that deletion markers get removed from the view as * part of the index segment merger logic (choose the most recent * write for each key, but if it is a delete then remove the entry * from the merge so that it does not appear in the temporary file * that is the input to the {@link IndexSegmentBuilder}). */ public Object call() throws Exception { // tmp file for the merge process. File tmpFile = File.createTempFile("merge", ".tmp", master.tmpDir); tmpFile.deleteOnExit(); // output file for the merged segment. File outFile = master.getSegmentFile(name, partId, segId); IResourceMetadata[] resources = getResources(); AbstractBTree[] srcs = new AbstractBTree[resources.length]; for(int i=0; i<srcs.length; i++) { // open the index - closed by the weak value cache. srcs[i] = master.getIndex(name, resources[i]); } // merge the data from the btree on the slave and the index // segment. MergedLeafIterator mergeItr = new IndexSegmentMerger( tmpFileBranchingFactor, srcs).merge(); // build the merged index segment. new IndexSegmentBuilder(outFile, null, mergeItr.nentries, new MergedEntryIterator(mergeItr), branchingFactor, valSer, useChecksum, recordCompressor, errorRate); // close the merged leaf iterator (and release its buffer/file). // @todo this should be automatic when the iterator is exhausted but // I am not seeing that. mergeItr.close(); /* * @todo Update the metadata index for this partition. This needs to * be an atomic operation so we have to synchronize on the metadata * index or simply move the operation into the MetadataIndex API and * let it encapsulate the problem. * * We mark the earlier index segment as "Dead" for this partition * since it has been replaced by the merged result. * * Note: it is a good idea to wait until you have opened the merged * index segment, and even until it has begun to serve data, before * deleting the old index segment that was an input to that merge! * The code currently waits until the next time a compacting merge * is performed for the partition and then deletes the Dead index * segment. */ final MetadataIndex mdi = master.getSlave().getMetadataIndex(name); final PartitionMetadata pmd = mdi.get(fromKey); // #of live segments for this partition. final int liveCount = pmd.getLiveCount(); // @todo assuming compacting merge each time. if(liveCount!=1) throw new UnsupportedOperationException(); // new segment definitions. final SegmentMetadata[] newSegs = new SegmentMetadata[2]; // assume only the last segment is live. final SegmentMetadata oldSeg = pmd.segs[pmd.segs.length-1]; newSegs[0] = new SegmentMetadata(oldSeg.filename, oldSeg.nbytes, ResourceState.Dead); newSegs[1] = new SegmentMetadata(outFile.toString(), outFile.length(), ResourceState.Live); mdi.put(fromKey, new PartitionMetadata(0, segId + 1, newSegs)); return null; } /** * The resources that comprise the view in reverse timestamp order * (increasing age). */ abstract protected IResourceMetadata[] getResources(); } /** * Task builds an {@link IndexSegment} using a compacting merge of two * resources having data for the same partition. Common use cases are a * journal and an index segment or two index segments. Only the most recent * writes will be retained for any given key (duplicate suppression). Since * this task does NOT provide a full compacting merge (it may be applied to * a subset of the resources required to materialize a view for a commit * time), deletion markers MAY be present in the resulting * {@link IndexSegment}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public static class MergeTask extends AbstractMergeTask { private final IResourceMetadata[] resources; /** * A compacting merge of two or more resources. * * @param srcs * The source resources, which must be given in reverse * timestamp order (increasing age). * @param segId * The output segment identifier. */ public MergeTask(PartitionedJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, IResourceMetadata[] resources, int segId) { super(master, name, branchingFactor, errorRate, partId, fromKey, toKey, segId, false); this.resources = resources; } protected IResourceMetadata[] getResources() { return resources; } } /** * Task builds an {@link IndexSegment} using a full compacting merge of all * resources having data for the same partition as of a specific commit * time. Only the most recent writes will be retained for any given key * (duplicate suppression). Deletion markers wil NOT be present in the * resulting {@link IndexSegment}. * <p> * Note: A full compacting merge does not necessarily result in only a * single {@link IndexSegment} for a partition since (a) it may be requested * for a historical commit time; and (b) subsequent writes may have * occurred, resulting in additional data on either the journal and/or new * {@link IndexSegment} that do not participate in the merge. * <p> * Note: in order to perform a full compacting merge the task MUST read from * all resources required to provide a consistent view for a partition * otherwise lost deletes may result. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public static class FullMergeTask extends AbstractMergeTask { private final long commitTime; /** * A full compacting merge of an index partition. * * @param segId * The output segment identifier. * * @param commitTime * The commit time for the view that is the input to the * merge operation. */ public FullMergeTask(PartitionedJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, long commitTime, int segId) { super(master, name, branchingFactor, errorRate, partId, fromKey, toKey, segId, true); this.commitTime = commitTime; } protected IResourceMetadata[] getResources() { final PartitionedIndex oldIndex = ((PartitionedIndex) master .getIndex(name, commitTime)); final IResourceMetadata[] resources = oldIndex.getResources(fromKey); return resources; } } // /** // * Task builds an {@link IndexSegment} by joining an existing // * {@link IndexSegment} for a partition with an existing // * {@link IndexSegment} for either the left or right sibling of that // * partition. // * // * @author <a href="mailto:tho...@us...">Bryan Thompson</a> // * @version $Id$ // */ // public static class JoinTask implements IPartitionTask { // // } // /** // * Task splits an {@link IndexSegment} into two new {@link IndexSegment}s. // * This task is executed when a partition is split in order to breakdown the // * {@link IndexSegment} for that partition into data for the partition and // * its new left/right sibling. // * // * @author <a href="mailto:tho...@us...">Bryan Thompson</a> // * @version $Id$ // */ // public static class SplitTask implements IPartitionTask { // // } // /** // * Update the metadata index to reflect the split of one index segment // * into two index segments. // * // * @param separatorKey // * Requests greater than or equal to the separatorKey (and // * less than the next largest separatorKey in the metadata // * index) are directed into seg2. Requests less than the // * separatorKey (and greated than any proceeding separatorKey // * in the metadata index) are directed into seg1. // * @param seg1 // * The metadata for the index segment that was split. // * @param seg2 // * The metadata for the right sibling of the split index // * segment in terms of the key range of the distributed // * index. // */ // public void split(Object separatorKey, PartitionMetadata md1, PartitionMetadata md2) { // // } // // /** // * @todo join of index segment with left or right sibling. unlike the // * nodes of a btree we merge nodes whenever a segment goes under // * capacity rather than trying to redistribute part of the key // * range from one index segment to another. // */ // public void join() { // // } } Index: PartitionedIndex.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/PartitionedIndex.java,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** PartitionedIndex.java 6 Mar 2007 20:38:06 -0000 1.7 --- PartitionedIndex.java 8 Mar 2007 18:14:05 -0000 1.8 *************** *** 48,52 **** package com.bigdata.scaleup; - import java.io.File; import java.util.HashMap; import java.util.Iterator; --- 48,51 ---- *************** *** 67,73 **** import com.bigdata.objndx.IIndex; import com.bigdata.objndx.IndexSegment; - import com.bigdata.objndx.IndexSegmentFileStore; - import com.bigdata.scaleup.PartitionedJournal.IViewMetadata; - import com.bigdata.scaleup.PartitionedJournal.JournalMetadata; /** --- 66,69 ---- *************** *** 114,120 **** --- 110,132 ---- /** * A cache of the fused views for in use partitions. + * + * @todo reconcile this with + * {@link PartitionedJournal#getIndex(String, IResourceMetadata)} + * which provides a weak value cache for index segments. */ private final Map<Integer,FusedView> views = new HashMap<Integer,FusedView>(); + public PartitionedJournal getMaster() { + + return getSlave().getMaster(); + + } + + public SlaveJournal getSlave() { + + return (SlaveJournal)getBTree().getStore(); + + } + /** * The mutable {@link BTree} used to absorb writes. *************** *** 135,160 **** * * @return The live {@link IndexSegment}s for that partition. - * - * @todo there needs to be synchronization with the - * {@link PartitionedJournal} so that index segments may be closed out - * as required but it would be very nice (or perhaps necessary) to be - * able to perform a compacting merge while an {@link IndexSegment} - * was open for reading. */ protected IndexSegment[] openIndexSegments(PartitionMetadata pmd) { ! String[] files = pmd.getLiveSegmentFiles(); ! IndexSegment[] segs = new IndexSegment[files.length]; ! for(int i=0; i<segs.length; i++) { ! File file = new File(files[i]); ! segs[i] = new IndexSegment(new IndexSegmentFileStore(file), btree ! .getNodeSerializer().getValueSerializer()); } return segs; --- 147,171 ---- * * @return The live {@link IndexSegment}s for that partition. */ protected IndexSegment[] openIndexSegments(PartitionMetadata pmd) { ! final int liveCount = pmd.getLiveCount(); ! IndexSegment[] segs = new IndexSegment[liveCount]; ! PartitionedJournal master = getMaster(); ! ! int n = 0; ! ! for(int i=0; i<pmd.segs.length; i++) { ! if(pmd.segs[i].state != ResourceState.Live) continue; ! segs[n++] = (IndexSegment) master.getIndex(getName(), pmd.segs[i]); } + assert n == liveCount; + return segs; *************** *** 263,270 **** * @todo reconcile with {@link #getView(byte[])}? */ ! protected IViewMetadata[] getResources(byte[] key) { JournalMetadata journalResource = new JournalMetadata((Journal) btree ! .getStore(), IndexSegmentLifeCycleEnum.LIVE); PartitionMetadata pmd = mdi.find(key); --- 274,281 ---- * @todo reconcile with {@link #getView(byte[])}? */ ! protected IResourceMetadata[] getResources(byte[] key) { JournalMetadata journalResource = new JournalMetadata((Journal) btree ! .getStore(), ResourceState.Live); PartitionMetadata pmd = mdi.find(key); *************** *** 272,276 **** final int liveCount = pmd.getLiveCount(); ! IViewMetadata[] resources = new IViewMetadata[liveCount + 1]; int n = 0; --- 283,287 ---- final int liveCount = pmd.getLiveCount(); ! IResourceMetadata[] resources = new IResourceMetadata[liveCount + 1]; int n = 0; *************** *** 282,286 **** SegmentMetadata seg = pmd.segs[i]; ! if (seg.state != IndexSegmentLifeCycleEnum.LIVE) continue; --- 293,297 ---- SegmentMetadata seg = pmd.segs[i]; ! if (seg.state != ResourceState.Live) continue; --- NEW FILE: ResourceState.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ package com.bigdata.scaleup; import com.bigdata.objndx.IndexSegment; /** * Enumeration of life-cycle states for {@link IndexSegment}s in a * partition. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public enum ResourceState { New("New",0), Live("Live",1), Dead("Dead",2); final private String name; final private int id; ResourceState(String name,int id) {this.name = name;this.id = id;} public String toString() {return name;} public int valueOf() {return id;} static public ResourceState valueOf(int id) { switch(id) { case 0: return New; case 1: return Live; case 2: return Dead; default: throw new IllegalArgumentException("Unknown: code="+id); } } } |
