Menu
▾
▴
[cweb-CVS] bigdata/src/java/com/bigdata/scaleup MasterJournal.java, 1.3, 1.4 SlaveJournal.java, 1.5, 1.6 PartitionedIndexView.java, 1.1, 1.2 MetadataIndex.java, 1.6, 1.7 AbstractPartitionTask.java, 1.2, 1.3
|
From: Bryan T. <tho...@us...> - 2007-03-27 14:34:58
|
Update of /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv6186/src/java/com/bigdata/scaleup Modified Files: MasterJournal.java SlaveJournal.java PartitionedIndexView.java MetadataIndex.java AbstractPartitionTask.java Log Message: Added indexUUID to AbstractBTree so that each scale-out index may have a unique indentifier. Modified the BTreeMetadata class and derived classes to use Externalizable, to support explicit versioning of the metadata record, and to have private fields since they can not be final with Externalizable. Index: AbstractPartitionTask.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/AbstractPartitionTask.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** AbstractPartitionTask.java 11 Mar 2007 11:42:42 -0000 1.2 --- AbstractPartitionTask.java 27 Mar 2007 14:34:22 -0000 1.3 *************** *** 45,48 **** --- 45,49 ---- import java.io.File; + import java.util.UUID; import java.util.concurrent.Executors; *************** *** 107,110 **** --- 108,112 ---- protected final double errorRate; protected final String name; + protected final UUID indexUUID; protected final int partId; protected final byte[] fromKey; *************** *** 172,176 **** */ public AbstractPartitionTask(MasterJournal master, String name, ! int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey) { --- 174,178 ---- */ public AbstractPartitionTask(MasterJournal master, String name, ! UUID indexUUID, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey) { *************** *** 179,182 **** --- 181,185 ---- this.errorRate = errorRate; this.name = name; + this.indexUUID = indexUUID; this.partId = partId; this.fromKey = fromKey; *************** *** 227,235 **** * The output segment identifier. */ ! public BuildTask(MasterJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, IResourceMetadata src, int segId) { ! super(master,name,branchingFactor,errorRate,partId,fromKey,toKey); this.src = src; --- 230,239 ---- * The output segment identifier. */ ! public BuildTask(MasterJournal master, String name, UUID indexUUID, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, IResourceMetadata src, int segId) { ! super(master, name, indexUUID, branchingFactor, errorRate, partId, ! fromKey, toKey); this.src = src; *************** *** 256,260 **** fromKey, toKey), src.rangeIterator(fromKey, toKey), branchingFactor, valSer, useChecksum, recordCompressor, ! errorRate); IResourceMetadata[] resources = new SegmentMetadata[] { new SegmentMetadata( --- 260,264 ---- fromKey, toKey), src.rangeIterator(fromKey, toKey), branchingFactor, valSer, useChecksum, recordCompressor, ! errorRate, indexUUID); IResourceMetadata[] resources = new SegmentMetadata[] { new SegmentMetadata( *************** *** 291,300 **** */ protected AbstractMergeTask(MasterJournal master, String name, ! int branchingFactor, double errorRate, int partId, ! byte[] fromKey, byte[] toKey, int segId, boolean fullCompactingMerge) { ! super(master, name, branchingFactor, errorRate, partId, fromKey, ! toKey); this.segId = segId; --- 295,304 ---- */ protected AbstractMergeTask(MasterJournal master, String name, ! UUID indexUUID, int branchingFactor, double errorRate, ! int partId, byte[] fromKey, byte[] toKey, int segId, boolean fullCompactingMerge) { ! super(master, name, indexUUID, branchingFactor, errorRate, partId, ! fromKey, toKey); this.segId = segId; *************** *** 343,347 **** new IndexSegmentBuilder(outFile, null, mergeItr.nentries, new MergedEntryIterator(mergeItr), branchingFactor, valSer, ! useChecksum, recordCompressor, errorRate); // close the merged leaf iterator (and release its buffer/file). --- 347,351 ---- new IndexSegmentBuilder(outFile, null, mergeItr.nentries, new MergedEntryIterator(mergeItr), branchingFactor, valSer, ! useChecksum, recordCompressor, errorRate, indexUUID); // close the merged leaf iterator (and release its buffer/file). *************** *** 429,439 **** * The output segment identifier. */ ! public MergeTask(MasterJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, IResourceMetadata[] resources, int segId) { ! super(master, name, branchingFactor, errorRate, partId, fromKey, ! toKey, segId, false); this.resources = resources; --- 433,443 ---- * The output segment identifier. */ ! public MergeTask(MasterJournal master, String name, UUID indexUUID, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, IResourceMetadata[] resources, int segId) { ! super(master, name, indexUUID, branchingFactor, errorRate, partId, ! fromKey, toKey, segId, false); this.resources = resources; *************** *** 483,492 **** * merge operation. */ ! public FullMergeTask(MasterJournal master, String name, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, long commitTime, int segId) { ! super(master, name, branchingFactor, errorRate, partId, fromKey, ! toKey, segId, true); this.commitTime = commitTime; --- 487,496 ---- * merge operation. */ ! public FullMergeTask(MasterJournal master, String name, UUID indexUUID, int branchingFactor, double errorRate, int partId, byte[] fromKey, byte[] toKey, long commitTime, int segId) { ! super(master, name, indexUUID, branchingFactor, errorRate, partId, ! fromKey, toKey, segId, true); this.commitTime = commitTime; Index: MetadataIndex.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/MetadataIndex.java,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** MetadataIndex.java 11 Mar 2007 11:42:42 -0000 1.6 --- MetadataIndex.java 27 Mar 2007 14:34:22 -0000 1.7 *************** *** 44,47 **** --- 44,55 ---- package com.bigdata.scaleup; + import java.io.Externalizable; + import java.io.IOException; + import java.io.ObjectInput; + import java.io.ObjectOutput; + import java.util.UUID; + + import org.CognitiveWeb.extser.LongPacker; + import com.bigdata.isolation.IsolatedBTree; import com.bigdata.journal.Journal; *************** *** 49,52 **** --- 57,61 ---- import com.bigdata.objndx.BTree; import com.bigdata.objndx.BTreeMetadata; + import com.bigdata.objndx.IIndex; import com.bigdata.objndx.IndexSegment; import com.bigdata.rawstore.IRawStore; *************** *** 59,65 **** * The values are {@link PartitionMetadata} objects. * - * @todo locator logic on a cluster (a socket address in addition to the other - * information). - * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ --- 68,71 ---- *************** *** 72,85 **** * transaction remains which can read those data. This metadata must be * restart-safe so that resources are eventually deleted. - * - * @todo define a UUID so that is at least possible to rename a partitioned - * index? the uuid would be store in the metadata record for the metadata - * index and in each index segment generated for that metadata index. we - * could also define a partition uuid. finally, each btree and index - * segment could have its own uuid. the index segment would also carry the - * uuid of the partition and the partitioned index. This would also make - * it possible to determine which index segments belong to which - * partitions of which partitioned indices and effectively reconstruct the - * metadata index for a partitioned index from the data on the ground. */ public class MetadataIndex extends BTree { --- 78,81 ---- *************** *** 88,91 **** --- 84,89 ---- * The name of the metadata index, which is the always the same as the name * under which the corresponding {@link PartitionedIndexView} was registered. + * + * @todo rename as managedIndexName (and the access method as well). */ private final String name; *************** *** 102,105 **** --- 100,137 ---- /** + * The unique identifier for the index whose data metadata is managed by + * this {@link MetadataIndex}. + * <p> + * When using a scale-out index the same <i>indexUUID</i> MUST be assigned + * to each mutable and immutable B+Tree having data for any partition of + * that scale-out index. This makes it possible to work backwards from the + * B+Tree data structures and identify the index to which they belong. This + * field is that UUID. Note that the inherited {@link #getIndexUUID()} + * method provides the UUID of the metadata index NOT the managed index! + * + * @see IIndex#getIndexUUID() + */ + protected final UUID managedIndexUUID; + + /** + * The unique identifier for the index whose data metadata is managed by + * this {@link MetadataIndex}. + * <p> + * When using a scale-out index the same <i>indexUUID</i> MUST be assigned + * to each mutable and immutable B+Tree having data for any partition of + * that scale-out index. This makes it possible to work backwards from the + * B+Tree data structures and identify the index to which they belong. This + * field is that UUID. Note that the inherited {@link #getIndexUUID()} + * method provides the UUID of the metadata index NOT the managed index! + * + * @see IIndex#getIndexUUID() + */ + public UUID getManagedIndexUUID() { + + return managedIndexUUID; + + } + + /** * Create a new {@link MetadataIndex}. * *************** *** 108,121 **** * @param branchingFactor * The branching factor. ! * @param name ! * The name of the metadata index - this MUST be the name under ! * which the corresponding {@link PartitionedIndexView} was ! * registered. */ ! public MetadataIndex(IRawStore store, int branchingFactor, String name) { ! super(store, branchingFactor, PartitionMetadata.Serializer.INSTANCE); ! this.name = name; } --- 140,160 ---- * @param branchingFactor * The branching factor. ! * @param indexUUID ! * The unique identifier for the metadata index. ! * @param managedIndexUUID ! * The unique identifier for the managed scale-out index. ! * @param managedIndexName ! * The name of the managed scale out index. */ ! public MetadataIndex(IRawStore store, int branchingFactor, UUID indexUUID, ! UUID managedIndexUUID, String managedIndexName) { ! super(store, branchingFactor, indexUUID, ! PartitionMetadata.Serializer.INSTANCE); ! this.name = managedIndexName; ! ! // ! this.managedIndexUUID = managedIndexUUID; } *************** *** 125,129 **** super(store, metadata); ! name = ((MetadataIndexMetadata)metadata).name; } --- 164,170 ---- super(store, metadata); ! name = ((MetadataIndexMetadata)metadata).getName(); ! ! managedIndexUUID = ((MetadataIndexMetadata)metadata).getManagedIndexUUID(); } *************** *** 142,154 **** * @version $Id$ */ ! public static class MetadataIndexMetadata extends BTreeMetadata { private static final long serialVersionUID = -7309267778881420043L; /** * The name of the metadata index, which is the always the same as the name * under which the corresponding {@link PartitionedIndexView} was registered. */ ! public final String name; /** --- 183,226 ---- * @version $Id$ */ ! public static class MetadataIndexMetadata extends BTreeMetadata implements Externalizable { private static final long serialVersionUID = -7309267778881420043L; + private String name; + private UUID managedIndexUUID; + /** * The name of the metadata index, which is the always the same as the name * under which the corresponding {@link PartitionedIndexView} was registered. */ ! public final String getName() { ! ! return name; ! ! } ! ! /** ! * The unique identifier for the index whose data metadata is managed by ! * this {@link MetadataIndex}. ! * <p> ! * When using a scale-out index the same <i>indexUUID</i> MUST be assigned ! * to each mutable and immutable B+Tree having data for any partition of ! * that scale-out index. This makes it possible to work backwards from the ! * B+Tree data structures and identify the index to which they belong. This ! * field is that UUID. Note that the inherited {@link #getIndexUUID()} ! * method provides the UUID of the metadata index NOT the managed index! ! */ ! public final UUID getManagedIndexUUID() { ! ! return managedIndexUUID; ! ! } ! ! /** ! * De-serialization constructor. ! */ ! public MetadataIndexMetadata() { ! ! } /** *************** *** 159,166 **** super(mdi); ! this.name = mdi.name; } } --- 231,274 ---- super(mdi); ! this.name = mdi.getName(); ! ! this.managedIndexUUID = mdi.getManagedIndexUUID(); } + private static final transient int VERSION0 = 0x0; + + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + + super.readExternal(in); + + final int version = (int)LongPacker.unpackLong(in); + + if (version != VERSION0) { + + throw new IOException("Unknown version: version=" + version); + + } + + name = in.readUTF(); + + managedIndexUUID = new UUID(in.readLong()/*MSB*/,in.readLong()/*LSB*/); + + } + + public void writeExternal(ObjectOutput out) throws IOException { + + super.writeExternal(out); + + LongPacker.packLong(out,VERSION0); + + out.writeUTF(name); + + out.writeLong(managedIndexUUID.getMostSignificantBits()); + + out.writeLong(managedIndexUUID.getLeastSignificantBits()); + + } + } Index: MasterJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/MasterJournal.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** MasterJournal.java 17 Mar 2007 23:14:58 -0000 1.3 --- MasterJournal.java 27 Mar 2007 14:34:22 -0000 1.4 *************** *** 777,783 **** */ final BTree newBTree = (oldBTree instanceof UnisolatedBTree ? new UnisolatedBTree( ! newJournal, oldBTree.getBranchingFactor()) : new BTree(newJournal, oldBTree.getBranchingFactor(), ! oldBTree.getNodeSerializer().getValueSerializer())); // Register the btree under the same name on the new slave. --- 777,785 ---- */ final BTree newBTree = (oldBTree instanceof UnisolatedBTree ? new UnisolatedBTree( ! newJournal, oldBTree.getBranchingFactor(), oldBTree ! .getIndexUUID()) : new BTree(newJournal, oldBTree.getBranchingFactor(), ! oldBTree.getIndexUUID(), oldBTree ! .getNodeSerializer().getValueSerializer())); // Register the btree under the same name on the new slave. *************** *** 920,924 **** .getEntryCount(), oldIndex.btree.getRoot().entryIterator(), mseg, Value.Serializer.INSTANCE, true/* useChecksum */, ! null/* new RecordCompressor() */, 0d); /* --- 922,927 ---- .getEntryCount(), oldIndex.btree.getRoot().entryIterator(), mseg, Value.Serializer.INSTANCE, true/* useChecksum */, ! null/* new RecordCompressor() */, 0d, oldIndex.btree ! .getIndexUUID()); /* *************** *** 960,968 **** // build the merged index segment. ! IndexSegmentBuilder builder = new IndexSegmentBuilder(outFile, ! null, mergeItr.nentries, new MergedEntryIterator(mergeItr), ! mseg, oldIndex.btree.getNodeSerializer() ! .getValueSerializer(), false/* useChecksum */, ! null/* recordCompressor */, 0d/* errorRate */); // close the merged leaf iterator (and release its buffer/file). --- 963,971 ---- // build the merged index segment. ! new IndexSegmentBuilder(outFile, null, mergeItr.nentries, ! new MergedEntryIterator(mergeItr), mseg, oldIndex.btree ! .getNodeSerializer().getValueSerializer(), ! false/* useChecksum */, null/* recordCompressor */, ! 0d/* errorRate */, oldIndex.btree.getIndexUUID()); // close the merged leaf iterator (and release its buffer/file). Index: SlaveJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/SlaveJournal.java,v retrieving revision 1.5 retrieving revision 1.6 diff -C2 -d -r1.5 -r1.6 *** SlaveJournal.java 15 Mar 2007 16:11:10 -0000 1.5 --- SlaveJournal.java 27 Mar 2007 14:34:22 -0000 1.6 *************** *** 46,49 **** --- 46,50 ---- import java.io.File; import java.util.Properties; + import java.util.UUID; import com.bigdata.isolation.IIsolatableIndex; *************** *** 188,198 **** public IIndex registerIndex(String name) { ! return registerIndex(name, new UnisolatedBTree(this)); } /** ! * Registers and returns a {@link PartitionedIndexView} under the given name and ! * assigns the supplied {@link IIndex} to absorb writes for that * {@link PartitionedIndexView}. * <p> --- 189,199 ---- public IIndex registerIndex(String name) { ! return registerIndex(name, new UnisolatedBTree(this, UUID.randomUUID())); } /** ! * Registers and returns a {@link PartitionedIndexView} under the given name ! * and assigns the supplied {@link IIndex} to absorb writes for that * {@link PartitionedIndexView}. * <p> *************** *** 207,216 **** * Note: You MUST {@link #commit()} before the registered index will be * either restart-safe or visible to new transactions. - * - * @todo use a prototype model so that the registered btree type is - * preserved? (Only the metadata extensions are preserved right now). - * One way to do this is by putting the constructor on the metadata - * object. Another is to make the btree Serializable and then just - * declare everything else as transient. */ public IIndex registerIndex(String name, IIndex btree) { --- 208,211 ---- *************** *** 236,242 **** } ! MetadataIndex mdi = new MetadataIndex(this, ! BTree.DEFAULT_BRANCHING_FACTOR, name); // create the initial partition which can accept any key. --- 231,244 ---- } ! ! /* ! * @todo the assigned random UUID for the metadata index must be used by ! * all B+Tree objects having data for the metadata index so once we ! * support partitions in the metadata index itself this UUID must be ! * propagated to all of those downstream objects. ! */ MetadataIndex mdi = new MetadataIndex(this, ! BTree.DEFAULT_BRANCHING_FACTOR, UUID.randomUUID(), btree ! .getIndexUUID(), name); // create the initial partition which can accept any key. Index: PartitionedIndexView.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/java/com/bigdata/scaleup/PartitionedIndexView.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** PartitionedIndexView.java 11 Mar 2007 11:42:38 -0000 1.1 --- PartitionedIndexView.java 27 Mar 2007 14:34:22 -0000 1.2 *************** *** 52,55 **** --- 52,56 ---- import java.util.Map; import java.util.NoSuchElementException; + import java.util.UUID; import com.bigdata.journal.ICommitter; *************** *** 62,70 **** import com.bigdata.objndx.BatchRemove; import com.bigdata.objndx.EmptyEntryIterator; - import com.bigdata.objndx.ReadOnlyFusedView; import com.bigdata.objndx.IEntryIterator; import com.bigdata.objndx.IFusedView; import com.bigdata.objndx.IIndex; import com.bigdata.objndx.IndexSegment; /** --- 63,71 ---- import com.bigdata.objndx.BatchRemove; import com.bigdata.objndx.EmptyEntryIterator; import com.bigdata.objndx.IEntryIterator; import com.bigdata.objndx.IFusedView; import com.bigdata.objndx.IIndex; import com.bigdata.objndx.IndexSegment; + import com.bigdata.objndx.ReadOnlyFusedView; /** *************** *** 347,350 **** --- 348,364 ---- } + + /* + * IIndex + */ + + /** + * The UUID for the scale-out index. + */ + public UUID getIndexUUID() { + + return mdi.getManagedIndexUUID(); + + } /* |
