Menu
▾
▴
[cweb-CVS] bigdata/src/test/com/bigdata/scaleup TestPartitionedIndex.java, 1.1, 1.2 TestMetadataIndex.java, 1.8, 1.9 TestPartitionedJournal.java, 1.4, 1.5
|
From: Bryan T. <tho...@us...> - 2007-03-08 18:14:12
|
Update of /cvsroot/cweb/bigdata/src/test/com/bigdata/scaleup In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv15000/src/test/com/bigdata/scaleup Modified Files: TestPartitionedIndex.java TestMetadataIndex.java TestPartitionedJournal.java Log Message: Updated the UML model and added a ZIP containing an HTML presentation of the model. Working on partitioned index support. Index: TestMetadataIndex.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/scaleup/TestMetadataIndex.java,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** TestMetadataIndex.java 6 Mar 2007 20:38:06 -0000 1.8 --- TestMetadataIndex.java 8 Mar 2007 18:14:06 -0000 1.9 *************** *** 162,166 **** PartitionMetadata part2 = new PartitionMetadata(partId0,1, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,IndexSegmentLifeCycleEnum.LIVE) }); assertEquals(part1,md.put(key0, part2)); --- 162,166 ---- PartitionMetadata part2 = new PartitionMetadata(partId0,1, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,ResourceState.Live) }); assertEquals(part1,md.put(key0, part2)); *************** *** 169,174 **** PartitionMetadata part3 = new PartitionMetadata(partId0,2, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,IndexSegmentLifeCycleEnum.LIVE), ! new SegmentMetadata("b", 20L,IndexSegmentLifeCycleEnum.LIVE) }); assertEquals(part2,md.put(key0, part3)); --- 169,174 ---- PartitionMetadata part3 = new PartitionMetadata(partId0,2, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,ResourceState.Live), ! new SegmentMetadata("b", 20L,ResourceState.Live) }); assertEquals(part2,md.put(key0, part3)); *************** *** 242,246 **** final int partId1 = 1; PartitionMetadata part1 = new PartitionMetadata(partId1,1, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,IndexSegmentLifeCycleEnum.LIVE) }); assertEquals(null,md.put(key1, part1)); assertEquals(part1,md.get(key1)); --- 242,246 ---- final int partId1 = 1; PartitionMetadata part1 = new PartitionMetadata(partId1,1, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,ResourceState.Live) }); assertEquals(null,md.put(key1, part1)); assertEquals(part1,md.get(key1)); *************** *** 248,253 **** final int partId2 = 2; PartitionMetadata part2 = new PartitionMetadata(partId2,2, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,IndexSegmentLifeCycleEnum.LIVE), ! new SegmentMetadata("b", 20L,IndexSegmentLifeCycleEnum.LIVE) }); assertEquals(null, md.put(key2, part2)); assertEquals(part2, md.get(key2)); --- 248,253 ---- final int partId2 = 2; PartitionMetadata part2 = new PartitionMetadata(partId2,2, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,ResourceState.Live), ! new SegmentMetadata("b", 20L,ResourceState.Live) }); assertEquals(null, md.put(key2, part2)); assertEquals(part2, md.get(key2)); *************** *** 330,334 **** final int partId1 = 1; PartitionMetadata part1 = new PartitionMetadata(partId1,1, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,IndexSegmentLifeCycleEnum.LIVE) }); assertEquals(null,md.put(key1, part1)); assertEquals(part1,md.get(key1)); --- 330,334 ---- final int partId1 = 1; PartitionMetadata part1 = new PartitionMetadata(partId1,1, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,ResourceState.Live) }); assertEquals(null,md.put(key1, part1)); assertEquals(part1,md.get(key1)); *************** *** 336,341 **** final int partId2 = 2; PartitionMetadata part2 = new PartitionMetadata(partId2,2, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,IndexSegmentLifeCycleEnum.LIVE), ! new SegmentMetadata("b", 20L,IndexSegmentLifeCycleEnum.LIVE) }); assertEquals(null, md.put(key2, part2)); assertEquals(part2,md.get(key2)); --- 336,341 ---- final int partId2 = 2; PartitionMetadata part2 = new PartitionMetadata(partId2,2, ! new SegmentMetadata[] { new SegmentMetadata("a", 10L,ResourceState.Live), ! new SegmentMetadata("b", 20L,ResourceState.Live) }); assertEquals(null, md.put(key2, part2)); assertEquals(part2,md.get(key2)); *************** *** 455,465 **** md.put(new byte[] {}, new PartitionMetadata(0,1, new SegmentMetadata[] { new SegmentMetadata("" + outFile, ! outFile.length(),IndexSegmentLifeCycleEnum.LIVE) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg = new IndexSegment(new IndexSegmentFileStore(outFile), ! btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree,seg); --- 455,465 ---- md.put(new byte[] {}, new PartitionMetadata(0,1, new SegmentMetadata[] { new SegmentMetadata("" + outFile, ! outFile.length(),ResourceState.Live) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg = new IndexSegmentFileStore(outFile).load(); ! // @todo btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree,seg); *************** *** 560,570 **** md.put(new byte[] {}, new PartitionMetadata(0,2, new SegmentMetadata[] { new SegmentMetadata("" + outFile01, ! outFile01.length(),IndexSegmentLifeCycleEnum.LIVE) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg01 = new IndexSegment(new IndexSegmentFileStore(outFile01), ! btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree,seg01); --- 560,570 ---- md.put(new byte[] {}, new PartitionMetadata(0,2, new SegmentMetadata[] { new SegmentMetadata("" + outFile01, ! outFile01.length(),ResourceState.Live) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg01 = new IndexSegmentFileStore(outFile01).load(); ! // @todo btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree,seg01); *************** *** 609,618 **** * update the metadata index for this partition. * ! * Note: We mark index segment 01 as "DEAD" for this partition since it * has been replaced by the merged result (index segment 02). */ md.put(new byte[] {}, new PartitionMetadata(0, 3, new SegmentMetadata[] { ! new SegmentMetadata("" + outFile01, outFile01.length(),IndexSegmentLifeCycleEnum.DEAD), ! new SegmentMetadata("" + outFile02, outFile02.length(),IndexSegmentLifeCycleEnum.LIVE) })); /* --- 609,618 ---- * update the metadata index for this partition. * ! * Note: We mark index segment 01 as "Dead" for this partition since it * has been replaced by the merged result (index segment 02). */ md.put(new byte[] {}, new PartitionMetadata(0, 3, new SegmentMetadata[] { ! new SegmentMetadata("" + outFile01, outFile01.length(),ResourceState.Dead), ! new SegmentMetadata("" + outFile02, outFile02.length(),ResourceState.Live) })); /* *************** *** 620,625 **** * data. */ ! IndexSegment seg02 = new IndexSegment(new IndexSegmentFileStore( ! outFile02), btree.getNodeSerializer().getValueSerializer()); assertSameIterator(new Object[] { v1, v2, v3, v4, v5, v6, v7, v8 }, --- 620,625 ---- * data. */ ! IndexSegment seg02 = new IndexSegmentFileStore(outFile02).load(); ! // @todo btree.getNodeSerializer().getValueSerializer()); assertSameIterator(new Object[] { v1, v2, v3, v4, v5, v6, v7, v8 }, *************** *** 829,839 **** new SegmentMetadata[] { new SegmentMetadata("" + outFile01, outFile01.length(), ! IndexSegmentLifeCycleEnum.LIVE) })); /* * open and verify the index segment against the btree data. */ ! seg = new IndexSegment(new IndexSegmentFileStore(outFile01), ! testData.getNodeSerializer().getValueSerializer()); if(validateEachTrial) assertSameBTree(testData, seg); --- 829,839 ---- new SegmentMetadata[] { new SegmentMetadata("" + outFile01, outFile01.length(), ! ResourceState.Live) })); /* * open and verify the index segment against the btree data. */ ! seg = new IndexSegmentFileStore(outFile01).load(); ! // @todo testData.getNodeSerializer().getValueSerializer()); if(validateEachTrial) assertSameBTree(testData, seg); *************** *** 885,891 **** * update the metadata index for this partition. * ! * @todo We could mark the earlier index segment as "DEAD" for * this partition since it has been replaced by the merged ! * result. We could then delete the files for "DEAD" index * segments after a suitable grace period. */ --- 885,891 ---- * update the metadata index for this partition. * ! * @todo We could mark the earlier index segment as "Dead" for * this partition since it has been replaced by the merged ! * result. We could then delete the files for "Dead" index * segments after a suitable grace period. */ *************** *** 894,898 **** new SegmentMetadata[] { new SegmentMetadata("" + outFile02, outFile02 ! .length(), IndexSegmentLifeCycleEnum.LIVE) })); /* --- 894,898 ---- new SegmentMetadata[] { new SegmentMetadata("" + outFile02, outFile02 ! .length(), ResourceState.Live) })); /* *************** *** 900,906 **** * expected data. */ ! IndexSegment seg02 = new IndexSegment( ! new IndexSegmentFileStore(outFile02), ! testData.getNodeSerializer().getValueSerializer()); if(validateEachTrial) assertSameBTree(groundTruth,seg02); --- 900,905 ---- * expected data. */ ! IndexSegment seg02 = new IndexSegmentFileStore(outFile02).load(); ! // @todo testData.getNodeSerializer().getValueSerializer()); if(validateEachTrial) assertSameBTree(groundTruth,seg02); *************** *** 1029,1039 **** md.put(new byte[] {}, new PartitionMetadata(0,1, new SegmentMetadata[] { new SegmentMetadata("" + outFile01, ! outFile01.length(),IndexSegmentLifeCycleEnum.LIVE) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg01 = new IndexSegment(new IndexSegmentFileStore(outFile01), ! btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree,seg01); --- 1028,1038 ---- md.put(new byte[] {}, new PartitionMetadata(0,1, new SegmentMetadata[] { new SegmentMetadata("" + outFile01, ! outFile01.length(),ResourceState.Live) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg01 = new IndexSegmentFileStore(outFile01).load(); ! // @todo btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree,seg01); *************** *** 1072,1083 **** */ md.put(new byte[] {}, new PartitionMetadata(0, 1, new SegmentMetadata[] { ! new SegmentMetadata("" + outFile01, outFile01.length(),IndexSegmentLifeCycleEnum.LIVE), ! new SegmentMetadata("" + outFile02, outFile02.length(),IndexSegmentLifeCycleEnum.LIVE) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg02 = new IndexSegment(new IndexSegmentFileStore( ! outFile02), btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree, seg02); --- 1071,1082 ---- */ md.put(new byte[] {}, new PartitionMetadata(0, 1, new SegmentMetadata[] { ! new SegmentMetadata("" + outFile01, outFile01.length(),ResourceState.Live), ! new SegmentMetadata("" + outFile02, outFile02.length(),ResourceState.Live) })); /* * open and verify the index segment against the btree data. */ ! IndexSegment seg02 = new IndexSegmentFileStore(outFile02).load(); ! // @todo btree.getNodeSerializer().getValueSerializer()); assertSameBTree(btree, seg02); Index: TestPartitionedIndex.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/scaleup/TestPartitionedIndex.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** TestPartitionedIndex.java 8 Feb 2007 21:32:15 -0000 1.1 --- TestPartitionedIndex.java 8 Mar 2007 18:14:06 -0000 1.2 *************** *** 48,56 **** --- 48,66 ---- package com.bigdata.scaleup; + import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.objndx.AbstractBTreeTestCase; + import com.bigdata.objndx.IndexSegmentBuilder; + import com.bigdata.objndx.IndexSegmentMerger; /** * Test suite for {@link PartitionedIndex}. * + * @todo This should be refactored to test when using {@link BTree} as well as + * {@link UnisolatedBTree}. The latter is more important for database + * operations as it supports isolation and column stores. Changing over + * from {@link BTree} to {@link UnisolatedBTree} highlights many assumptions + * in {@link IndexSegmentBuilder}, {@link IndexSegmentMerger}, and the + * code that handles {@link PartitionedJournal#overflow()}. + * * @todo where possible, it would be nice to leverage the unit tests for the * mutable btree. note that rangeCount does not have the same behavior for Index: TestPartitionedJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/scaleup/TestPartitionedJournal.java,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** TestPartitionedJournal.java 22 Feb 2007 16:59:35 -0000 1.4 --- TestPartitionedJournal.java 8 Mar 2007 18:14:06 -0000 1.5 *************** *** 54,64 **** import junit.framework.TestCase2; import com.bigdata.journal.Journal; import com.bigdata.objndx.AbstractBTreeTestCase; - import com.bigdata.objndx.BTree; import com.bigdata.objndx.BatchInsert; import com.bigdata.objndx.IIndex; import com.bigdata.objndx.KeyBuilder; - import com.bigdata.objndx.SimpleEntry; import com.bigdata.rawstore.SimpleMemoryRawStore; import com.bigdata.scaleup.PartitionedJournal.MergePolicy; --- 54,63 ---- import junit.framework.TestCase2; + import com.bigdata.isolation.UnisolatedBTree; import com.bigdata.journal.Journal; import com.bigdata.objndx.AbstractBTreeTestCase; import com.bigdata.objndx.BatchInsert; import com.bigdata.objndx.IIndex; import com.bigdata.objndx.KeyBuilder; import com.bigdata.rawstore.SimpleMemoryRawStore; import com.bigdata.scaleup.PartitionedJournal.MergePolicy; *************** *** 144,148 **** final String name = "abc"; ! IIndex index = new BTree(journal, 3, SimpleEntry.Serializer.INSTANCE); assertNull(journal.getIndex(name)); --- 143,147 ---- final String name = "abc"; ! IIndex index = new UnisolatedBTree(journal); assertNull(journal.getIndex(name)); *************** *** 160,164 **** final byte[] k0 = new byte[]{0}; ! final Object v0 = new SimpleEntry(0); index.insert( k0, v0); --- 159,163 ---- final byte[] k0 = new byte[]{0}; ! final byte[] v0 = new byte[]{0}; index.insert( k0, v0); *************** *** 182,186 **** assertNotNull("btree", index); assertEquals("entryCount", 1, ((PartitionedIndex)index).getBTree().getEntryCount()); ! assertEquals(v0, index.lookup(k0)); journal.dropIndex(name); --- 181,185 ---- assertNotNull("btree", index); assertEquals("entryCount", 1, ((PartitionedIndex)index).getBTree().getEntryCount()); ! assertEquals(v0, (byte[])index.lookup(k0)); journal.dropIndex(name); *************** *** 210,215 **** final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new BTree(journal, 3, ! SimpleEntry.Serializer.INSTANCE))); journal.overflow(); --- 209,213 ---- final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new UnisolatedBTree(journal))); journal.overflow(); *************** *** 245,250 **** final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new BTree(journal, 3, ! SimpleEntry.Serializer.INSTANCE))); final TestData data = new TestData(journal.migrationThreshold-1); --- 243,247 ---- final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new UnisolatedBTree(journal, 3))); final TestData data = new TestData(journal.migrationThreshold-1); *************** *** 296,301 **** final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new BTree(journal, 3, ! SimpleEntry.Serializer.INSTANCE))); final TestData data = new TestData(journal.migrationThreshold); --- 293,297 ---- final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new UnisolatedBTree(journal))); final TestData data = new TestData(journal.migrationThreshold); *************** *** 362,370 **** final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new BTree(journal, 3, ! SimpleEntry.Serializer.INSTANCE))); ! final BTree groundTruth = new BTree(new SimpleMemoryRawStore(), ! BTree.DEFAULT_BRANCHING_FACTOR, SimpleEntry.Serializer.INSTANCE); final int ntrials = 10; --- 358,365 ---- final String name = "abc"; ! assertNotNull(journal.registerIndex(name, new UnisolatedBTree(journal))); ! final UnisolatedBTree groundTruth = new UnisolatedBTree( ! new SimpleMemoryRawStore()); final int ntrials = 10; *************** *** 404,411 **** assertEquals("#segments",2,pmd.segs.length); ! assertEquals("state", IndexSegmentLifeCycleEnum.DEAD, pmd.segs[0].state); ! assertEquals("state", IndexSegmentLifeCycleEnum.LIVE, pmd.segs[1].state); --- 399,406 ---- assertEquals("#segments",2,pmd.segs.length); ! assertEquals("state", ResourceState.Dead, pmd.segs[0].state); ! assertEquals("state", ResourceState.Live, pmd.segs[1].state); *************** *** 461,465 **** public final byte[][] keys; ! public final SimpleEntry[] vals; /** --- 456,460 ---- public final byte[][] keys; ! public final byte[][] vals; /** *************** *** 477,481 **** keys = new byte[nrecords][]; ! vals = new SimpleEntry[nrecords]; int key = 0; --- 472,476 ---- keys = new byte[nrecords][]; ! vals = new byte[nrecords][]; int key = 0; *************** *** 490,495 **** keys[i] = keyBuilder.reset().append(key).getKey(); ! ! vals[i] = new SimpleEntry(key); } --- 485,490 ---- keys[i] = keyBuilder.reset().append(key).getKey(); ! ! vals[i] = keys[i].clone(); } *************** *** 508,512 **** for(int i=0; i<nrecords; i++) { ! SimpleEntry actual = (SimpleEntry)ndx.lookup(keys[i]); assertEquals(vals[i],actual); --- 503,507 ---- for(int i=0; i<nrecords; i++) { ! byte[] actual = (byte[])ndx.lookup(keys[i]); assertEquals(vals[i],actual); |
