Menu
▾
▴
[cweb-CVS] bigdata/src/test/com/bigdata/objndx TestReopen.java, NONE, 1.1 TestIndexSegmentWithBloomFilter.java, 1.9, 1.10 TestIndexSegmentBuilderWithLargeTrees.java, 1.11, 1.12 TestAll.java, 1.31, 1.32
|
From: Bryan T. <tho...@us...> - 2007-03-15 16:11:46
|
Update of /cvsroot/cweb/bigdata/src/test/com/bigdata/objndx In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv10595/src/test/com/bigdata/objndx Modified Files: TestIndexSegmentWithBloomFilter.java TestIndexSegmentBuilderWithLargeTrees.java TestAll.java Added Files: TestReopen.java Log Message: Refactoring to define service apis (data service, transaction manager service) and some approximate implementations of those services (not supporting service discovery, network protocol, or service robustness). Copied in the UML model so that it will actually get committed to CVS.... --- NEW FILE: TestReopen.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Nov 17, 2006 */ package com.bigdata.objndx; import com.bigdata.rawstore.IRawStore; import com.bigdata.rawstore.SimpleMemoryRawStore; /** * Unit tests for the close/reopen protocol designed to manage the resource * burden of indices without invalidating the index objects (indices opens can * be reopened as long as their backing store remains available). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public class TestReopen extends AbstractBTreeTestCase { /** * */ public TestReopen() { } /** * @param name */ public TestReopen(String name) { super(name); } /** * Test close on a new tree - should force the root to the store since a new * root is dirty (if empty). reopen should then reload the empty root and on * life goes. */ public void test_reopen01() { BTree btree = getBTree(3); assertTrue(btree.isOpen()); btree.close(); assertFalse(btree.isOpen()); try { btree.close(); fail("Expecting: " + IllegalStateException.class); } catch (IllegalStateException ex) { System.err.println("Ignoring expected exception: " + ex); } assertNotNull(btree.getRoot()); assertTrue(btree.isOpen()); } /** * Stress test comparison with ground truth btree when {@link BTree#close()} * is randomly invoked during mutation operations. */ public void test_reopen02() { IRawStore store = new SimpleMemoryRawStore(); /* * The btree under test. * * Note: the fixture factory is NOT used since this node evictions will * be forced when this tree is closed (node evictions are not permitted * by the default fixture factory). */ BTree btree = new BTree(store, 3, SimpleEntry.Serializer.INSTANCE); /* * The btree used to maintain ground truth. * * Note: the fixture factory is NOT used here since the stress test will * eventually overflow the hard reference queue and begin evicting nodes * and leaves onto the store. */ BTree groundTruth = new BTree(store, 3, SimpleEntry.Serializer.INSTANCE); final int limit = 10000; final int keylen = 6; for (int i = 0; i < limit; i++) { int n = r.nextInt(100); if (n < 5) { if(btree.isOpen()) btree.close(); } else if (n < 20) { byte[] key = new byte[keylen]; r.nextBytes(key); btree.remove(key); groundTruth.remove(key); } else { byte[] key = new byte[keylen]; r.nextBytes(key); SimpleEntry value = new SimpleEntry(i); btree.insert(key, value); groundTruth.insert(key, value); } } assertSameBTree(groundTruth, btree); } } Index: TestIndexSegmentBuilderWithLargeTrees.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/objndx/TestIndexSegmentBuilderWithLargeTrees.java,v retrieving revision 1.11 retrieving revision 1.12 diff -C2 -d -r1.11 -r1.12 *** TestIndexSegmentBuilderWithLargeTrees.java 8 Mar 2007 18:14:05 -0000 1.11 --- TestIndexSegmentBuilderWithLargeTrees.java 15 Mar 2007 16:11:09 -0000 1.12 *************** *** 52,56 **** import java.util.Properties; - import com.bigdata.cache.HardReferenceQueue; import com.bigdata.journal.BufferMode; import com.bigdata.journal.Journal; --- 52,55 ---- *************** *** 61,72 **** * and then compares the trees for the same total ordering. * - * @todo do a variant that only evicts index segments when the journal overflows - * and that merges segments as we go. While we can limit the size of the - * journal to provoke overflow, for this test will be unable to compare to - * the data in a single btree. Instead we have to compare against ground - * truth in either a generated or external dataset. This will also test - * the logic that directs queries and scans to the appropriate segment of - * a segmented index. - * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ --- 60,63 ---- *************** *** 210,214 **** doBuildIndexSegmentAndCompare( doSplitWithRandomDenseKeySequence( getBTree(m), m, m*m*m ) ); ! // @todo overflows the initial journal extent. // doBuildIndexSegmentAndCompare( doSplitWithRandomDenseKeySequence( getBTree(m), m, m*m*m*m ) ); --- 201,205 ---- doBuildIndexSegmentAndCompare( doSplitWithRandomDenseKeySequence( getBTree(m), m, m*m*m ) ); ! // Note: overflows the initial journal extent. // doBuildIndexSegmentAndCompare( doSplitWithRandomDenseKeySequence( getBTree(m), m, m*m*m*m ) ); *************** *** 239,243 **** doBuildIndexSegmentAndCompare( doInsertRandomSparseKeySequenceTest(m,m*m*m,trace) ); ! //@todo overflows the initial journal extent. // doBuildIndexSegmentAndCompare( doInsertRandomSparseKeySequenceTest(m,m*m*m*m,trace) ); --- 230,234 ---- doBuildIndexSegmentAndCompare( doInsertRandomSparseKeySequenceTest(m,m*m*m,trace) ); ! // Note: overflows the initial journal extent. // doBuildIndexSegmentAndCompare( doInsertRandomSparseKeySequenceTest(m,m*m*m*m,trace) ); *************** *** 277,293 **** + "), out(m=" + m + ")"); ! IndexSegmentBuilder builder = new IndexSegmentBuilder(outFile, ! tmpDir, btree, m, 0. ! /* ! * @todo pass in btree.nodeSer.valueSerializer ! */ ! ); /* ! * Verify can load the index file and that the metadata ! * associated with the index file is correct (we are only ! * checking those aspects that are easily defined by the test ! * case and not, for example, those aspects that depend on the ! * specifics of the length of serialized nodes or leaves). */ System.err.println("Opening index segment."); --- 268,279 ---- + "), out(m=" + m + ")"); ! new IndexSegmentBuilder(outFile, tmpDir, btree, m, 0.); /* ! * Verify can load the index file and that the metadata associated ! * with the index file is correct (we are only checking those ! * aspects that are easily defined by the test case and not, for ! * example, those aspects that depend on the specifics of the length ! * of serialized nodes or leaves). */ System.err.println("Opening index segment."); *************** *** 301,304 **** --- 287,302 ---- System.err.println("Closing index segment."); seg.close(); + + /* + * Note: close() is a reversable operation. This verifies that by + * immediately re-verifying the index segment. The index segment + * SHOULD be transparently re-opened for this operation. + */ + System.err.println("Re-verifying index segment."); + assertSameBTree(btree, seg); + + // Close again so that we can delete the backing file. + System.err.println("Re-closing index segment."); + seg.close(); if (!outFile.delete()) { Index: TestIndexSegmentWithBloomFilter.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/objndx/TestIndexSegmentWithBloomFilter.java,v retrieving revision 1.9 retrieving revision 1.10 diff -C2 -d -r1.9 -r1.10 *** TestIndexSegmentWithBloomFilter.java 8 Mar 2007 18:14:05 -0000 1.9 --- TestIndexSegmentWithBloomFilter.java 15 Mar 2007 16:11:09 -0000 1.10 *************** *** 58,62 **** import com.bigdata.journal.Journal; import com.bigdata.journal.Options; - import com.bigdata.rawstore.Bytes; /** --- 58,61 ---- *************** *** 284,291 **** System.err.println("Opening index segment w/ bloom filter."); final IndexSegment seg2 = new IndexSegmentFileStore(outFile2).load(); - // // setup reference queue. - // new HardReferenceQueue<PO>(new DefaultEvictionListener(), - // 100, 20), - // @todo btree.nodeSer.valueSerializer); /* --- 283,286 ---- *************** *** 379,386 **** System.err.println("Opening index segment w/o bloom filter."); final IndexSegment seg = new IndexSegmentFileStore(outFile).load(); - // // setup reference queue. - // new HardReferenceQueue<PO>(new DefaultEvictionListener(), - // 100, 20), - // @todo btree.nodeSer.valueSerializer); /* --- 374,377 ---- *************** *** 393,400 **** System.err.println("Opening index segment w/ bloom filter."); final IndexSegment seg2 = new IndexSegmentFileStore(outFile2).load(); - // // setup reference queue. - // new HardReferenceQueue<PO>(new DefaultEvictionListener(), - // 100, 20), - // @todo btree.nodeSer.valueSerializer); /* --- 384,387 ---- *************** *** 426,429 **** --- 413,417 ---- assertSameBTree(btree, seg); assertSameBTree(btree, seg2); + seg2.close(); // close seg w/ bloom filter and the verify with implicit reopen. assertSameBTree(seg, seg2); Index: TestAll.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/objndx/TestAll.java,v retrieving revision 1.31 retrieving revision 1.32 diff -C2 -d -r1.31 -r1.32 *** TestAll.java 8 Mar 2007 18:14:05 -0000 1.31 --- TestAll.java 15 Mar 2007 16:11:10 -0000 1.32 *************** *** 112,115 **** --- 112,117 ---- // verify that a store is restart-safe iff it commits. suite.addTestSuite( TestRestartSafe.class ); + // test the close/reopen protocol for releasing index buffers. + suite.addTestSuite( TestReopen.class ); /* *************** *** 132,136 **** /* ! * use of btree to support column store. * * @todo handle column names and timestamp as part of the key. --- 134,138 ---- /* ! * @todo use of btree to support column store (in another package) * * @todo handle column names and timestamp as part of the key. |
