Menu
▾
▴
cweb-cvs
Update of /cvsroot/cweb/bigdata/src/test/com/bigdata/journal In directory sc8-pr-cvs4.sourceforge.net:/tmp/cvs-serv5461/src/test/com/bigdata/journal Modified Files: TestDiskJournal.java AbstractRestartSafeTestCase.java TestMappedJournal.java TestTemporaryStore.java TestDirectJournal.java AbstractTestCase.java BenchmarkJournalWriteRate.java AbstractCommitRecordTestCase.java AbstractBufferStrategyTestCase.java TestTransientJournal.java TestTx.java StressTestConcurrent.java Added Files: AbstractMROWTestCase.java ComparisonTestDriver.java AbstractBTreeWithJournalTestCase.java Log Message: Further work supporting transactional isolation. --- NEW FILE: AbstractBTreeWithJournalTestCase.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Nov 17, 2006 */ package com.bigdata.journal; import java.util.Properties; import com.bigdata.objndx.AbstractBTreeTestCase; import com.bigdata.objndx.BTree; import com.bigdata.objndx.SimpleEntry; /** * Stress tests of the {@link BTree} writing on the {@link Journal}. This suite * simply contains stress tests of the btree operations at larger scale and * including incremental writes against the store. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ abstract public class AbstractBTreeWithJournalTestCase extends AbstractBTreeTestCase { public AbstractBTreeWithJournalTestCase() { } public AbstractBTreeWithJournalTestCase(String name) { super(name); } abstract public BufferMode getBufferMode(); public Properties getProperties() { if (properties == null) { properties = super.getProperties(); properties.setProperty(Options.BUFFER_MODE, getBufferMode().toString() ); } return properties; } private Properties properties; /** * Return a btree backed by a journal with the indicated branching factor. * The serializer requires that values in leaves are {@link SimpleEntry} * objects. * * @param branchingFactor * The branching factor. * * @return The btree. */ public BTree getBTree(int branchingFactor) { Properties properties = getProperties(); Journal journal = new Journal(properties); BTree btree = new BTree(journal, branchingFactor, SimpleEntry.Serializer.INSTANCE); return btree; } /* * @todo try large branching factors, but limit the total #of keys inserted * or the running time will be too long (I am using an expontential #of keys * by default). * * Note: For sequential keys, m=128 causes the journal to exceed its initial * extent. */ int[] branchingFactors = new int[]{3,4,5,10};//,20,64};//,128};//,512}; /** * A stress test for sequential key insertion that runs with a variety of * branching factors and #of keys to insert. */ public void test_splitRootLeaf_increasingKeySequence() { for(int i=0; i<branchingFactors.length; i++) { int m = branchingFactors[i]; doSplitWithIncreasingKeySequence( getBTree(m), m, m ); doSplitWithIncreasingKeySequence( getBTree(m), m, m*m ); doSplitWithIncreasingKeySequence( getBTree(m), m, m*m*m ); doSplitWithIncreasingKeySequence( getBTree(m), m, m*m*m*m ); } } /** * A stress test for sequential decreasing key insertions that runs with a * variety of branching factors and #of keys to insert. */ public void test_splitRootLeaf_decreasingKeySequence() { for(int i=0; i<branchingFactors.length; i++) { int m = branchingFactors[i]; doSplitWithDecreasingKeySequence( getBTree(m), m, m ); doSplitWithDecreasingKeySequence( getBTree(m), m, m*m ); doSplitWithDecreasingKeySequence( getBTree(m), m, m*m*m ); doSplitWithDecreasingKeySequence( getBTree(m), m, m*m*m*m ); } } /** * A stress test for random key insertion using a that runs with a variety * of branching factors and #of keys to insert. */ public void test_splitRootLeaf_randomKeySequence() { for(int i=0; i<branchingFactors.length; i++) { int m = branchingFactors[i]; doSplitWithRandomDenseKeySequence( getBTree(m), m, m ); doSplitWithRandomDenseKeySequence( getBTree(m), m, m*m ); doSplitWithRandomDenseKeySequence( getBTree(m), m, m*m*m ); // This case overflows the default journal extent. // doSplitWithRandomKeySequence( getBTree(m), m, m*m*m*m ); } } /** * Stress test inserts random permutations of keys into btrees of order m * for several different btrees, #of keys to be inserted, and permutations * of keys. */ public void test_stress_split() { for(int i=0; i<branchingFactors.length; i++) { int m = branchingFactors[i]; doSplitTest( m, 0 ); } } /** * Stress test of insert, removal and lookup of keys in the tree (allows * splitting of the root leaf). */ public void test_insertLookupRemoveKeyTreeStressTest() { int nkeys = 2000; int ntrials = 25000; for(int i=0; i<branchingFactors.length; i++) { int m = branchingFactors[i]; doInsertLookupRemoveStressTest(m, nkeys, ntrials); } } /** * Stress test for building up a tree and then removing all keys in a random * order. */ public void test_stress_removeStructure() { int nkeys = 5000; for(int i=0; i<branchingFactors.length; i++) { int m = branchingFactors[i]; doRemoveStructureStressTest(m,nkeys); } } } Index: TestDiskJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/TestDiskJournal.java,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** TestDiskJournal.java 8 Feb 2007 21:32:08 -0000 1.6 --- TestDiskJournal.java 21 Feb 2007 20:17:20 -0000 1.7 *************** *** 91,94 **** --- 91,100 ---- suite.addTestSuite( TestRawStore.class ); + // test suite for MROW correctness. + suite.addTestSuite( TestMROW.class ); + + // test suite for btree on the journal. + suite.addTestSuite( TestBTree.class ); + /* * Pickup the basic journal test suite. This is a proxied test suite, so *************** *** 133,136 **** --- 139,143 ---- assertTrue("isStable",bufferStrategy.isStable()); + assertFalse("isFullyBuffered",bufferStrategy.isFullyBuffered()); assertEquals(Options.FILE, properties.getProperty(Options.FILE), bufferStrategy.file.toString()); assertEquals(Options.INITIAL_EXTENT, Options.DEFAULT_INITIAL_EXTENT, *************** *** 175,177 **** --- 182,232 ---- } + /** + * Test suite integration for {@link AbstractMROWTestCase}. + * + * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ + */ + public static class TestMROW extends AbstractMROWTestCase { + + public TestMROW() { + super(); + } + + public TestMROW(String name) { + super(name); + } + + protected BufferMode getBufferMode() { + + return BufferMode.Disk; + + } + + } + + /** + * Test suite integration for {@link AbstractBTreeWithJournalTestCase}. + * + * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ + */ + public static class TestBTree extends AbstractBTreeWithJournalTestCase { + + public TestBTree() { + super(); + } + + public TestBTree(String name) { + super(name); + } + + public BufferMode getBufferMode() { + + return BufferMode.Disk; + + } + + } + } Index: AbstractCommitRecordTestCase.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/AbstractCommitRecordTestCase.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** AbstractCommitRecordTestCase.java 17 Feb 2007 21:34:13 -0000 1.1 --- AbstractCommitRecordTestCase.java 21 Feb 2007 20:17:20 -0000 1.2 *************** *** 94,98 **** final long timestamp = System.currentTimeMillis(); ! final int n = ICommitRecord.MAX_ROOT_ADDRS; --- 94,101 ---- final long timestamp = System.currentTimeMillis(); ! ! // using the clock for this as well so that it is an ascending value. ! final long commitCounter = System.currentTimeMillis(); ! final int n = ICommitRecord.MAX_ROOT_ADDRS; *************** *** 107,111 **** } ! return new CommitRecord(timestamp,roots); } --- 110,114 ---- } ! return new CommitRecord(timestamp,commitCounter,roots); } --- NEW FILE: AbstractMROWTestCase.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Feb 20, 2007 */ package com.bigdata.journal; import java.io.File; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import com.bigdata.rawstore.Addr; import com.bigdata.util.concurrent.DaemonThreadFactory; /** * Test suite for MROW (Multiple Readers, One Writer) support. * <p> * Supporting MROW is easy for a fully buffered implementation since it need * only use a read-only view for readers. If the implementation is not fully * buffered, e.g., {@link DiskOnlyStrategy}, then it needs to serialize reads * that are not buffered. The exception as always is the * {@link MappedBufferStrategy} - since this uses the nio * {@link MappedByteBuffer} it supports concurrent readers using the same * approach as a fully buffered strategy even though data may not always reside * in memory. * * @todo This test suite could also be used to tune AIO (asynchronous IO) * support for the {@link DirectBufferStrategy} and the * {@link DiskOnlyStrategy}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ abstract public class AbstractMROWTestCase extends AbstractBufferStrategyTestCase { /** * */ public AbstractMROWTestCase() { } /** * @param name */ public AbstractMROWTestCase(String name) { super(name); } /** * Correctness/stress test verifies that the implementation supports * Multiple Readers One Writer (MROW). */ public void test_mrow() throws Exception { IBufferStrategy store = ((Journal)getStore()).getBufferStrategy(); final long timeout = 5; final int nclients = 20; final int nwrites = 10000; final int writeDelayMillis = 1; final int ntrials = 10000; final int reclen = 128; final int nreads = 100; doMROWTest(store, nwrites, writeDelayMillis, timeout, nclients, ntrials, reclen, nreads); } /** * A correctness/stress/performance test with a pool of concurrent clients * designed to verify MROW operations. If the store passes these tests, then * {@link StressTestConcurrent} is designed to reveal concurrency problems * in the higher level data structures (transaction process and especially * the indices). * * @param store * The store. * * @param nwrites * The #of records to write. * * @param writeDelayMillis * The #of milliseconds delay between writes. * * @param timeout * The timeout (seconds). * * @param nclients * The #of concurrent clients. * * @param ntrials * The #of distinct client trials to execute. * * @param reclen * The length of the random byte[] records used in the * operations. * * @param nreads * The #of operations to be performed in each transaction. */ static public void doMROWTest(IBufferStrategy store, int nwrites, long writeDelayMillis, long timeout, int nclients, int ntrials, int reclen, int nreads) throws Exception { // A single-threaded writer. ExecutorService writerExecutor = Executors .newSingleThreadExecutor(DaemonThreadFactory .defaultThreadFactory()); WriterTask writerTask = new WriterTask(store, reclen, nwrites, writeDelayMillis); /* * Pre-write 25% of the records so that clients have something to * choose from when they start running. */ final int npreWrites = nwrites/4; for( int i=0; i<npreWrites; i++) { // write a single record. writerTask.write(); } System.err.println("Pre-wrote "+npreWrites+" records"); // start the writer. writerExecutor.submit(writerTask); // Concurrent readers. ExecutorService readerExecutor = Executors.newFixedThreadPool( nclients, DaemonThreadFactory.defaultThreadFactory()); // Setup readers queue. Collection<Callable<Long>> tasks = new HashSet<Callable<Long>>(); for(int i=0; i<ntrials; i++) { tasks.add(new ReaderTask(store, writerTask, nreads)); } /* * Run the M trials on N clients. */ final long begin = System.currentTimeMillis(); // start readers. List<Future<Long>> results = readerExecutor.invokeAll(tasks, timeout, TimeUnit.SECONDS); final long elapsed = System.currentTimeMillis() - begin; // force the writer to terminate. writerExecutor.shutdownNow(); // force the reads to terminate. readerExecutor.shutdownNow(); if(!writerExecutor.awaitTermination(1, TimeUnit.SECONDS)) { System.err.println("Writer did not terminate."); } if (!readerExecutor.awaitTermination(1, TimeUnit.SECONDS)) { /* * Note: if readers do not terminate within the timeout then an * IOException MAY be reported by disk-backed stores if the store is * closed while readers are still attempting to resolve records on * disk. */ System.err.println("Reader(s) did not terminate."); } // #of records actually written. final int nwritten = writerTask.nrecs; Iterator<Future<Long>> itr = results.iterator(); int nok = 0; // #of trials that successfully committed. int ncancelled = 0; // #of trials that did not complete in time. int nerr = 0; Throwable[] errors = new Throwable[ntrials]; while(itr.hasNext()) { Future<Long> future = itr.next(); if(future.isCancelled()) { ncancelled++; continue; } try { future.get(); // ignore the return (always zero). nok++; } catch(ExecutionException ex ) { System.err.println("Not expecting: "+ex); errors[nerr++] = ex.getCause(); } } System.err.println("mode=" + store.getBufferMode() + ", #clients=" + nclients + ", ntrials=" + ntrials + ", nok=" + nok + ", ncancelled=" + ncancelled + ", nerrors=" + nerr + " in " + elapsed + "ms (" + nok * 1000 / elapsed + " reads per second); nwritten=" + nwritten); } /** * A ground truth record as generated by a {@link WriterTask}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public static class Record { public final long addr; public final byte[] data; public Record(long addr, byte[] data) { assert addr != 0L; assert data != null; this.addr = addr; this.data = data; } }; /** * Run a writer. * <p> * The writer exposes state to the readers so that they can perform reads on * written records and so that the can validate those reads against ground * truth. */ public static class WriterTask implements Callable<Integer> { private final IBufferStrategy store; private final int reclen; private final int nwrites; private final long writeDelayMillis; /** * The #of records in {@link #records}. */ private volatile int nrecs = 0; /** * The ground truth data written so far. */ private volatile Record[] records; final Random r = new Random(); /** * Returns random data that will fit in N bytes. N is choosen randomly in * 1:<i>reclen</i>. * * @return A new {@link ByteBuffer} wrapping a new <code>byte[]</code> of * random length and having random contents. */ private ByteBuffer getRandomData() { final int nbytes = r.nextInt(reclen) + 1; byte[] bytes = new byte[nbytes]; r.nextBytes(bytes); return ByteBuffer.wrap(bytes); } public WriterTask(IBufferStrategy store, int reclen, int nwrites, long writeDelayMillis) { this.store = store; this.reclen = reclen; this.nwrites = nwrites; this.writeDelayMillis = writeDelayMillis; this.records = new Record[nwrites]; } /** * Return a randomly choosen ground truth record. */ public Record getRandomGroundTruthRecord() { int index = r.nextInt(nrecs); return records[ index ]; } /** * Writes any remaining records (starts from nrecs and runs to nwrites * so we can pre-write some records first). * * @return The #of records written. */ public Integer call() throws Exception { for (int i = nrecs; i < nwrites; i++) { write(); /* * Note: it is difficult to get this task to yield such that a * large #of records are written, but not before the readers * even get a chance to start executing. You may have to adjust * this by hand for different JVM/OS combinations! */ Thread.sleep(0, 1); // Thread.yield(); // Thread.sleep(writeDelayMillis,writeDelayNanos); // long begin = System.nanoTime(); // long elapsed = 0L; // while(elapsed<1000) { // // Thread.yield(); // elapsed = System.nanoTime() - begin; // // } } System.err.println("Writer done: nwritten="+nrecs); return nrecs; } /** * Write a random record and record it in {@link #records}. */ public void write() { ByteBuffer data = getRandomData(); final long addr = store.write(data); records[nrecs] = new Record(addr, data.array()); nrecs++; } } /** * Run a reader. */ public static class ReaderTask implements Callable<Long> { private final IBufferStrategy store; private final WriterTask writer; private final int nops; final Random r = new Random(); /** * * @param store * @param writer * @param nwrites #of reads to perform. */ public ReaderTask(IBufferStrategy store, WriterTask writer, int nops) { this.store = store; this.writer = writer; this.nops = nops; } /** * Executes random reads and validates against ground truth. */ public Long call() throws Exception { // Random reads. for (int i = 0; i < nops; i++) { // Thread.yield(); Record record = writer.getRandomGroundTruthRecord(); ByteBuffer buf; if (r.nextInt(100) > 30) { buf = store.read(record.addr); } else { buf = ByteBuffer.allocate(Addr.getByteCount(record.addr)); buf = store.read(record.addr); } assertEquals(record.data, buf); } return 0L; } } /** * Correctness/stress/performance test for MROW behavior. */ public static void main(String[] args) throws Exception { // timeout in seconds. final long timeout = 10; final int nclients = 20; final int nwrites = 10000; final int writeDelayMillis = 1; final int ntrials = 100000; final int reclen = 1024; final int nreads = 100; Properties properties = new Properties(); // properties.setProperty(Options.USE_DIRECT_BUFFERS,"false"); // properties.setProperty(Options.BUFFER_MODE, BufferMode.Transient.toString()); // properties.setProperty(Options.USE_DIRECT_BUFFERS,"true"); // properties.setProperty(Options.BUFFER_MODE, BufferMode.Transient.toString()); // properties.setProperty(Options.USE_DIRECT_BUFFERS,"false"); // properties.setProperty(Options.BUFFER_MODE, BufferMode.Direct.toString()); properties.setProperty(Options.USE_DIRECT_BUFFERS,"true"); properties.setProperty(Options.BUFFER_MODE, BufferMode.Direct.toString()); // properties.setProperty(Options.BUFFER_MODE, BufferMode.Mapped.toString()); // properties.setProperty(Options.BUFFER_MODE, BufferMode.Disk.toString()); properties.setProperty(Options.SEGMENT, "0"); File file = File.createTempFile("bigdata", ".jnl"); file.deleteOnExit(); if(!file.delete()) fail("Could not remove temp file before test"); properties.setProperty(Options.FILE, file.toString()); Journal journal = new Journal(properties); doMROWTest(journal.getBufferStrategy(), nwrites, writeDelayMillis, timeout, nclients, ntrials, reclen, nreads); journal.shutdown(); } } Index: AbstractBufferStrategyTestCase.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/AbstractBufferStrategyTestCase.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** AbstractBufferStrategyTestCase.java 15 Feb 2007 22:01:18 -0000 1.2 --- AbstractBufferStrategyTestCase.java 21 Feb 2007 20:17:20 -0000 1.3 *************** *** 59,65 **** /** * * @todo write tests for ! * {@link IBufferStrategy#transferTo(java.io.RandomAccessFile)}. This * code is currently getting "checked" by the {@link IndexSegmentBuilder}. * --- 59,67 ---- /** + * Base class for writing test cases for the different {@link IBufferStrategy} + * implementations. * * @todo write tests for ! * {@link IBufferStrategy#transferTo(java.io.RandomAccessFile)}. This * code is currently getting "checked" by the {@link IndexSegmentBuilder}. * *************** *** 271,275 **** assertEquals("userExtent",userExtent, bufferStrategy.getUserExtent()); ! assertEquals(b, bufferStrategy.read(addr, null)); /* --- 273,277 ---- assertEquals("userExtent",userExtent, bufferStrategy.getUserExtent()); ! assertEquals(b, bufferStrategy.read(addr)); /* *************** *** 296,303 **** // verify data written before we overflowed the buffer. ! assertEquals(b, bufferStrategy.read(addr, null)); // verify data written after we overflowed the buffer. ! assertEquals(b2, bufferStrategy.read(addr2, null)); store.close(); --- 298,305 ---- // verify data written before we overflowed the buffer. ! assertEquals(b, bufferStrategy.read(addr)); // verify data written after we overflowed the buffer. ! assertEquals(b2, bufferStrategy.read(addr2)); store.close(); --- NEW FILE: ComparisonTestDriver.java --- /** The Notice below must appear in each file of the Source Code of any copy you distribute of the Licensed Product. Contributors to any Modifications may add their own copyright notices to identify their own contributions. License: The contents of this file are subject to the CognitiveWeb Open Source License Version 1.1 (the License). You may not copy or use this file, in either source code or executable form, except in compliance with the License. You may obtain a copy of the License from http://www.CognitiveWeb.org/legal/license/ Software distributed under the License is distributed on an AS IS basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. Copyrights: Portions created by or assigned to CognitiveWeb are Copyright (c) 2003-2003 CognitiveWeb. All Rights Reserved. Contact information for CognitiveWeb is available at http://www.CognitiveWeb.org Portions Copyright (c) 2002-2003 Bryan Thompson. Acknowledgements: Special thanks to the developers of the Jabber Open Source License 1.0 (JOSL), from which this License was derived. This License contains terms that differ from JOSL. Special thanks to the CognitiveWeb Open Source Contributors for their suggestions and support of the Cognitive Web. Modifications: */ /* * Created on Feb 21, 2007 */ package com.bigdata.journal; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Properties; import com.bigdata.journal.StressTestConcurrent.TestOptions; import com.bigdata.rawstore.Bytes; /** * A harness for running comparison of different journal configurations. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public class ComparisonTestDriver { /** * Interface for tests that can be run by {@link ComparisonTestDriver}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ public static interface IComparisonTest { /** * Run a test. * * @param properties * The properties used to configure the test. * * @return The test result to report. */ public String doComparisonTest(Properties properties) throws Exception; } /** * A name-value pair used to override {@link Properties} for a {@link Condition}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ protected static class NV { public final String name; public final String value; public NV(String name,Object value) { this.name = name; this.value = value.toString(); } public int hashCode() { return name.hashCode(); } public boolean equals(NV o) { return name.equals(o.name) && value.equals(o.value); } } /** * An experimental condition. * * @todo record the results. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ protected static class Condition { public final String name; public final Properties properties; public String result; public Condition(String name,Properties properties) { this.name = name; this.properties = properties; } } /** * Return a {@link Properties} object that inherits defaults from * <i>properties</i> and sets/overrides properties identified in <i>entries</i>. * * @param properties * The inherited properties (this object is NOT modified). * @param entries * The overriden properties. * * @return A new {@link Properties}. */ protected static Condition getCondition(Properties properties, NV[] entries) throws IOException { properties = new Properties(properties); StringBuilder sb = new StringBuilder(); // sb.append("{"); for(int i=0; i<entries.length; i++) { if(i>0) sb.append("; "); // Note: Not a comma since CSV delimited. sb.append(entries[i].name+"="+entries[i].value); properties.setProperty(entries[i].name,entries[i].value); } // sb.append("}"); String name = sb.toString(); /* * Create a temporary file for the journal. Note that you must delete * the temporary file before starting the journal since it is empty and * just a placeholder for a unique filename. */ File file = File.createTempFile("bigdata", ".jnl"); file.deleteOnExit(); properties.setProperty(Options.FILE, file.toString()); return new Condition(name,properties); } static public List<Condition> getBasicConditions(Properties properties, NV[] params) throws Exception { properties = new Properties(properties); for(int i=0; i<params.length; i++) { properties.setProperty(params[i].name,params[i].value); } Condition[] conditions = new Condition[] { // getCondition(properties, new NV[] { // new NV(Options.BUFFER_MODE, BufferMode.Transient), // }), // // getCondition( // properties, // new NV[] { // // new NV(Options.BUFFER_MODE, // BufferMode.Transient), // // new NV(Options.USE_DIRECT_BUFFERS, Boolean.TRUE) // // }), // getCondition(properties, new NV[] { // new NV(Options.BUFFER_MODE, BufferMode.Direct), // }), // // getCondition( // properties, // new NV[] { // // new NV(Options.BUFFER_MODE, BufferMode.Direct), // // new NV(Options.USE_DIRECT_BUFFERS, Boolean.TRUE) // // }), // getCondition(properties, new NV[] { // new NV(Options.BUFFER_MODE, BufferMode.Direct), // new NV(Options.FORCE_ON_COMMIT, ForceEnum.No) // }), // getCondition(properties, new NV[] { // new NV(Options.BUFFER_MODE, BufferMode.Disk), // }), // getCondition(properties, new NV[] { // new NV(Options.BUFFER_MODE, BufferMode.Disk), // new NV(Options.FORCE_ON_COMMIT, ForceEnum.No) // }), // }; return Arrays.asList(conditions); } /** * Runs a comparison of various an {@link IComparisonTest} under various * conditions and writes out a summary of the reported results. * * @param args * The name of a class that implements IComparisonTest. * * @todo this is not really parameterized very well for the className since * some of the options are specific to the test class. * * @todo it would be nice to factor out the column names for the results. * Perhaps change from a String result to a NVPair<String,String> and * add something to {@link IComparisonTest} to declare the headings? * * @todo Optional name of a properties file to be read. the properties will * be used as the basis for all conditions. */ public static void main(String[] args) throws Exception { String className = args[0]; if(className==null) { className = StressTestConcurrent.class.getName(); } File outFile = new File(className+".comparison.csv"); if(outFile.exists()) throw new IOException("File exists: "+outFile.getAbsolutePath()); Properties properties = new Properties(); // force delete of the files on close of the journal under test. properties.setProperty(Options.DELETE_ON_CLOSE,"true"); properties.setProperty(Options.SEGMENT, "0"); // avoids journal overflow when running out to 60 seconds. properties.setProperty(Options.MAXIMUM_EXTENT, ""+Bytes.megabyte32*400); properties.setProperty(TestOptions.TIMEOUT,"30"); List<Condition>conditions = new ArrayList<Condition>(); conditions.addAll(getBasicConditions(properties, new NV[] { new NV( TestOptions.NCLIENTS, "1") })); conditions.addAll(getBasicConditions(properties, new NV[] { new NV( TestOptions.NCLIENTS, "2") })); conditions.addAll(getBasicConditions(properties, new NV[] { new NV( TestOptions.NCLIENTS, "10") })); conditions.addAll(getBasicConditions(properties, new NV[] { new NV( TestOptions.NCLIENTS, "20") })); conditions.addAll(getBasicConditions(properties, new NV[] { new NV( TestOptions.NCLIENTS, "100") })); conditions.addAll(getBasicConditions(properties, new NV[] { new NV( TestOptions.NCLIENTS, "200") })); // properties.setProperty(Options.BUFFER_MODE, // BufferMode.Mapped.toString()); final int nconditions = conditions.size(); { FileWriter writer = new FileWriter(outFile); System.err.println("Running comparison of " + nconditions + " conditions for " + className); Class cl = Class.forName(className); Iterator<Condition> itr = conditions.iterator(); while (itr.hasNext()) { Condition condition = itr.next(); File file = new File(condition.properties .getProperty(Options.FILE)); if (!file.delete()) throw new AssertionError( "Could not remove temp file before test"); IComparisonTest test = (IComparisonTest) cl.newInstance(); System.err.println("Running: "+ condition.name); try { condition.result = test .doComparisonTest(condition.properties); } catch (Exception ex) { condition.result = ex.getMessage(); } System.err.println(condition.result + ", " + condition.name); writer.write(condition.result + ", " + condition.name+"\n"); } writer.flush(); writer.close(); } { System.err.println("Result summary:"); Iterator<Condition> itr = conditions.iterator(); while (itr.hasNext()) { Condition condition = itr.next(); System.err.println(condition.result + ", " + condition.name); } } } } Index: TestTx.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/TestTx.java,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** TestTx.java 19 Feb 2007 19:00:18 -0000 1.17 --- TestTx.java 21 Feb 2007 20:17:20 -0000 1.18 *************** *** 421,425 **** */ final byte[] id0 = new byte[] { 1 }; ! final byte[] v0 = getRandomData(journal).array(); journal.getIndex(name,tx0).insert(id0, v0); assertEquals(v0, journal.getIndex(name,tx0).lookup(id0)); --- 421,425 ---- */ final byte[] id0 = new byte[] { 1 }; ! final byte[] v0 = getRandomData().array(); journal.getIndex(name,tx0).insert(id0, v0); assertEquals(v0, journal.getIndex(name,tx0).lookup(id0)); *************** *** 445,449 **** * Test write after delete (succeeds, returning null). */ ! final byte[] v1 = getRandomData(journal).array(); assertNull(journal.getIndex(name,tx0).insert(id0, v1)); --- 445,449 ---- * Test write after delete (succeeds, returning null). */ ! final byte[] v1 = getRandomData().array(); assertNull(journal.getIndex(name,tx0).insert(id0, v1)); *************** *** 512,516 **** */ final byte[] id0 = new byte[] { 1 }; ! final byte[] v0 = getRandomData(journal).array(); journal.getIndex(name,tx0).insert(id0, v0); assertEquals(v0, journal.getIndex(name,tx0).lookup(id0)); --- 512,516 ---- */ final byte[] id0 = new byte[] { 1 }; ! final byte[] v0 = getRandomData().array(); journal.getIndex(name,tx0).insert(id0, v0); assertEquals(v0, journal.getIndex(name,tx0).lookup(id0)); *************** *** 536,540 **** * Test write after delete (succeeds, returning null). */ ! final byte[] v1 = getRandomData(journal).array(); assertNull(journal.getIndex(name,tx0).insert(id0, v1)); --- 536,540 ---- * Test write after delete (succeeds, returning null). */ ! final byte[] v1 = getRandomData().array(); assertNull(journal.getIndex(name,tx0).insert(id0, v1)); *************** *** 621,625 **** */ final byte[] id0 = new byte[] { 1 }; ! final byte[] v0 = getRandomData(journal).array(); journal.getIndex(name,tx0).insert(id0, v0); assertEquals(v0, journal.getIndex(name,tx0).lookup(id0)); --- 621,625 ---- */ final byte[] id0 = new byte[] { 1 }; ! final byte[] v0 = getRandomData().array(); journal.getIndex(name,tx0).insert(id0, v0); assertEquals(v0, journal.getIndex(name,tx0).lookup(id0)); *************** *** 640,644 **** * write(id0,v1) in tx1. */ ! final byte[] v1 = getRandomData(journal).array(); assertNull(journal.getIndex(name,tx1).insert(id0, v1)); --- 640,644 ---- * write(id0,v1) in tx1. */ ! final byte[] v1 = getRandomData().array(); assertNull(journal.getIndex(name,tx1).insert(id0, v1)); *************** *** 707,711 **** // // Write a random data version for id 0. // final int id0 = 1; ! // final ByteBuffer expected_id0_v0 = getRandomData(journal); // journal.write( id0, expected_id0_v0); // assertEquals(expected_id0_v0.array(),journal.read( id0, null)); --- 707,711 ---- // // Write a random data version for id 0. // final int id0 = 1; ! // final ByteBuffer expected_id0_v0 = getRandomData(); // journal.write( id0, expected_id0_v0); // assertEquals(expected_id0_v0.array(),journal.read( id0, null)); *************** *** 734,738 **** // * NOT be visible to either transaction. // */ ! // final ByteBuffer expected_id0_v1 = getRandomData(journal); // journal.write( id0, expected_id0_v1); //// final ISlotAllocation slots_v1 = journal.objectIndex.getSlots(0); --- 734,738 ---- // * NOT be visible to either transaction. // */ ! // final ByteBuffer expected_id0_v1 = getRandomData(); // journal.write( id0, expected_id0_v1); //// final ISlotAllocation slots_v1 = journal.objectIndex.getSlots(0); *************** *** 768,772 **** // * show up either on the journal or in tx1. // */ ! // final ByteBuffer expected_tx1_id0_v0 = getRandomData(journal); // tx1.write(id0, expected_tx1_id0_v0); // assertDeleted(journal, id0); --- 768,772 ---- // * show up either on the journal or in tx1. // */ ! // final ByteBuffer expected_tx1_id0_v0 = getRandomData(); // tx1.write(id0, expected_tx1_id0_v0); // assertDeleted(journal, id0); *************** *** 780,784 **** // * show up either on the journal or in tx1. // */ ! // final ByteBuffer expected_tx0_id0_v0 = getRandomData(journal); // tx0.write(id0, expected_tx0_id0_v0); // assertDeleted(journal, id0); --- 780,784 ---- // * show up either on the journal or in tx1. // */ ! // final ByteBuffer expected_tx0_id0_v0 = getRandomData(); // tx0.write(id0, expected_tx0_id0_v0); // assertDeleted(journal, id0); *************** *** 789,793 **** // * Write a 2nd version on tx0 and reverify. // */ ! // final ByteBuffer expected_tx0_id0_v1 = getRandomData(journal); // tx0.write(id0, expected_tx0_id0_v1); // assertDeleted(journal, id0); --- 789,793 ---- // * Write a 2nd version on tx0 and reverify. // */ ! // final ByteBuffer expected_tx0_id0_v1 = getRandomData(); // tx0.write(id0, expected_tx0_id0_v1); // assertDeleted(journal, id0); *************** *** 798,802 **** // * Write a 2nd version on tx1 and reverify. // */ ! // final ByteBuffer expected_tx1_id0_v1 = getRandomData(journal); // tx1.write(id0, expected_tx1_id0_v1); // assertDeleted(journal, id0); --- 798,802 ---- // * Write a 2nd version on tx1 and reverify. // */ ! // final ByteBuffer expected_tx1_id0_v1 = getRandomData(); // tx1.write(id0, expected_tx1_id0_v1); // assertDeleted(journal, id0); *************** *** 815,819 **** // * Write a 3rd version on tx0 and reverify. // */ ! // final ByteBuffer expected_tx0_id0_v2 = getRandomData(journal); // tx0.write(id0, expected_tx0_id0_v2); // assertDeleted(journal, id0); --- 815,819 ---- // * Write a 3rd version on tx0 and reverify. // */ ! // final ByteBuffer expected_tx0_id0_v2 = getRandomData(); // tx0.write(id0, expected_tx0_id0_v2); // assertDeleted(journal, id0); *************** *** 876,890 **** // // // pre-existing version of id0. ! // final ByteBuffer expected_preExistingVersion = getRandomData(journal); // // // Two versions of id0 written during tx0. ! // final ByteBuffer expected0v0 = getRandomData(journal); ! // final ByteBuffer expected0v1 = getRandomData(journal); // // // Three versions of id1. // final int id1 = 2; ! // final ByteBuffer expected1v0 = getRandomData(journal); ! // final ByteBuffer expected1v1 = getRandomData(journal); ! // final ByteBuffer expected1v2 = getRandomData(journal); // // // Write pre-existing version of id0 onto the journal. --- 876,890 ---- // // // pre-existing version of id0. ! // final ByteBuffer expected_preExistingVersion = getRandomData(); // // // Two versions of id0 written during tx0. ! // final ByteBuffer expected0v0 = getRandomData(); ! // final ByteBuffer expected0v1 = getRandomData(); // // // Three versions of id1. // final int id1 = 2; ! // final ByteBuffer expected1v0 = getRandomData(); ! // final ByteBuffer expected1v1 = getRandomData(); ! // final ByteBuffer expected1v2 = getRandomData(); // // // Write pre-existing version of id0 onto the journal. *************** *** 1152,1156 **** final byte[] id1 = new byte[]{1}; ! final byte[] v0 = getRandomData(journal).array(); // write data version on tx1 --- 1152,1156 ---- final byte[] id1 = new byte[]{1}; ! final byte[] v0 = getRandomData().array(); // write data version on tx1 *************** *** 1230,1234 **** final byte[] id0 = new byte[]{1}; ! final byte[] v0 = getRandomData(journal).array(); // data version not visible in global scope. --- 1230,1234 ---- final byte[] id0 = new byte[]{1}; ! final byte[] v0 = getRandomData().array(); // data version not visible in global scope. *************** *** 1312,1317 **** // // // create random data for versions. ! // ByteBuffer expected_id0_v0 = getRandomData(journal); ! // ByteBuffer expected_id0_v1 = getRandomData(journal); // // // data version not visible in global scope. --- 1312,1317 ---- // // // create random data for versions. ! // ByteBuffer expected_id0_v0 = getRandomData(); ! // ByteBuffer expected_id0_v1 = getRandomData(); // // // data version not visible in global scope. Index: TestMappedJournal.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/TestMappedJournal.java,v retrieving revision 1.7 retrieving revision 1.8 diff -C2 -d -r1.7 -r1.8 *** TestMappedJournal.java 8 Feb 2007 21:32:09 -0000 1.7 --- TestMappedJournal.java 21 Feb 2007 20:17:20 -0000 1.8 *************** *** 92,95 **** --- 92,101 ---- suite.addTestSuite( TestRawStore.class ); + // test suite for MROW correctness. + suite.addTestSuite( TestMROW.class ); + + // test suite for BTree on the journal. + suite.addTestSuite( TestBTree.class ); + /* * Pickup the basic journal test suite. This is a proxied test suite, so *************** *** 134,137 **** --- 140,144 ---- assertTrue("isStable",bufferStrategy.isStable()); + assertFalse("isFullyBuffered",bufferStrategy.isFullyBuffered()); assertEquals(Options.FILE, properties.getProperty(Options.FILE), bufferStrategy.file.toString()); assertEquals(Options.INITIAL_EXTENT, Options.DEFAULT_INITIAL_EXTENT, *************** *** 181,183 **** --- 188,238 ---- } + /** + * Test suite integration for {@link AbstractMROWTestCase}. + * + * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ + */ + public static class TestMROW extends AbstractMROWTestCase { + + public TestMROW() { + super(); + } + + public TestMROW(String name) { + super(name); + } + + protected BufferMode getBufferMode() { + + return BufferMode.Mapped; + + } + + } + + /** + * Test suite integration for {@link AbstractBTreeWithJournalTestCase}. + * + * @author <a href="mailto:tho...@us...">Bryan Thompson</a> + * @version $Id$ + */ + public static class TestBTree extends AbstractBTreeWithJournalTestCase { + + public TestBTree() { + super(); + } + + public TestBTree(String name) { + super(name); + } + + public BufferMode getBufferMode() { + + return BufferMode.Mapped; + + } + + } + } Index: TestTemporaryStore.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/TestTemporaryStore.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** TestTemporaryStore.java 15 Feb 2007 20:59:21 -0000 1.1 --- TestTemporaryStore.java 21 Feb 2007 20:17:20 -0000 1.2 *************** *** 57,61 **** /** ! * Test suite for {@link TemporaryStore}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> --- 57,61 ---- /** ! * Test suite for {@link TemporaryRawStore}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> *************** *** 79,83 **** protected IRawStore getStore() { ! return new TemporaryStore(); } --- 79,83 ---- protected IRawStore getStore() { ! return new TemporaryRawStore(); } *************** *** 90,94 **** public void test_overflow() { ! TemporaryStore store = (TemporaryStore) getStore(); AbstractBufferStrategy bufferStrategy = (AbstractBufferStrategy) store --- 90,94 ---- public void test_overflow() { ! TemporaryRawStore store = (TemporaryRawStore) getStore(); AbstractBufferStrategy bufferStrategy = (AbstractBufferStrategy) store *************** *** 126,130 **** public void test_writeNoExtend() { ! TemporaryStore store = (TemporaryStore) getStore(); AbstractBufferStrategy bufferStrategy = (AbstractBufferStrategy) store --- 126,130 ---- public void test_writeNoExtend() { ! TemporaryRawStore store = (TemporaryRawStore) getStore(); AbstractBufferStrategy bufferStrategy = (AbstractBufferStrategy) store *************** *** 167,171 **** public void test_writeWithExtend() { ! TemporaryStore store = (TemporaryStore) getStore(); AbstractBufferStrategy bufferStrategy = (AbstractBufferStrategy) store --- 167,171 ---- public void test_writeWithExtend() { ! TemporaryRawStore store = (TemporaryRawStore) getStore(); AbstractBufferStrategy bufferStrategy = (AbstractBufferStrategy) store *************** *** 206,210 **** assertEquals("userExtent",userExtent, bufferStrategy.getUserExtent()); ! assertEquals(b, bufferStrategy.read(addr, null)); /* --- 206,210 ---- assertEquals("userExtent",userExtent, bufferStrategy.getUserExtent()); ! assertEquals(b, bufferStrategy.read(addr)); /* *************** *** 231,238 **** // verify data written before we overflowed the buffer. ! assertEquals(b, bufferStrategy.read(addr, null)); // verify data written after we overflowed the buffer. ! assertEquals(b2, bufferStrategy.read(addr2, null)); store.close(); --- 231,238 ---- // verify data written before we overflowed the buffer. ! assertEquals(b, bufferStrategy.read(addr)); // verify data written after we overflowed the buffer. ! assertEquals(b2, bufferStrategy.read(addr2)); store.close(); *************** *** 257,261 **** * correctness. */ ! TemporaryStore store = new TemporaryStore(Bytes.kilobyte*10, Bytes.kilobyte * 100, false); --- 257,261 ---- * correctness. */ ! TemporaryRawStore store = new TemporaryRawStore(Bytes.kilobyte*10, Bytes.kilobyte * 100, false); *************** *** 347,351 **** // verify the data. ! assertEquals(b, store.read(addr, null)); } --- 347,351 ---- // verify the data. ! assertEquals(b, store.read(addr)); } *************** *** 383,390 **** // verify data written before we overflowed the buffer. ! assertEquals(b, store.read(addr, null)); // verify data written after we overflowed the buffer. ! assertEquals(b2, store.read(addr2, null)); // the name of the on-disk file. --- 383,390 ---- // verify data written before we overflowed the buffer. ! assertEquals(b, store.read(addr)); // verify data written after we overflowed the buffer. ! assertEquals(b2, store.read(addr2)); // the name of the on-disk file. Index: AbstractTestCase.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/AbstractTestCase.java,v retrieving revision 1.17 retrieving revision 1.18 diff -C2 -d -r1.17 -r1.18 *** AbstractTestCase.java 17 Feb 2007 21:34:12 -0000 1.17 --- AbstractTestCase.java 21 Feb 2007 20:17:20 -0000 1.18 *************** *** 316,320 **** * Buffer. */ ! public void assertEquals(byte[] expected, ByteBuffer actual ) { if( expected == null ) throw new IllegalArgumentException(); --- 316,320 ---- * Buffer. */ ! public static void assertEquals(byte[] expected, ByteBuffer actual ) { if( expected == null ) throw new IllegalArgumentException(); *************** *** 322,325 **** --- 322,333 ---- if( actual == null ) fail("actual is null"); + if( actual.hasArray() && actual.arrayOffset() == 0 ) { + + assertEquals(expected,actual.array()); + + return; + + } + /* Create a read-only view on the buffer so that we do not mess with * its position, mark, or limit. *************** *** 349,353 **** * random length and having random contents. */ ! public ByteBuffer getRandomData(IJournal journal) { final int nbytes = r.nextInt(1024) + 1; --- 357,361 ---- * random length and having random contents. */ ! public ByteBuffer getRandomData() { final int nbytes = r.nextInt(1024) + 1; Index: AbstractRestartSafeTestCase.java =================================================================== RCS file: /cvsroot/cweb/bigdata/src/test/com/bigdata/journal/AbstractRestartSafeTestCase.java,v retrieving revision 1.3 retrieving revision 1.4 diff -C2 -d -r1.3 -r1.4 *** AbstractRestartSafeTestCase.java 9 Feb 2007 16:13:17 -0000 1.3 --- AbstractRestartSafeTestCase.java 21 Feb 2007 20:17:20 -0000 1.4 *************** *** 119,123 **** // read the data back. ! ByteBuffer actual = store.read(addr1,null); assertEquals(expected,actual); --- 119,123 ---- // read the data back. ! ByteBuffer actual = store.read(addr1); assertEquals(expected,actual); *************** *** 149,153 **** */ try { ! actual = store.read(addr1,null); fail("Expecting: "+IllegalArgumentException.class); }catch(IllegalArgumentException ex) {... [truncated message content] |
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X