From: <btm...@us...> - 2010-07-30 13:51:55
|
Revision: 3372 http://bigdata.svn.sourceforge.net/bigdata/?rev=3372&view=rev Author: btmurphy Date: 2010-07-30 13:51:43 +0000 (Fri, 30 Jul 2010) Log Message: ----------- merge -r:3358:HEAD(3370) ~/bigdata/trunk ~/bigdata/branches/dev-btm [trunk --> branch dev-btm] Modified Paths: -------------- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java branches/dev-btm/bigdata/src/test/com/bigdata/service/TestBasicIndexStuff.java branches/dev-btm/bigdata/src/test/com/bigdata/service/TestDistributedTransactionService.java branches/dev-btm/bigdata/src/test/com/bigdata/sparse/TestKeyEncodeDecode.java branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSidsWithoutInlining.java branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataConnectionTest.java branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java branches/dev-btm/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataStoreTest.java Removed Paths: ------------- branches/dev-btm/CVSROOT/ branches/dev-btm/bigdata-master-pom/ Property Changed: ---------------- branches/dev-btm/ branches/dev-btm/bigdata-jini/src/java/com/bigdata/attr/ branches/dev-btm/bigdata-jini/src/java/com/bigdata/disco/ branches/dev-btm/bigdata-jini/src/java/com/bigdata/util/config/ branches/dev-btm/bigdata-perf/ branches/dev-btm/bigdata-perf/lubm/lib/ branches/dev-btm/bigdata-perf/lubm/src/resources/ branches/dev-btm/bigdata-rdf/src/java/com/bigdata/rdf/util/ branches/dev-btm/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/ branches/dev-btm/dsi-utils/LEGAL/ branches/dev-btm/dsi-utils/lib/ branches/dev-btm/dsi-utils/src/ branches/dev-btm/dsi-utils/src/test/ branches/dev-btm/dsi-utils/src/test/it/ branches/dev-btm/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/ branches/dev-btm/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/ branches/dev-btm/osgi/ Property changes on: branches/dev-btm ___________________________________________________________________ Modified: svn:mergeinfo - /branches/BTREE_BUFFER_BRANCH:2004-2045 /branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH:2633-3304 /branches/bugfix-btm:2594-3237 /branches/fko:3150-3194 /trunk:2575-2594,2596-2877,2882-2903,2910-3358 + /branches/BTREE_BUFFER_BRANCH:2004-2045 /branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782 /branches/LEXICON_REFACTOR_BRANCH:2633-3304 /branches/bugfix-btm:2594-3237 /branches/fko:3150-3194 /trunk:2575-2594,2596-2877,2882-2903,2910-3370 Modified: branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java =================================================================== --- branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/java/com/bigdata/bfs/BlobOverflowHandler.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -6,8 +6,10 @@ import com.bigdata.btree.IOverflowHandler; import com.bigdata.btree.ITuple; +import com.bigdata.btree.keys.IKeyBuilder; import com.bigdata.btree.keys.KeyBuilder; import com.bigdata.io.DataOutputBuffer; +import com.bigdata.rawstore.Bytes; import com.bigdata.rawstore.IBlock; import com.bigdata.rawstore.IRawStore; @@ -33,7 +35,7 @@ } - DataOutputBuffer buf; + private transient DataOutputBuffer buf; public void close() { @@ -62,6 +64,8 @@ } + final IKeyBuilder keyBuilder = new KeyBuilder(Bytes.SIZEOF_LONG); + if (addr == 0L) { /* @@ -69,7 +73,7 @@ * their address. */ - return KeyBuilder.asSortKey(0L); + return keyBuilder.append(0L).getKey(); } @@ -143,7 +147,7 @@ } // the address of the block on the target store. - return KeyBuilder.asSortKey(addr2); + return keyBuilder.append(addr2).getKey(); } Modified: branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java =================================================================== --- branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -1081,17 +1081,25 @@ * Note: This method is thread-safe. * <p> * Note: Strings are Unicode safe for the default locale. See - * {@link Locale#getDefault()}. If you require a specific local or - * different locals at different times or for different indices then you - * MUST provision and apply your own {@link KeyBuilder}. + * {@link Locale#getDefault()}. If you require a specific local or different + * locals at different times or for different indices then you MUST + * provision and apply your own {@link KeyBuilder}. * * @param val * An application key. * - * @return The unsigned byte[] equivilent of that key. This will be - * <code>null</code> iff the <i>key</i> is <code>null</code>. - * If the <i>key</i> is a byte[], then the byte[] itself will be - * returned. + * @return The unsigned byte[] equivalent of that key. This will be + * <code>null</code> iff the <i>key</i> is <code>null</code>. If the + * <i>key</i> is a byte[], then the byte[] itself will be returned. + * + * @deprecated This method circumvents explicit configuration of the + * {@link KeyBuilder} and is used nearly exclusively by unit + * tests. While explicit configuration is not required for keys + * which do not include Unicode sort key components, this method + * also relies on a single global {@link KeyBuilder} instance + * protected by a lock. That lock is therefore a bottleneck. The + * correct practice is to use thread-local or per task + * {@link IKeyBuilder}s to avoid lock contention. */ @SuppressWarnings("unchecked") public static final byte[] asSortKey(Object val) { Modified: branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java =================================================================== --- branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -57,7 +57,8 @@ * iterator. When compared to the {@link HardReferenceGlobalLRURecycler}, this * implementation has approximately 10% higher throughput. * - * @version $Id$ + * @version $Id: HardReferenceGlobalLRU.java 2799 2010-05-11 21:04:43Z + * thompsonbry $ * @author <a href="mailto:tho...@us...">Bryan Thompson * </a> * @param <K> @@ -69,6 +70,8 @@ * {@link IDataRecordAccess} since we can not measure the bytesInMemory * for those objects and hence the LRU eviction policy will not account * for their memory footprint? + * + * @deprecated This implementation is not used. */ public class HardReferenceGlobalLRU<K, V> implements IHardReferenceGlobalLRU<K, V> { Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java =================================================================== --- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -73,6 +73,9 @@ * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ + * + * @todo The key is now 100% decodable. The package should be updated to take + * advantage of that. */ public class KeyDecoder { @@ -157,6 +160,22 @@ return a; } + + /** + * Return the schema name. + * + * @throws UnsupportedOperationException + * unless {@link SparseRowStore#schemaNameUnicodeClean} is + * <code>true</code>. + */ + public String getSchemaName() { + + if(!SparseRowStore.schemaNameUnicodeClean) + throw new UnsupportedOperationException(); + + return new String(getSchemaBytes()); + + } /** * The decoded {@link KeyType} for the primary key. @@ -485,10 +504,17 @@ */ public String toString() { - return "KeyDecoder{primaryKeyType=" + primaryKeyType + ",col=" + col - + ",timestamp=" + timestamp + ",key=" + BytesUtil.toString(key) + return "KeyDecoder{" + + (SparseRowStore.schemaNameUnicodeClean ? "schema=" + + getSchemaName() + "," : "")// + + "primaryKeyType="+ primaryKeyType// + + (SparseRowStore.primaryKeyUnicodeClean ? ",primaryKey=" + + getPrimaryKey() : "")// + + ",col=" + col // + + ",timestamp=" + timestamp // + + ",key=" + BytesUtil.toString(key) // + "}"; } - + } Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java =================================================================== --- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/Schema.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -128,14 +128,22 @@ if (schemaBytes == null) { - /* - * One time encoding of the schema name as a Unicode sort key. - */ - - schemaBytes = KeyBuilder.asSortKey(name); + if (SparseRowStore.schemaNameUnicodeClean) { + /* + * One time encoding of the schema name as UTF8. + */ + try { + schemaBytes = name.getBytes(SparseRowStore.UTF8); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } else { + /* + * One time encoding of the schema name as a Unicode sort key. + */ + schemaBytes = KeyBuilder.asSortKey(name); + } -// schemaBytes = KeyBuilder.newInstance().append(name).append("\0").getKey(); - } return schemaBytes; Modified: branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java =================================================================== --- branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -1048,11 +1048,35 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ public interface Options { /** + * The schema name was originally written using a Unicode sort key. + * However, the JDK can generate Unicode sort keys with embedded nuls + * which in turn will break the logic to detect the end of the schema + * name in the key. In order to accommodate this behavior, the schema + * name is now encoded as UTF8 which also has the advantage that we can + * decode the schema name. Standard prefix compression on the B+Tree + * should make up for the larger representation of the schema name in + * the B+Tree. + * <p> + * This change was introduced on 7/29/2010 in the trunk. When this + * property is <code>true</code> it breaks compatibility with earlier + * revisions of the {@link SparseRowStore}. This flag may be set to + * <code>false</code> for backward compatibility. + * + * @see #DEFAULT_SCHEMA_NAME_UNICODE_CLEAN + */ + String SCHEMA_NAME_UNICODE_CLEAN = Schema.class.getName() + + ".schemaName.unicodeClean"; + + /** + * @see https://sourceforge.net/apps/trac/bigdata/ticket/107 + */ + String DEFAULT_SCHEMA_NAME_UNICODE_CLEAN = "false"; + + /** * The primary key was originally written using a Unicode sort key. * However, the JDK generates Unicode sort keys with embedded nuls and * that broke the logic to detect the end of the Unicode primary keys. @@ -1083,6 +1107,17 @@ * This is a global option since it was always <code>false</code> for * historical stores. * + * @see Options#SCHEMA_NAME_UNICODE_CLEAN + */ + final static transient boolean schemaNameUnicodeClean = Boolean + .valueOf(System.getProperty( + SparseRowStore.Options.SCHEMA_NAME_UNICODE_CLEAN, + SparseRowStore.Options.DEFAULT_SCHEMA_NAME_UNICODE_CLEAN)); + + /** + * This is a global option since it was always <code>false</code> for + * historical stores. + * * @see Options#PRIMARY_KEY_UNICODE_CLEAN */ final static transient boolean primaryKeyUnicodeClean = Boolean Modified: branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java =================================================================== --- branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/test/com/bigdata/cache/TestAll.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -76,7 +76,8 @@ suite.addTestSuite(TestStoreAndAddressLRUCache.class); - suite.addTestSuite(TestHardReferenceGlobalLRU.class); + // Note: This implementation is not used. +// suite.addTestSuite(TestHardReferenceGlobalLRU.class); suite.addTestSuite(TestHardReferenceGlobalLRURecycler.class); Modified: branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java =================================================================== --- branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -971,190 +971,193 @@ } - /** - * Test verifies that an {@link ITx#UNISOLATED} task failure does not cause - * concurrent writers to abort. The test also verifies that the - * {@link Checkpoint} record for the named index is NOT updated since none - * of the tasks write anything on the index. - * - * @todo The assumptions for this test may have been invalidated by the - * recent (4/29) changes to the group commit and task commit protocol - * and this test might need to be reworked or rewritten. + /* + * @todo revisit this unit test. It's semantics appear to have aged. */ - public void test_writeService001() throws Exception { - - final Journal journal = new Journal(getProperties()); +// /** +// * Test verifies that an {@link ITx#UNISOLATED} task failure does not cause +// * concurrent writers to abort. The test also verifies that the +// * {@link Checkpoint} record for the named index is NOT updated since none +// * of the tasks write anything on the index. +// * +// * @todo The assumptions for this test may have been invalidated by the +// * recent (4/29) changes to the group commit and task commit protocol +// * and this test might need to be reworked or rewritten. +// */ +// public void test_writeService001() throws Exception { +// +// final Journal journal = new Journal(getProperties()); +// +// try { +// +// final String name = "test"; +// +// // Note: checkpoint for the newly registered index. +// final long checkpointAddr0; +// { +// +// journal.registerIndex(name,new IndexMetadata(name,UUID.randomUUID())); +// +// journal.commit(); +// +// checkpointAddr0 = journal.getIndex(name).getCheckpoint() +// .getCheckpointAddr(); +// +// } +// +// // the list of tasks to be run. +// final List<AbstractTask<Object>> tasks = new LinkedList<AbstractTask<Object>>(); +// +// // NOP +// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) { +// protected String getTaskName() { +// return "a"; +// } +// protected Object doTask() throws Exception { +// assertEquals(checkpointAddr0, ((BTree) getIndex(name)) +// .getCheckpoint().getCheckpointAddr()); +// return null; +// } +// }); +// +// // throws exception. +// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) { +// protected String getTaskName() { +// return "b"; +// } +// protected Object doTask() throws Exception { +// assertEquals(checkpointAddr0, ((BTree) getIndex(name)) +// .getCheckpoint().getCheckpointAddr()); +// throw new ForcedAbortException(); +// } +// }); +// +// // NOP +// tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) { +// protected String getTaskName() { +// return "c"; +// } +// protected Object doTask() throws Exception { +// assertEquals(checkpointAddr0, ((BTree) getIndex(name)) +// .getCheckpoint().getCheckpointAddr()); +// return null; +// } +// }); +// +// // the commit counter before we submit the tasks. +// final long commitCounter0 = journal.getRootBlockView() +// .getCommitCounter(); +// +// // the write service on which the tasks execute. +// final WriteExecutorService writeService = journal +// .getConcurrencyManager().getWriteService(); +// +// // the group commit count before we submit the tasks. +// final long groupCommitCount0 = writeService.getGroupCommitCount(); +// +// // the abort count before we submit the tasks. +// final long abortCount0 = writeService.getAbortCount(); +// +// // the #of failed tasks before we submit the tasks. +// final long failedTaskCount0 = writeService.getTaskFailedCount(); +// +// // the #of successfully tasks before we submit the tasks. +// final long successTaskCount0 = writeService.getTaskSuccessCount(); +// +// // the #of successfully committed tasks before we submit the tasks. +// final long committedTaskCount0 = writeService.getTaskCommittedCount(); +// +// // submit the tasks and await their completion. +// final List<Future<Object>> futures = journal.invokeAll( tasks ); +// +// /* +// * verify the #of commits on the journal is unchanged since nothing +// * is written by any of these tasks. +// * +// * The expectation is that the tasks that succeed make it into the +// * same commit group while the task that throws an exception does +// * not cause the commit group to be aborted. +// * +// * Note: The tasks will make it into the same commit group iff the +// * first task that completes is willing to wait for the others to +// * join the commit group. +// * +// * Note: The tasks have a dependency on the same resource so they +// * will be serialized (executed in a strict sequence). +// */ +// assertEquals("commitCounter", commitCounter0, journal +// .getRootBlockView().getCommitCounter()); +// +// // however, a group commit SHOULD have been performed. +// assertEquals("groupCommitCount", groupCommitCount0 + 1, writeService +// .getGroupCommitCount()); +// +// // NO aborts should have been performed. +// assertEquals("aboutCount", abortCount0, writeService.getAbortCount()); +// +// // ONE(1) tasks SHOULD have failed. +// assertEquals("failedTaskCount", failedTaskCount0 + 1, writeService. +// getTaskFailedCount()); +// +// // TWO(2) tasks SHOULD have succeeded. +// assertEquals("successTaskCount", successTaskCount0 + 2, writeService +// .getTaskSuccessCount()); +// +// // TWO(2) successfull tasks SHOULD have been committed. +// assertEquals("committedTaskCount", committedTaskCount0 + 2, writeService +// .getTaskCommittedCount()); +// +// assertEquals( 3, futures.size()); +// +// // tasks[0] +// { +// +// Future f = futures.get(0); +// +// assertTrue(f.isDone()); +// +// f.get(); // No exception expected. +// +// } +// +// // tasks[2] +// { +// +// Future f = futures.get(2); +// +// assertTrue(f.isDone()); +// +// f.get(); // No exception expected. +// +// } +// +// // tasks[1] +// { +// +// Future f = futures.get(1); +// +// assertTrue(f.isDone()); +// +// try { +// f.get(); +// fail("Expecting exception"); +// } catch(ExecutionException ex) { +// assertTrue(InnerCause.isInnerCause(ex, ForcedAbortException.class)); +// } +// +// } +// +// assertEquals(checkpointAddr0, journal.getIndex(name) +// .getCheckpoint().getCheckpointAddr()); +// +// } finally { +// +// journal.destroy(); +// +// } +// +// } - try { - - final String name = "test"; - - // Note: checkpoint for the newly registered index. - final long checkpointAddr0; - { - - journal.registerIndex(name,new IndexMetadata(name,UUID.randomUUID())); - - journal.commit(); - - checkpointAddr0 = journal.getIndex(name).getCheckpoint() - .getCheckpointAddr(); - - } - - // the list of tasks to be run. - final List<AbstractTask<Object>> tasks = new LinkedList<AbstractTask<Object>>(); - - // NOP - tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) { - protected String getTaskName() { - return "a"; - } - protected Object doTask() throws Exception { - assertEquals(checkpointAddr0, ((BTree) getIndex(name)) - .getCheckpoint().getCheckpointAddr()); - return null; - } - }); - - // throws exception. - tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) { - protected String getTaskName() { - return "b"; - } - protected Object doTask() throws Exception { - assertEquals(checkpointAddr0, ((BTree) getIndex(name)) - .getCheckpoint().getCheckpointAddr()); - throw new ForcedAbortException(); - } - }); - - // NOP - tasks.add(new AbstractTask(journal, ITx.UNISOLATED, name) { - protected String getTaskName() { - return "c"; - } - protected Object doTask() throws Exception { - assertEquals(checkpointAddr0, ((BTree) getIndex(name)) - .getCheckpoint().getCheckpointAddr()); - return null; - } - }); - - // the commit counter before we submit the tasks. - final long commitCounter0 = journal.getRootBlockView() - .getCommitCounter(); - - // the write service on which the tasks execute. - final WriteExecutorService writeService = journal - .getConcurrencyManager().getWriteService(); - - // the group commit count before we submit the tasks. - final long groupCommitCount0 = writeService.getGroupCommitCount(); - - // the abort count before we submit the tasks. - final long abortCount0 = writeService.getAbortCount(); - - // the #of failed tasks before we submit the tasks. - final long failedTaskCount0 = writeService.getTaskFailedCount(); - - // the #of successfully tasks before we submit the tasks. - final long successTaskCount0 = writeService.getTaskSuccessCount(); - - // the #of successfully committed tasks before we submit the tasks. - final long committedTaskCount0 = writeService.getTaskCommittedCount(); - - // submit the tasks and await their completion. - final List<Future<Object>> futures = journal.invokeAll( tasks ); - - /* - * verify the #of commits on the journal is unchanged since nothing - * is written by any of these tasks. - * - * The expectation is that the tasks that succeed make it into the - * same commit group while the task that throws an exception does - * not cause the commit group to be aborted. - * - * Note: The tasks will make it into the same commit group iff the - * first task that completes is willing to wait for the others to - * join the commit group. - * - * Note: The tasks have a dependency on the same resource so they - * will be serialized (executed in a strict sequence). - */ - assertEquals("commitCounter", commitCounter0, journal - .getRootBlockView().getCommitCounter()); - - // however, a group commit SHOULD have been performed. - assertEquals("groupCommitCount", groupCommitCount0 + 1, writeService - .getGroupCommitCount()); - - // NO aborts should have been performed. - assertEquals("aboutCount", abortCount0, writeService.getAbortCount()); - - // ONE(1) tasks SHOULD have failed. - assertEquals("failedTaskCount", failedTaskCount0 + 1, writeService. - getTaskFailedCount()); - - // TWO(2) tasks SHOULD have succeeded. - assertEquals("successTaskCount", successTaskCount0 + 2, writeService - .getTaskSuccessCount()); - - // TWO(2) successfull tasks SHOULD have been committed. - assertEquals("committedTaskCount", committedTaskCount0 + 2, writeService - .getTaskCommittedCount()); - - assertEquals( 3, futures.size()); - - // tasks[0] - { - - Future f = futures.get(0); - - assertTrue(f.isDone()); - - f.get(); // No exception expected. - - } - - // tasks[2] - { - - Future f = futures.get(2); - - assertTrue(f.isDone()); - - f.get(); // No exception expected. - - } - - // tasks[1] - { - - Future f = futures.get(1); - - assertTrue(f.isDone()); - - try { - f.get(); - fail("Expecting exception"); - } catch(ExecutionException ex) { - assertTrue(InnerCause.isInnerCause(ex, ForcedAbortException.class)); - } - - } - - assertEquals(checkpointAddr0, journal.getIndex(name) - .getCheckpoint().getCheckpointAddr()); - - } finally { - - journal.destroy(); - - } - - } - /** * Test verifies that a write on an index will cause the index to be * checkpointed when the task completes. @@ -1206,262 +1209,265 @@ } } - - /** - * Test verifies that a task failure causes accessed indices to be rolled - * back to their last checkpoint. - * - * FIXME write test where a task registers an index and then throws an - * exception. This will cause the index to have a checkpoint record that - * does not agree with {@link Name2Addr} for the last commit point. Verify - * that the index is not in fact available to another task that is executed - * after the failed task (it will be if we merely close the index and then - * re-open it since it will reopen from the last checkpoint NOT from the - * last commit point). - * - * FIXME write test where a tasks (a), (b) and (c) are submitted with - * invokeAll() in that order and require a lock on the same index. Task (a) - * writes on an existing index and completes normally. The index SHOULD be - * checkpointed and task (b) SHOULD be able to read the data written in task - * (a) and SHOULD be run in the same commit group. Task (b) then throws an - * exception. Verify that the index is rolledback to the checkpoint for (a) - * (vs the last commit point) using task (c) which will read on the same - * index looking for the correct checkpoint record and data in the index. - * This test will fail if (b) is not reading from the checkpoint written by - * (a) or if (c) reads from the last commit point rather than the checkpoint - * written by (a). - * - * FIXME write tests to verify that an {@link #abort()} causes all running - * tasks to be interrupted and have their write sets discarded (should it? - * Should an abort just be an shutdownNow() in response to some truely nasty - * problem?) + + /* + * @todo revisit this unit test. It's semantics appear to have aged. */ - public void test_writeService002()throws Exception { - - final Properties properties = new Properties(getProperties()); - - /* - * Note: restricting the thread pool size does not give us the control - * that we need because it results in each task running as its own - * commit group. - */ +// /** +// * Test verifies that a task failure causes accessed indices to be rolled +// * back to their last checkpoint. +// * +// * FIXME write test where a task registers an index and then throws an +// * exception. This will cause the index to have a checkpoint record that +// * does not agree with {@link Name2Addr} for the last commit point. Verify +// * that the index is not in fact available to another task that is executed +// * after the failed task (it will be if we merely close the index and then +// * re-open it since it will reopen from the last checkpoint NOT from the +// * last commit point). +// * +// * FIXME write test where a tasks (a), (b) and (c) are submitted with +// * invokeAll() in that order and require a lock on the same index. Task (a) +// * writes on an existing index and completes normally. The index SHOULD be +// * checkpointed and task (b) SHOULD be able to read the data written in task +// * (a) and SHOULD be run in the same commit group. Task (b) then throws an +// * exception. Verify that the index is rolledback to the checkpoint for (a) +// * (vs the last commit point) using task (c) which will read on the same +// * index looking for the correct checkpoint record and data in the index. +// * This test will fail if (b) is not reading from the checkpoint written by +// * (a) or if (c) reads from the last commit point rather than the checkpoint +// * written by (a). +// * +// * FIXME write tests to verify that an {@link #abort()} causes all running +// * tasks to be interrupted and have their write sets discarded (should it? +// * Should an abort just be an shutdownNow() in response to some truely nasty +// * problem?) +// */ +// public void test_writeService002()throws Exception { +// +// final Properties properties = new Properties(getProperties()); +// // /* -// * Note: Force the write service to be single threaded so that we can -// * control the order in which the tasks start by the order in which they -// * are submitted. +// * Note: restricting the thread pool size does not give us the control +// * that we need because it results in each task running as its own +// * commit group. // */ -// properties.setProperty(Options.WRITE_SERVICE_CORE_POOL_SIZE,"1"); -// properties.setProperty(Options.WRITE_SERVICE_MAXIMUM_POOL_SIZE,"1"); - - final Journal journal = new Journal(properties); - - try { - - final String name = "test"; - - // Note: checkpoint for the newly registered index. - final long checkpointAddr0; - { - - // register - journal.registerIndex(name); - - // commit. - journal.commit(); - - // note checkpoint for index. - checkpointAddr0 = journal.getIndex(name).getCheckpoint() - .getCheckpointAddr(); - - } - - // Note: commit counter before we invoke the tasks. - final long commitCounter = journal.getRootBlockView() - .getCommitCounter(); - - final WriteExecutorService writeService = journal - .getConcurrencyManager().getWriteService(); - - // Note: group commit counter before we invoke the tasks. - final long groupCommitCount0 = writeService.getGroupCommitCount(); - - // Note: #of failed tasks before we submit the tasks. - final long failedTaskCount0 = writeService.getTaskFailedCount(); - final long successTaskCount0 = writeService.getTaskSuccessCount(); - final long committedTaskCount0 = writeService.getTaskCommittedCount(); - - // Note: set by one of the tasks below. - final AtomicLong checkpointAddr2 = new AtomicLong(0L); - - final AtomicReference<Future<? extends Object>> futureB = new AtomicReference<Future<? extends Object>>(); - final AtomicReference<Future<? extends Object>> futureC = new AtomicReference<Future<? extends Object>>(); - final AtomicReference<Future<? extends Object>> futureD = new AtomicReference<Future<? extends Object>>(); - - /* - * Note: the setup for this test is a PITA. In order to exert full - * control over the order in which the tasks begin to execute we - * need to have each task submit the next itself. This is because it - * is possible for any of these tasks to be the first one to grab - * the exclusive lock on the necessary resource [name]. We can't - * solve this problem by restricting the #of threads that can run - * the tasks since that limits the size of the commit group. So we - * are stuck imposing serial execution using the behavior of the - * tasks themselves. - * - * Create the task objects in the reverse order of their execution. - */ - - // task (d) verifies expected rollback checkpoint was restored. - final AbstractTask d = new AbstractTask(journal,ITx.UNISOLATED,name){ - protected String getTaskName() {return "d";} - protected Object doTask() throws Exception { - // commit counter unchanged. - assertEquals("commitCounter", commitCounter, getJournal() - .getRootBlockView().getCommitCounter()); - if(checkpointAddr2.get()==0L) { - fail("checkpointAddr2 was not set"); - } - // lookup index. - BTree ndx = (BTree)getIndex(name); - final long newCheckpointAddr =ndx.getCheckpoint().getCheckpointAddr(); - // verify checkpoint != last committed checkpoint. - assertNotSame(checkpointAddr0,newCheckpointAddr); - // verify checkpoint == last rollback checkpoint. - assertEquals(checkpointAddr2.get(),newCheckpointAddr); - return null; - } - }; - - /* - * task (c) notes the last checkpoint, writes on the index, and then - * fails. This is designed to trigger rollback of the index to the - * last checkpoint, which is the checkpoint that we note at the - * start of this task. - */ - final AbstractTask c = new AbstractTask(journal,ITx.UNISOLATED,name){ - protected String getTaskName() {return "c";} - protected Object doTask() throws Exception { - // commit counter unchanged. - assertEquals("commitCounter", commitCounter, getJournal() - .getRootBlockView().getCommitCounter()); - // lookup index. - BTree ndx = (BTree)getIndex(name); - // note the last checkpoint written. - final long newCheckpointAddr = ndx.getCheckpoint().getCheckpointAddr(); - assertNotSame(0L,newCheckpointAddr); - assertNotSame(checkpointAddr0,newCheckpointAddr); - // make note of the checkpoint before we force an abort. - assertTrue("checkpointAddr2 already set?",checkpointAddr2.compareAndSet(0L, newCheckpointAddr)); - // write another record on the index. - ndx.insert(new byte[]{3}, new byte[]{3}); - // run task (d) next. - assertTrue(futureD.compareAndSet(null,journal.submit(d))); - // force task to about with dirty index. - throw new ForcedAbortException(); - } - }; - - // task (b) writes another record on the index. - final AbstractTask b = new AbstractTask(journal,ITx.UNISOLATED,name){ - protected String getTaskName() {return "b";} - protected Object doTask() throws Exception { - // commit counter unchanged. - assertEquals("commitCounter", commitCounter, getJournal() - .getRootBlockView().getCommitCounter()); - // lookup index. - BTree ndx = (BTree)getIndex(name); - // verify checkpoint was updated. - assertNotSame(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr()); - // write another record on the index. - ndx.insert(new byte[]{2}, new byte[]{2}); - // run task (c) next. - assertTrue(futureC.compareAndSet(null,journal.submit(c))); - return null; - } - }; - - // task (a) writes on index. - final AbstractTask a = new AbstractTask(journal,ITx.UNISOLATED,name){ - protected String getTaskName() {return "a";} - protected Object doTask() throws Exception { - // commit counter unchanged. - assertEquals("commitCounter", commitCounter, getJournal() - .getRootBlockView().getCommitCounter()); - // group commit counter unchanged. - assertEquals("groupCommitCounter", groupCommitCount0, - writeService.getGroupCommitCount()); - // lookup index. - BTree ndx = (BTree)getIndex(name); - // verify same checkpoint. - assertEquals(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr()); - // write record on the index. - ndx.insert(new byte[]{1}, new byte[]{1}); - // run task (b) next. - assertTrue(futureB.compareAndSet(null,journal.submit(b))); - return null; - } - }; - -// final List<AbstractTask> tasks = Arrays.asList(new AbstractTask[] { -// a,b,c,d -// }); +//// /* +//// * Note: Force the write service to be single threaded so that we can +//// * control the order in which the tasks start by the order in which they +//// * are submitted. +//// */ +//// properties.setProperty(Options.WRITE_SERVICE_CORE_POOL_SIZE,"1"); +//// properties.setProperty(Options.WRITE_SERVICE_MAXIMUM_POOL_SIZE,"1"); +// +// final Journal journal = new Journal(properties); +// +// try { +// +// final String name = "test"; +// +// // Note: checkpoint for the newly registered index. +// final long checkpointAddr0; +// { +// +// // register +// journal.registerIndex(name); +// +// // commit. +// journal.commit(); +// +// // note checkpoint for index. +// checkpointAddr0 = journal.getIndex(name).getCheckpoint() +// .getCheckpointAddr(); +// +// } +// +// // Note: commit counter before we invoke the tasks. +// final long commitCounter = journal.getRootBlockView() +// .getCommitCounter(); +// +// final WriteExecutorService writeService = journal +// .getConcurrencyManager().getWriteService(); +// +// // Note: group commit counter before we invoke the tasks. +// final long groupCommitCount0 = writeService.getGroupCommitCount(); +// +// // Note: #of failed tasks before we submit the tasks. +// final long failedTaskCount0 = writeService.getTaskFailedCount(); +// final long successTaskCount0 = writeService.getTaskSuccessCount(); +// final long committedTaskCount0 = writeService.getTaskCommittedCount(); +// +// // Note: set by one of the tasks below. +// final AtomicLong checkpointAddr2 = new AtomicLong(0L); +// +// final AtomicReference<Future<? extends Object>> futureB = new AtomicReference<Future<? extends Object>>(); +// final AtomicReference<Future<? extends Object>> futureC = new AtomicReference<Future<? extends Object>>(); +// final AtomicReference<Future<? extends Object>> futureD = new AtomicReference<Future<? extends Object>>(); // -// final List<Future<Object>> futures = journal.invokeAll( tasks ); - - final Future<? extends Object> futureA = journal.submit( a ); - - /* - * wait for (a). if all tasks are in the same commit group then all - * tasks will be done once we have the future for (a). - */ - futureA.get(); // task (a) - - /* - * The expectation is that the tasks that succeed make it into the - * same commit group while the task that throws an exception does - * not cause the commit group to be aborted. Therefore there should - * be ONE (1) commit more than when we submitted the tasks. - * - * Note: The tasks will make it into the same commit group iff the - * first task that completes is willing to wait for the others to - * join the commit group. - * - * Note: The tasks have a dependency on the same resource so they - * will be serialized (executed in a strict sequence). - */ - assertEquals("failedTaskCount", failedTaskCount0 + 1, - writeService.getTaskFailedCount()); - assertEquals("successTaskCount", successTaskCount0 + 3, - writeService.getTaskSuccessCount()); - assertEquals("committedTaskCount", committedTaskCount0 + 3, - writeService.getTaskCommittedCount()); - assertEquals("groupCommitCount", groupCommitCount0 + 1, - writeService.getGroupCommitCount()); - assertEquals("commitCounter", commitCounter + 1, journal - .getRootBlockView().getCommitCounter()); - -// assertEquals( 4, futures.size()); - - futureB.get().get(); // task (b) - { - // task (c) did the abort. - Future f = futureC.get(); - try {f.get(); fail("Expecting exception");} - catch(ExecutionException ex) { - if(!InnerCause.isInnerCause(ex, ForcedAbortException.class)) { - fail("Expecting "+ForcedAbortException.class+", not "+ex, ex); - } - } - } - futureD.get().get(); // task (d) - - } finally { - - journal.destroy(); - - } - - } +// /* +// * Note: the setup for this test is a PITA. In order to exert full +// * control over the order in which the tasks begin to execute we +// * need to have each task submit the next itself. This is because it +// * is possible for any of these tasks to be the first one to grab +// * the exclusive lock on the necessary resource [name]. We can't +// * solve this problem by restricting the #of threads that can run +// * the tasks since that limits the size of the commit group. So we +// * are stuck imposing serial execution using the behavior of the +// * tasks themselves. +// * +// * Create the task objects in the reverse order of their execution. +// */ +// +// // task (d) verifies expected rollback checkpoint was restored. +// final AbstractTask d = new AbstractTask(journal,ITx.UNISOLATED,name){ +// protected String getTaskName() {return "d";} +// protected Object doTask() throws Exception { +// // commit counter unchanged. +// assertEquals("commitCounter", commitCounter, getJournal() +// .getRootBlockView().getCommitCounter()); +// if(checkpointAddr2.get()==0L) { +// fail("checkpointAddr2 was not set"); +// } +// // lookup index. +// BTree ndx = (BTree)getIndex(name); +// final long newCheckpointAddr =ndx.getCheckpoint().getCheckpointAddr(); +// // verify checkpoint != last committed checkpoint. +// assertNotSame(checkpointAddr0,newCheckpointAddr); +// // verify checkpoint == last rollback checkpoint. +// assertEquals(checkpointAddr2.get(),newCheckpointAddr); +// return null; +// } +// }; +// +// /* +// * task (c) notes the last checkpoint, writes on the index, and then +// * fails. This is designed to trigger rollback of the index to the +// * last checkpoint, which is the checkpoint that we note at the +// * start of this task. +// */ +// final AbstractTask c = new AbstractTask(journal,ITx.UNISOLATED,name){ +// protected String getTaskName() {return "c";} +// protected Object doTask() throws Exception { +// // commit counter unchanged. +// assertEquals("commitCounter", commitCounter, getJournal() +// .getRootBlockView().getCommitCounter()); +// // lookup index. +// BTree ndx = (BTree)getIndex(name); +// // note the last checkpoint written. +// final long newCheckpointAddr = ndx.getCheckpoint().getCheckpointAddr(); +// assertNotSame(0L,newCheckpointAddr); +// assertNotSame(checkpointAddr0,newCheckpointAddr); +// // make note of the checkpoint before we force an abort. +// assertTrue("checkpointAddr2 already set?",checkpointAddr2.compareAndSet(0L, newCheckpointAddr)); +// // write another record on the index. +// ndx.insert(new byte[]{3}, new byte[]{3}); +// // run task (d) next. +// assertTrue(futureD.compareAndSet(null,journal.submit(d))); +// // force task to about with dirty index. +// throw new ForcedAbortException(); +// } +// }; +// +// // task (b) writes another record on the index. +// final AbstractTask b = new AbstractTask(journal,ITx.UNISOLATED,name){ +// protected String getTaskName() {return "b";} +// protected Object doTask() throws Exception { +// // commit counter unchanged. +// assertEquals("commitCounter", commitCounter, getJournal() +// .getRootBlockView().getCommitCounter()); +// // lookup index. +// BTree ndx = (BTree)getIndex(name); +// // verify checkpoint was updated. +// assertNotSame(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr()); +// // write another record on the index. +// ndx.insert(new byte[]{2}, new byte[]{2}); +// // run task (c) next. +// assertTrue(futureC.compareAndSet(null,journal.submit(c))); +// return null; +// } +// }; +// +// // task (a) writes on index. +// final AbstractTask a = new AbstractTask(journal,ITx.UNISOLATED,name){ +// protected String getTaskName() {return "a";} +// protected Object doTask() throws Exception { +// // commit counter unchanged. +// assertEquals("commitCounter", commitCounter, getJournal() +// .getRootBlockView().getCommitCounter()); +// // group commit counter unchanged. +// assertEquals("groupCommitCounter", groupCommitCount0, +// writeService.getGroupCommitCount()); +// // lookup index. +// BTree ndx = (BTree)getIndex(name); +// // verify same checkpoint. +// assertEquals(checkpointAddr0,ndx.getCheckpoint().getCheckpointAddr()); +// // write record on the index. +// ndx.insert(new byte[]{1}, new byte[]{1}); +// // run task (b) next. +// assertTrue(futureB.compareAndSet(null,journal.submit(b))); +// return null; +// } +// }; +// +//// final List<AbstractTask> tasks = Arrays.asList(new AbstractTask[] { +//// a,b,c,d +//// }); +//// +//// final List<Future<Object>> futures = journal.invokeAll( tasks ); +// +// final Future<? extends Object> futureA = journal.submit( a ); +// +// /* +// * wait for (a). if all tasks are in the same commit group then all +// * tasks will be done once we have the future for (a). +// */ +// futureA.get(); // task (a) +// +// /* +// * The expectation is that the tasks that succeed make it into the +// * same commit group while the task that throws an exception does +// * not cause the commit group to be aborted. Therefore there should +// * be ONE (1) commit more than when we submitted the tasks. +// * +// * Note: The tasks will make it into the same commit group iff the +// * first task that completes is willing to wait for the others to +// * join the commit group. +// * +// * Note: The tasks have a dependency on the same resource so they +// * will be serialized (executed in a strict sequence). +// */ +// assertEquals("failedTaskCount", failedTaskCount0 + 1, +// writeService.getTaskFailedCount()); +// assertEquals("successTaskCount", successTaskCount0 + 3, +// writeService.getTaskSuccessCount()); +// assertEquals("committedTaskCount", committedTaskCount0 + 3, +// writeService.getTaskCommittedCount()); +// assertEquals("groupCommitCount", groupCommitCount0 + 1, +// writeService.getGroupCommitCount()); +// assertEquals("commitCounter", commitCounter + 1, journal +// .getRootBlockView().getCommitCounter()); +// +//// assertEquals( 4, futures.size()); +// +// futureB.get().get(); // task (b) +// { +// // task (c) did the abort. +// Future f = futureC.get(); +// try {f.get(); fail("Expecting exception");} +// catch(ExecutionException ex) { +// if(!InnerCause.isInnerCause(ex, ForcedAbortException.class)) { +// fail("Expecting "+ForcedAbortException.class+", not "+ex, ex); +// } +// } +// } +// futureD.get().get(); // task (d) +// +// } finally { +// +// journal.destroy(); +// +// } +// +// } /** * A class used to force aborts on tasks and then recognize the abort by the Modified: branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java =================================================================== --- branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/test/com/bigdata/journal/TestJournalBasics.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -104,8 +104,9 @@ // test basics of the concurrent task execution. suite.addTestSuite(TestConcurrentJournal.class); - // test tasks to add and drop named indices. - suite.addTestSuite(TestAddDropIndexTask.class); +// test tasks to add and drop named indices. +// This has been commented out since the unit test has dated semantics. +// suite.addTestSuite(TestAddDropIndexTask.class); // test writing on one or more unisolated indices and verify read back after the commit. suite.addTestSuite(TestUnisolatedWriteTasks.class); // stress test of throughput when lock contention serializes unisolated writers. Modified: branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java =================================================================== --- branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -163,9 +163,12 @@ namespace, ITx.UNISOLATED)) == mockRelation); /* - * the read-committed view still does not see the relation since - * there has not been a commit yet after the index was created. + * @todo The read-committed view still does not see the relation + * since there has not been a commit yet after the index was + * created. */ + if(false) { + assertNull(((MockRelation) store.getResourceLocator().locate( namespace, ITx.READ_COMMITTED))); @@ -207,6 +210,8 @@ assertTrue(readCommittedView2 == (MockRelation) store .getResourceLocator().locate(namespace, ITx.READ_COMMITTED)); + + } } Modified: branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java =================================================================== --- branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java 2010-07-30 13:51:36 UTC (rev 3371) +++ branches/dev-btm/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java 2010-07-30 13:51:43 UTC (rev 3372) @@ -183,74 +183,77 @@ + BytesUtil.toString(b)); } - - /** - * @todo this test needs to populate an index with terms that would match if - * we were allowing a prefix match and then verify that the terms are - * NOT matched. it should also verify that terms that are exact - * matches are matched. - * - * @todo also test ability to extract the docId and fieldId from the key. - * - * @todo refactor into an {@link ITupleSerializer}. - * - * @todo make the fieldId optional in the key. this needs to be part o... [truncated message content] |