|
From: <tho...@us...> - 2014-04-11 12:24:57
|
Revision: 8106
http://sourceforge.net/p/bigdata/code/8106
Author: thompsonbry
Date: 2014-04-11 12:24:53 +0000 (Fri, 11 Apr 2014)
Log Message:
-----------
Removing test cases that did not actually compare against ground truth. Both of these tickets have test cases against ground truth in TestUnions of the data driven AST evaluation test suite.
See #831
See #874
Removed Paths:
-------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket831.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket874.java
Deleted: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket831.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket831.java 2014-04-11 12:13:24 UTC (rev 8105)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket831.java 2014-04-11 12:24:53 UTC (rev 8106)
@@ -1,177 +0,0 @@
-/**
-Copyright (C) SYSTAP, LLC 2011. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-
-package com.bigdata.rdf.sail;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.Properties;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.log4j.Logger;
-import org.openrdf.model.impl.URIImpl;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.QueryLanguage;
-import org.openrdf.query.TupleQueryResult;
-import org.openrdf.query.impl.BindingImpl;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.sail.SailTupleQuery;
-import org.openrdf.rio.RDFFormat;
-
-import com.bigdata.rdf.axioms.NoAxioms;
-import com.bigdata.rdf.vocab.NoVocabulary;
-
-/**
- * Unit test template for use in submission of bugs.
- * <p>
- * This test case will delegate to an underlying backing store. You can
- * specify this store via a JVM property as follows:
- * <code>-DtestClass=com.bigdata.rdf.sail.TestBigdataSailWithQuads</code>
- * <p>
- * There are three possible configurations for the testClass:
- * <ul>
- * <li>com.bigdata.rdf.sail.TestBigdataSailWithQuads (quads mode)</li>
- * <li>com.bigdata.rdf.sail.TestBigdataSailWithoutSids (triples mode)</li>
- * <li>com.bigdata.rdf.sail.TestBigdataSailWithSids (SIDs mode)</li>
- * </ul>
- * <p>
- * The default for triples and SIDs mode is for inference with truth maintenance
- * to be on. If you would like to turn off inference, make sure to do so in
- * {@link #getProperties()}.
- *
- * @author <a href="mailto:mrp...@us...">Mike Personick</a>
- * @version $Id$
- */
-public class TestTicket831 extends ProxyBigdataSailTestCase {
-
- protected static final Logger log = Logger.getLogger(TestTicket831.class);
-
- /**
- * Please set your database properties here, except for your journal file,
- * please DO NOT SPECIFY A JOURNAL FILE.
- */
- @Override
- public Properties getProperties() {
-
- Properties props = super.getProperties();
-
- /*
- * For example, here is a set of five properties that turns off
- * inference, truth maintenance, and the free text index.
- */
- props.setProperty(BigdataSail.Options.AXIOMS_CLASS, NoAxioms.class.getName());
- props.setProperty(BigdataSail.Options.VOCABULARY_CLASS, NoVocabulary.class.getName());
- props.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
- props.setProperty(BigdataSail.Options.JUSTIFY, "false");
- props.setProperty(BigdataSail.Options.TEXT_INDEX, "false");
-
- return props;
-
- }
-
- public TestTicket831() {
- }
-
- public TestTicket831(String arg0) {
- super(arg0);
- }
-
- public void testBug1() throws Exception {
-
- /*
- * The bigdata store, backed by a temporary journal file.
- */
- final BigdataSail bigdataSail = getSail();
-
- /*
- * Data file containing the data demonstrating your bug.
- */
- final String data = "831.ttl";
- final String baseURI = "";
- final RDFFormat format = RDFFormat.TURTLE;
-
- try {
-
- bigdataSail.initialize();
-
- final BigdataSailRepository bigdataRepo = new BigdataSailRepository(bigdataSail);
-
- { // load the data into the bigdata store
-
- final RepositoryConnection cxn = bigdataRepo.getConnection();
- try {
- cxn.setAutoCommit(false);
- cxn.add(getClass().getResourceAsStream(data), baseURI, format);
-// cxn.add(data);
- cxn.commit();
- } finally {
- cxn.close();
- }
-
- }
-
- {
-// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
-// answer.add(createBindingSet(
-// new BindingImpl("sub", new URIImpl("http://example.org/B"))
-// ));
-
- final String query = IOUtils.toString(getClass().getResourceAsStream("831.rq"));
-
- if (log.isInfoEnabled()) {
- log.info("running query:\n" + query);
- }
-
- /*
- * Run the problem query using the bigdata store and then compare
- * the answer.
- */
- final RepositoryConnection cxn = bigdataRepo.getReadOnlyConnection();
- try {
-
- final SailTupleQuery tupleQuery = (SailTupleQuery)
- cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
- tupleQuery.setIncludeInferred(false /* includeInferred */);
-
- final TupleQueryResult result = tupleQuery.evaluate();
-// compare(result, answer);
-
- while (result.hasNext()) {
- log.info(result.next());
- }
-
- } finally {
- cxn.close();
- }
-
- }
-
- } finally {
-
- bigdataSail.__tearDownUnitTest();
-
- }
-
- }
-
-}
Deleted: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket874.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket874.java 2014-04-11 12:13:24 UTC (rev 8105)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestTicket874.java 2014-04-11 12:24:53 UTC (rev 8106)
@@ -1,177 +0,0 @@
-/**
-Copyright (C) SYSTAP, LLC 2011. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-
-package com.bigdata.rdf.sail;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.Properties;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.log4j.Logger;
-import org.openrdf.model.impl.URIImpl;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.QueryLanguage;
-import org.openrdf.query.TupleQueryResult;
-import org.openrdf.query.impl.BindingImpl;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.sail.SailTupleQuery;
-import org.openrdf.rio.RDFFormat;
-
-import com.bigdata.rdf.axioms.NoAxioms;
-import com.bigdata.rdf.vocab.NoVocabulary;
-
-/**
- * Unit test template for use in submission of bugs.
- * <p>
- * This test case will delegate to an underlying backing store. You can
- * specify this store via a JVM property as follows:
- * <code>-DtestClass=com.bigdata.rdf.sail.TestBigdataSailWithQuads</code>
- * <p>
- * There are three possible configurations for the testClass:
- * <ul>
- * <li>com.bigdata.rdf.sail.TestBigdataSailWithQuads (quads mode)</li>
- * <li>com.bigdata.rdf.sail.TestBigdataSailWithoutSids (triples mode)</li>
- * <li>com.bigdata.rdf.sail.TestBigdataSailWithSids (SIDs mode)</li>
- * </ul>
- * <p>
- * The default for triples and SIDs mode is for inference with truth maintenance
- * to be on. If you would like to turn off inference, make sure to do so in
- * {@link #getProperties()}.
- *
- * @author <a href="mailto:mrp...@us...">Mike Personick</a>
- * @version $Id$
- */
-public class TestTicket874 extends ProxyBigdataSailTestCase {
-
- protected static final Logger log = Logger.getLogger(TestTicket874.class);
-
- /**
- * Please set your database properties here, except for your journal file,
- * please DO NOT SPECIFY A JOURNAL FILE.
- */
- @Override
- public Properties getProperties() {
-
- Properties props = super.getProperties();
-
- /*
- * For example, here is a set of five properties that turns off
- * inference, truth maintenance, and the free text index.
- */
- props.setProperty(BigdataSail.Options.AXIOMS_CLASS, NoAxioms.class.getName());
- props.setProperty(BigdataSail.Options.VOCABULARY_CLASS, NoVocabulary.class.getName());
- props.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
- props.setProperty(BigdataSail.Options.JUSTIFY, "false");
- props.setProperty(BigdataSail.Options.TEXT_INDEX, "false");
-
- return props;
-
- }
-
- public TestTicket874() {
- }
-
- public TestTicket874(String arg0) {
- super(arg0);
- }
-
- public void testBug1() throws Exception {
-
- /*
- * The bigdata store, backed by a temporary journal file.
- */
- final BigdataSail bigdataSail = getSail();
-
- /*
- * Data file containing the data demonstrating your bug.
- */
- final String data = "874.ttl";
- final String baseURI = "";
- final RDFFormat format = RDFFormat.TURTLE;
-
- try {
-
- bigdataSail.initialize();
-
- final BigdataSailRepository bigdataRepo = new BigdataSailRepository(bigdataSail);
-
- { // load the data into the bigdata store
-
- final RepositoryConnection cxn = bigdataRepo.getConnection();
- try {
- cxn.setAutoCommit(false);
- cxn.add(getClass().getResourceAsStream(data), baseURI, format);
-// cxn.add(data);
- cxn.commit();
- } finally {
- cxn.close();
- }
-
- }
-
- {
-// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
-// answer.add(createBindingSet(
-// new BindingImpl("sub", new URIImpl("http://example.org/B"))
-// ));
-
- final String query = IOUtils.toString(getClass().getResourceAsStream("874.rq"));
-
- if (log.isInfoEnabled()) {
- log.info("running query:\n" + query);
- }
-
- /*
- * Run the problem query using the bigdata store and then compare
- * the answer.
- */
- final RepositoryConnection cxn = bigdataRepo.getReadOnlyConnection();
- try {
-
- final SailTupleQuery tupleQuery = (SailTupleQuery)
- cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
- tupleQuery.setIncludeInferred(false /* includeInferred */);
-
- final TupleQueryResult result = tupleQuery.evaluate();
-// compare(result, answer);
-
- while (result.hasNext()) {
- log.info(result.next());
- }
-
- } finally {
- cxn.close();
- }
-
- }
-
- } finally {
-
- bigdataSail.__tearDownUnitTest();
-
- }
-
- }
-
-}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2014-06-04 15:37:18
|
Revision: 8445
http://sourceforge.net/p/bigdata/code/8445
Author: thompsonbry
Date: 2014-06-04 15:37:13 +0000 (Wed, 04 Jun 2014)
Log Message:
-----------
We have extensively modified the TestMROWTransaction test suite. This class is designed to test for problems where there is a single writer and concurrent readers. We have modified the class to force spurious failures in the BTree.writeNodeOrLeaf() method. These failures directly simulate the behavior on the system of an exception in DefaultNodeCoder.encodeLive(). That method does not have a side-effect. It either succeeds, in which case the caller applies the side-effect, or it fails, in which case there is no side effect. We have demonstrated that the Sail level rollback correctly discards the partial update of the index and that new writer threads continue to make progress and that readers do not observe errors. This effectively disproves the hypothesis that rollback() was failing to discard some state.
See #855 (AssertionError: Child does not have persistent identity)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactionsNoHistory.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactionsWithHistory.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java 2014-06-04 10:55:01 UTC (rev 8444)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java 2014-06-04 15:37:13 UTC (rev 8445)
@@ -1,460 +1,652 @@
-package com.bigdata.rdf.sail;
-
-import java.util.Properties;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.log4j.Logger;
-import org.openrdf.model.URI;
-import org.openrdf.model.impl.URIImpl;
-
-import com.bigdata.btree.IndexMetadata;
-import com.bigdata.counters.CAT;
-import com.bigdata.journal.BufferMode;
-import com.bigdata.journal.ITx;
-import com.bigdata.journal.Journal;
-import com.bigdata.rdf.axioms.NoAxioms;
-import com.bigdata.rdf.sail.BigdataSail.Options;
-import com.bigdata.rdf.store.AbstractTripleStore;
-import com.bigdata.rdf.store.BD;
-import com.bigdata.rdf.store.BigdataStatementIterator;
-import com.bigdata.rdf.vocab.NoVocabulary;
-import com.bigdata.service.AbstractTransactionService;
-import com.bigdata.util.InnerCause;
-import com.bigdata.util.concurrent.DaemonThreadFactory;
-
-abstract public class TestMROWTransactions extends ProxyBigdataSailTestCase {
-
- private static final Logger txLog = Logger.getLogger("com.bigdata.txLog");
-
- TestMROWTransactions() {
- }
-
- TestMROWTransactions(String arg0) {
- super(arg0);
- }
-
- void domultiple_csem_transaction_onethread(final int retentionMillis) throws Exception {
-
- domultiple_csem_transaction_onethread(retentionMillis, 2000, 50);
-
- }
-
- void domultiple_csem_transaction(final int retentionMillis) throws Exception {
-
- domultiple_csem_transaction2(retentionMillis, 2/* nreaderThreads */,
- 1000/* nwriters */, 20 * 1000/* nreaders */);
-
- }
-
- /**
- *
- * @param retentionMillis
- * The retention time (milliseconds).
- * @param nreaderThreads
- * The #of threads running reader tasks. Increase nreaderThreads
- * to increase chance startup condition and decrement to increase
- * chance of commit point with no open read-only transaction (no
- * sessions). Value is in [1:...].
- * @param nwriters
- * The #of writer tasks (there is only one writer thread).
- * @param nreaders
- * The #of reader tasks.
- *
- * @throws Exception
- */
- void domultiple_csem_transaction2(final int retentionMillis,
- final int nreaderThreads, final int nwriters, final int nreaders)
- throws Exception {
-
- /**
- * The most likely problem is related to the session protection in the
- * RWStore. In development we saw problems when concurrent transactions
- * had reduced the open/active transactions to zero, therefore releasing
- * session protection. If the protocol works correctly we should never
- * release session protection if any transaction has been initialized.
- *
- * The message of "invalid address" would be generated if an allocation
- * has been freed and is no longer protected from recycling when an
- * attempt is made to read from it.
- *
- * TODO Experiment with different values of [nthreads] for the with and
- * w/o history variations of this test. Consider lifting that parameter
- * into the signature of this method.
- */
- final int nuris = 2000; // number of unique subject/objects
- final int npreds = 50; //
- // final PseudoRandom r = new PseudoRandom(2000);
- // r.next(1500);
- final Random r = new Random();
-
- final CAT commits = new CAT();
- final CAT nreadersDone = new CAT();
- final AtomicReference<Throwable> failex = new AtomicReference<Throwable>(null);
- // Set [true] iff there are no failures by the time we cancel the running tasks.
- final AtomicBoolean success = new AtomicBoolean(false);
- final BigdataSail sail = getSail(getProperties(retentionMillis));
- // log.warn("Journal: "+sail.getDatabase().getIndexManager()+", file="+((Journal)sail.getDatabase().getIndexManager()).getFile());
- try {
-
- sail.initialize();
- final BigdataSailRepository repo = new BigdataSailRepository(sail);
- final AbstractTripleStore origStore = repo.getDatabase();
-
- final URI[] subs = new URI[nuris];
- for (int i = 0; i < nuris; i++) {
- subs[i] = uri("uri:" + i);
- }
- final URI[] preds = new URI[npreds];
- for (int i = 0; i < npreds; i++) {
- preds[i] = uri("pred:" + i);
- }
-
- // Writer task adds nwrites statements then commits
- class Writer implements Callable<Long> {
- final int nwrites;
-
- Writer(final int nwrites) {
- this.nwrites = nwrites;
- }
-
- public Long call() throws Exception {
- try {
- final boolean isQuads = origStore.isQuads();
- // Thread.sleep(r.nextInt(2000) + 500);
- try {
-
- for (int i = 0; i < nwrites; i++) {
- origStore
- .addStatement(
- subs[r.nextInt(nuris)],
- preds[r.nextInt(npreds)],
- subs[r.nextInt(nuris)],
- isQuads ? subs[r.nextInt(nuris)]
- : null);
- // System.out.print('.');
- }
- // System.out.println("\n");
- commits.increment();
-
- } finally {
- origStore.commit();
- if (log.isInfoEnabled()) {
- log.info("Commit #" + commits);
- }
- }
- } catch (Throwable ise) {
- if (!InnerCause.isInnerCause(ise,
- InterruptedException.class)) {
- if (failex
- .compareAndSet(null/* expected */, ise/* newValue */)) {
- log.error("firstCause:" + ise, ise);
- } else {
- if (log.isInfoEnabled())
- log.info("Other error: " + ise, ise);
- }
- } else {
- // Ignore.
- }
- }
- return null;
- }
-
- }
-
- // ReaderTask makes nreads and closes
- class Reader implements Callable<Long> {
- final int nreads;
-
- Reader(final int nwrites) {
- this.nreads = nwrites;
- }
-
- public Long call() throws Exception {
- try {
- final Long txId = ((Journal) origStore
- .getIndexManager()).newTx(ITx.READ_COMMITTED);
-
- try {
- /*
- * Note: This sleep makes it much easier to hit the
- * bug documented here. However, the sleep can also
- * cause the test to really stretch out. So the
- * sleep is only used until the writers are done.
- *
- * https://sourceforge.net/apps/trac/bigdata/ticket/467
- */
- if (commits.get() < nwriters)
- Thread.sleep(2000/* millis */);
- txLog.info("Reading with tx: " + txId);
- final AbstractTripleStore readstore = (AbstractTripleStore) origStore
- .getIndexManager().getResourceLocator()
- .locate(origStore.getNamespace(), txId);
-
- for (int i = 0; i < nreads; i++) {
- final BigdataStatementIterator stats = readstore
- .getStatements(subs[r.nextInt(nuris)],
- null, null);
- try {
- while (stats.hasNext()) {
- stats.next();
- }
- } finally {
- stats.close();
- }
- }
-
- txLog.info("Finished with tx: " + txId);
- } catch (IllegalStateException ise) {
- txLog.info("IllegalStateException tx: " + txId);
- failex.compareAndSet(null, ise);
- } catch (Exception e) {
- txLog.info("UnexpectedException tx: " + txId);
- failex.compareAndSet(null, e);
- throw e;
- } finally {
- txLog.info("Aborting tx: " + txId);
- ((Journal) origStore.getIndexManager()).abort(txId);
- nreadersDone.increment();
- }
- } catch (Throwable ise) {
- if (!InnerCause.isInnerCause(ise,
- InterruptedException.class)) {
- if (failex
- .compareAndSet(null/* expected */, ise/* newValue */)) {
- log.error("firstCause:" + ise, ise);
- } else {
- if (log.isInfoEnabled())
- log.info("Other error: " + ise, ise);
- }
- } else {
- // Ignore.
- }
- }
- return null;
- }
-
- }
-
- ExecutorService writers = null;
- ExecutorService readers = null;
- try {
-
- writers = Executors
- .newSingleThreadExecutor(new DaemonThreadFactory(
- "test-writer-pool"));
-
- readers = Executors.newFixedThreadPool(nreaderThreads,
- new DaemonThreadFactory("test-reader-pool"));
-
- // let's schedule a few writers and readers (more than needed)
- // writers.submit(new Writer(5000000/* nwrite */));
- Future<Long> lastWriterFuture = null;
- Future<Long> lastReaderFuture = null;
- for (int i = 0; i < nwriters; i++) {
- lastWriterFuture = writers
- .submit(new Writer(500/* nwrite */));
- }
- for (int rdrs = 0; rdrs < nreaders; rdrs++) {
- lastReaderFuture = readers
- .submit(new Reader(60/* nread */));
- }
-
- // let the writers run riot for a time, checking for failure
- while (true) {
- final boolean bothDone = lastWriterFuture.isDone()
- && lastReaderFuture.isDone();
- if (bothDone)
- break;
- if (failex.get() != null) {
- // Something errored.
- break;
- }
- Thread.sleep(1000/* ms */);
- }
- // for (int i = 0; i < 600; i++) {
- // Thread.sleep(1000);
- // if (failex.get() != null)
- // break;
- // }
- if (failex.get() == null) {
- /*
- * Note whether or not there are failures before we
- * interrupt the running tasks.
- */
- success.set(true);
- }
- writers.shutdownNow();
- readers.shutdownNow();
- writers.awaitTermination(5, TimeUnit.SECONDS);
- readers.awaitTermination(5, TimeUnit.SECONDS);
- if (!success.get()) {
- final Throwable ex = failex.get();
- if (ex != null) {
- fail("Test failed: firstCause=" + ex
- + ", retentionMillis=" + retentionMillis
- + ", nreaderThreads=" + nreaderThreads
- + ", nwriters=" + nwriters + ", nreaders="
- + nreaders + ", indexManager="
- + repo.getDatabase().getIndexManager(), ex);
- }
- }
- if (log.isInfoEnabled())
- log.info("Writers committed: " + commits.get()
- + ", readers done: " + nreadersDone.get());
- } finally {
- if (writers != null)
- writers.shutdownNow();
- if (readers != null)
- readers.shutdownNow();
- }
- } finally {
-
- sail.__tearDownUnitTest();
-
- }
-
- }
-
- void domultiple_csem_transaction_onethread(final int retention, final int nuris, final int npreds) throws Exception {
-
- // final PseudoRandom r = new PseudoRandom(20000 /*10000*/);
- final Random r = new Random();
-
- final CAT writes = new CAT();
- final CAT reads = new CAT();
-// final AtomicReference<Throwable> failex = new AtomicReference<Throwable>(null);
- // Set [true] iff there are no failures by the time we cancel the
- // running tasks.
- // final AtomicBoolean success = new AtomicBoolean(false);
- final BigdataSail sail = getSail(getProperties(retention));
- try {
-
- sail.initialize();
- final BigdataSailRepository repo = new BigdataSailRepository(sail);
- final AbstractTripleStore origStore = repo.getDatabase();
-
- final URI[] subs = new URI[nuris];
- for (int i = 0; i < nuris; i++) {
- subs[i] = uri("uri:" + i);
- }
- final URI[] preds = new URI[npreds + 20];
- for (int i = 0; i < npreds; i++) {
- preds[i] = uri("pred:" + i);
- }
- final int nwrites = 600;
- final int nreads = 50;
- final int ntrials = 20;
- final boolean isQuads = origStore.isQuads();
-
- for (int loop = 0; loop < ntrials; loop++) {
- final Long txId = ((Journal) origStore.getIndexManager())
- .newTx(ITx.READ_COMMITTED);
- try {
- // System.err.println("READ_STATE: " + txId);
- final AbstractTripleStore readstore = (AbstractTripleStore) origStore
- .getIndexManager().getResourceLocator()
- .locate(origStore.getNamespace(), txId);
- for (int i = 0; i < nreads; i++) {
- final BigdataStatementIterator stats = readstore
- // .getStatements(subs[nuris/2 + loop], null,
- // null);
- .getStatements(subs[r.nextInt(nuris)], null,
- null);
- try {
- while (stats.hasNext()) {
- stats.next();
- reads.increment();
- }
- } finally {
- stats.close();
- }
- }
-
- // Thread.sleep(r.nextInt(1000) + 500);
- try {
-
- for (int i = 0; i < nwrites; i++) {
- origStore.addStatement(subs[r.nextInt(nuris)],
- preds[r.nextInt(npreds)],
- subs[r.nextInt(nuris)],
- isQuads ? subs[r.nextInt(nuris)] : null);
- // origStore.addStatement(subs[nuris/2 + loop],
- // preds[npreds/2 + loop],
- // subs[nuris/2 - loop],
- // isQuads ? subs[nuris/2 + loop] : null);
- writes.increment();
- // System.out.print('.');
- }
- // System.out.println("\n");
-
- } finally {
- origStore.commit();
- log.warn("Commit: " + loop);
- // if (log.isInfoEnabled())
- // log.info("Commit");
- }
- // Close Read Connection
- ((Journal) readstore.getIndexManager()).abort(txId);
-
- } catch (Throwable ise) {
- log.error("firstCause:" + ise, ise);
- throw new Exception(ise);
- }
- }
-
- } finally {
-
- sail.__tearDownUnitTest();
-
- }
-
- }
-
- protected URI uri(String s) {
- return new URIImpl(BD.NAMESPACE + s);
- }
-
- @Override
- public Properties getProperties() {
-
- Properties props = super.getProperties();
-
- props.setProperty(BigdataSail.Options.ISOLATABLE_INDICES, "true");
- props.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
- props.setProperty(BigdataSail.Options.AXIOMS_CLASS, NoAxioms.class.getName());
- props.setProperty(BigdataSail.Options.VOCABULARY_CLASS, NoVocabulary.class.getName());
- props.setProperty(BigdataSail.Options.JUSTIFY, "false");
- props.setProperty(BigdataSail.Options.TEXT_INDEX, "false");
- // props.setProperty(Options.WRITE_CACHE_BUFFER_COUNT, "3");
-
- // ensure using RWStore
- props.setProperty(Options.BUFFER_MODE, BufferMode.DiskRW.toString());
- // props.setProperty(RWStore.Options.MAINTAIN_BLACKLIST, "false");
- // props.setProperty(RWStore.Options.OVERWRITE_DELETE, "true");
- // props.setProperty(Options.CREATE_TEMP_FILE, "false");
- // props.setProperty(Options.FILE, "/Volumes/SSDData/csem.jnl");
-
- // props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY, "20");
- // props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_SCAN, "0");
- props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY, "500");
- props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_SCAN, "10");
-
- return props;
-
- }
-
- protected Properties getProperties(int retention) {
- final Properties props = getProperties();
- props.setProperty(AbstractTransactionService.Options.MIN_RELEASE_AGE, "" + retention);
-
- return props;
- }
-
-}
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Dec 19, 2006
+ */
+package com.bigdata.rdf.sail;
+
+import info.aduna.iteration.CloseableIteration;
+
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.sail.SailException;
+
+import com.bigdata.btree.AbstractNode;
+import com.bigdata.btree.BTree;
+import com.bigdata.btree.Checkpoint;
+import com.bigdata.btree.IndexMetadata;
+import com.bigdata.counters.CAT;
+import com.bigdata.journal.BufferMode;
+import com.bigdata.rawstore.IRawStore;
+import com.bigdata.rdf.axioms.NoAxioms;
+import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
+import com.bigdata.rdf.sail.BigdataSail.Options;
+import com.bigdata.rdf.store.BD;
+import com.bigdata.rdf.vocab.NoVocabulary;
+import com.bigdata.service.AbstractTransactionService;
+import com.bigdata.util.InnerCause;
+import com.bigdata.util.concurrent.DaemonThreadFactory;
+
+/**
+ * TestCase to test single writer/mutiple transaction committed readers with
+ * SAIL interface.
+ *
+ * @author Martyn Cutcher
+ */
+abstract public class TestMROWTransactions extends ProxyBigdataSailTestCase {
+
+// private static final Logger txLog = Logger.getLogger("com.bigdata.txLog");
+
+ TestMROWTransactions() {
+ }
+
+ TestMROWTransactions(final String arg0) {
+ super(arg0);
+ }
+
+// void domultiple_csem_transaction_onethread(final int retentionMillis) throws Exception {
+//
+// domultiple_csem_transaction_onethread(retentionMillis, 2000, 50);
+//
+// }
+//
+// void domultiple_csem_transaction(final int retentionMillis) throws Exception {
+//
+// domultiple_csem_transaction2(retentionMillis, 2/* nreaderThreads */,
+// 1000/* nwriters */, 20 * 1000/* nreaders */);
+//
+// }
+
+ /**
+ *
+ * @param retentionMillis
+ * The retention time (milliseconds).
+ * @param nreaderThreads
+ * The #of threads running reader tasks. Increase nreaderThreads
+ * to increase chance startup condition and decrement to increase
+ * chance of commit point with no open read-only transaction (no
+ * sessions). Value is in [1:...].
+ * @param nwriters
+ * The #of writer tasks (there is only one writer thread).
+ * @param nreaders
+ * The #of reader tasks.
+ * @param isolatableIndice
+ * When <code>true</code> the writers will use read/write
+ * transactions. Otherwise they will use the unisolated
+ * connection.
+ * @throws Exception
+ */
+ void domultiple_csem_transaction2(final int retentionMillis,
+ final int nreaderThreads, final int nwriters, final int nreaders,
+ final boolean isolatableIndices) throws Exception {
+
+ if (log.isInfoEnabled()) {
+ log.info("=================================================================================");
+ log.info("retentionMillis=" + retentionMillis + ", nreaderThreads="
+ + nreaderThreads + ", nwriters=" + nwriters + ", nreaders="
+ + nreaders + ", isolatableIndices=" + isolatableIndices);
+ log.info("=================================================================================");
+ }
+
+ /**
+ * The most likely problem is related to the session protection in the
+ * RWStore. In development we saw problems when concurrent transactions
+ * had reduced the open/active transactions to zero, therefore releasing
+ * session protection. If the protocol works correctly we should never
+ * release session protection if any transaction has been initialized.
+ *
+ * The message of "invalid address" would be generated if an allocation
+ * has been freed and is no longer protected from recycling when an
+ * attempt is made to read from it.
+ *
+ * TODO Experiment with different values of [nthreads] for the with and
+ * w/o history variations of this test. Consider lifting that parameter
+ * into the signature of this method.
+ */
+ final int nuris = 2000; // number of unique subject/objects
+ final int npreds = 50; //
+ // final PseudoRandom r = new PseudoRandom(2000);
+ // r.next(1500);
+ final Random r = new Random();
+
+ final int maxAborts = 100;
+
+ final CAT commits = new CAT();
+ final CAT aborts = new CAT();
+ final CAT nreadersDone = new CAT();
+ final AtomicReference<Throwable> failex = new AtomicReference<Throwable>(null);
+ // Set [true] iff there are no failures by the time we cancel the running tasks.
+ final AtomicBoolean success = new AtomicBoolean(false);
+ final BigdataSail sail = getSail(getProperties(retentionMillis,
+ isolatableIndices));
+ // log.warn("Journal: "+sail.getDatabase().getIndexManager()+", file="+((Journal)sail.getDatabase().getIndexManager()).getFile());
+ try {
+
+ sail.initialize();
+ // TODO Force an initial commit?
+
+// final BigdataSailRepository repo = new BigdataSailRepository(sail);
+// final AbstractTripleStore origStore = repo.getDatabase();
+
+ final URI[] subs = new URI[nuris];
+ for (int i = 0; i < nuris; i++) {
+ subs[i] = uri("uri:" + i);
+ }
+ final URI[] preds = new URI[npreds];
+ for (int i = 0; i < npreds; i++) {
+ preds[i] = uri("pred:" + i);
+ }
+
+ ExecutorService writers = null;
+ ExecutorService readers = null;
+ try {
+
+ writers = Executors
+ .newSingleThreadExecutor(new DaemonThreadFactory(
+ "test-writer-pool"));
+
+ readers = Executors.newFixedThreadPool(nreaderThreads,
+ new DaemonThreadFactory("test-reader-pool"));
+
+ // let's schedule a few writers and readers (more than needed)
+ // writers.submit(new Writer(5000000/* nwrite */));
+ Future<Long> lastWriterFuture = null;
+ @SuppressWarnings("unused")
+ Future<Long> lastReaderFuture = null;
+
+ for (int i = 0; i < nwriters; i++) {
+
+ lastWriterFuture = writers.submit(new Writer(r,
+ 500/* nwrites */, sail, commits, aborts,
+ maxAborts, failex, subs, preds));
+
+ }
+
+ for (int rdrs = 0; rdrs < nreaders; rdrs++) {
+
+ lastReaderFuture = readers.submit(new Reader(r,
+ 60/* nread */, nwriters, sail, failex,
+ commits, nreadersDone, subs));
+
+ }
+
+ // let the writers run riot for a time, checking for failure
+ while (true) {
+// final boolean bothDone = lastWriterFuture.isDone()
+// && lastReaderFuture.isDone();
+// if (bothDone)
+// ...
[truncated message content] |
|
From: <mar...@us...> - 2014-06-13 14:06:26
|
Revision: 8475
http://sourceforge.net/p/bigdata/code/8475
Author: martyncutcher
Date: 2014-06-13 14:06:22 +0000 (Fri, 13 Jun 2014)
Log Message:
-----------
Added main() to com.bigdata.rdf.sail.TestMROWTransactions to simplify long running tests
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/AbstractBigdataSailTestCase.java
branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/AbstractBigdataSailTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/AbstractBigdataSailTestCase.java 2014-06-12 10:43:06 UTC (rev 8474)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/AbstractBigdataSailTestCase.java 2014-06-13 14:06:22 UTC (rev 8475)
@@ -125,10 +125,11 @@
* returned by this method and the appropriate properties must be provided
* either through the command line or in a properties file.
* </p>
+ * @param storeFile
*
* @return A new properties object.
*/
- public Properties getProperties() {
+ public Properties getProperties(final String storeFile) {
if( m_properties == null ) {
@@ -149,12 +150,16 @@
// transient means that there is nothing to delete after the test.
// m_properties.setProperty(Options.BUFFER_MODE,BufferMode.Transient.toString());
m_properties.setProperty(Options.BUFFER_MODE,BufferMode.Disk.toString());
+
+ if (storeFile != null) { // overrides if one set by super class
+ m_properties.setProperty(Options.FILE,storeFile);
+ }
/*
* If an explicit filename is not specified...
*/
if(m_properties.get(Options.FILE)==null) {
-
+
/*
* Use a temporary file for the test. Such files are always deleted when
* the journal is closed or the VM exits.
@@ -172,6 +177,10 @@
}
+ public Properties getProperties() {
+ return getProperties(null);
+ }
+
/**
* This method is invoked from methods that MUST be proxied to this class.
* {@link GenericProxyTestCase} extends this class, as do the concrete
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java 2014-06-12 10:43:06 UTC (rev 8474)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMROWTransactions.java 2014-06-13 14:06:22 UTC (rev 8475)
@@ -28,6 +28,7 @@
import info.aduna.iteration.CloseableIteration;
+import java.io.FileInputStream;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.Callable;
@@ -113,7 +114,6 @@
void domultiple_csem_transaction2(final int retentionMillis,
final int nreaderThreads, final int nwriters, final int nreaders,
final boolean isolatableIndices) throws Exception {
-
if (log.isInfoEnabled()) {
log.info("=================================================================================");
log.info("retentionMillis=" + retentionMillis + ", nreaderThreads="
@@ -122,6 +122,15 @@
log.info("=================================================================================");
}
+ final BigdataSail sail = getSail(getProperties(retentionMillis,
+ isolatableIndices));
+
+ domultiple_csem_transaction2(sail, nreaderThreads, nwriters, nreaders, true);
+ }
+
+ static void domultiple_csem_transaction2( final BigdataSail sail,
+ final int nreaderThreads, final int nwriters, final int nreaders, final boolean teardown) throws Exception {
+
/**
* The most likely problem is related to the session protection in the
* RWStore. In development we saw problems when concurrent transactions
@@ -151,8 +160,6 @@
final AtomicReference<Throwable> failex = new AtomicReference<Throwable>(null);
// Set [true] iff there are no failures by the time we cancel the running tasks.
final AtomicBoolean success = new AtomicBoolean(false);
- final BigdataSail sail = getSail(getProperties(retentionMillis,
- isolatableIndices));
// log.warn("Journal: "+sail.getDatabase().getIndexManager()+", file="+((Journal)sail.getDatabase().getIndexManager()).getFile());
try {
@@ -235,7 +242,6 @@
final Throwable ex = failex.get();
if (ex != null) {
fail("Test failed: firstCause=" + ex
- + ", retentionMillis=" + retentionMillis
+ ", nreaderThreads=" + nreaderThreads
+ ", nwriters=" + nwriters + ", nreaders="
+ nreaders + ", indexManager="
@@ -253,17 +259,19 @@
readers.shutdownNow();
}
} finally {
- try {
- sail.__tearDownUnitTest();
- } catch (Throwable t) {
- /*
- * FIXME The test helper tear down should not throw anything,
- * but it can do so if a tx has been asynchronously closed. This
- * has to do with the logic that openrdf uses to close open
- * transactions when the sail is shutdown by the caller.
- */
- log.error("Problem with test shutdown: " + t, t);
- }
+ if (teardown) {
+ try {
+ sail.__tearDownUnitTest();
+ } catch (Throwable t) {
+ /*
+ * FIXME The test helper tear down should not throw anything,
+ * but it can do so if a tx has been asynchronously closed. This
+ * has to do with the logic that openrdf uses to close open
+ * transactions when the sail is shutdown by the caller.
+ */
+ log.error("Problem with test shutdown: " + t, t);
+ }
+ }
}
@@ -326,13 +334,13 @@
log.info("Commit #" + commits);
} catch (Throwable ise) {
- log.warn(ise, ise);
if (InnerCause.isInnerCause(ise, InterruptedException.class)) {
// ignore
} else if (InnerCause.isInnerCause(ise, MyBTreeException.class)
&& aborts.get() < maxAborts) {
// ignore
} else {
+ log.warn(ise, ise);
// Set the first cause (but not for the forced abort).
if (failex
.compareAndSet(null/* expected */, ise/* newValue */)) {
@@ -532,7 +540,7 @@
//
// }
- protected URI uri(String s) {
+ protected static URI uri(String s) {
return new URIImpl(BD.NAMESPACE + s);
}
@@ -568,7 +576,33 @@
final boolean isolatableIndices) {
final Properties props = getProperties();
+
+ props.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
+ props.setProperty(BigdataSail.Options.AXIOMS_CLASS, NoAxioms.class.getName());
+ props.setProperty(BigdataSail.Options.VOCABULARY_CLASS, NoVocabulary.class.getName());
+ props.setProperty(BigdataSail.Options.JUSTIFY, "false");
+ props.setProperty(BigdataSail.Options.TEXT_INDEX, "false");
+ // props.setProperty(Options.WRITE_CACHE_BUFFER_COUNT, "3");
+ // ensure using RWStore
+ props.setProperty(Options.BUFFER_MODE, BufferMode.DiskRW.toString());
+ // props.setProperty(RWStore.Options.MAINTAIN_BLACKLIST, "false");
+ // props.setProperty(RWStore.Options.OVERWRITE_DELETE, "true");
+ // props.setProperty(Options.CREATE_TEMP_FILE, "false");
+ // props.setProperty(Options.FILE, "/Volumes/SSDData/csem.jnl");
+
+ // props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY, "20");
+ // props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_SCAN, "0");
+ props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY, "500");
+ props.setProperty(IndexMetadata.Options.WRITE_RETENTION_QUEUE_SCAN, "10");
+
+ setProperties(props, retention, isolatableIndices);
+
+ return props;
+ }
+
+ static void setProperties(final Properties props, final int retention,
+ final boolean isolatableIndices) {
props.setProperty(BigdataSail.Options.ISOLATABLE_INDICES,
Boolean.toString(isolatableIndices));
@@ -599,8 +633,9 @@
+ ".com.bigdata.btree.BTree.className",
MyBTree.class.getName());
}
- return props;
}
+
+
/**
* Helper class for force abort of a B+Tree write.
@@ -649,4 +684,73 @@
}
+ /** utilities for main subclass support **/
+ static long getLongArg(final String[] args, final String arg, final long def) {
+ final String sv = getArg(args, arg, null);
+
+ return sv == null ? def : Long.parseLong(sv);
+ }
+
+ static String getArg(final String[] args, final String arg, final String def) {
+ for (int p = 0; p < args.length; p+=2) {
+ if (arg.equals(args[p]))
+ return args[p+1];
+ }
+
+ return def;
+ }
+
+ /**
+ * Command line variant to allow stress testing without JUnit support
+ *
+ * Invokes the same domultiple_csem_transaction2 method.
+ *
+ * A property file is required. Note that if a file is specified then
+ * it will be re-opened and not removed for each run as specified by
+ * nruns.
+ *
+ * Optional arguments
+ * -nruns - number of runs through the test
+ * -nreaderthreads - reader threads
+ * -nwriters - writer tasks
+ * -nreaders - reader tasks
+ */
+ public static void main(String[] args) throws Exception {
+
+ final String propertyFile = getArg(args, "-propertyfile", null);
+ if (propertyFile == null) {
+ System.out.println("-propertyfile <properties> must be specified");
+ return;
+ }
+
+
+ final Properties props = new Properties();
+
+ props.load(new FileInputStream(propertyFile));
+
+ BigdataSail sail = new BigdataSail(props);
+
+ final int nreaderThreads = (int) getLongArg(args, "-nreaderthreads", 20); // 20
+
+ final long nwriters = getLongArg(args, "-nwriters", 100); // 1000000;
+
+ final long nreaders = getLongArg(args, "-nreaders", 400); // 100000;
+
+ final long nruns = getLongArg(args, "-nruns", 1); // 1000;
+
+ for (int i = 0; i < nruns; i++) {
+ domultiple_csem_transaction2(sail, (int) nreaderThreads,
+ (int) nwriters, (int) nreaders, false /*no tear down*/);
+
+ // reopen for second run - should be open if !teardown
+ if (sail.isOpen())
+ sail.shutDown();
+
+ sail = new BigdataSail(props);
+
+ System.out.println("Completed run: " + i);
+ }
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|