Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[3782] branches/QUADS_QUERY_BRANCH/bigdata
|
From: <tho...@us...> - 2010-10-13 13:46:43
|
Revision: 3782
http://bigdata.svn.sourceforge.net/bigdata/?rev=3782&view=rev
Author: thompsonbry
Date: 2010-10-13 13:46:30 +0000 (Wed, 13 Oct 2010)
Log Message:
-----------
Fixed test failure on "normalization" by appropriately configuring Unicode collation for that test.
Fixed (or at least worked around) test failures with two data services by not destroying the triple store in tearDown(). tearDown() is somehow being invoked before the query runs to completion which was causing those "no such index" exceptions to be thrown.
Modified the DataServer to use a single proxy for its QueryEngine for its live span. Added support for the proxy of the query engine to the MetadataServer, through we are not yet using query against the MDS.
Simplified the query invocation in BigdataEvaluationStrategyImpl. It now uses QueryEngine.eval/1.
Modified the SAIL to use full read-only transactions when the caller requests a read-only connection. This asserts the appropriate read lock on the transaction server and prevents commit points against which the query is running from being aged out of the RWStore or the federation. (This was not an issue in the SPARQL queries, but it is one of the things that I explored.)
Javadoc changes in FederatedRunningQuery.
Log statement changed in Algorithm_NestedLocatorScan.
Added some logic to dump out the partition locators in AccessPath if the index could not be found. This was eventually traced to the test tear down so the logic is commented out in AccessPath.
Added an explicit getQueryConnection() method to SPARQLQueryTest.
Various changes to the test harness to get things running properly.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/DataServer.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -46,6 +46,7 @@
import com.bigdata.bop.engine.IQueryClient;
import com.bigdata.bop.engine.IQueryPeer;
import com.bigdata.bop.engine.LocalChunkMessage;
+import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.engine.RunningQuery;
import com.bigdata.bop.fed.shards.MapBindingSetsOverShardsBuffer;
import com.bigdata.io.DirectBufferPool;
@@ -313,19 +314,20 @@
}
- /**
- * Resolve the proxy for an {@link IQueryPeer}. This is special cased for
- * both <i>this</i> service (the actual reference is returned) and the query
- * controller (we use an alternative path to discover the query controller
- * since it might not be registered against a lookup service if it is not a
- * data service).
- *
- * @param serviceUUID
- * The service identifier for the peer.
- *
- * @return The proxy for the service or <code>null</code> if the service
- * could not be discovered.
- */
+ /**
+ * Resolve the proxy for an {@link IQueryPeer}. This is special cased for
+ * both <i>this</i> service (the actual reference is returned) and the query
+ * controller (we use an alternative path to discover the query controller
+ * since it might not be registered against a lookup service if it is not a
+ * data service).
+ *
+ * @param serviceUUID
+ * The service identifier for the peer.
+ *
+ * @return The proxy for the service, the actual {@link QueryEngine}
+ * reference if the identified service is <i>this</i> service, or
+ * <code>null</code> if the service could not be discovered.
+ */
protected IQueryPeer getQueryPeer(final UUID serviceUUID) {
if (serviceUUID == null)
@@ -334,9 +336,16 @@
final IQueryPeer queryPeer;
if(serviceUUID.equals(getQueryEngine().getServiceUUID())) {
-
- // Return a hard reference to this query engine (NOT a proxy).
- return getQueryEngine();
+
+ /*
+ * Return a hard reference to this query engine (NOT a proxy).
+ *
+ * Note: This is used to avoid RMI when the message will be consumed
+ * by the service which produced that message. This is a deliberate
+ * performance optimization which is supported by all of the data
+ * structures involved.
+ */
+ queryPeer = getQueryEngine();
} else if (serviceUUID.equals(queryControllerUUID)) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -45,9 +45,10 @@
final PartitionLocator locator = itr.next();
if (log.isTraceEnabled())
- log.trace("adding bindingSet to buffer" + ": partitionId="
- + locator.getPartitionId() + ", bindingSet="
- + bundle.bindingSet);
+ log.trace("adding bindingSet to buffer" + ": partitionId="
+ + locator.getPartitionId() + "dataService="
+ + locator.getDataServiceUUID() + ", bindingSet="
+ + bundle.bindingSet);
final IBuffer<IBindingSet[]> sink = op.getBuffer(locator);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -390,6 +390,11 @@
if (ndx == null) {
+// // For debugging only - comment this out.
+// dumpMDI((AbstractScaleOutFederation<?>) relation
+// .getIndexManager(), relation.getNamespace(), timestamp,
+// keyOrder);
+
throw new RuntimeException("No such index: relation="
+ relation.getNamespace() + ", timestamp=" + timestamp
+ ", keyOrder=" + keyOrder + ", pred=" + predicate
@@ -1688,4 +1693,33 @@
*/
private static final DiskCostModel diskCostModel = DiskCostModel.DEFAULT;
+// /**
+// * Dumps the locators for an index of a relation.
+// *
+// * @param fed
+// * @param namespace
+// * The relation namespace.
+// * @param timestamp
+// * The timestamp of the view.
+// * @param keyOrder
+// * The index.
+// */
+// private static void dumpMDI(AbstractScaleOutFederation<?> fed,
+// final String namespace, final long timestamp,
+// final IKeyOrder<?> keyOrder) {
+//
+// final String name = namespace + "." + keyOrder.getIndexName();
+//
+// final Iterator<PartitionLocator> itr = fed
+// .locatorScan(name, timestamp, new byte[] {}/* fromKey */,
+// null/* toKey */, false/* reverseScan */);
+//
+// System.err.println("name=" + name + " @ "
+// + TimestampUtility.toString(timestamp));
+// while (itr.hasNext()) {
+// System.err.println(itr.next());
+// }
+//
+// }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/DataServer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/DataServer.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/DataServer.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -372,17 +372,27 @@
}
+ /**
+ * Extends the base behavior to return an RMI compatible proxy for the
+ * {@link IQueryEngine}.
+ */
@Override
public IQueryPeer getQueryEngine() {
- /*
- * Note: DGC is not necessary since the DataService has a hard
- * reference to the QueryEngine.
- */
- return getFederation()
- .getProxy(super.getQueryEngine(), false/* enableDGC */);
-
+ synchronized (this) {
+ if (queryPeerProxy == null) {
+ /*
+ * Note: DGC is not necessary since the DataService has a
+ * hard reference to the QueryEngine.
+ */
+ queryPeerProxy = getFederation().getProxy(
+ super.getQueryEngine(), false/* enableDGC */);
+ }
+ return queryPeerProxy;
+ }
+
}
+ private IQueryPeer queryPeerProxy = null;
/**
* Extends the base behavior to return a {@link Name} of the service
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -43,9 +43,9 @@
import org.apache.log4j.MDC;
+import com.bigdata.bop.engine.IQueryPeer;
import com.bigdata.btree.proc.IIndexProcedure;
import com.bigdata.service.IDataService;
-import com.bigdata.service.IMetadataService;
import com.bigdata.service.MetadataService;
import com.bigdata.service.DataService.DataServiceFederationDelegate;
import com.sun.jini.start.LifeCycle;
@@ -302,6 +302,28 @@
/**
* Extends the base behavior to return an RMI compatible proxy for the
+ * {@link IQueryEngine}.
+ */
+ @Override
+ public IQueryPeer getQueryEngine() {
+
+ synchronized (this) {
+ if (queryPeerProxy == null) {
+ /*
+ * Note: DGC is not necessary since the DataService has a
+ * hard reference to the QueryEngine.
+ */
+ queryPeerProxy = getFederation().getProxy(
+ super.getQueryEngine(), false/* enableDGC */);
+ }
+ return queryPeerProxy;
+ }
+
+ }
+ private IQueryPeer queryPeerProxy = null;
+
+ /**
+ * Extends the base behavior to return an RMI compatible proxy for the
* {@link Future}.
*/
@Override
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -14,7 +14,6 @@
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -55,7 +54,6 @@
import com.bigdata.BigdataStatics;
import com.bigdata.bop.BOp;
import com.bigdata.bop.Constant;
-import com.bigdata.bop.HashBindingSet;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
@@ -70,7 +68,6 @@
import com.bigdata.bop.constraint.NE;
import com.bigdata.bop.constraint.NEConstant;
import com.bigdata.bop.constraint.OR;
-import com.bigdata.bop.engine.LocalChunkMessage;
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.engine.RunningQuery;
import com.bigdata.bop.solutions.ISortOrder;
@@ -103,7 +100,6 @@
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.IBuffer;
import com.bigdata.relation.accesspath.IElementFilter;
-import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
import com.bigdata.relation.rule.IAccessPathExpander;
import com.bigdata.relation.rule.IProgram;
import com.bigdata.relation.rule.IQueryOptions;
@@ -1718,12 +1714,14 @@
}
- final UUID queryId = UUID.randomUUID();
- final RunningQuery runningQuery = queryEngine.eval(queryId, query,
- new LocalChunkMessage<IBindingSet>(queryEngine, queryId,
- startId, -1/* partitionId */,
- newBindingSetIterator(new HashBindingSet())));
-
+// final UUID queryId = UUID.randomUUID();
+// final RunningQuery runningQuery = queryEngine.eval(queryId, query,
+// new LocalChunkMessage<IBindingSet>(queryEngine, queryId,
+// startId, -1/* partitionId */,
+// newBindingSetIterator(new HashBindingSet())));
+
+ final RunningQuery runningQuery = queryEngine.eval(query);
+
final IAsynchronousIterator<IBindingSet[]> it1 =
runningQuery.iterator();
@@ -1817,21 +1815,21 @@
}
- /**
- * Return an {@link IAsynchronousIterator} that will read a single,
- * empty {@link IBindingSet}.
- *
- * @param bindingSet
- * the binding set.
- */
- protected ThickAsynchronousIterator<IBindingSet[]> newBindingSetIterator(
- final IBindingSet bindingSet) {
+// /**
+// * Return an {@link IAsynchronousIterator} that will read a single,
+// * empty {@link IBindingSet}.
+// *
+// * @param bindingSet
+// * the binding set.
+// */
+// protected ThickAsynchronousIterator<IBindingSet[]> newBindingSetIterator(
+// final IBindingSet bindingSet) {
+//
+// return new ThickAsynchronousIterator<IBindingSet[]>(
+// new IBindingSet[][] { new IBindingSet[] { bindingSet } });
+//
+// }
- return new ThickAsynchronousIterator<IBindingSet[]>(
- new IBindingSet[][] { new IBindingSet[] { bindingSet } });
-
- }
-
@SuppressWarnings("serial")
private class UnknownOperatorException extends RuntimeException {
private TupleExpr operator;
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -114,11 +114,11 @@
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.fed.QueryEngineFactory;
+import com.bigdata.journal.AbstractJournal;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITransactionService;
import com.bigdata.journal.ITx;
import com.bigdata.journal.Journal;
-import com.bigdata.journal.TimestampUtility;
import com.bigdata.rdf.axioms.NoAxioms;
import com.bigdata.rdf.inf.TruthMaintenance;
import com.bigdata.rdf.internal.IV;
@@ -153,6 +153,7 @@
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IElementFilter;
import com.bigdata.relation.rule.IRule;
+import com.bigdata.service.AbstractFederation;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.striterator.CloseableIteratorWrapper;
import com.bigdata.striterator.IChunkedIterator;
@@ -1163,7 +1164,7 @@
public BigdataSailConnection getReadOnlyConnection() {
final long timestamp = database.getIndexManager().getLastCommitTime();
-
+
return getReadOnlyConnection(timestamp);
}
@@ -1178,15 +1179,118 @@
*
* @return The view.
*/
- public BigdataSailConnection getReadOnlyConnection(long timestamp) {
+ public BigdataSailConnection getReadOnlyConnection(final long timestamp) {
- AbstractTripleStore view = (AbstractTripleStore) database
- .getIndexManager().getResourceLocator().locate(
- database.getNamespace(),
- TimestampUtility.asHistoricalRead(timestamp));
+// AbstractTripleStore view = (AbstractTripleStore) database
+// .getIndexManager().getResourceLocator().locate(
+// database.getNamespace(),
+// TimestampUtility.asHistoricalRead(timestamp));
+//
+// return new BigdataSailConnection(view, null);
- return new BigdataSailConnection(view, null);
+ try {
+ return _getReadOnlyConnection(timestamp);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ /**
+ * Return a read-only connection backed by a read-only transaction. The
+ * transaction will be closed when the connection is closed.
+ * @param timestamp The timestamp.
+ * @return The transaction.
+ * @throws IOException
+ * @see ITransactionService#newTx(long)
+ */
+ private BigdataSailConnection _getReadOnlyConnection(final long timestamp) throws IOException {
+
+ final String namespace = database.getNamespace();
+
+ final IIndexManager indexManager = database.getIndexManager();
+
+ final ITransactionService txService = getTxService();
+
+ return new BigdataSailConnection(null/*lock*/) {
+
+ /**
+ * The transaction id.
+ */
+ private long tx;
+
+ /**
+ * Constructor starts a new transaction.
+ */
+ {
+ newTx();
+ }
+
+ /**
+ * Obtain a new read-only transaction from the journal's
+ * transaction service, and attach this SAIL connection to the new
+ * view of the database.
+ */
+ protected void newTx() throws IOException {
+
+ this.tx = txService.newTx(timestamp);
+
+ final AbstractTripleStore txView = (AbstractTripleStore) indexManager
+ .getResourceLocator().locate(namespace, tx);
+
+ attach(txView);
+
+ }
+
+ /**
+ * NOP
+ */
+ @Override
+ public synchronized void commit() throws SailException {
+
+ // NOP.
+
+ }
+
+ /**
+ * NOP
+ */
+ @Override
+ public synchronized void rollback() throws SailException {
+
+ // NOP
+
+ }
+
+ /**
+ * A specialized close that will also abort the current read-only
+ * transaction.
+ */
+ @Override
+ public synchronized void close() throws SailException {
+
+ if (!isOpen()) {
+
+ return;
+
+ }
+
+ super.close();
+
+ try {
+
+ txService.abort(tx);
+
+ } catch(IOException ex) {
+
+ throw new SailException(ex);
+
+ }
+
+ }
+
+ };
+
}
/**
@@ -1204,10 +1308,14 @@
final IIndexManager indexManager = database.getIndexManager();
- // @todo no way to get the txService here w/o a cast?
- final ITransactionService txService = ((Journal) indexManager)
- .getTransactionManager().getTransactionService();
+ if(indexManager instanceof IBigdataFederation<?>) {
+
+ throw new UnsupportedOperationException("Read/write transactions are not yet supported in scale-out.");
+
+ }
+ final ITransactionService txService = getTxService();
+
final String namespace = database.getNamespace();
final Lock readLock = lock.readLock();
@@ -1344,13 +1452,37 @@
}
+ /**
+ * Return the {@link ITransactionService}.
+ */
+ protected ITransactionService getTxService() {
+
+ final IIndexManager indexManager = database.getIndexManager();
+
+ final ITransactionService txService;
+
+ if (indexManager instanceof AbstractJournal) {
+
+ txService = ((Journal) indexManager).getTransactionManager()
+ .getTransactionService();
+
+ } else {
+
+ txService = ((AbstractFederation<?>) indexManager)
+ .getTransactionService();
+
+ }
+
+ return txService;
+
+ }
+
public QueryEngine getQueryEngine() {
return queryEngine;
}
-
/**
* Inner class implements the {@link SailConnection}. Some additional
* functionality is available on this class, including
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -35,12 +35,16 @@
import org.openrdf.query.parser.sparql.ManifestTest;
import org.openrdf.query.parser.sparql.SPARQLQueryTest;
import org.openrdf.repository.Repository;
+import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.dataset.DatasetRepository;
+import com.bigdata.btree.keys.CollatorEnum;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.btree.keys.StrengthEnum;
import com.bigdata.journal.ITx;
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSailRepository;
+import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
import com.bigdata.rdf.sail.BigdataSail.Options;
import com.bigdata.rdf.store.ScaleOutTripleStore;
import com.bigdata.service.jini.JiniClient;
@@ -52,6 +56,7 @@
* in its own bigdata namespace.
*
* @author <a href="mailto:dm...@us...">David MacMillan</a>
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
public class BigdataFederationSparqlTest extends SPARQLQueryTest
@@ -123,14 +128,29 @@
throws Exception
{
super.tearDown () ;
- _ts.destroy () ;
- _ts = null ;
- }
+ /*
+ * @todo We should destroy the triple store here, but this is causing
+ * problems with tear down of the query while it is still running. Once
+ * that issue has been fixed, uncomment both the line to destroy the
+ * triple store and the line to shutdown the federation (the latter is
+ * really optional - it should be Ok to leave the federation up across
+ * the test runs, but then we will never take it down cleanly when the
+ * test suite is done. Again, that should be Ok.)
+ */
+ if (_ts != null) {
+// _ts.destroy();
+ _ts = null;
+ }
+// if (_fed != null) {
+// _fed.shutdownNow();
+// _fed = null;
+// }
+ }
@Override protected Repository newRepository ()
throws Exception
{
- return new DatasetRepository ( new BigdataSailRepository ( new BigdataSail ( newTripleStore () ) ) ) ;
+ return new DatasetRepository ( new BigdataSailRepository ( _sail = new BigdataSail ( newTripleStore () ) ) ) ;
}
@Override
@@ -140,6 +160,15 @@
return repo;
}
+ protected RepositoryConnection getQueryConnection(Repository dataRep)
+ throws Exception {
+ // return dataRep.getConnection();
+ final BigdataSailRepositoryConnection con = new BigdataSailRepositoryConnection(new BigdataSailRepository(
+ _sail), _sail.getReadOnlyConnection());
+ System.err.println(_sail.getDatabase().dumpStore());
+ return con;
+ }
+
private ScaleOutTripleStore newTripleStore ()
throws Exception
{
@@ -176,163 +205,105 @@
/**
* Configuration options for the KB instances used to run the SPARQL
* compliance test suite.
+ * <p>
+ * Note: These properties can not be cached across tests since they have to
+ * be slightly different for some of the tests to handle things like tests
+ * which will fail with inlining enabled ot tests which require Unicode
+ * collation strength of IDENTICAL.
*/
private Properties getProperties () throws Exception
{
-// Note: This approach does not work because we are using a different namespace for each test.
-// /*
-// * Pick up properties configured for the client as defaults.
-// *
-// * You can specify those properties using NV[] for the component.
-// */
-// final String component = System.getProperty ( COMPONENT_PROPERTY, DEFAULT_COMPONENT_PROPERTY ) ;
-// final Properties properties = getFederation().getClient().getProperties(
-// component);
-// return properties;
- if ( null == _properties )
- {
-
-// /* Multiplier for the scatter effect.
-// */
-// final int scatterFactor = 1;
-// final int scatterFactor_term2id = 1;
-// final int dataServiceCount = 2;
-//
-// /* The #of index partitions to allocate on a scatter split. ZERO
-// * (0) means that 2 index partitions will be allocated per
-// * data service which partiticpates in the scatter split.
-// * Non-zero values directly give the #of index partitions to
-// * create.
-// */
-// final int scatterSplitIndexPartitionCount = ConfigMath.multiply
-// ( scatterFactor,
-// dataServiceCount
-// );
-// final int scatterSplitIndexPartitionCount_term2id = ConfigMath.multiply
-// ( scatterFactor_term2id,
-// dataServiceCount
-// );
-//
-// // Use all discovered data services when scattering an index.
-// final int scatterSplitDataServiceCount = 0;
-//
-// /* Scatter split trigger point. The scatter split will not be
-// * triggered until the initial index partition has reached
-// * this percentage of a nominal index partition in size.
-// */
-// final double scatterSplitPercentOfSplitThreshold = 0.5;//was .5
-//
-// /*
-// * Multipliers that compensate for the consumer/producer ratio for
-// * the asynchronous index write API. These are empirical factors
-// * based on observing the ratio (chunkWritingTime/chunkWaitingTime).
-// * Assuming a constant chunk writing time, if the chunk size for each
-// * index is adjusted by its multiplier then this ratio would be 1:1.
-// * In practice, the chunk writing time is not a linear function of
-// * the chunk size, which is one reason why we prefer larger chunks
-// * and why the asynchronous write API is a win.
-// *
-// * Note: These factors were set relative to TERM2ID. However, when
-// * I reduced the scatterFactor for TERM2ID by 1/2, I doubled its
-// * chunk size to keep up the same throughput so it is now at 2.00
-// * rather than 1.00.
-// */
-// final double chunkSizeFactor_id2term = 1.79;
-// final double chunkSizeFactor_term2id = 2.00;
-// final double chunkSizeFactor_stmts = 8.00;
-//
-// /* The nominal sink chunk size. For each index, this is adjusted
-// * by the factor specified above.
-// */
-//// static private sinkChunkSize = 10000;
-// final int sinkChunkSize = 1000;
-
- /*
- * Specify / override some triple store properties.
- *
- * Note: You must reference this object in the section for the
- * component which will actually create the KB instance, e.g.,
- * either the RDFDataLoadMaster or the LubmGeneratorMaster.
- */
- _properties = new Properties ();
-
- /*
- * Setup for quads.
- */
- _properties.put ( BigdataSail.Options.QUADS_MODE, "true" );
- _properties.put ( BigdataSail.Options.TRUTH_MAINTENANCE, "false" );
- _properties.put ( BigdataSail.Options.QUERY_TIME_EXPANDER, "false" );
-
- /*
- * The Sesame TCK forces statement level connection auto-commit so
- * we set a flag to permit that here. However, auto-commit and this
- * flag SHOULD NOT be used outside of the test suite as they provide
- * an extreme performance penalty.
- */
- //_properties.put ( BigdataSail.Options.ALLOW_AUTO_COMMIT, "true" ) ;
+ final Properties _properties;
- /*
- * Provide Unicode support for keys with locale-based string
- * collation. This is more expensive in key creation during loading,
- * but allows key comparison and sorting in the specified locale in
- * queries.
- *
- * @see com.bigdata.btree.keys.CollatorEnum
- */
- _properties.put(KeyBuilder.Options.COLLATOR,"ICU");
- _properties.put(KeyBuilder.Options.USER_LANGUAGE,"en");
- _properties.put(KeyBuilder.Options.USER_COUNTRY,"US");
- _properties.put(KeyBuilder.Options.USER_VARIANT,"");
+ /*
+ * Specify / override some triple store properties.
+ *
+ * Note: You must reference this object in the section for the component
+ * which will actually create the KB instance, e.g., either the
+ * RDFDataLoadMaster or the LubmGeneratorMaster.
+ */
+ _properties = new Properties();
- /*
- * Turn off the full text index (search for literals by keyword).
- */
- _properties.put(BigdataSail.Options.TEXT_INDEX, "false");
+ /*
+ * Setup for quads.
+ */
+ _properties.put(BigdataSail.Options.QUADS_MODE, "true");
+ _properties.put(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
+ _properties.put(BigdataSail.Options.QUERY_TIME_EXPANDER, "false");
- /*
- * Turn on bloom filter for the SPO index (good up to ~2M index
- * entries for scale-up -or- for any size index for scale-out). This
- * is a big win for some queries on scale-out indices since we can
- * avoid touching the disk if the bloom filter reports "false" for a
- * key.
- */
- _properties.put(BigdataSail.Options.BLOOM_FILTER, "true");
+ /*
+ * The Sesame TCK forces statement level connection auto-commit so we
+ * set a flag to permit that here. However, auto-commit and this flag
+ * SHOULD NOT be used outside of the test suite as they provide an
+ * extreme performance penalty.
+ */
+ // _properties.put ( BigdataSail.Options.ALLOW_AUTO_COMMIT, "true" ) ;
- /*
- * The #of low order bits from the TERM2ID index partition local
- * counter that will be reversed and written into the high-order
- * bits of the term identifier. This has a strong effect on the
- * distribution of bulk index read/write operations for the triple
- * store. For a given value of N, a bulk write will tend to touch
- * 2^N index partitions. Therefore if this is even roughly on the
- * order of the number of index partitions, each bulk write will
- * tend to be scattered to all index partitions.
- *
- * Note: If this value is too large then the writes WITHIN the index
- * partitions will become uniformly distributed, which will
- * negatively impact index performance.
- */
- _properties.put(BigdataSail.Options.TERMID_BITS_TO_REVERSE,"0"); // was 2.
-
- /*
- * Option may be enabled to store blank nodes such that they are
- * stable (they are not stored by default).
- */
- // new NV(BigdataSail.Options.STORE_BLANK_NODES,"true");
+ /*
+ * Provide Unicode support for keys with locale-based string collation.
+ * This is more expensive in key creation during loading, but allows key
+ * comparison and sorting in the specified locale in queries.
+ *
+ * @see com.bigdata.btree.keys.CollatorEnum
+ */
+ _properties.put(KeyBuilder.Options.COLLATOR, "ICU");
+ _properties.put(KeyBuilder.Options.USER_LANGUAGE, "en");
+ _properties.put(KeyBuilder.Options.USER_COUNTRY, "US");
+ _properties.put(KeyBuilder.Options.USER_VARIANT, "");
- }
+ /*
+ * Turn off the full text index (search for literals by keyword).
+ */
+ _properties.put(BigdataSail.Options.TEXT_INDEX, "false");
/*
- * Turn inlining on or off depending on _this_ test. This is outside of
- * the if block above because _properties is cached.
+ * Turn on bloom filter for the SPO index (good up to ~2M index entries
+ * for scale-up -or- for any size index for scale-out). This is a big
+ * win for some queries on scale-out indices since we can avoid touching
+ * the disk if the bloom filter reports "false" for a key.
*/
- if (BigdataSparqlTest.cannotInlineTests.contains(testURI)) {
- _properties.setProperty(Options.INLINE_LITERALS, "false");
- } else {
- _properties.setProperty(Options.INLINE_LITERALS, "true");
- }
-
- return _properties ;
+ _properties.put(BigdataSail.Options.BLOOM_FILTER, "true");
+
+ /*
+ * The #of low order bits from the TERM2ID index partition local counter
+ * that will be reversed and written into the high-order bits of the
+ * term identifier. This has a strong effect on the distribution of bulk
+ * index read/write operations for the triple store. For a given value
+ * of N, a bulk write will tend to touch 2^N index partitions. Therefore
+ * if this is even roughly on the order of the number of index
+ * partitions, each bulk write will tend to be scattered to all index
+ * partitions.
+ *
+ * Note: If this value is too large then the writes WITHIN the index
+ * partitions will become uniformly distributed, which will negatively
+ * impact index performance.
+ */
+ _properties.put(BigdataSail.Options.TERMID_BITS_TO_REVERSE, "0");
+
+ /*
+ * Option may be enabled to store blank nodes such that they are stable
+ * (they are not stored by default).
+ */
+ // new NV(BigdataSail.Options.STORE_BLANK_NODES,"true");
+
+ /*
+ * Turn inlining on or off depending on _this_ test.
+ */
+ if (BigdataSparqlTest.cannotInlineTests.contains(testURI)) {
+ _properties.setProperty(Options.INLINE_LITERALS, "false");
+ } else {
+ _properties.setProperty(Options.INLINE_LITERALS, "true");
+ }
+
+ if (BigdataSparqlTest.unicodeStrengthIdentical.contains(testURI)) {
+ // Force identical Unicode comparisons.
+ _properties.setProperty(Options.COLLATOR, CollatorEnum.JDK
+ .toString());
+ _properties.setProperty(Options.STRENGTH, StrengthEnum.Identical
+ .toString());
+ }
+
+ return _properties;
}
/**
@@ -340,21 +311,11 @@
*/
public static final String CONFIG_PROPERTY = "bigdata.configuration";
-// /**
-// * The name of the component in that configuration file whose "properties"
-// * field will be used to initialize the KB.
-// */
-// public static final String COMPONENT_PROPERTY = "bigdata.component" ;
-//
-// /**
-// * The default value for {@link #COMPONENT_PROPERTY}.
-// */
-// public static final String DEFAULT_COMPONENT_PROPERTY = "SparqlQuadsTestSuite";
-
private static final Logger _logger = Logger.getLogger ( BigdataFederationSparqlTest.class ) ;
private static JiniFederation<Object> _fed = null ;
- private static Properties _properties = null ;
private ScaleOutTripleStore _ts = null ;
+ private BigdataSail _sail = null;
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -87,6 +87,15 @@
"http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#no-distinct-9",
"http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-9",
});
+
+ /**
+ * The following tests require Unicode configuration for identical
+ * comparisons. This appears to work with {ASCII,IDENTICAL} or
+ * {JDK,IDENTICAL} but not with {ICU,IDENTICAL} for some reason.
+ */
+ static final Collection<String> unicodeStrengthIdentical = Arrays.asList(new String[] {
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/i18n/manifest#normalization-1"
+ });
// private static String datasetTests = "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset";
@@ -176,9 +185,9 @@
"http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-004",
*/
// Dataset crap
- // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/i18n/manifest#normalization-1"
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/i18n/manifest#normalization-1",
- // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset/manifest#dawg-dataset-01"
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset/manifest#dawg-dataset-01",
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2//manifest#",
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-str-1",
@@ -191,6 +200,13 @@
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-1",
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-2",
+ /*
+ * Tests which fail with 2 data services.
+ */
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#bgp-no-match",//Ok
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#prefix-name-1",//OK
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#spoo-1",//BOOM
+
});
/**
@@ -351,10 +367,10 @@
// exact size only there for TCK
props.setProperty(Options.EXACT_SIZE, "true");
- props.setProperty(Options.COLLATOR, CollatorEnum.ASCII.toString());
+// props.setProperty(Options.COLLATOR, CollatorEnum.ASCII.toString());
// Force identical unicode comparisons (assuming default COLLATOR setting).
- props.setProperty(Options.STRENGTH, StrengthEnum.Identical.toString());
+// props.setProperty(Options.STRENGTH, StrengthEnum.Identical.toString());
// disable read/write transactions
props.setProperty(Options.ISOLATABLE_INDICES, "false");
@@ -372,9 +388,17 @@
if (true) {
final Properties props = getProperties();
- if (cannotInlineTests.contains(testURI))
+ if (cannotInlineTests.contains(testURI)){
+ // The test can not be run using XSD inlining.
props.setProperty(Options.INLINE_LITERALS, "false");
+ }
+ if(unicodeStrengthIdentical.contains(testURI)) {
+ // Force identical Unicode comparisons.
+ props.setProperty(Options.COLLATOR, CollatorEnum.JDK.toString());
+ props.setProperty(Options.STRENGTH, StrengthEnum.Identical.toString());
+ }
+
final BigdataSail sail = new BigdataSail(props);
return new DatasetRepository(new BigdataSailRepository(sail));
} else {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java 2010-10-13 13:27:48 UTC (rev 3781)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java 2010-10-13 13:46:30 UTC (rev 3782)
@@ -62,6 +62,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.bigdata.rdf.sail.BigdataSailQuery;
+import com.bigdata.rdf.sail.BigdataSailRepository;
public abstract class SPARQLQueryTest extends TestCase {
@@ -164,11 +165,23 @@
}
}
+ /**
+ * Return the connection which will be used to query the repository.
+ * @param dataRep The repository.
+ * @return The connection.
+ * @throws Exception
+ */
+ protected RepositoryConnection getQueryConnection(Repository dataRep)
+ throws Exception
+ {
+ return dataRep.getConnection();
+ }
+
@Override
protected void runTest()
throws Exception
{
- RepositoryConnection con = dataRep.getConnection();
+ RepositoryConnection con = getQueryConnection(dataRep);
try {
String queryString = readQueryString();
Query query = con.prepareQuery(QueryLanguage.SPARQL, queryString, queryFileURL);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
