Menu
▾
▴
bigdata-commit
|
From: <mrp...@us...> - 2012-04-02 09:15:59
|
Revision: 6245
http://bigdata.svn.sourceforge.net/bigdata/?rev=6245&view=rev
Author: mrpersonick
Date: 2012-04-02 09:15:46 +0000 (Mon, 02 Apr 2012)
Log Message:
-----------
integrated the subject-centric full text index
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -1,3 +1,4 @@
+
/**
Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
@@ -77,6 +78,7 @@
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.IResourceLock;
import com.bigdata.journal.ITx;
+import com.bigdata.journal.NoSuchIndexException;
import com.bigdata.journal.TimestampUtility;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.internal.IDatatypeURIResolver;
@@ -99,6 +101,7 @@
import com.bigdata.rdf.model.BigdataValueFactoryImpl;
import com.bigdata.rdf.model.BigdataValueSerializer;
import com.bigdata.rdf.rio.StatementBuffer;
+import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.vocab.NoVocabulary;
import com.bigdata.rdf.vocab.Vocabulary;
@@ -141,6 +144,11 @@
private final AtomicReference<IValueCentricTextIndexer<?>> viewRef = new AtomicReference<IValueCentricTextIndexer<?>>();
/**
+ * A new one for the subject-centric full text index.
+ */
+ private final AtomicReference<ISubjectCentricTextIndexer<?>> viewRef2 = new AtomicReference<ISubjectCentricTextIndexer<?>>();
+
+ /**
* Note: This is a stateless class.
*/
private final BlobsIndexHelper h = new BlobsIndexHelper();
@@ -196,6 +204,32 @@
}
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ protected Class<ISubjectCentricTextIndexer> determineSubjectCentricTextIndexerClass() {
+
+ final String className = getProperty(
+ AbstractTripleStore.Options.SUBJECT_CENTRIC_TEXT_INDEXER_CLASS,
+ AbstractTripleStore.Options.DEFAULT_SUBJECT_CENTRIC_TEXT_INDEXER_CLASS);
+
+ final Class<?> cls;
+ try {
+ cls = Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Bad option: "
+ + AbstractTripleStore.Options.SUBJECT_CENTRIC_TEXT_INDEXER_CLASS, e);
+ }
+
+ if (!ISubjectCentricTextIndexer.class.isAssignableFrom(cls)) {
+ throw new RuntimeException(
+ AbstractTripleStore.Options.SUBJECT_CENTRIC_TEXT_INDEXER_CLASS
+ + ": Must implement: "
+ + ISubjectCentricTextIndexer.class.getName());
+ }
+
+ return (Class<ISubjectCentricTextIndexer>) cls;
+
+ }
+
@SuppressWarnings("unchecked")
protected Class<IExtensionFactory> determineExtensionFactoryClass() {
@@ -283,6 +317,13 @@
}
+ // just for now while I am testing, don't feel like rebuilding
+ // the entire journal
+ this.subjectCentricTextIndex = textIndex;
+// this.subjectCentricTextIndex = Boolean.parseBoolean(getProperty(
+// AbstractTripleStore.Options.SUBJECT_CENTRIC_TEXT_INDEX,
+// AbstractTripleStore.Options.DEFAULT_SUBJECT_CENTRIC_TEXT_INDEXER_CLASS));
+
}
this.storeBlankNodes = Boolean.parseBoolean(getProperty(
@@ -705,6 +746,13 @@
private final boolean textIndex;
/**
+ * When <code>true</code> a secondary subject-centric full text index is
+ * maintained.
+ *
+ * @see AbstractTripleStore.Options#SUBJECT_CENTRIC_TEXT_INDEX
+ */
+ private final boolean subjectCentricTextIndex;
+ /**
* When <code>true</code> the kb is using told blank nodes semantics.
*
* @see AbstractTripleStore.Options#STORE_BLANK_NODES
@@ -851,6 +899,17 @@
}
+ /**
+ * <code>true</code> iff the subject-centric full text index is enabled.
+ *
+ * @see AbstractTripleStore.Options#SUBJECT_CENTRIC_TEXT_INDEX
+ */
+ final public boolean isSubjectCentricTextIndex() {
+
+ return subjectCentricTextIndex;
+
+ }
+
/**
* Overridden to use local cache of the index reference.
*/
@@ -1068,6 +1127,61 @@
}
/**
+ * A factory returning the softly held singleton for the
+ * {@link FullTextIndex} representing the subject-centric full text index.
+ *
+ * @see AbstractTripleStore.Options#TEXT_INDEX
+ *
+ * @todo replace with the use of the {@link IResourceLocator} since it
+ * already imposes a canonicalizing mapping within for the index name
+ * and timestamp inside of a JVM.
+ */
+ public ISubjectCentricTextIndexer<?> getSubjectCentricSearchEngine() {
+
+ if (!subjectCentricTextIndex)
+ return null;
+
+ /*
+ * Note: Double-checked locking pattern requires [volatile] variable or
+ * AtomicReference. This uses the AtomicReference since that gives us a
+ * lock object which is specific to this request.
+ */
+ if (viewRef2.get() == null) {
+
+ synchronized (viewRef2) {// NB: Ignore find bugs complaint per above.
+
+ if (viewRef2.get() == null) {
+
+ final ISubjectCentricTextIndexer<?> tmp;
+ try {
+ final Class<?> vfc = determineSubjectCentricTextIndexerClass();
+ final Method gi = vfc.getMethod("getInstance",
+ IIndexManager.class, String.class, Long.class,
+ Properties.class);
+ tmp = (ISubjectCentricTextIndexer<?>) gi.invoke(null/* object */,
+ getIndexManager(), getNamespace(),
+ getTimestamp(), getProperties());
+ if(tmp instanceof ILocatableResource<?>) {
+ ((ILocatableResource<?>)tmp).init();
+ }
+ viewRef2.set(tmp);
+ } catch (Throwable e) {
+ throw new IllegalArgumentException(
+ AbstractTripleStore.Options.SUBJECT_CENTRIC_TEXT_INDEXER_CLASS,
+ e);
+ }
+
+ }
+
+ }
+
+ }
+
+ return viewRef2.get();
+
+ }
+
+ /**
* Return the {@link IndexMetadata} for the TERM2ID index.
*
* @param name
@@ -2002,6 +2116,299 @@
}
+ /**
+ * Utility method to (re-)build the subject-based full text index. This is a
+ * high latency operation for a database of any significant size. You must
+ * be using the unisolated view of the {@link AbstractTripleStore} for this
+ * operation. {@link AbstractTripleStore.Options#TEXT_INDEX} must be
+ * enabled. This operation is only supported when the {@link ITextIndexer}
+ * uses the {@link FullTextIndex} class.
+ * <p>
+ * The subject-based full text index is one that rolls up normal
+ * object-based full text index into a similarly structured index that
+ * captures relevancy across subjects. Instead of
+ *
+ * (t,s) => s.len, termWeight
+ *
+ * Where s is the subject's IV. The term weight has the same
+ * interpretation, but it is across all literals which are linked to that
+ * subject and which contain the given token. This index basically
+ * pre-computes the (?s ?p ?o) join that sometimes follows the (?o
+ * bd:search "xyz") request.
+ * <p>
+ * Truth Maintenance
+ * <p>
+ * We will need to perform truth maintenance on the subject-centric text
+ * index, that is - the index will need to be updated as statements are
+ * added and removed (to the extent that those statements involving a
+ * literal in the object position). Adding a statement is the easier
+ * case because we will never need to remove entries from the index, we
+ * can simply write over them with new relevance values. All that is
+ * involved with truth maintenance for adding a statement is taking a post-
+ * commit snapshot of the subject in the statement and running it through
+ * the indexer (a "subject-refresh").
+ * <p>
+ * The same "subject-refresh" will be necessary for truth maintenance for
+ * removal, but an additional step will be necessary beforehand - the index
+ * entries associated with the deleted subject/object (tokens+subject) will
+ * need to be removed in case the token appears only in the removed literal.
+ * After this pruning step the subject can be refreshed in the index exactly
+ * the same as for truth maintenance on add.
+ * <p>
+ * It looks like the right place to hook in truth maintenance for add is
+ * {@link AbstractTripleStore#addStatements(AbstractTripleStore, boolean, IChunkedOrderedIterator, com.bigdata.relation.accesspath.IElementFilter)}
+ * after the ISPOs are added to the SPORelation.
+ * Likewise, the place to hook in truth maintenance for delete is
+ * {@link AbstractTripleStore#removeStatements(IChunkedOrderedIterator, boolean)}
+ * after the ISPOs are removed from the SPORelation.
+ */
+ @SuppressWarnings("unchecked")
+ public void buildSubjectCentricTextIndex() {
+
+ if (getTimestamp() != ITx.UNISOLATED)
+ throw new UnsupportedOperationException();
+
+ if (!subjectCentricTextIndex)
+ throw new UnsupportedOperationException();
+
+ final ISubjectCentricTextIndexer<?> textIndexer = getSubjectCentricSearchEngine();
+
+ try {
+
+ // destroy the existing text index.
+ textIndexer.destroy();
+
+ } catch (NoSuchIndexException ex) {
+
+ if (log.isInfoEnabled())
+ log.info("could not destroy subject-centric full text index, does not currently exist");
+
+ }
+
+ // create a new index.
+ textIndexer.create();
+
+ // TermIVs
+ {
+ // The index to scan for the individual subjects and their literal
+ // values.
+ final IIndex spoNdx = getContainer().getSPORelation().getPrimaryIndex();
+
+ /*
+ * For each S in SPO, collect up O values and pass this information
+ * to the subject-centric text indexer for indexing.
+ */
+
+ // used to decode the
+ @SuppressWarnings("rawtypes")
+ final ITupleSerializer tupSer = spoNdx.getIndexMetadata()
+ .getTupleSerializer();
+
+ /*
+ * Visit all plain, language code, and datatype literals in the
+ * object position of the primary statement index.
+ *
+ * Note: This uses a filter on the ITupleIterator in order to filter
+ * out non-literal terms before they are shipped from a remote index
+ * shard.
+ */
+ final Iterator<ISPO> itr = new Striterator(
+ spoNdx.rangeIterator(null/* fromKey */, null/* toKey */,
+ 0/* capacity */, IRangeQuery.DEFAULT,
+ new TupleFilter<ISPO>() {
+ private static final long serialVersionUID = 1L;
+
+ protected boolean isValid(
+ final ITuple<ISPO> obj) {
+ final ISPO spo = (ISPO) tupSer
+ .deserializeKey(obj);
+ if (spo.o().isLiteral()) {
+ return true;
+ }
+ return false;
+ }
+ })).addFilter(new Resolver() {
+ private static final long serialVersionUID = 1L;
+
+ protected Object resolve(final Object obj) {
+ final ISPO spo = (ISPO) tupSer
+ .deserializeKey((ITuple<?>) obj);
+ return spo;
+ }
+ });
+
+ /*
+ * Keep track of the current subject being indexed.
+ */
+ IV<?,?> s = null;
+
+ /*
+ * Keep a collection of literals to be indexed for that subject.
+ */
+ final Collection<IV<?,?>> literals = new LinkedList<IV<?,?>>();
+
+ long subjectCount = 0;
+ long statementCount = 0;
+
+ final boolean l = log.isInfoEnabled();
+
+ while (itr.hasNext()) {
+
+ final ISPO spo = itr.next();
+
+ if (!spo.s().equals(s)) {
+
+ // flush the old s to the text index if != null
+
+ if (s != null) {
+
+ textIndexer.index(s, getTerms(literals).values().iterator());
+
+ subjectCount++;
+ statementCount += literals.size();
+
+ if (l && subjectCount % 1000 == 0) {
+ log.info("indexed " + subjectCount + " subjects, " + statementCount + " statements");
+ }
+
+ }
+
+ // set the current s and clear the literals
+
+ s = spo.s();
+
+ literals.clear();
+
+ }
+
+ literals.add(spo.o());
+
+ }
+
+ if (s != null) {
+
+ // flush the last subject
+ textIndexer.index(s, getTerms(literals).values().iterator());
+
+ subjectCount++;
+ statementCount += literals.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("indexed " + subjectCount + " subjects, " + statementCount + " statements");
+ }
+
+ }
+
+ }
+
+ }
+
+// @SuppressWarnings("unchecked")
+// public void refreshSubjectCentricTextIndex(final Set<IV<?,?>> subjects) {
+//
+// if (getTimestamp() != ITx.UNISOLATED)
+// throw new UnsupportedOperationException();
+//
+// if (!subjectCentricTextIndex)
+// throw new UnsupportedOperationException();
+//
+// final ISubjectCentricTextIndexer<?> textIndexer = getSubjectCentricSearchEngine();
+//
+// final AbstractTripleStore db = getContainer();
+//
+// /*
+// * Keep a collection of literals to be indexed for each subject.
+// */
+// final Collection<IV<?,?>> literals = new LinkedList<IV<?,?>>();
+//
+// for (IV<?,?> s : subjects) {
+//
+// literals.clear();
+//
+// /*
+// * Visit all plain, language code, and datatype literals in the
+// * object position of the primary statement index.
+// *
+// * Note: This uses a filter on the ITupleIterator in order to filter
+// * out non-literal terms before they are shipped from a remote index
+// * shard.
+// */
+// final Iterator<ISPO> itr = db.getAccessPath(s, null, null, new SPOFilter<ISPO>() {
+// private static final long serialVersionUID = 1L;
+// @Override
+// public boolean isValid(Object e) {
+// return ((ISPO)e).o().isLiteral();
+// }
+// }).iterator();
+//
+// while (itr.hasNext()) {
+//
+// final ISPO spo = itr.next();
+//
+// literals.add(spo.o());
+//
+// }
+//
+// // flush the last subject
+// textIndexer.index(s, getTerms(literals).values().iterator());
+//
+// }
+//
+// }
+//
+// @SuppressWarnings("unchecked")
+// public void refreshSubjectCentricTextIndex(final Set<ISPO> removed) {
+//
+// if (getTimestamp() != ITx.UNISOLATED)
+// throw new UnsupportedOperationException();
+//
+// if (!subjectCentricTextIndex)
+// throw new UnsupportedOperationException();
+//
+// final ISubjectCentricTextIndexer<?> textIndexer = getSubjectCentricSearchEngine();
+//
+// final AbstractTripleStore db = getContainer();
+//
+// /*
+// * Keep a collection of literals to be indexed for each subject.
+// */
+// final Collection<IV<?,?>> literals = new LinkedList<IV<?,?>>();
+//
+// for (ISPO spo : removed) {
+//
+// literals.clear();
+//
+// /*
+// * Visit all plain, language code, and datatype literals in the
+// * object position of the primary statement index.
+// *
+// * Note: This uses a filter on the ITupleIterator in order to filter
+// * out non-literal terms before they are shipped from a remote index
+// * shard.
+// */
+// final Iterator<ISPO> itr = db.getAccessPath(s, null, null, new SPOFilter<ISPO>() {
+// private static final long serialVersionUID = 1L;
+// @Override
+// public boolean isValid(Object e) {
+// return ((ISPO)e).o().isLiteral();
+// }
+// }).iterator();
+//
+// while (itr.hasNext()) {
+//
+// final ISPO spo = itr.next();
+//
+// literals.add(spo.o());
+//
+// }
+//
+// // flush the last subject
+//// textIndexer.index(s, getTerms(literals).values().iterator());
+//
+// }
+//
+// }
+
/**
* Batch resolution of internal values to {@link BigdataValue}s.
*
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -104,6 +104,8 @@
set.add(BD.MAX_RELEVANCE);
set.add(BD.MIN_RELEVANCE);
set.add(BD.MATCH_ALL_TERMS);
+ set.add(BD.SUBJECT_SEARCH);
+ set.add(BD.SEARCH_TIMEOUT);
searchUris = Collections.unmodifiableSet(set);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -47,6 +47,7 @@
import com.bigdata.bop.bindingSet.ListBindingSet;
import com.bigdata.rdf.internal.constraints.RangeBOp;
import com.bigdata.rdf.internal.impl.literal.XSDNumericIV;
+import com.bigdata.rdf.lexicon.ITextIndexer;
import com.bigdata.rdf.lexicon.IValueCentricTextIndexer;
import com.bigdata.rdf.sparql.ast.ConstantNode;
import com.bigdata.rdf.sparql.ast.GroupNodeBase;
@@ -282,6 +283,14 @@
assertObjectIsLiteral(sp);
+ } else if(uri.equals(BD.SUBJECT_SEARCH)) {
+
+ assertObjectIsLiteral(sp);
+
+ } else if(uri.equals(BD.SEARCH_TIMEOUT)) {
+
+ assertObjectIsLiteral(sp);
+
} else {
throw new AssertionError("Unverified search predicate: " + sp);
@@ -339,6 +348,8 @@
private final Literal minRelevance;
private final Literal maxRelevance;
private final boolean matchAllTerms;
+ private final boolean subjectSearch;
+ private final Literal searchTimeout;
public SearchCall(
final AbstractTripleStore store,
@@ -382,6 +393,8 @@
Literal minRelevance = null;
Literal maxRelevance = null;
boolean matchAllTerms = false;
+ boolean subjectSearch = false;
+ Literal searchTimeout = null;
for (StatementPatternNode meta : statementPatterns.values()) {
@@ -407,6 +420,10 @@
maxRelevance = (Literal) oVal;
} else if (BD.MATCH_ALL_TERMS.equals(p)) {
matchAllTerms = ((Literal) oVal).booleanValue();
+ } else if (BD.SUBJECT_SEARCH.equals(p)) {
+ subjectSearch = ((Literal) oVal).booleanValue();
+ } else if (BD.SEARCH_TIMEOUT.equals(p)) {
+ searchTimeout = (Literal) oVal;
}
}
@@ -421,15 +438,22 @@
this.minRelevance = minRelevance;
this.maxRelevance = maxRelevance;
this.matchAllTerms = matchAllTerms;
+ this.subjectSearch = subjectSearch;
+ this.searchTimeout = searchTimeout;
}
@SuppressWarnings({ "rawtypes", "unchecked" })
private Hiterator<IHit<?>> getHiterator() {
- final IValueCentricTextIndexer<IHit> textIndex = (IValueCentricTextIndexer) store
- .getLexiconRelation().getSearchEngine();
+// final IValueCentricTextIndexer<IHit> textIndex = (IValueCentricTextIndexer) store
+// .getLexiconRelation().getSearchEngine();
+ final ITextIndexer<IHit> textIndex = (ITextIndexer)
+ (this.subjectSearch ?
+ store.getLexiconRelation().getSubjectCentricSearchEngine() :
+ store.getLexiconRelation().getSearchEngine());
+
if (textIndex == null)
throw new UnsupportedOperationException("No free text index?");
@@ -450,7 +474,7 @@
minRank == null ? BD.DEFAULT_MIN_RANK/*1*/ : minRank.intValue()/* minRank */,
maxRank == null ? BD.DEFAULT_MAX_RANK/*Integer.MAX_VALUE*/ : maxRank.intValue()/* maxRank */,
matchAllTerms,
- BD.DEFAULT_TIMEOUT/*0L*//* timeout */,
+ searchTimeout == null ? BD.DEFAULT_TIMEOUT/*0L*/ : searchTimeout.longValue()/* timeout */,
TimeUnit.MILLISECONDS);
}
@@ -481,11 +505,11 @@
if (bindingsClause.length == 1 && !bindingsClause[0].isEmpty()) {
/*
- * FIXME This case is not supported. We need to propagate
- * the bindings into the search engine.
+ * Fixed this by putting the ASTBindingAssigner before the
+ * ASTSearchOptimizer in the DefaultOptimizerList.
*/
- throw new UnsupportedOperationException();
+// throw new UnsupportedOperationException();
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -200,20 +200,6 @@
add(new ASTEmptyGroupOptimizer());
/**
- * Translate {@link BD#SEARCH} and associated magic predicates into a a
- * {@link ServiceNode}. If there are multiple searches in the query,
- * then each is translated into its own {@link ServiceNode}. The magic
- * predicates identify the bindings to be projected out of the named
- * subquery (rank, cosine, etc).
- * <p>
- * Note: Search is most efficiently expressed within named subqueries.
- * This let's you perform additional joins against the solutions from
- * the search service before the results are materialized on a hash
- * index to be joined into the main query.
- */
- add(new ASTSearchOptimizer());
-
- /**
* Rewrites any {@link ProjectionNode} with a wild card into the set of
* variables visible to the {@link QueryBase} having that projection.
* This is done first for the {@link NamedSubqueriesNode} and then
@@ -249,6 +235,20 @@
add(new ASTBindingAssigner());
/**
+ * Translate {@link BD#SEARCH} and associated magic predicates into a a
+ * {@link ServiceNode}. If there are multiple searches in the query,
+ * then each is translated into its own {@link ServiceNode}. The magic
+ * predicates identify the bindings to be projected out of the named
+ * subquery (rank, cosine, etc).
+ * <p>
+ * Note: Search is most efficiently expressed within named subqueries.
+ * This let's you perform additional joins against the solutions from
+ * the search service before the results are materialized on a hash
+ * index to be joined into the main query.
+ */
+ add(new ASTSearchOptimizer());
+
+ /**
* Imposes a LIMIT of ONE for a non-aggregation ASK query.
*/
add(new AskOptimizer());
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -99,8 +99,10 @@
import com.bigdata.rdf.internal.VTE;
import com.bigdata.rdf.internal.impl.BlobIV;
import com.bigdata.rdf.internal.impl.extensions.XSDStringExtension;
+import com.bigdata.rdf.lexicon.BigdataSubjectCentricFullTextIndex;
import com.bigdata.rdf.lexicon.BigdataValueCentricFullTextIndex;
import com.bigdata.rdf.lexicon.ITermIndexCodes;
+import com.bigdata.rdf.lexicon.ITextIndexer;
import com.bigdata.rdf.lexicon.IValueCentricTextIndexer;
import com.bigdata.rdf.lexicon.LexiconKeyOrder;
import com.bigdata.rdf.lexicon.LexiconRelation;
@@ -862,6 +864,20 @@
String DEFAULT_TEXT_INDEX = "true";
+ /**
+ * Boolean option (default <code>true</code>) enables support for a
+ * full text index that may be used to lookup literals by tokens found
+ * in the text of those literals.
+ *
+ * @see #TEXT_INDEXER_CLASS
+ * @see #TEXT_INDEX_DATATYPE_LITERALS
+ * @see #INLINE_TEXT_LITERALS
+ * @see #MAX_INLINE_TEXT_LENGTH
+ */
+ String SUBJECT_CENTRIC_TEXT_INDEX = AbstractTripleStore.class.getName() + ".subjectCentricTextIndex";
+
+ String DEFAULT_SUBJECT_CENTRIC_TEXT_INDEX = "false";
+
/**
* Boolean option enables support for a full text index that may be used
* to lookup datatype literals by tokens found in the text of those
@@ -894,6 +910,25 @@
String DEFAULT_TEXT_INDEXER_CLASS = BigdataValueCentricFullTextIndex.class
.getName();
+ /**
+ * The name of the {@link ITextIndexer} class. The implementation MUST
+ * declare a method with the following signature which will be used to
+ * locate instances of that class.
+ *
+ * <pre>
+ * static public ITextIndexer getInstance(final IIndexManager indexManager,
+ * final String namespace, final Long timestamp,
+ * final Properties properties)
+ * </pre>
+ *
+ * @see #DEFAULT_TEXT_INDEXER_CLASS
+ */
+ String SUBJECT_CENTRIC_TEXT_INDEXER_CLASS = AbstractTripleStore.class.getName()
+ + ".subjectCentricTextIndexerClass";
+
+ String DEFAULT_SUBJECT_CENTRIC_TEXT_INDEXER_CLASS = BigdataSubjectCentricFullTextIndex.class
+ .getName();
+
/*
* Inlining options.
*/
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -284,6 +284,45 @@
final boolean DEFAULT_PREFIX_MATCH = false;
/**
+ * Magic predicate used to query for free text search metadata. Use
+ * in conjunction with {@link #SEARCH} as follows:
+ * <p>
+ * <pre>
+ *
+ * select ?s
+ * where {
+ * ?s bd:search "scale-out RDF triplestore" .
+ * ?s bd:subjectSearch "true" .
+ * }
+ *
+ * </pre>
+ * <p>
+ * The subject-centric search index must be enabled via
+ * {@link AbstractTripleStore.Options#SUBJECT_CENTRIC_TEXT_INDEX}.
+ */
+ final URI SUBJECT_SEARCH = new URIImpl(SEARCH_NAMESPACE+"subjectSearch");
+
+ final boolean DEFAULT_SUBJECT_SEARCH = false;
+
+ /**
+ * Magic predicate used to query for free text search metadata. Use
+ * in conjunction with {@link #SEARCH} as follows:
+ * <p>
+ * <pre>
+ *
+ * select ?s
+ * where {
+ * ?s bd:search "scale-out RDF triplestore" .
+ * ?s bd:searchTimeout "5000" .
+ * }
+ *
+ * </pre>
+ * <p>
+ * Timeout specified in milliseconds.
+ */
+ final URI SEARCH_TIMEOUT = new URIImpl(SEARCH_NAMESPACE+"searchTimeout");
+
+ /**
* The default timeout for a free text search (milliseconds).
*/
final long DEFAULT_TIMEOUT = Long.MAX_VALUE;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAll.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestAll.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -108,6 +108,9 @@
// test suite for access paths reading on the TERMS index.
suite.addTestSuite(TestAccessPaths.class);
+ // test suite for subject-centric text index
+ suite.addTestSuite(TestSubjectCentricFullTextIndex.class);
+
return suite;
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -0,0 +1,541 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Dec 19, 2007
+ */
+
+package com.bigdata.rdf.lexicon;
+
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.AssertionFailedError;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.model.vocabulary.RDFS;
+import org.openrdf.model.vocabulary.XMLSchema;
+
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.spo.TestSPOKeyOrder;
+import com.bigdata.rdf.store.AbstractTripleStore;
+import com.bigdata.rdf.store.AbstractTripleStoreTestCase;
+import com.bigdata.rdf.store.BD;
+import com.bigdata.rdf.store.BigdataValueIteratorImpl;
+import com.bigdata.search.Hit;
+import com.bigdata.search.Hiterator;
+import com.bigdata.striterator.ChunkedWrappedIterator;
+import com.bigdata.striterator.ICloseableIterator;
+import com.bigdata.striterator.Resolver;
+import com.bigdata.striterator.Striterator;
+
+/**
+ * Test of adding terms with the full text index enabled and of lookup of terms
+ * by tokens which appear within those terms.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class TestSubjectCentricFullTextIndex extends AbstractTripleStoreTestCase {
+
+ private static final transient Logger log = Logger.getLogger(TestSubjectCentricFullTextIndex.class);
+
+ /**
+ *
+ */
+ public TestSubjectCentricFullTextIndex() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestSubjectCentricFullTextIndex(String name) {
+ super(name);
+ }
+
+ public Properties getProperties() {
+
+ Properties properties = new Properties(super.getProperties());
+
+ // enable the full text index.
+ properties.setProperty(AbstractTripleStore.Options.TEXT_INDEX,"true");
+ properties.setProperty(AbstractTripleStore.Options.SUBJECT_CENTRIC_TEXT_INDEX,"true");
+
+ return properties;
+
+ }
+
+// /**
+// * Test helper verifies that the term is not in the lexicon, adds the term
+// * to the lexicon, verifies that the term can be looked up by its assigned
+// * term identifier, verifies that the term is now in the lexicon, and
+// * verifies that adding the term again returns the same term identifier.
+// *
+// * @param term
+// * The term.
+// */
+// protected void doAddTermTest(final AbstractTripleStore store,
+// final BigdataValue term) {
+//
+// assertEquals(NULL, store.getIV(term));
+//
+// final IV<?,?> id = store.addTerm(term);
+//
+// assertNotSame(NULL, id);
+//
+// assertEquals(id, store.getIV(term));
+//
+// assertEquals(term, store.getTerm(id));
+//
+// assertEquals(id, store.addTerm(term));
+//
+// }
+
+ private void assertExpectedHits(final AbstractTripleStore store,
+ final String query, final String languageCode,
+ final BigdataValue[] expected) {
+
+ assertExpectedHits(store, query, languageCode, 0f/* minCosine */,
+ expected);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private void assertExpectedHits(final AbstractTripleStore store,
+ final String query, final String languageCode,
+ final float minCosine, final BigdataValue[] expected) {
+
+ final Hiterator hitr = store.getLexiconRelation().getSubjectCentricSearchEngine()
+ .search(query, languageCode, false/* prefixMatch */,
+ minCosine, 1.0d/* maxCosine */,
+ 1/* minRank */, Integer.MAX_VALUE/* maxRank */,
+ false/* matchAllTerms */,
+ Long.MAX_VALUE,//2L/* timeout */,
+ TimeUnit.MILLISECONDS// TimeUnit.SECONDS
+ );
+
+ // assertEquals("#hits", (long) expected.length, itr.size());
+
+ final ICloseableIterator<BigdataValue> itr2 = new BigdataValueIteratorImpl(
+ store, new ChunkedWrappedIterator<IV>(new Striterator(hitr)
+ .addFilter(new Resolver() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected Object resolve(Object e) {
+ final Hit hit = (Hit) e;
+ if (log.isDebugEnabled()) {
+ log.debug(hit);
+ }
+ return hit.getDocId();
+ }
+ })));
+
+ try {
+
+ TestSPOKeyOrder.assertSameIteratorAnyOrder(expected, itr2);
+
+ } catch (AssertionFailedError ex) {
+
+ fail("minCosine=" + minCosine + ", expected="
+ + Arrays.toString(expected) + ", actual=" + hitr, ex);
+
+ } finally {
+
+ itr2.close();
+
+ }
+
+ }
+
+ private LiteralImpl getLargeLiteral(final AbstractTripleStore store) {
+
+ final int len = store.getLexiconRelation().getBlobsThreshold();
+
+ final StringBuilder sb = new StringBuilder(len);
+
+ final String[] tokens = new String[] {
+ "apple",
+ "mary",
+ "john",
+ "barley",
+ "mellow",
+ "pudding",
+ "fries",
+ "peal",
+ "gadzooks"
+ };
+
+ for (int i = 0; sb.length() < len; i++) {
+
+ sb.append(tokens[(i % tokens.length)]);
+
+ sb.append(" ");
+
+ }
+
+ final String s = sb.toString();
+
+ if (log.isInfoEnabled())
+ log.info("length(s)=" + s.length());
+
+ return new LiteralImpl(s);
+
+ }
+
+ public void test_SingleSubject() {
+
+ AbstractTripleStore store = getStore();
+
+ try {
+
+ assertNotNull(store.getLexiconRelation().getSearchEngine());
+
+ final BigdataValueFactory f = store.getValueFactory();
+
+ final BigdataURI s = f.createURI(BD.NAMESPACE+"s");
+
+ final BigdataURI p = f.createURI(BD.NAMESPACE+"p");
+
+ final LiteralImpl largeLiteral = getLargeLiteral(store);
+
+ final BigdataValue[] terms = new BigdataValue[] {//
+ f.createLiteral("abc"),//
+ f.createLiteral("abc", "en"),//
+ f.createLiteral("good day", "en"),//
+ f.createLiteral("gutten tag", "de"),//
+ f.createLiteral("tag team", "en"),//
+ f.createLiteral("the first day", "en"),// // 'the' is a stopword.
+
+ f.createURI("http://www.bigdata.com"),//
+ f.asValue(RDF.TYPE),//
+ f.asValue(RDFS.SUBCLASSOF),//
+ f.asValue(XMLSchema.DECIMAL),//
+
+ f.createBNode(UUID.randomUUID().toString()),//
+ f.createBNode("a12"),//
+
+ f.asValue(largeLiteral),//
+
+ };
+
+ for (BigdataValue o : terms) {
+
+ store.addStatement(s, p, o);
+
+ }
+
+ store.commit();
+
+ // build the subject-centric full text index.
+ store.getLexiconRelation().buildSubjectCentricTextIndex();
+
+ if (log.isInfoEnabled()) {
+ log.info("\n"+store.dumpStore(true, false, false));
+ }
+
+ /*
+ * Note: the language code is only used when tokenizing literals. It
+ * IS NOT applied as a filter to the recovered literals.
+ */
+
+ assertExpectedHits(store, "abc", null/* languageCode */,
+ new BigdataValue[] {
+ s
+ });
+
+ assertExpectedHits(store, "tag", "en", new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "tag", "de", new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "GOOD DAY", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "GOOD DAY", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "day", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s
+ });
+
+ // 'the' is a stopword, so there are no hits.
+ assertExpectedHits(store, "the", "en", new BigdataValue[] {});
+
+ // BLOB
+ assertExpectedHits(store, largeLiteral.getLabel(), null/*lang*/, //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s
+ });
+
+ /*
+ * re-open the store before search to verify that the data were made
+ * restart safe.
+ */
+ if (store.isStable()) {
+
+ store.commit();
+
+ store = reopenStore(store);
+
+ }
+
+ // re-verify the full text index.
+ {
+
+ assertNotNull(store.getLexiconRelation().getSubjectCentricSearchEngine());
+
+ assertExpectedHits(store, "abc", null/* languageCode */,
+ new BigdataValue[] { //
+ s
+ });
+
+ assertExpectedHits(store, "tag", "en", new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "tag", "de", new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "GOOD DAY", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "GOOD DAY", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {//
+ s
+ });
+
+ assertExpectedHits(store, "day", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s
+ });
+
+ // BLOB
+ assertExpectedHits(store, largeLiteral.getLabel(), null/*lang*/, //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s
+ });
+
+ }
+
+ } finally {
+
+ store.__tearDownUnitTest();
+
+ }
+
+ }
+
+ public void test_MultiSubject() {
+
+ AbstractTripleStore store = getStore();
+
+ try {
+
+ assertNotNull(store.getLexiconRelation().getSearchEngine());
+
+ final BigdataValueFactory f = store.getValueFactory();
+
+ final BigdataURI s1 = f.createURI(BD.NAMESPACE+"s1");
+
+ final BigdataURI s2 = f.createURI(BD.NAMESPACE+"s2");
+
+ final BigdataURI s3 = f.createURI(BD.NAMESPACE+"s3");
+
+ final BigdataURI p = f.createURI(BD.NAMESPACE+"p");
+
+ final LiteralImpl largeLiteral = getLargeLiteral(store);
+
+ final BigdataValue[] terms = new BigdataValue[] {//
+ f.createLiteral("abc"),//
+ f.createLiteral("abc", "en"),//
+ f.createLiteral("good day", "en"),//
+ f.createLiteral("gutten tag", "de"),//
+ f.createLiteral("tag team", "en"),//
+ f.createLiteral("the first day", "en"),// // 'the' is a stopword.
+
+ f.createURI("http://www.bigdata.com"),//
+ f.asValue(RDF.TYPE),//
+ f.asValue(RDFS.SUBCLASSOF),//
+ f.asValue(XMLSchema.DECIMAL),//
+
+ f.createBNode(UUID.randomUUID().toString()),//
+ f.createBNode("a12"),//
+
+ f.asValue(largeLiteral),//
+
+ };
+
+ for (BigdataValue o : terms) {
+
+ store.addStatement(s1, p, o);
+
+ }
+
+ for (int i = 0; i < 3; i++) {
+
+ store.addStatement(s2, p, terms[i]);
+
+ }
+
+ for (int i = 3; i < 6; i++) {
+
+ store.addStatement(s3, p, terms[i]);
+
+ }
+
+ store.commit();
+
+ // build the subject-centric full text index.
+ store.getLexiconRelation().buildSubjectCentricTextIndex();
+
+ if (log.isInfoEnabled()) {
+ log.info("\n"+store.dumpStore(true, false, false));
+ }
+
+ /*
+ * Note: the language code is only used when tokenizing literals. It
+ * IS NOT applied as a filter to the recovered literals.
+ */
+
+ assertExpectedHits(store, "abc", null/* languageCode */,
+ new BigdataValue[] {
+ s1, s2
+ });
+
+ assertExpectedHits(store, "tag", "en", new BigdataValue[] {//
+ s1, s3
+ });
+
+ assertExpectedHits(store, "tag", "de", new BigdataValue[] {//
+ s1, s3
+ });
+
+ assertExpectedHits(store, "GOOD DAY", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {//
+ s1, s2, s3
+ });
+
+ assertExpectedHits(store, "day", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s1, s2, s3
+ });
+
+ // 'the' is a stopword, so there are no hits.
+ assertExpectedHits(store, "the", "en", new BigdataValue[] {});
+
+ // BLOB
+ assertExpectedHits(store, largeLiteral.getLabel(), null/*lang*/, //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s1
+ });
+
+ /*
+ * re-open the store before search to verify that the data were made
+ * restart safe.
+ */
+ if (store.isStable()) {
+
+ store.commit();
+
+ store = reopenStore(store);
+
+ }
+
+ // re-verify the full text index.
+ {
+
+ assertNotNull(store.getLexiconRelation().getSubjectCentricSearchEngine());
+
+ assertExpectedHits(store, "abc", null/* languageCode */,
+ new BigdataValue[] { //
+ s1, s2
+ });
+
+ assertExpectedHits(store, "tag", "en", new BigdataValue[] {//
+ s1, s3
+ });
+
+ assertExpectedHits(store, "tag", "de", new BigdataValue[] {//
+ s1, s3
+ });
+
+ assertExpectedHits(store, "GOOD DAY", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {//
+ s1, s2, s3
+ });
+
+ assertExpectedHits(store, "day", "en", //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s1, s2, s3
+ });
+
+ // BLOB
+ assertExpectedHits(store, largeLiteral.getLabel(), null/*lang*/, //
+ .0f, // minCosine
+ new BigdataValue[] {
+ s1
+ });
+
+ }
+
+ } finally {
+
+ store.__tearDownUnitTest();
+
+ }
+
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java 2012-04-01 17:36:23 UTC (rev 6244)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java 2012-04-02 09:15:46 UTC (rev 6245)
@@ -60,7 +60,9 @@
import com.bigdata.journal.BufferMode;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.lexicon.ITextIndexer;
import com.bigdata.rdf.lexicon.IValueCentricTextIndexer;
+import com.bigdata.rdf.model.BigdataURI;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.sail.BigdataSail.Options;
import com.bigdata.rdf.sparql.ast.eval.service.TestSearch;
@@ -87,21 +89,18 @@
super(name);
}
-// /**
-// * Overriden to use a persistent backing store.
-// */
-// public Properties getProperties() {
-//
-// Properties properties = super.getProperties();
-//
-// // use a disk-based mode since we will re-open the store to test restart safety.
-// properties.setProperty(Options.BUFFER_MODE,BufferMode.Disk.toString());
-//
-// properties.setProperty(Options.FILE,file.toString());
-//
-// return properties;
-//
-// }
+ /**
+ * Overriden to allow the subject-centric full text index.
+ */
+ public Properties getProperties() {
+
+ Properties properties = super.getProperties();
+
+ properties.setProperty(Options.SUBJECT_CENTRIC_TEXT_INDEX, "true");
+
+ return properties;
+
+ }
// /**
// * Overriden to cause the backing store to be deleted.
@@ -789,7 +788,7 @@
Collection<BindingSet> answer = new LinkedList<BindingSet>();
- final IValueCentricTextIndexer search =
+ final ITextIndexer search =
sail.getDatabase().getLexiconRelation().getSearchEngine();
final Hiterator<IHit> hits =
search.search(searchQuery,
@@ -1360,6 +1359,848 @@
}
+ public void testSubjectSearch() throws Exception {
+
+ final BigdataSail sail = getSail();
+ try {
+
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+ final BigdataSailRepositoryConnection cxn =
+ (BigdataSailRepositoryConnection) repo.getConnection();
+
+ try {
+
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ final URI s1 = vf.createURI(BD.NAMESPACE+"s1");
+ final URI s2 = vf.createURI(BD.NAMESPACE+"s2");
+ final URI s3 = vf.createURI(BD.NAMESPACE+"s3");
+ final URI s4 = vf.createURI(BD.NAMESPACE+"s4");
+ final URI s5 = vf.createURI(BD.NAMESPACE+"s5");
+ final URI s6 = vf.createURI(BD.NAMESPACE+"s6");
+ final URI s7 = vf.createURI(BD.NAMESPACE+"s7");
+ final URI s8 = vf.createURI(BD.NAMESPACE+"s8");
+ final Literal l1 = vf.createLiteral("how");
+ final Literal l2 = vf.createLiteral("now");
+ final Literal l3 = vf.createLiteral("brown");
+ final Literal l4 = vf.createLiteral("cow");
+ final Literal l5 = vf.createLiteral("how now");
+ final Literal l6 = vf.createLiteral("brown cow");
+ final Literal l7 = vf.createLiteral("how now brown cow");
+ final Literal l8 = vf.createLiteral("toilet");
+
+ cxn.add(s1, RDFS.LABEL, l1);
+ cxn.add(s2, RDFS.LABEL, l2);
+ cxn.add(s3, RDFS.LABEL, l3);
+ cxn.add(s4, RDFS.LABEL, l4);
+ cxn.add(s5, RDFS.LABEL, l5);
+ cxn.add(s6, RDFS.LABEL, l6);
+ cxn.add(s7, RDFS.LABEL, l7);
+ cxn.add(s8, RDFS.LABEL, l8);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ sail.getDatabase().getLexiconRelation().buildSubjectCentricTextIndex();
+
+ final Map<IV, Literal> literals = new LinkedHashMap<IV, Literal>();
+ literals.put(((BigdataURI)s1).getIV(), l1);
+ literals.put(((BigdataURI)s2).getIV(), l2);
+ literals.put(((BigdataURI)s3).getIV(), l3);
+ literals.put(((BigdataURI)s4).getIV(), l4);
+ literals.put(((BigdataURI)s5).getIV(), l5);
+ literals.put(((BigdataURI)s6).getIV(), l6);
+ literals.put(((BigdataURI)s7).getIV(), l7);
+ literals.put(((BigdataURI)s8).getIV(), l8);
+
+ final Map<IV, URI> uris = new LinkedHashMap<IV, URI>();
+ uris.put(((BigdataURI)s1).getIV(), s1);
+ uris.put(((BigdataURI)s2).getIV(), s2);
+ uris.put(((BigdataURI)s3).getIV(), s3);
+ uris.put(((BigdataURI)s4).getIV(), s4);
+ uris.put(((BigdataURI)s5).getIV(), s5);
+ uris.put(((BigdataURI)s6).getIV(), s6);
+ uris.put(((BigdataURI)s7).getIV(), s7);
+ uris.put(((BigdataURI)s8).getIV(), s8);
+
+/**/
+ if (log.isInfoEnabled()) {
+ log.info("\n" + sail.getDatabase().dumpStore());
+ }
+
+ {
+ final String searchQuery = "how now brown cow";
+
+ final String query =
+ "select ?s ?score " +
+ "where " +
+ "{ " +
+ " ?s <"+BD.SEARCH+"> \""+searchQuery+"\" . " +
+ " ?s <"+BD.RELEVANCE+"> ?score . " +
+ " ?s <"+BD.SUBJECT_SEARCH+"> true . " +
+ "} " +
+ "order by desc(?score)";
+
+ final TupleQuery tupleQuery =
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+ TupleQueryResult result = tupleQuery.evaluate();
+
+ int i = 0;
+ while (result.hasNext()) {
+ final BindingSet tmp = result.next();
+ if (log.isInfoEnabled())
+ log.info(i + ": " + tmp.toString());
+ i++;
+ }
+ assertEquals("wrong # of results", 7, i);
+
+ result = tupleQuery.evaluate();
+
+ Collection<BindingSet> answer = new LinkedList<BindingSet>();
+
+ final ITextIndexer search =
+ sail.getDatabase().getLexiconRelation().getSubjectCentricSearchEngine();
+ final Hiterator<IHit> hits =
+ search.search(searchQuery,
+ null, // languageCode
+ BD.DEFAULT_PREFIX_MATCH,//false, // prefixMatch
+ BD.DEFAULT_MIN_RELEVANCE,//0d, // minCosine
+ BD.DEFAULT_MAX_RELEVANCE,//1.0d, // maxCosine
+ BD.DEFAULT_MIN_RANK,//1
+ BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
+ BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ BD.DEFAULT_TIMEOUT,//1000L, // timeout
+ TimeUnit.MILLISECONDS // unit
+ );
+
+ while (hits.hasNext()) {
+ final IHit hit = hits.next();
+ final IV id = (IV)hit.getDocId();
+ final Literal score = vf.createLiteral(hit.getCosine());
+ final URI s = uris.get(id);
+ final BindingSet bs = createBindingSet(
+ new BindingImpl("s", s),
+ new BindingImpl("score", score));
+ if(log.isInfoEnabled())
+ log.info(bs);
+ answer.add(bs);
+ }
+
+ compare(result, answer);
+
+ }
+
+ {
+ final String searchQuery = "how now brown cow";
+ final int maxHits = 5;
+
+ final String query =
+ "select ?s ?score " +
+ "where " +
+ "{ " +
+ " ?s <"+BD.SEARCH+"> \""+searchQuery+"\" . " +
+ " ?s <"+BD.RELEVANCE+"> ?score . " +
+ " ?s <"+BD.MAX_RANK+"> \""+maxHits+"\" . " +
+ " ?s <"+BD.SUBJECT_SEARCH+"> true . " +
+ "} " +
+ "order by desc(?score)";
+
+ final TupleQuery tupleQuery =
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+ TupleQueryResult result = tupleQuery.evaluate();
+
+ int i = 0;
+ while (result.hasNext()) {
+ final BindingSet tmp = result.next();
+ if (log.isInfoEnabled())
+ log.info(i + ": " + tmp.toString());
+ i++;
+ }
+ assertEquals("wrong # of results", 5, i);
+
+ result = tupleQuery.evaluate();
+
+ Collection<BindingSet> answer = new LinkedList<BindingSet>();
+
+ final ITextIndexer search =
+ sail.getDatabase().getLexiconRelation().getSubjectCentricSearchEngine();
+ final Hiterator<IHit> hits =
+ search.search(searchQuery,
+ null, // languageCode
+ BD.DEFAULT_PREFIX_MATCH,//false, // prefixMatch
+ ...
[truncated message content] |
|
From: <tho...@us...> - 2012-04-02 13:32:32
|
Revision: 6247
http://bigdata.svn.sourceforge.net/bigdata/?rev=6247&view=rev
Author: thompsonbry
Date: 2012-04-02 13:32:21 +0000 (Mon, 02 Apr 2012)
Log Message:
-----------
Added the CustomServiceFactory which extends ServiceFactory and provides a hook for services to notice mutable BigdataSailConnection starts. This makes it possible for a custom service to register an IChangeLog listener and thereby notice database updates (statements added or removed).
Added a test suite for the ServiceRegistry.
@see https://sourceforge.net/apps/mediawiki/bigdata/index.php?title=FederatedQuery
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/CustomServiceFactory.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java
branches/BIGDATA_RELEASE_1_2_0/ctc-striterators/src/java/cutthecrap/utils/striterators/ReadOnlyIterator.java
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/CustomServiceFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/CustomServiceFactory.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/CustomServiceFactory.java 2012-04-02 13:32:21 UTC (rev 6247)
@@ -0,0 +1,56 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2012. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Apr 2, 2012
+ */
+
+package com.bigdata.rdf.sparql.ast.service;
+
+import com.bigdata.rdf.changesets.IChangeLog;
+import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
+
+/**
+ * Interface for custom services.
+ *
+ * @see <a
+ * href="https://sourceforge.net/apps/mediawiki/bigdata/index.php?title=FederatedQuery">
+ * Federated Query and Custom Services</a>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public interface CustomServiceFactory extends ServiceFactory {
+
+ /**
+ * Callback is used to notify the {@link ServiceFactory} of connection
+ * starts. If the service wishes to be notified of updates, then it must
+ * return a {@link IChangeLog} implementation here. The listener will be
+ * obtained when the {@link ServiceFactory} is registered.
+ *
+ * @return The {@link IChangeLog} listener -or- <code>null</code> if the
+ * service does not want to observe changes.
+ */
+ void startConnection(BigdataSailConnection conn);
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/CustomServiceFactory.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java 2012-04-02 09:19:51 UTC (rev 6246)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java 2012-04-02 13:32:21 UTC (rev 6247)
@@ -1,7 +1,9 @@
package com.bigdata.rdf.sparql.ast.service;
+import java.util.Iterator;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.http.conn.ClientConnectionManager;
@@ -11,8 +13,14 @@
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.store.BD;
+import cutthecrap.utils.striterators.ReadOnlyIterator;
+
/**
* Registry for service calls.
+ *
+ * @see <a
+ * href="https://sourceforge.net/apps/mediawiki/bigdata/index.php?title=FederatedQuery">
+ * Federated Query and Custom Services</a>
*/
public class ServiceRegistry {
@@ -27,17 +35,37 @@
}
- private ConcurrentMap<URI, ServiceFactory> services;
- private ConcurrentMap<URI/*from*/, URI/*to*/> aliases;
+ /**
+ * Primary {@link ServiceFactory} registration.
+ */
+ private final ConcurrentMap<URI, ServiceFactory> services;
+ /**
+ * Aliases for registered {@link ServiceFactory}s.
+ */
+ private final ConcurrentMap<URI/* from */, URI/* to */> aliases;
+
+ /**
+ * The set of registered {@link ServiceFactory}s is also maintained here for
+ * fast, safe iteration by {@link #services()}.
+ */
+ private final CopyOnWriteArrayList<CustomServiceFactory> customServices;
+
+ /**
+ * The default {@link ServiceFactory} used for REMOTE SPARQL SERVICE end
+ * points which are not otherwise registered.
+ */
private AtomicReference<ServiceFactory> defaultServiceFactoryRef;
protected ServiceRegistry() {
services = new ConcurrentHashMap<URI, ServiceFactory>();
+ customServices = new CopyOnWriteArrayList<CustomServiceFactory>();
+
aliases = new ConcurrentHashMap<URI, URI>();
+ // Add the Bigdata search service.
add(BD.SEARCH, new SearchServiceFactory());
defaultServiceFactoryRef = new AtomicReference<ServiceFactory>(
@@ -74,18 +102,28 @@
public final void add(final URI serviceURI, final ServiceFactory factory) {
- if (aliases.containsKey(serviceURI)) {
+ synchronized (this) {
- throw new UnsupportedOperationException("Already declared.");
+ if (aliases.containsKey(serviceURI)) {
- }
+ throw new UnsupportedOperationException("Already declared.");
- if (services.putIfAbsent(serviceURI, factory) != null) {
+ }
- throw new UnsupportedOperationException("Already declared.");
+ if (services.putIfAbsent(serviceURI, factory) != null) {
+ throw new UnsupportedOperationException("Already declared.");
+
+ }
+
+ if (factory instanceof CustomServiceFactory) {
+
+ customServices.add((CustomServiceFactory) factory);
+
+ }
+
}
-
+
}
/**
@@ -100,16 +138,30 @@
boolean modified = false;
- if (aliases.remove(serviceURI) != null) {
+ synchronized (this) {
- modified = true;
+ if (aliases.remove(serviceURI) != null) {
- }
+ // removed an alias.
+ modified = true;
- if (services.remove(serviceURI) != null) {
+ }
- modified = true;
+ // Remove the factory.
+ final ServiceFactory factory = services.remove(serviceURI);
+ if (factory != null) {
+
+ modified = true;
+
+ if(factory instanceof CustomServiceFactory) {
+
+ customServices.remove(factory);
+
+ }
+
+ }
+
}
return modified;
@@ -117,11 +169,22 @@
}
/**
+ * Register one URI as an alias for another.
*
* @param serviceURI
- * The URI of a service which is already declared.
+ * The URI of a service. It is expressly permitted to register an
+ * alias for a URI which does not have a registered
+ * {@link ServiceFactory}. This may be used to alias a remote URI
+ * which you want to intercept locally.
* @param aliasURI
* The URI of an alias under which that service may be accessed.
+ * @throws IllegalStateException
+ * if the <i>serviceURI</i> has already been registered as a
+ * alias (you must {@link #remove(URI)} the old alias before you
+ * can map it against a different <i>serviceURI</i>).
+ * @throws IllegalStateException
+ * if the <i>aliasURI</i> has already been registered as a
+ * service (you can not mask an existing service registration).
*/
public final void addAlias(final URI serviceURI, final URI aliasURI) {
@@ -131,18 +194,61 @@
if (aliasURI == null)
throw new IllegalArgumentException();
- if (services.containsKey(serviceURI)) {
+ synchronized (this) {
- throw new UnsupportedOperationException("ServiceURI:" + serviceURI
- + " already registered.");
+ /*
+ * Note: it is expressly permitted to register an alias for a URI
+ * which does not have a registered ServiceFactory. This may be used
+ * to alias a remote URI which you want to intercept locally.
+ */
+
+// // Lookup the service.
+// final ServiceFactory service = services.get(serviceURI);
+//
+// if (service == null) {
+//
+// throw new IllegalStateException("No such service: uri="
+// + serviceURI);
+//
+// }
+
+ if (services.containsKey(aliasURI)) {
+ throw new IllegalStateException(
+ "Alias already registered as service: uri=" + aliasURI);
+
+ }
+
+ if (aliases.containsKey(aliasURI)) {
+
+ throw new IllegalStateException(
+ "Alias already registered: uri=" + aliasURI);
+
+ }
+
+ aliases.put(aliasURI, serviceURI);
+
}
- aliases.put(aliasURI, serviceURI);
-
}
/**
+ * Return an {@link Iterator} providing a read-only view of the registered
+ * {@link CustomServiceFactory}s.
+ */
+ public Iterator<CustomServiceFactory> customServices() {
+
+ /*
+ * Note: This relies on the copy-on-write array list for fast and
+ * efficient traversal with snapshot isolation.
+ */
+
+ return new ReadOnlyIterator<CustomServiceFactory>(
+ customServices.iterator());
+
+ }
+
+ /**
* Return the {@link ServiceFactory} for that URI. If the {@link URI} is a
* known alias, then it is resolved before looking up the
* {@link ServiceFactory}.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestAll.java 2012-04-02 09:19:51 UTC (rev 6246)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestAll.java 2012-04-02 13:32:21 UTC (rev 6247)
@@ -108,7 +108,12 @@
* Note: See the NanoSparqlServer test suite for REMOTE SPARQL SERVICE
* evaluation against embedded HTTP end points.
*/
-
+
+ /*
+ * Test suite for registering and managing services.
+ */
+ suite.addTestSuite(TestServiceRegistry.class);
+
return suite;
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java 2012-04-02 13:32:21 UTC (rev 6247)
@@ -0,0 +1,439 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+package com.bigdata.rdf.sparql.ast.eval.service;
+
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.log4j.Logger;
+import org.openrdf.model.URI;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.sail.SailException;
+
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.bop.fed.QueryEngineFactory;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.sail.BigdataSail;
+import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
+import com.bigdata.rdf.sail.sparql.AbstractBigdataExprBuilderTestCase;
+import com.bigdata.rdf.sparql.ast.ConstantNode;
+import com.bigdata.rdf.sparql.ast.JoinGroupNode;
+import com.bigdata.rdf.sparql.ast.service.CustomServiceFactory;
+import com.bigdata.rdf.sparql.ast.service.IServiceOptions;
+import com.bigdata.rdf.sparql.ast.service.OpenrdfNativeServiceOptions;
+import com.bigdata.rdf.sparql.ast.service.RemoteServiceFactoryImpl;
+import com.bigdata.rdf.sparql.ast.service.RemoteServiceOptions;
+import com.bigdata.rdf.sparql.ast.service.ServiceCall;
+import com.bigdata.rdf.sparql.ast.service.ServiceCallCreateParams;
+import com.bigdata.rdf.sparql.ast.service.ServiceFactory;
+import com.bigdata.rdf.sparql.ast.service.ServiceNode;
+import com.bigdata.rdf.sparql.ast.service.ServiceRegistry;
+import com.bigdata.rdf.store.AbstractTripleStore;
+
+/**
+ * Test suite for the {@link ServiceRegistry}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class TestServiceRegistry extends AbstractBigdataExprBuilderTestCase {
+
+ private static final Logger log = Logger
+ .getLogger(TestServiceRegistry.class);
+
+ public TestServiceRegistry() {
+
+ }
+
+ public TestServiceRegistry(final String name) {
+ super(name);
+ }
+
+ /**
+ * Unit test for adding, resolving, and removing a {@link ServiceFactory}.
+ */
+ public void test_addGetRemove() {
+
+ final URI serviceURI1 = new URIImpl("http://www.bigdata.com/myService");
+
+ final URI serviceURI2 = new URIImpl("http://www.bigdata.com/myService2");
+
+ final RemoteServiceOptions options = new RemoteServiceOptions();
+
+ final RemoteServiceFactoryImpl serviceFactory = new RemoteServiceFactoryImpl(
+ options);
+
+ try {
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ // Register.
+ ServiceRegistry.getInstance().add(serviceURI1, serviceFactory);
+
+ // Verify discovery.
+ assertNotNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ // Verify same object.
+ assertTrue(serviceFactory == ServiceRegistry.getInstance().get(
+ serviceURI1));
+
+ } finally {
+
+ // De-register.
+ ServiceRegistry.getInstance().remove(serviceURI1);
+
+ }
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ }
+
+ /**
+ * Unit test service alias.
+ */
+ public void test_serviceAlias() {
+
+ final URI serviceURI1 = new URIImpl("http://www.bigdata.com/myService");
+
+ final URI serviceURI2 = new URIImpl("http://www.bigdata.com/myService2");
+
+ final RemoteServiceOptions options = new RemoteServiceOptions();
+
+ final RemoteServiceFactoryImpl serviceFactory = new RemoteServiceFactoryImpl(
+ options);
+
+ try {
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ // Register.
+ ServiceRegistry.getInstance().add(serviceURI1, serviceFactory);
+
+ // Verify discovery.
+ assertNotNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ // Verify same object.
+ assertTrue(serviceFactory == ServiceRegistry.getInstance().get(
+ serviceURI1));
+
+ /*
+ * Can not register the serviceURI as an alias (for any registered
+ * service URI).
+ */
+ try {
+ ServiceRegistry.getInstance()
+ .addAlias(serviceURI1, serviceURI1);
+ fail("Expecting: " + IllegalStateException.class);
+ } catch (IllegalStateException ex) {
+ if (log.isInfoEnabled())
+ log.info("Ignoring expected exception: " + ex);
+ }
+
+ /*
+ * Now alias the service to the 2nd URI.
+ */
+
+ // Register alias.
+ ServiceRegistry.getInstance().addAlias(serviceURI1, serviceURI2);
+
+ // Verify discovery.
+ assertNotNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNotNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ // Verify same object.
+ assertTrue(serviceFactory == ServiceRegistry.getInstance().get(
+ serviceURI1));
+ assertTrue(serviceFactory == ServiceRegistry.getInstance().get(
+ serviceURI2));
+
+ /*
+ * Can not re-register the same alias.
+ */
+ try {
+ ServiceRegistry.getInstance()
+ .addAlias(serviceURI1, serviceURI2);
+ fail("Expecting: " + IllegalStateException.class);
+ } catch (IllegalStateException ex) {
+ if (log.isInfoEnabled())
+ log.info("Ignoring expected exception: " + ex);
+ }
+
+ } finally {
+
+ // De-register.
+ ServiceRegistry.getInstance().remove(serviceURI1);
+ ServiceRegistry.getInstance().remove(serviceURI2);
+
+ }
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ }
+
+ /**
+ * Unit test verifies that you MAY register an alias for a URI which is NOT
+ * associated with an explicitly registered service.
+ */
+ public void test_serviceAlias2() {
+
+ final AbstractTripleStore store = getStore(getProperties());
+
+ try {
+
+ /*
+ * Declare terms that we will need.
+ */
+
+ final BigdataValueFactory f = store.getValueFactory();
+
+ final BigdataURI serviceURI1 = f
+ .createURI("http://www.bigdata.com/myService");
+
+ final BigdataURI serviceURI2 = f
+ .createURI("http://www.bigdata.com/myService2");
+
+ final BigdataValue[] values = new BigdataValue[] { //
+ serviceURI1,//
+ serviceURI2,//
+ };
+
+ store.getLexiconRelation()
+ .addTerms(values, values.length, false/* readOnly */);
+
+ try {
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ // Register alias.
+ ServiceRegistry.getInstance()
+ .addAlias(serviceURI1, serviceURI2);
+
+ // Verify discovery (both null since neither paired with Service).
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+ assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+
+ /*
+ * Verify create for both end points. One is an alias. Neither
+ * is explicitly paired with a ServiceFactory.
+ */
+ final QueryEngine queryEngine = QueryEngineFactory
+ .getQueryController(store.getIndexManager());
+
+ final ClientConnectionManager cm = queryEngine
+ .getClientConnectionManager();
+
+ final JoinGroupNode groupNode = new JoinGroupNode();
+
+ final ServiceNode serviceNode = new ServiceNode(
+ new ConstantNode(serviceURI1.getIV()), groupNode);
+
+ // the end point which was aliased.
+ {
+
+ final ServiceCall<?> serviceCall = ServiceRegistry
+ .getInstance().toServiceCall(store, cm,
+ serviceURI1, serviceNode);
+
+ assertNotNull(serviceCall);
+
+ }
+
+ // the end point which we did not alias.
+ {
+
+ final ServiceCall<?> serviceCall = ServiceRegistry
+ .getInstance().toServiceCall(store, cm,
+ serviceURI2, serviceNode);
+
+ assertNotNull(serviceCall);
+
+ }
+
+ } finally {
+
+ // De-register alias
+ ServiceRegistry.getInstance().remove(serviceURI1);
+
+ }
+
+ } finally {
+
+ store.destroy();
+
+ }
+
+ }
+
+
+ /**
+ * Unit test a {@link CustomServiceFactory} which hooks the connection start
+ * for the {@link BigdataSail}.
+ *
+ * @throws SailException
+ */
+ public void test_customService() throws SailException {
+
+ final URI serviceURI1 = new URIImpl("http://www.bigdata.com/myService");
+
+ final MyCustomServiceFactory serviceFactory = new MyCustomServiceFactory(
+ new OpenrdfNativeServiceOptions());
+
+ try {
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+
+ // Register.
+ ServiceRegistry.getInstance().add(serviceURI1, serviceFactory);
+
+ // Verify discovery.
+ assertNotNull(ServiceRegistry.getInstance().get(serviceURI1));
+
+ // Verify same object.
+ assertTrue(serviceFactory == ServiceRegistry.getInstance().get(
+ serviceURI1));
+
+ /*
+ * Verify custom service visitation.
+ */
+ {
+
+ final Iterator<CustomServiceFactory> itr = ServiceRegistry
+ .getInstance().customServices();
+
+ boolean found = false;
+
+ while (itr.hasNext()) {
+
+ final CustomServiceFactory t = itr.next();
+
+ if (t == serviceFactory) {
+
+ found = true;
+
+ }
+
+ }
+
+ assertTrue(found);
+
+ }
+
+ /*
+ * Verify hooked on connection start.
+ */
+ {
+ final AbstractTripleStore store = getStore(getProperties());
+ try {
+
+ final BigdataSail sail = new BigdataSail(store);
+ try {
+ sail.initialize();
+ // Nothing started yet.
+ assertEquals("nstarted", 0,
+ serviceFactory.nstarted.get());
+ final BigdataSailConnection conn = sail.getConnection();
+ try {
+ // Verify the service was notified.
+ assertEquals("nstarted", 1,
+ serviceFactory.nstarted.get());
+ } finally {
+ conn.close();
+ }
+
+ } finally {
+ sail.shutDown();
+ }
+
+ } finally {
+ store.destroy();
+ }
+ }
+
+ // Verify the service was notified just once.
+ assertEquals("nstarted", 1, serviceFactory.nstarted.get());
+
+ } finally {
+
+ // De-register.
+ ServiceRegistry.getInstance().remove(serviceURI1);
+
+ }
+
+ // Verify not registered.
+ assertNull(ServiceRegistry.getInstance().get(serviceURI1));
+
+ }
+
+ /**
+ * Private helper class used to verify that new mutable connections are
+ * hooked.
+ */
+ private static class MyCustomServiceFactory implements CustomServiceFactory {
+
+ public final AtomicInteger nstarted = new AtomicInteger();
+
+ private final IServiceOptions serviceOptions;
+
+ public MyCustomServiceFactory(final IServiceOptions serviceOptions) {
+
+ this.serviceOptions = serviceOptions;
+
+ }
+
+ @Override
+ public IServiceOptions getServiceOptions() {
+ return serviceOptions;
+ }
+
+ @Override
+ public ServiceCall<?> create(ServiceCallCreateParams params) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void startConnection(BigdataSailConnection conn) {
+
+ nstarted.incrementAndGet();
+
+ }
+
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2012-04-02 09:19:51 UTC (rev 6246)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2012-04-02 13:32:21 UTC (rev 6247)
@@ -120,6 +120,8 @@
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.QueryRoot;
import com.bigdata.rdf.sparql.ast.eval.ASTEvalHelper;
+import com.bigdata.rdf.sparql.ast.service.CustomServiceFactory;
+import com.bigdata.rdf.sparql.ast.service.ServiceRegistry;
import com.bigdata.rdf.spo.ExplicitSPOFilter;
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.spo.InferredSPOFilter;
@@ -1141,20 +1143,53 @@
protected NotifyingSailConnection getConnectionInternal()
throws SailException {
+ final BigdataSailConnection conn;
try {
// if we have isolatable indices then use a read/write transaction
// @todo finish testing so we can enable this
if (isolatable) {
- return getReadWriteConnection();
+ conn = getReadWriteConnection();
} else {
- return getUnisolatedConnection();
+ conn = getUnisolatedConnection();
+
+ }
+
+ if (!conn.isReadOnly()) {
+
+ /*
+ * Give each registered ServiceFactory instance an opportunity
+ * to intercept the start of this connection.
+ */
+ final Iterator<CustomServiceFactory> itr = ServiceRegistry
+ .getInstance().customServices();
+
+ while (itr.hasNext()) {
+
+ final CustomServiceFactory f = itr.next();
+
+ try {
+
+ f.startConnection(conn);
+
+ } catch (Throwable t) {
+
+ log.error(t, t);
+
+ continue;
+
+ }
+
+ }
+
}
+ return conn;
+
} catch (Exception ex) {
throw new SailException(ex);
Added: branches/BIGDATA_RELEASE_1_2_0/ctc-striterators/src/java/cutthecrap/utils/striterators/ReadOnlyIterator.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/ctc-striterators/src/java/cutthecrap/utils/striterators/ReadOnlyIterator.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/ctc-striterators/src/java/cutthecrap/utils/striterators/ReadOnlyIterator.java 2012-04-02 13:32:21 UTC (rev 6247)
@@ -0,0 +1,66 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2012. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Apr 2, 2012
+ */
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * An iterator wrapper which does not support {@link #remove()}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class ReadOnlyIterator<E> implements Iterator<E> {
+
+ private final Iterator<E> src;
+
+ public ReadOnlyIterator(final Iterator<E> src) {
+
+ if (src == null)
+ throw new IllegalArgumentException();
+
+ this.src = src;
+
+ }
+
+ @Override
+ public boolean hasNext() {
+ return src.hasNext();
+ }
+
+ @Override
+ public E next() {
+ return src.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/ctc-striterators/src/java/cutthecrap/utils/striterators/ReadOnlyIterator.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-04-02 15:46:44
|
Revision: 6248
http://bigdata.svn.sourceforge.net/bigdata/?rev=6248&view=rev
Author: thompsonbry
Date: 2012-04-02 15:46:33 +0000 (Mon, 02 Apr 2012)
Log Message:
-----------
Modified the SPARQL UPDATE code paths to pass back the commitTime.
Modified the NSS to report the commit time.
Added a FORM to the NSS index.html page for SPARQL UPDATE in addition to the SPARQL Query FORM.
Fixed various validation errors in the index.html page.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailUpdate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-war/src/html/index.html
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-02 15:46:33 UTC (rev 6248)
@@ -270,6 +270,8 @@
final long commitTime = context.conn.commit2();
+ context.setCommitTime(commitTime);
+
if (log.isDebugEnabled())
log.debug("COMMIT: commitTime=" + commitTime);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdateContext.java 2012-04-02 15:46:33 UTC (rev 6248)
@@ -27,6 +27,8 @@
package com.bigdata.rdf.sparql.ast.eval;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.openrdf.sail.SailException;
import com.bigdata.journal.ITx;
@@ -53,7 +55,13 @@
public final BigdataSailRepositoryConnection conn;
private boolean includeInferred;
-
+
+ /**
+ * The timestamp associated with the commit point for the update and
+ * <code>-1</code> if until there is a commit.
+ */
+ private AtomicLong commitTime = new AtomicLong(-1);
+
public final boolean isIncludeInferred() {
return includeInferred;
@@ -67,6 +75,22 @@
}
/**
+ * The timestamp associated with the commit point for the update and
+ * <code>-1</code> if until there is a commit.
+ */
+ public long getCommitTime() {
+
+ return commitTime.get();
+
+ }
+
+ public void setCommitTime(final long commitTime) {
+
+ this.commitTime.set(commitTime);
+
+ }
+
+ /**
* @param astContainer
* @param db
*
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java 2012-04-02 15:46:33 UTC (rev 6248)
@@ -663,12 +663,14 @@
* sequence (optional).
* @param includeInferred
* if inferences should be included in various operations.
+ *
+ * @return The timestamp of the commit point.
*
* @throws SailException
*
* TODO timeout for update?
*/
- static public void executeUpdate(//
+ static public long executeUpdate(//
final BigdataSailRepositoryConnection conn,//
final ASTContainer astContainer,//
final Dataset dataset,
@@ -711,6 +713,8 @@
*/
AST2BOpUpdate.convertUpdate(ctx);
+ return ctx.getCommitTime();
+
} catch (Exception ex) {
throw new UpdateExecutionException(ex);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailUpdate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailUpdate.java 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailUpdate.java 2012-04-02 15:46:33 UTC (rev 6248)
@@ -119,7 +119,20 @@
@Override
public void execute() throws UpdateExecutionException {
+
+ execute2();
+
+ }
+ /**
+ * Execute a SPARQL UPDATE request.
+ *
+ * @return The timestamp of the commit point for that UPDATE.
+ *
+ * @throws UpdateExecutionException
+ */
+ public long execute2() throws UpdateExecutionException {
+
// final QueryRoot originalQuery = astContainer.getOriginalAST();
//
//// if (getMaxQueryTime() > 0)
@@ -128,7 +141,7 @@
//
// originalQuery.setIncludeInferred(getIncludeInferred());
- ASTEvalHelper.executeUpdate(
+ return ASTEvalHelper.executeUpdate(
((BigdataSailRepositoryConnection) getConnection()),
astContainer,//
dataset,//
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2012-04-02 15:46:33 UTC (rev 6248)
@@ -23,6 +23,8 @@
package com.bigdata.rdf.sail.webapp;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.LinkedList;
@@ -554,7 +556,7 @@
this.update = true;
this.queryType = null;
this.mimeType = null;
- this.charset = null;
+ this.charset = Charset.forName("UTF-8");
this.fileExt = null;
this.req = req;
this.explain = req.getParameter(EXPLAIN) != null;
@@ -982,8 +984,14 @@
/**
* Executes a SPARQL UPDATE.
*/
- private class UpdateTask extends AbstractQueryTask {
+ class UpdateTask extends AbstractQueryTask {
+ /**
+ * The timestamp for the commit point associated with the update and
+ * <code>-1</code> if the commit point has not yet been assigned.
+ */
+ public AtomicLong commitTime = new AtomicLong(-1);
+
public UpdateTask(final String namespace, final long timestamp,
final String baseURI, final ASTContainer astContainer,
final HttpServletRequest req, final OutputStream os) {
@@ -1004,8 +1012,63 @@
final BigdataSailUpdate update = setupUpdate(cxn);
- update.execute();
+ this.commitTime.set(update.execute2());
+ /*
+ * Setup the response headers.
+ */
+
+// // No caching for UPDATE.
+// resp.addHeader("Cache-Control", "no-cache");
+
+// final Charset charset = updateTask.charset;
+
+ final Writer w = new OutputStreamWriter(os, charset);
+
+ try {
+
+ final HTMLBuilder doc = new HTMLBuilder(charset.name(), w);
+
+ {
+
+ XMLBuilder.Node current = doc.root("html");
+ {
+ current = current.node("head");
+ current.node("meta")
+ .attr("http-equiv", "Content-Type")
+ .attr("content",
+ "text/html;charset=" + charset.name())
+ .close();
+ current.node("title").textNoEncode("bigdata®")
+ .close();
+ current = current.close();// close the head.
+ }
+
+ // open the body
+ current = current.node("body");
+
+ current.node("p")//
+ .text("commitTime=" + commitTime.get())//
+ .close();
+
+ // current.node("p")//
+ // .text("commitTime=" + updateTask.commitTime.get())//
+ // .close();
+
+ doc.closeAll(current);
+
+ }
+
+ w.flush();
+
+ } finally {
+
+ w.close();
+ // os.flush();
+ // os.close();
+
+ }
+
}
}
@@ -1056,7 +1119,7 @@
*/
final String baseURI = req.getRequestURL().toString();
- if(update) {
+ if (update) {
/*
* Parse the query so we can figure out how it will need to be executed.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2012-04-02 15:46:33 UTC (rev 6248)
@@ -28,6 +28,7 @@
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
+import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
@@ -70,6 +71,7 @@
import com.bigdata.rdf.sail.sparql.ast.SimpleNode;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.AbstractQueryTask;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.RunningQuery;
+import com.bigdata.rdf.sail.webapp.BigdataRDFContext.UpdateTask;
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.QueryRoot;
import com.bigdata.rdf.store.AbstractTripleStore;
@@ -318,15 +320,15 @@
// final boolean explain =
// req.getParameter(BigdataRDFContext.EXPLAIN) != null;
- final AbstractQueryTask queryTask;
+ final UpdateTask updateTask;
try {
/*
* Attempt to construct a task which we can use to evaluate the
* query.
*/
- queryTask = context
- .getQueryTask(namespace, timestamp, updateStr,
- null/* acceptOverride */, req, os, true/* update */);
+ updateTask = (UpdateTask) context.getQueryTask(namespace,
+ timestamp, updateStr, null/* acceptOverride */, req,
+ os, true/* update */);
} catch (MalformedQueryException ex) {
/*
* Send back a BAD REQUEST (400) along with the text of the
@@ -337,21 +339,11 @@
return;
}
- final FutureTask<Void> ft = new FutureTask<Void>(queryTask);
+ final FutureTask<Void> ft = new FutureTask<Void>(updateTask);
if (log.isTraceEnabled())
log.trace("Will run update: " + updateStr);
- /*
- * Setup the response headers.
- */
-
- resp.setStatus(HTTP_OK);
- resp.setContentType(BigdataServlet.MIME_TEXT_PLAIN);
-
- // No caching for UPDATE.
- resp.addHeader("Cache-Control", "no-cache");
-
// if (queryTask.explain) {
// resp.setContentType(BigdataServlet.MIME_TEXT_HTML);
// final Writer w = new OutputStreamWriter(os, queryTask.charset);
@@ -427,9 +419,7 @@
// Wait for the Future.
ft.get();
-//
-// }
-
+
} catch (Throwable e) {
try {
throw BigdataRDFServlet.launderThrowable(e, resp, updateStr);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-war/src/html/index.html
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-war/src/html/index.html 2012-04-02 13:32:21 UTC (rev 6247)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-war/src/html/index.html 2012-04-02 15:46:33 UTC (rev 6248)
@@ -37,10 +37,10 @@
URL. For that reason ONLY this operation defaults to a POST. You SHOULD
use GET for database queries since they are, by and large, idempotent.
-->
-<h2>Query Data:</h2>
+<h2><a href="http://www.w3.org/TR/sparql11-query/"> SPARQL Query </a></h2>
<FORM action="sparql" method="post">
<P>
- <TEXTAREA name="query" rows="30" cols="100" title="Enter SPARQL Query."
+ <TEXTAREA name="query" rows="10" cols="80" title="Enter SPARQL Query."
>SELECT * { ?s ?p ?o } LIMIT 1</TEXTAREA>
</P><P>
<INPUT type="submit" value="Send" title="Submit query.">
@@ -52,17 +52,42 @@
> Analytic
</P>
</FORM>
+<h2><a href="http://www.w3.org/TR/sparql11-update/">SPARQL Update</a></h2>
+<FORM action="sparql" method="post">
+ <P>
+ <TEXTAREA name="update" rows="10" cols="80" title="Enter SPARQL Update."
+ >
+PREFIX dc: <http://purl.org/dc/elements/1.1/>
+INSERT DATA
+{
+ <http://example/book1> dc:title "A new book" ;
+ dc:creator "A.N.Other" .
+}</TEXTAREA>
+ </P><P>
+ <INPUT type="submit" value="Send" title="Submit Update.">
+ <!--INPUT type="checkbox" name="explain" value="true"
+ title="Explain query plan rather than returning the query results."
+ > Explain-->
+ <INPUT type="checkbox" name="analytic" value="true"
+ title="Enable the analytic query package."
+ > Analytic
+ </P>
+</FORM>
<p>
-Here are some useful namespaces for queries:
+<!-- Note: Some common characters need to be escaped here and also in the SPARQL
+ examples above.
+ -->
+Here are some useful namespaces:
</p>
-<dl>
-<dt>prefix dc: <http://purl.org/dc/elements/1.1/></dt>
-<dt>prefix xsd: <http://www.w3.org/2001/XMLSchema#></dt>
-<dt>prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#></dt>
-<dt>prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#></dt>
-<dt>prefix owl: <http://www.w3.org/2002/07/owl#></dt>
-</dl>
-
+<pre>
+prefix dc: <http://purl.org/dc/elements/1.1/>
+prefix xsd: <http://www.w3.org/2001/XMLSchema#>
+prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+prefix owl: <http://www.w3.org/2002/07/owl#>
+prefix hint: <http://www.bigdata.com/queryHints#>
+</pre>
+<!-- Note: Use SPARQL Update "LOAD" instead.
<h2>Upload Data (URL):</h2>
<form action="sparql" method="post">
<p>
@@ -71,6 +96,6 @@
<input type="submit" value="Upload">
</p>
</form>
-
+-->
</body>
</html>
\ No newline at end of file
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-04-02 23:08:52
|
Revision: 6251
http://bigdata.svn.sourceforge.net/bigdata/?rev=6251&view=rev
Author: thompsonbry
Date: 2012-04-02 23:08:45 +0000 (Mon, 02 Apr 2012)
Log Message:
-----------
Fixed TestFederatedQuery which was failing to remove the alias.
Fixed TestServiceRegistry which was not verifying that the alias, once removed, could be re-aliased (it could and can).
Javadoc on ServiceRegistry.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestFederatedQuery.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java 2012-04-02 18:57:46 UTC (rev 6250)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/service/ServiceRegistry.java 2012-04-02 23:08:45 UTC (rev 6251)
@@ -130,7 +130,8 @@
* Remove a service from the registry and/or set of known aliases.
*
* @param serviceURI
- * The service URI.
+ * The URI of the service -or- the URI of an alias registered
+ * using {@link #addAlias(URI, URI)}.
*
* @return <code>true</code> iff a service for that URI was removed.
*/
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java 2012-04-02 18:57:46 UTC (rev 6250)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/service/TestServiceRegistry.java 2012-04-02 23:08:45 UTC (rev 6251)
@@ -202,6 +202,17 @@
assertNull(ServiceRegistry.getInstance().get(serviceURI1));
assertNull(ServiceRegistry.getInstance().get(serviceURI2));
+ /*
+ * Verify that we can re-register the alias.
+ */
+ try {
+ // Register alias.
+ ServiceRegistry.getInstance().addAlias(serviceURI1, serviceURI2);
+ } finally {
+ // Remove alias.
+ ServiceRegistry.getInstance().remove(serviceURI2);
+ }
+
}
/**
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestFederatedQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestFederatedQuery.java 2012-04-02 18:57:46 UTC (rev 6250)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestFederatedQuery.java 2012-04-02 23:08:45 UTC (rev 6251)
@@ -485,9 +485,9 @@
} finally {
- // De-register those service URIs.
- ServiceRegistry.getInstance().remove(serviceURI1);
- ServiceRegistry.getInstance().remove(serviceURI2);
+ // De-register those service aliases.
+ ServiceRegistry.getInstance().remove(serviceURI1_alias);
+ ServiceRegistry.getInstance().remove(serviceURI2_alias);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2012-04-03 16:38:49
|
Revision: 6253
http://bigdata.svn.sourceforge.net/bigdata/?rev=6253&view=rev
Author: mrpersonick
Date: 2012-04-03 16:38:42 +0000 (Tue, 03 Apr 2012)
Log Message:
-----------
fixed ticket 539 NotMaterializedException for REGEX on Vocab item
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestRegexBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -304,7 +304,7 @@
}
- if (!iv.isInline() || iv.isExtension() || materializeAll) {
+ if (iv.needsMaterialization() || materializeAll) {
ids.add(iv);
@@ -331,7 +331,7 @@
}
- if (!iv.isInline() || iv.isExtension() || materializeAll) {
+ if (iv.needsMaterialization() || materializeAll) {
ids.add(iv);
@@ -414,7 +414,7 @@
final BigdataValue value = terms.get(iv);
- if (value == null && (iv.isExtension() || !iv.isInline())) {
+ if (value == null && iv.needsMaterialization()) {
throw new RuntimeException("Could not resolve: iv=" + iv);
@@ -457,7 +457,7 @@
final BigdataValue value = terms.get(iv);
- if (value == null && (iv.isExtension() || !iv.isInline())) {
+ if (value == null && iv.needsMaterialization()) {
throw new RuntimeException("Could not resolve: iv=" + iv);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -372,7 +372,7 @@
// final BigdataValueFactory vf = getValueFactory();
- if (iv.isInline() && !iv.isExtension()) {
+ if (iv.isInline() && !iv.needsMaterialization()) {
// if (iv instanceof Literal) {
@@ -405,7 +405,7 @@
* @param iv
* The {@link IV}.
*
- * @return The {@link Literal}.
+ * @return The {@link Value}.
*
* @throws SparqlTypeErrorException
* if the argument is <code>null</code>.
@@ -419,7 +419,7 @@
if (iv == null)
throw new SparqlTypeErrorException();
- if (iv.isInline() && !iv.isExtension()) {
+ if (iv.isInline() && !iv.needsMaterialization()) {
return (Value) iv;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -143,10 +143,10 @@
public boolean accept(final IBindingSet bs) {
@SuppressWarnings("rawtypes")
- final IV var = get(0).get(bs);
+ final Value var = asValue(getAndCheckBound(0, bs));
@SuppressWarnings("rawtypes")
- final IV pattern = get(1).get(bs);
+ final IV pattern = getAndCheckBound(1, bs);
@SuppressWarnings("rawtypes")
final IV flags = arity() > 2 ? get(2).get(bs) : null;
@@ -157,11 +157,7 @@
log.debug("regex flags: " + flags);
}
- // not yet bound
- if (var == null || pattern == null)
- throw new SparqlTypeErrorException();
-
- return accept(var.getValue(), pattern.getValue(),
+ return accept(var, pattern.getValue(),
flags != null ? flags.getValue() : null);
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestRegexBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestRegexBOp.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestRegexBOp.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -128,4 +128,5 @@
}
}
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -77,20 +77,8 @@
@Override
public TupleQueryResult evaluate() throws QueryEvaluationException {
- final QueryRoot originalQuery = astContainer.getOriginalAST();
+ return evaluate((BindingsClause) null);
- if (getMaxQueryTime() > 0)
- originalQuery.setTimeout(TimeUnit.SECONDS
- .toMillis(getMaxQueryTime()));
-
- originalQuery.setIncludeInferred(getIncludeInferred());
-
- final TupleQueryResult queryResult = ASTEvalHelper.evaluateTupleQuery(
- getTripleStore(), astContainer, new QueryBindingSet(
- getBindings()));
-
- return queryResult;
-
}
public TupleQueryResult evaluate(final BindingsClause bc)
@@ -98,7 +86,8 @@
final QueryRoot originalQuery = astContainer.getOriginalAST();
- originalQuery.setBindingsClause(bc);
+ if (bc != null)
+ originalQuery.setBindingsClause(bc);
if (getMaxQueryTime() > 0)
originalQuery.setTimeout(TimeUnit.SECONDS
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -113,7 +113,8 @@
suite.addTestSuite(com.bigdata.rdf.sail.DavidsTestBOps.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
-
+ suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
+
// The Sesame TCK, including the SPARQL test suite.
{
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -124,6 +124,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.DavidsTestBOps.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
// The Sesame TCK, including the SPARQL test suite.
{
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -100,6 +100,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.TestTicket422.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
return suite;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java 2012-04-03 16:33:46 UTC (rev 6252)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -95,6 +95,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.TestTicket473.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
return suite;
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java 2012-04-03 16:38:42 UTC (rev 6253)
@@ -0,0 +1,169 @@
+/**
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.sail;
+
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.model.vocabulary.RDFS;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.sail.SailTupleQuery;
+
+import com.bigdata.rdf.axioms.NoAxioms;
+import com.bigdata.rdf.internal.XSD;
+import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.sparql.ast.QueryOptimizerEnum;
+import com.bigdata.rdf.store.BD;
+import com.bigdata.rdf.vocab.NoVocabulary;
+
+public class TestRegex extends ProxyBigdataSailTestCase {
+
+ protected static final Logger log = Logger.getLogger(TestRegex.class);
+
+ protected static final boolean INFO = log.isInfoEnabled();
+
+ /**
+ *
+ */
+ public TestRegex() {
+ }
+
+ /**
+ * @param arg0
+ */
+ public TestRegex(String arg0) {
+ super(arg0);
+ }
+
+ @Override
+ public Properties getProperties() {
+
+ Properties props = super.getProperties();
+
+ props.setProperty(BigdataSail.Options.TEXT_INDEX, "true");
+
+ return props;
+
+ }
+
+ public void testRegex() throws Exception {
+
+ final BigdataSail sail = getSail();
+ try {
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+
+ final RepositoryConnection cxn = repo.getConnection();
+
+ try {
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ /*
+ * Create some terms.
+ */
+ final URI X = vf.createURI(BD.NAMESPACE + "X");
+ final Literal label = vf.createLiteral("John");
+
+ /*
+ * Create some statements.
+ */
+ cxn.add(X, RDF.TYPE, RDFS.RESOURCE);
+ cxn.add(X, RDFS.LABEL, label);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ if (log.isInfoEnabled()) {
+ log.info(sail.getDatabase().dumpStore());
+ }
+
+ {
+
+ String query =
+ "select * where { ?s ?p ?o . FILTER (regex(?o,\"John\",\"i\")) }";
+
+ final SailTupleQuery tupleQuery = (SailTupleQuery)
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+
+ if (log.isInfoEnabled()) {
+
+ log.info(query);
+
+// final BigdataSailTupleQuery bdTupleQuery =
+// (BigdataSailTupleQuery) tupleQuery;
+// final QueryRoot root = (QueryRoot) bdTupleQuery.getTupleExpr();
+// final Projection p = (Projection) root.getArg();
+// final TupleExpr tupleExpr = p.getArg();
+// final SOpTreeBuilder stb = new SOpTreeBuilder();
+// final SOpTree tree = stb.collectSOps(tupleExpr);
+
+// log.info(tree);
+// log.info(query);
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ while (result.hasNext()) {
+ log.info(result.next());
+ }
+
+ }
+
+// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
+// answer.add(createBindingSet(
+// new BindingImpl("a", paul),
+// new BindingImpl("b", mary)
+// ));
+// answer.add(createBindingSet(
+// new BindingImpl("a", brad),
+// new BindingImpl("b", john)
+// ));
+//
+// final TupleQueryResult result = tupleQuery.evaluate();
+// compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ }
+ } finally {
+ if (sail instanceof BigdataSail)
+ ((BigdataSail)sail).__tearDownUnitTest();//shutDown();
+ }
+
+ }
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2012-04-05 08:39:51
|
Revision: 6256
http://bigdata.svn.sourceforge.net/bigdata/?rev=6256&view=rev
Author: mrpersonick
Date: 2012-04-05 08:39:41 +0000 (Thu, 05 Apr 2012)
Log Message:
-----------
fixed more materialization problems
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/XsdStrBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMaterialization.java
Removed Paths:
-------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RegexBOp.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -64,25 +64,35 @@
final IValueExpression<? extends IV> pattern,
final IValueExpression<? extends IV> flags) {
- if (pattern instanceof IConstant &&
- (flags == null || flags instanceof IConstant)) {
+ try {
- final IV parg = ((IConstant<IV>) pattern).get();
-
- final IV farg = flags != null ?
- ((IConstant<IV>) flags).get() : null;
-
- if (parg.hasValue() && (farg == null || farg.hasValue())) {
-
- final Value pargVal = parg.getValue();
-
- final Value fargVal = farg != null ? farg.getValue() : null;
-
- return NV.asMap(
- new NV(Annotations.PATTERN,
- getPattern(pargVal, fargVal)));
+ if (pattern instanceof IConstant &&
+ (flags == null || flags instanceof IConstant)) {
- }
+ final IV parg = ((IConstant<IV>) pattern).get();
+
+ final IV farg = flags != null ?
+ ((IConstant<IV>) flags).get() : null;
+
+ if (parg.hasValue() && (farg == null || farg.hasValue())) {
+
+ final Value pargVal = parg.getValue();
+
+ final Value fargVal = farg != null ? farg.getValue() : null;
+
+ return NV.asMap(
+ new NV(Annotations.PATTERN,
+ getPattern(pargVal, fargVal)));
+
+ }
+
+ }
+
+ } catch (Exception ex) {
+
+ if (log.isInfoEnabled()) {
+ log.info("could not create pattern for: " + pattern + ", " + flags);
+ }
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBOp.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrBOp.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -27,18 +27,18 @@
import java.util.Map;
import org.apache.log4j.Logger;
+import org.openrdf.model.Literal;
import org.openrdf.model.URI;
+import org.openrdf.model.Value;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
-import com.bigdata.bop.NV;
import com.bigdata.rdf.error.SparqlTypeErrorException;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.impl.literal.AbstractLiteralIV;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataValueFactory;
-import com.bigdata.rdf.model.BigdataValueFactoryImpl;
import com.bigdata.rdf.sparql.ast.GlobalAnnotations;
/**
@@ -86,27 +86,22 @@
public IV get(final IBindingSet bs) {
final IV iv = getAndCheckBound(0, bs);
+
+ if (log.isDebugEnabled()) {
+ log.debug(iv);
+ }
+
+ final Value val = asValue(iv);
+ if (log.isDebugEnabled()) {
+ log.debug(val);
+ }
+
// use to create my simple literals
final BigdataValueFactory vf = getValueFactory();
- if (iv.isInline() && !iv.isExtension()) {
- if(iv.isLiteral()){
- return super.asIV(vf.createLiteral(
- ((AbstractLiteralIV)iv).getLabel()), bs);
- }else{
- return super.asIV(vf.createLiteral(iv
- .getInlineValue().toString()), bs);
- }
- }
-
- if (iv.isURI()) {
- // return new simple literal using URI label
- final URI uri = (URI) iv.getValue();
- final BigdataLiteral str = vf.createLiteral(uri.toString());
- return super.asIV(str, bs);
- } else if (iv.isLiteral()) {
- final BigdataLiteral lit = (BigdataLiteral) iv.getValue();
+ if (val instanceof Literal) {
+ final Literal lit = (Literal) val;
if (lit.getDatatype() == null && lit.getLanguage() == null) {
// if simple literal return it
return iv;
@@ -116,9 +111,43 @@
final BigdataLiteral str = vf.createLiteral(lit.getLabel());
return super.asIV(str, bs);
}
+ } else if (val instanceof URI) {
+ // return new simple literal using URI label
+ final BigdataLiteral str = vf.createLiteral(val.stringValue());
+ return super.asIV(str, bs);
} else {
throw new SparqlTypeErrorException();
}
+
+// if (iv.isInline() && !iv.isExtension()) {
+// if(iv.isLiteral()){
+// return super.asIV(vf.createLiteral(
+// ((AbstractLiteralIV)iv).getLabel()), bs);
+// }else{
+// return super.asIV(vf.createLiteral(iv
+// .getInlineValue().toString()), bs);
+// }
+// }
+//
+// if (iv.isURI()) {
+// // return new simple literal using URI label
+// final URI uri = (URI) iv.getValue();
+// final BigdataLiteral str = vf.createLiteral(uri.toString());
+// return super.asIV(str, bs);
+// } else if (iv.isLiteral()) {
+// final BigdataLiteral lit = (BigdataLiteral) iv.getValue();
+// if (lit.getDatatype() == null && lit.getLanguage() == null) {
+// // if simple literal return it
+// return iv;
+// }
+// else {
+// // else return new simple literal using Literal.getLabel
+// final BigdataLiteral str = vf.createLiteral(lit.getLabel());
+// return super.asIV(str, bs);
+// }
+// } else {
+// throw new SparqlTypeErrorException();
+// }
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/XsdStrBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/XsdStrBOp.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/XsdStrBOp.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -27,7 +27,9 @@
import java.util.Map;
import org.apache.log4j.Logger;
+import org.openrdf.model.Literal;
import org.openrdf.model.URI;
+import org.openrdf.model.Value;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
@@ -86,27 +88,22 @@
public IV get(final IBindingSet bs) {
final IV iv = getAndCheckBound(0, bs);
+
+ if (log.isDebugEnabled()) {
+ log.debug(iv);
+ }
+
+ final Value val = asValue(iv);
+ if (log.isDebugEnabled()) {
+ log.debug(val);
+ }
+
// use to create my simple literals
final BigdataValueFactory vf = getValueFactory();
- if (iv.isInline() && !iv.isExtension()) {
- if(iv.isLiteral()){
- return super.asIV(vf.createLiteral(
- ((AbstractLiteralIV)iv).getLabel(), XSD.STRING), bs);
- }else{
- return super.asIV(vf.createLiteral(iv
- .getInlineValue().toString(), XSD.STRING), bs);
- }
- }
-
- if (iv.isURI()) {
- // return new xsd:string literal using URI label
- final URI uri = (URI) iv.getValue();
- final BigdataLiteral str = vf.createLiteral(uri.toString(), XSD.STRING);
- return super.asIV(str, bs);
- } else if (iv.isLiteral()) {
- final BigdataLiteral lit = (BigdataLiteral) iv.getValue();
+ if (val instanceof Literal) {
+ final Literal lit = (Literal) val;
if (lit.getDatatype() != null && lit.getDatatype().equals(XSD.STRING)) {
// if xsd:string literal return it
return iv;
@@ -116,9 +113,48 @@
final BigdataLiteral str = vf.createLiteral(lit.getLabel(), XSD.STRING);
return super.asIV(str, bs);
}
+ } else if (val instanceof URI) {
+ // return new simple literal using URI label
+ final BigdataLiteral str = vf.createLiteral(val.stringValue(), XSD.STRING);
+ return super.asIV(str, bs);
} else {
throw new SparqlTypeErrorException();
}
+
+// final IV iv = getAndCheckBound(0, bs);
+//
+// // use to create my simple literals
+// final BigdataValueFactory vf = getValueFactory();
+//
+// if (iv.isInline() && !iv.isExtension()) {
+// if(iv.isLiteral()){
+// return super.asIV(vf.createLiteral(
+// ((AbstractLiteralIV)iv).getLabel(), XSD.STRING), bs);
+// }else{
+// return super.asIV(vf.createLiteral(iv
+// .getInlineValue().toString(), XSD.STRING), bs);
+// }
+// }
+//
+// if (iv.isURI()) {
+// // return new xsd:string literal using URI label
+// final URI uri = (URI) iv.getValue();
+// final BigdataLiteral str = vf.createLiteral(uri.toString(), XSD.STRING);
+// return super.asIV(str, bs);
+// } else if (iv.isLiteral()) {
+// final BigdataLiteral lit = (BigdataLiteral) iv.getValue();
+// if (lit.getDatatype() != null && lit.getDatatype().equals(XSD.STRING)) {
+// // if xsd:string literal return it
+// return iv;
+// }
+// else {
+// // else return new xsd:string literal using Literal.getLabel
+// final BigdataLiteral str = vf.createLiteral(lit.getLabel(), XSD.STRING);
+// return super.asIV(str, bs);
+// }
+// } else {
+// throw new SparqlTypeErrorException();
+// }
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -113,7 +113,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.DavidsTestBOps.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
- suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestMaterialization.class);
// The Sesame TCK, including the SPARQL test suite.
{
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -124,7 +124,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.DavidsTestBOps.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
- suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestMaterialization.class);
// The Sesame TCK, including the SPARQL test suite.
{
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithSids.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -100,7 +100,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.TestTicket422.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
- suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestMaterialization.class);
return suite;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithoutSids.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -95,7 +95,7 @@
suite.addTestSuite(com.bigdata.rdf.sail.TestTicket473.class);
suite.addTestSuite(com.bigdata.rdf.sail.TestLexJoinOps.class);
- suite.addTestSuite(com.bigdata.rdf.sail.TestRegex.class);
+ suite.addTestSuite(com.bigdata.rdf.sail.TestMaterialization.class);
return suite;
Copied: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMaterialization.java (from rev 6253, branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java)
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMaterialization.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMaterialization.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -0,0 +1,328 @@
+/**
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.sail;
+
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.model.vocabulary.RDFS;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.sail.SailTupleQuery;
+
+import com.bigdata.rdf.axioms.NoAxioms;
+import com.bigdata.rdf.internal.XSD;
+import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.sparql.ast.QueryOptimizerEnum;
+import com.bigdata.rdf.store.BD;
+import com.bigdata.rdf.vocab.NoVocabulary;
+
+public class TestMaterialization extends ProxyBigdataSailTestCase {
+
+ protected static final Logger log = Logger.getLogger(TestMaterialization.class);
+
+ protected static final boolean INFO = log.isInfoEnabled();
+
+ /**
+ *
+ */
+ public TestMaterialization() {
+ }
+
+ /**
+ * @param arg0
+ */
+ public TestMaterialization(String arg0) {
+ super(arg0);
+ }
+
+ @Override
+ public Properties getProperties() {
+
+ Properties props = super.getProperties();
+
+ props.setProperty(BigdataSail.Options.TEXT_INDEX, "true");
+
+ return props;
+
+ }
+
+ public void testRegex() throws Exception {
+
+ final BigdataSail sail = getSail();
+ try {
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+
+ final RepositoryConnection cxn = repo.getConnection();
+
+ try {
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ /*
+ * Create some terms.
+ */
+ final URI X = vf.createURI(BD.NAMESPACE + "X");
+ final Literal label = vf.createLiteral("John");
+
+ /*
+ * Create some statements.
+ */
+ cxn.add(X, RDF.TYPE, RDFS.RESOURCE);
+ cxn.add(X, RDFS.LABEL, label);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ if (log.isInfoEnabled()) {
+ log.info(sail.getDatabase().dumpStore());
+ }
+
+ {
+
+ String query =
+ "select * where { ?s ?p ?o . FILTER (regex(?o,\"John\",\"i\")) }";
+
+ final SailTupleQuery tupleQuery = (SailTupleQuery)
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+
+ if (log.isInfoEnabled()) {
+
+ log.info(query);
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ while (result.hasNext()) {
+ log.info(result.next());
+ }
+
+ }
+
+// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
+// answer.add(createBindingSet(
+// new BindingImpl("a", paul),
+// new BindingImpl("b", mary)
+// ));
+// answer.add(createBindingSet(
+// new BindingImpl("a", brad),
+// new BindingImpl("b", john)
+// ));
+//
+// final TupleQueryResult result = tupleQuery.evaluate();
+// compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ }
+ } finally {
+ if (sail instanceof BigdataSail)
+ ((BigdataSail)sail).__tearDownUnitTest();//shutDown();
+ }
+
+ }
+
+ public void testStr() throws Exception {
+
+ final BigdataSail sail = getSail();
+ try {
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+
+ final RepositoryConnection cxn = repo.getConnection();
+
+ try {
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ /*
+ * Create some terms.
+ */
+ final URI X = vf.createURI(BD.NAMESPACE + "X");
+ final Literal label = vf.createLiteral("John");
+
+ /*
+ * Create some statements.
+ */
+ cxn.add(X, RDF.TYPE, RDFS.RESOURCE);
+ cxn.add(X, RDFS.LABEL, label);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ if (log.isInfoEnabled()) {
+ log.info(sail.getDatabase().dumpStore());
+ }
+
+ {
+
+ String query =
+ "select * where { ?s ?p ?o . FILTER (str(?o) = \""+RDF.PROPERTY+"\") }";
+
+ final SailTupleQuery tupleQuery = (SailTupleQuery)
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+
+ if (log.isInfoEnabled()) {
+
+ log.info(query);
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ if (result.hasNext()) {
+ while (result.hasNext())
+ log.info(result.next());
+ } else {
+ fail("expecting result from the vocab");
+ }
+
+ }
+
+// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
+// answer.add(createBindingSet(
+// new BindingImpl("a", paul),
+// new BindingImpl("b", mary)
+// ));
+// answer.add(createBindingSet(
+// new BindingImpl("a", brad),
+// new BindingImpl("b", john)
+// ));
+ //
+// final TupleQueryResult result = tupleQuery.evaluate();
+// compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ }
+ } finally {
+ if (sail instanceof BigdataSail)
+ ((BigdataSail)sail).__tearDownUnitTest();//shutDown();
+ }
+
+ }
+
+ public void testXsdStr() throws Exception {
+
+ final BigdataSail sail = getSail();
+ try {
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+
+ final RepositoryConnection cxn = repo.getConnection();
+
+ try {
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ /*
+ * Create some terms.
+ */
+ final URI X = vf.createURI(BD.NAMESPACE + "X");
+ final Literal label = vf.createLiteral("John");
+
+ /*
+ * Create some statements.
+ */
+ cxn.add(X, RDF.TYPE, RDFS.RESOURCE);
+ cxn.add(X, RDFS.LABEL, label);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ if (log.isInfoEnabled()) {
+ log.info(sail.getDatabase().dumpStore());
+ }
+
+ {
+
+ String query =
+ "select * where { ?s ?p ?o . FILTER (xsd:string(?o) = \""+RDF.PROPERTY+"\"^^xsd:string) }";
+
+ final SailTupleQuery tupleQuery = (SailTupleQuery)
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+
+ if (log.isInfoEnabled()) {
+
+ log.info(query);
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ if (result.hasNext()) {
+ while (result.hasNext())
+ log.info(result.next());
+ } else {
+ fail("expecting result from the vocab");
+ }
+
+ }
+
+// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
+// answer.add(createBindingSet(
+// new BindingImpl("a", paul),
+// new BindingImpl("b", mary)
+// ));
+// answer.add(createBindingSet(
+// new BindingImpl("a", brad),
+// new BindingImpl("b", john)
+// ));
+ //
+// final TupleQueryResult result = tupleQuery.evaluate();
+// compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ }
+ } finally {
+ if (sail instanceof BigdataSail)
+ ((BigdataSail)sail).__tearDownUnitTest();//shutDown();
+ }
+
+ }
+
+}
Deleted: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestRegex.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -1,169 +0,0 @@
-/**
-Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-
-package com.bigdata.rdf.sail;
-
-import java.util.Properties;
-
-import org.apache.log4j.Logger;
-import org.openrdf.model.Literal;
-import org.openrdf.model.URI;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.vocabulary.RDF;
-import org.openrdf.model.vocabulary.RDFS;
-import org.openrdf.query.QueryLanguage;
-import org.openrdf.query.TupleQueryResult;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.sail.SailTupleQuery;
-
-import com.bigdata.rdf.axioms.NoAxioms;
-import com.bigdata.rdf.internal.XSD;
-import com.bigdata.rdf.sparql.ast.QueryHints;
-import com.bigdata.rdf.sparql.ast.QueryOptimizerEnum;
-import com.bigdata.rdf.store.BD;
-import com.bigdata.rdf.vocab.NoVocabulary;
-
-public class TestRegex extends ProxyBigdataSailTestCase {
-
- protected static final Logger log = Logger.getLogger(TestRegex.class);
-
- protected static final boolean INFO = log.isInfoEnabled();
-
- /**
- *
- */
- public TestRegex() {
- }
-
- /**
- * @param arg0
- */
- public TestRegex(String arg0) {
- super(arg0);
- }
-
- @Override
- public Properties getProperties() {
-
- Properties props = super.getProperties();
-
- props.setProperty(BigdataSail.Options.TEXT_INDEX, "true");
-
- return props;
-
- }
-
- public void testRegex() throws Exception {
-
- final BigdataSail sail = getSail();
- try {
- sail.initialize();
- final BigdataSailRepository repo = new BigdataSailRepository(sail);
-
- final RepositoryConnection cxn = repo.getConnection();
-
- try {
- cxn.setAutoCommit(false);
-
- final ValueFactory vf = sail.getValueFactory();
-
- /*
- * Create some terms.
- */
- final URI X = vf.createURI(BD.NAMESPACE + "X");
- final Literal label = vf.createLiteral("John");
-
- /*
- * Create some statements.
- */
- cxn.add(X, RDF.TYPE, RDFS.RESOURCE);
- cxn.add(X, RDFS.LABEL, label);
-
- /*
- * Note: The either flush() or commit() is required to flush the
- * statement buffers to the database before executing any operations
- * that go around the sail.
- */
- cxn.commit();
-
- if (log.isInfoEnabled()) {
- log.info(sail.getDatabase().dumpStore());
- }
-
- {
-
- String query =
- "select * where { ?s ?p ?o . FILTER (regex(?o,\"John\",\"i\")) }";
-
- final SailTupleQuery tupleQuery = (SailTupleQuery)
- cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
- tupleQuery.setIncludeInferred(true /* includeInferred */);
-
- if (log.isInfoEnabled()) {
-
- log.info(query);
-
-// final BigdataSailTupleQuery bdTupleQuery =
-// (BigdataSailTupleQuery) tupleQuery;
-// final QueryRoot root = (QueryRoot) bdTupleQuery.getTupleExpr();
-// final Projection p = (Projection) root.getArg();
-// final TupleExpr tupleExpr = p.getArg();
-// final SOpTreeBuilder stb = new SOpTreeBuilder();
-// final SOpTree tree = stb.collectSOps(tupleExpr);
-
-// log.info(tree);
-// log.info(query);
-
- final TupleQueryResult result = tupleQuery.evaluate();
- while (result.hasNext()) {
- log.info(result.next());
- }
-
- }
-
-// final Collection<BindingSet> answer = new LinkedList<BindingSet>();
-// answer.add(createBindingSet(
-// new BindingImpl("a", paul),
-// new BindingImpl("b", mary)
-// ));
-// answer.add(createBindingSet(
-// new BindingImpl("a", brad),
-// new BindingImpl("b", john)
-// ));
-//
-// final TupleQueryResult result = tupleQuery.evaluate();
-// compare(result, answer);
-
- }
-
- } finally {
- cxn.close();
- }
- } finally {
- if (sail instanceof BigdataSail)
- ((BigdataSail)sail).__tearDownUnitTest();//shutDown();
- }
-
- }
-
-}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2012-04-05 08:37:22 UTC (rev 6255)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2012-04-05 08:39:41 UTC (rev 6256)
@@ -229,6 +229,7 @@
*/
static final Collection<String> testURIs = Arrays.asList(new String[] {
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-datatype-2"
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/cast/manifest#cast-dT"
// 8, 9, 14-19, 23-30
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2012-04-05 11:17:04
|
Revision: 6257
http://bigdata.svn.sourceforge.net/bigdata/?rev=6257&view=rev
Author: mrpersonick
Date: 2012-04-05 11:16:55 +0000 (Thu, 05 Apr 2012)
Log Message:
-----------
fixing more materialization problems
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/aggregate/AggregateBase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/AVERAGE.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/GROUP_CONCAT.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/SUM.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateTimeUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NumericBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/SameTermBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/StrdtBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/literal/XSDDecimalIV.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/literal/XSDIntegerIV.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/literal/XSDNumericIV.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionRegistry.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/literal/NumericIV.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/aggregate/AggregateBase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/aggregate/AggregateBase.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/aggregate/AggregateBase.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -11,7 +11,7 @@
import com.bigdata.bop.ImmutableBOp;
import com.bigdata.bop.NV;
import com.bigdata.bop.Var;
-import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.impl.literal.NumericIV;
import com.bigdata.rdf.internal.impl.literal.XSDNumericIV;
import com.bigdata.rdf.model.BigdataLiteral;
@@ -193,7 +193,7 @@
/** A ZERO. */
- protected static final transient IV ZERO = new XSDNumericIV<BigdataLiteral>(0);
+ protected static final transient NumericIV ZERO = new XSDNumericIV<BigdataLiteral>(0);
/** An empty string. */
protected static final transient Literal EMPTY_LITERAL = new LiteralImpl("");
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/AVERAGE.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/AVERAGE.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/AVERAGE.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -38,8 +38,11 @@
import com.bigdata.rdf.internal.IVUtility;
import com.bigdata.rdf.internal.NotMaterializedException;
import com.bigdata.rdf.internal.constraints.INeedsMaterialization;
+import com.bigdata.rdf.internal.constraints.IVValueExpression;
import com.bigdata.rdf.internal.constraints.MathBOp;
+import com.bigdata.rdf.internal.constraints.MathUtility;
import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
+import com.bigdata.rdf.internal.impl.literal.NumericIV;
import com.bigdata.rdf.internal.impl.literal.XSDNumericIV;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataValue;
@@ -81,7 +84,7 @@
* Note: This field is guarded by the monitor on the {@link AVERAGE}
* instance.
*/
- private transient IV aggregated = ZERO;
+ private transient NumericIV aggregated = ZERO;
/**
* The #of observed values.
@@ -114,7 +117,7 @@
if(n == 0)
return ZERO;
- return IVUtility.numericalMath(aggregated,
+ return MathUtility.literalMath(aggregated,
new XSDNumericIV<BigdataLiteral>(n), MathBOp.MathOp.DIVIDE);
}
@@ -148,36 +151,51 @@
if (iv != null) {
/*
- * Aggregate non-null values.
+ * Aggregate non-null literal values.
*/
+
+ /*
+ * Aggregate non-null literal values.
+ */
- if (iv.isInline()) {
+ final Literal lit = IVValueExpression.asLiteral(iv);
+
+ if (!MathUtility.checkNumericDatatype(lit))
+ throw new SparqlTypeErrorException();
+
+ aggregated =
+ MathUtility.literalMath(aggregated, lit, MathOp.PLUS);
- // Two IVs.
- aggregated = IVUtility.numericalMath(iv, aggregated,
- MathOp.PLUS);
+// if (!iv.isLiteral())
+// throw new SparqlTypeErrorException();
+//
+// if (iv.isInline()) {
+//
+// // Two IVs.
+// aggregated = IVUtility.numericalMath(iv, aggregated,
+// MathOp.PLUS);
+//
+// } else {
+//
+// // One IV and one Literal.
+// final BigdataValue val1 = iv.getValue();
+//
+// if (val1 == null)
+// throw new NotMaterializedException();
+//
+//// if (!(val1 instanceof Literal))
+//// throw new SparqlTypeErrorException();
+//
+// // Only numeric value can be used in math expressions
+// final URI dt1 = ((Literal)val1).getDatatype();
+// if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1))
+// throw new SparqlTypeErrorException();
+//
+// aggregated = IVUtility.numericalMath((Literal) val1,
+// aggregated, MathOp.PLUS);
+//
+// }
- } else {
-
- // One IV and one Literal.
- final BigdataValue val1 = iv.getValue();
-
- if (val1 == null)
- throw new NotMaterializedException();
-
- if (!(val1 instanceof Literal))
- throw new SparqlTypeErrorException();
-
- // Only numeric value can be used in math expressions
- final URI dt1 = ((Literal)val1).getDatatype();
- if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1))
- throw new SparqlTypeErrorException();
-
- aggregated = IVUtility.numericalMath((Literal) val1,
- aggregated, MathOp.PLUS);
-
- }
-
n++;
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/GROUP_CONCAT.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/GROUP_CONCAT.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/GROUP_CONCAT.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -33,6 +33,7 @@
import com.bigdata.bop.solutions.PipelinedAggregationOp;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.constraints.INeedsMaterialization;
+import com.bigdata.rdf.internal.constraints.IVValueExpression;
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.model.BigdataValueFactoryImpl;
import com.bigdata.rdf.sparql.ast.DummyConstantNode;
@@ -279,12 +280,17 @@
if (iv != null && !done) {
- final String str;
- if (iv.isInline() && !iv.isExtension()) {
- str = iv.getInlineValue().toString();
- } else {
- str = iv.getValue().stringValue();
- }
+ final String str = IVValueExpression.asValue(iv).stringValue();
+ /*
+ * This kind of logic needs to be hidden. It causes too many
+ * problems. Right now IVValueExpression.asValue(IV) knows how
+ * to handle this correctly.
+ */
+// if (iv.isInline() && !iv.isExtension()) {
+// str = iv.getInlineValue().toString();
+// } else {
+// str = iv.getValue().stringValue();
+// }
if (aggregated == null)
aggregated = new StringBuilder(str);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/SUM.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/SUM.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/aggregate/SUM.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -38,7 +38,10 @@
import com.bigdata.rdf.internal.IVUtility;
import com.bigdata.rdf.internal.NotMaterializedException;
import com.bigdata.rdf.internal.constraints.INeedsMaterialization;
+import com.bigdata.rdf.internal.constraints.IVValueExpression;
import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
+import com.bigdata.rdf.internal.constraints.MathUtility;
+import com.bigdata.rdf.internal.impl.literal.NumericIV;
import com.bigdata.rdf.model.BigdataValue;
/**
@@ -78,7 +81,7 @@
* Note: This field is guarded by the monitor on the {@link SUM} instance.
*/
@SuppressWarnings("rawtypes")
- private transient IV aggregated = ZERO;
+ private transient NumericIV aggregated = ZERO;
/**
* The first error encountered since the last {@link #reset()}.
@@ -141,36 +144,44 @@
if (iv != null) {
/*
- * Aggregate non-null values.
+ * Aggregate non-null literal values.
*/
- if (iv.isInline()) {
+ final Literal lit = IVValueExpression.asLiteral(iv);
+
+ if (!MathUtility.checkNumericDatatype(lit))
+ throw new SparqlTypeErrorException();
+
+ aggregated =
+ MathUtility.literalMath(aggregated, lit, MathOp.PLUS);
+
+// if (iv.isInline()) {
+//
+// // Two IVs.
+// aggregated = IVUtility.numericalMath(iv, aggregated,
+// MathOp.PLUS);
+//
+// } else {
+//
+// // One IV and one Literal.
+// final BigdataValue val1 = iv.getValue();
+//
+// if (val1 == null)
+// throw new NotMaterializedException();
+//
+//// if (!(val1 instanceof Literal))
+//// throw new SparqlTypeErrorException();
+//
+// // Only numeric value can be used in math expressions
+// final URI dt1 = ((Literal) val1).getDatatype();
+// if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1))
+// throw new SparqlTypeErrorException();
+//
+// aggregated = IVUtility.numericalMath((Literal) val1,
+// aggregated, MathOp.PLUS);
+//
+// }
- // Two IVs.
- aggregated = IVUtility.numericalMath(iv, aggregated,
- MathOp.PLUS);
-
- } else {
-
- // One IV and one Literal.
- final BigdataValue val1 = iv.getValue();
-
- if (val1 == null)
- throw new NotMaterializedException();
-
- if (!(val1 instanceof Literal))
- throw new SparqlTypeErrorException();
-
- // Only numeric value can be used in math expressions
- final URI dt1 = ((Literal) val1).getDatatype();
- if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1))
- throw new SparqlTypeErrorException();
-
- aggregated = IVUtility.numericalMath((Literal) val1,
- aggregated, MathOp.PLUS);
-
- }
-
}
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -35,22 +35,14 @@
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
-import java.math.RoundingMode;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
-import org.openrdf.model.Literal;
-import org.openrdf.model.URI;
-import org.openrdf.model.datatypes.XMLDatatypeUtil;
import org.openrdf.model.impl.URIImpl;
-import org.openrdf.model.vocabulary.XMLSchema;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
-import com.bigdata.rdf.error.SparqlTypeErrorException;
-import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
-import com.bigdata.rdf.internal.constraints.NumericBOp.NumericOp;
import com.bigdata.rdf.internal.impl.AbstractIV;
import com.bigdata.rdf.internal.impl.AbstractInlineIV;
import com.bigdata.rdf.internal.impl.BlobIV;
@@ -137,502 +129,7 @@
}
- public static IV literalMath(final Literal l1, final Literal l2,
- final MathOp op)
- {
- final URI dt1 = l1.getDatatype();
- final URI dt2 = l2.getDatatype();
-
- // Only numeric value can be used in math expressions
- if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1)) {
- throw new IllegalArgumentException("Not a number: " + l1);
- }
- if (dt2 == null || !XMLDatatypeUtil.isNumericDatatype(dt2)) {
- throw new IllegalArgumentException("Not a number: " + l2);
- }
-
- // Determine most specific datatype that the arguments have in common,
- // choosing from xsd:integer, xsd:decimal, xsd:float and xsd:double as
- // per the SPARQL/XPATH spec
- URI commonDatatype;
-
- if (dt1.equals(XMLSchema.DOUBLE) || dt2.equals(XMLSchema.DOUBLE)) {
- commonDatatype = XMLSchema.DOUBLE;
- } else if (dt1.equals(XMLSchema.FLOAT) || dt2.equals(XMLSchema.FLOAT)) {
- commonDatatype = XMLSchema.FLOAT;
- } else if (dt1.equals(XMLSchema.DECIMAL) || dt2.equals(XMLSchema.DECIMAL)) {
- commonDatatype = XMLSchema.DECIMAL;
- } else if (op == MathOp.DIVIDE) {
- // Result of integer divide is decimal and requires the arguments to
- // be handled as such, see for details:
- // http://www.w3.org/TR/xpath-functions/#func-numeric-divide
- commonDatatype = XMLSchema.DECIMAL;
- } else {
- commonDatatype = XMLSchema.INTEGER;
- }
-
- // Note: Java already handles cases like divide-by-zero appropriately
- // for floats and doubles, see:
- // http://www.particle.kth.se/~lindsey/JavaCourse/Book/Part1/Tech/
- // Chapter02/floatingPt2.html
-
- try {
- if (commonDatatype.equals(XMLSchema.DOUBLE)) {
- double left = l1.doubleValue();
- double right = l2.doubleValue();
- return IVUtility.numericalMath(left, right, op);
- }
- else if (commonDatatype.equals(XMLSchema.FLOAT)) {
- float left = l1.floatValue();
- float right = l2.floatValue();
- return IVUtility.numericalMath(left, right, op);
- }
- else if (commonDatatype.equals(XMLSchema.DECIMAL)) {
- BigDecimal left = l1.decimalValue();
- BigDecimal right = l2.decimalValue();
- return IVUtility.numericalMath(left, right, op);
- }
- else { // XMLSchema.INTEGER
- BigInteger left = l1.integerValue();
- BigInteger right = l2.integerValue();
- return IVUtility.numericalMath(left, right, op);
- }
- } catch (NumberFormatException e) {
- throw new SparqlTypeErrorException();
- } catch (ArithmeticException e) {
- throw new SparqlTypeErrorException();
- }
-
- }
-
- public static final IV numericalMath(final Literal l1, final IV iv2,
- final MathOp op) {
-
- final URI dt1 = l1.getDatatype();
-
- // Only numeric value can be used in math expressions
- if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1)) {
- throw new IllegalArgumentException("Not a number: " + l1);
- }
-
- if (!iv2.isInline())
- throw new IllegalArgumentException(
- "right term is not inline: left=" + l1 + ", right=" + iv2);
-
- if (!iv2.isLiteral())
- throw new IllegalArgumentException(
- "right term is not literal: left=" + l1 + ", right=" + iv2);
-
- final DTE dte2 = iv2.getDTE();
-
- if (!dte2.isNumeric())
- throw new IllegalArgumentException(
- "right term is not numeric: left=" + l1 + ", right=" + iv2);
-
- final AbstractLiteralIV<BigdataLiteral, ?> num2 = (AbstractLiteralIV<BigdataLiteral, ?>) iv2;
-
- // Determine most specific datatype that the arguments have in common,
- // choosing from xsd:integer, xsd:decimal, xsd:float and xsd:double as
- // per the SPARQL/XPATH spec
-
- if (dte2 == DTE.XSDDouble || dt1.equals(XMLSchema.DOUBLE)) {
- return numericalMath(l1.doubleValue(), num2.doubleValue(), op);
- } else if (dte2 == DTE.XSDFloat || dt1.equals(XMLSchema.FLOAT)) {
- return numericalMath(l1.floatValue(), num2.floatValue(), op);
- } else if (dte2 == DTE.XSDDecimal || dt1.equals(XMLSchema.DECIMAL)) {
- return numericalMath(l1.decimalValue(), num2.decimalValue(), op);
- } else if (op == MathOp.DIVIDE) {
- // Result of integer divide is decimal and requires the arguments to
- // be handled as such, see for details:
- // http://www.w3.org/TR/xpath-functions/#func-numeric-divide
- return numericalMath(l1.decimalValue(), num2.decimalValue(), op);
- } else {
- return numericalMath(l1.integerValue(), num2.integerValue(), op);
- }
-
- }
-
- public static final IV numericalMath(final Literal l1, final Literal l2,
- final MathOp op) {
-
- final URI dt1 = l1.getDatatype();
-
- // Only numeric value can be used in math expressions
- if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1)) {
- throw new IllegalArgumentException("Not a number: " + l1);
- }
-
- final URI dt2 = l2.getDatatype();
-
- // Only numeric value can be used in math expressions
- if (dt2 == null || !XMLDatatypeUtil.isNumericDatatype(dt2)) {
- throw new IllegalArgumentException("Not a number: " + l2);
- }
-
- // Determine most specific datatype that the arguments have in common,
- // choosing from xsd:integer, xsd:decimal, xsd:float and xsd:double as
- // per the SPARQL/XPATH spec
-
- if (dt2.equals(XMLSchema.DOUBLE)|| dt1.equals(XMLSchema.DOUBLE)) {
- return numericalMath(l1.doubleValue(), l2.doubleValue(), op);
- } else if ( dt2.equals(XMLSchema.FLOAT)|| dt1.equals(XMLSchema.FLOAT)) {
- return numericalMath(l1.floatValue(), l2.floatValue(), op);
- } else if (dt2.equals(XMLSchema.DECIMAL)|| dt1.equals(XMLSchema.DECIMAL)) {
- return numericalMath(l1.decimalValue(), l2.decimalValue(), op);
- } else if (op == MathOp.DIVIDE) {
- // Result of integer divide is decimal and requires the arguments to
- // be handled as such, see for details:
- // http://www.w3.org/TR/xpath-functions/#func-numeric-divide
- return numericalMath(l1.decimalValue(), l2.decimalValue(), op);
- } else {
- return numericalMath(l1.integerValue(), l2.integerValue(), op);
- }
-
- }
-
- public static final IV numericalMath(final IV iv1, final IV iv2,
- final MathOp op) {
-
- if (!iv1.isInline())
- throw new IllegalArgumentException(
- "left term is not inline: left=" + iv1 + ", right=" + iv2);
-
- if (!iv2.isInline())
- throw new IllegalArgumentException(
- "right term is not inline: left=" + iv1 + ", right=" + iv2);
-
- if (!iv1.isLiteral())
- throw new IllegalArgumentException(
- "left term is not literal: left=" + iv1 + ", right=" + iv2);
-
- if (!iv2.isLiteral())
- throw new IllegalArgumentException(
- "right term is not literal: left=" + iv1 + ", right=" + iv2);
-
- final DTE dte1 = iv1.getDTE();
- final DTE dte2 = iv2.getDTE();
-
- if (!dte1.isNumeric())
- throw new IllegalArgumentException(
- "right term is not numeric: left=" + iv1 + ", right=" + iv2);
-
- if (!dte2.isNumeric())
- throw new IllegalArgumentException(
- "left term is not numeric: left=" + iv1 + ", right=" + iv2);
-
- final Literal num1 = (Literal) iv1;
- final Literal num2 = (Literal) iv2;
-
- // Determine most specific datatype that the arguments have in common,
- // choosing from xsd:integer, xsd:decimal, xsd:float and xsd:double as
- // per the SPARQL/XPATH spec
-
- if (dte1 == DTE.XSDDouble || dte2 == DTE.XSDDouble) {
- return numericalMath(num1.doubleValue(), num2.doubleValue(), op);
- } else if (dte1 == DTE.XSDFloat || dte2 == DTE.XSDFloat) {
- return numericalMath(num1.floatValue(), num2.floatValue(), op);
- } if (dte1 == DTE.XSDDecimal || dte2 == DTE.XSDDecimal) {
- return numericalMath(num1.decimalValue(), num2.decimalValue(), op);
- } if (op == MathOp.DIVIDE) {
- // Result of integer divide is decimal and requires the arguments to
- // be handled as such, see for details:
- // http://www.w3.org/TR/xpath-functions/#func-numeric-divide
- return numericalMath(num1.decimalValue(), num2.decimalValue(), op);
- } else {
- return numericalMath(num1.integerValue(), num2.integerValue(), op);
- }
-
-// // if one's a BigDecimal we should use the BigDecimal comparator for both
-// if (dte1 == DTE.XSDDecimal || dte2 == DTE.XSDDecimal) {
-// return numericalMath(num1.decimalValue(), num2.decimalValue(), op);
-// }
-//
-// // same for BigInteger
-// if (dte1 == DTE.XSDInteger || dte2 == DTE.XSDInteger) {
-// return numericalMath(num1.integerValue(), num2.integerValue(), op);
-// }
-//
-// // fixed length numerics
-// if (dte1.isFloatingPointNumeric() || dte2.isFloatingPointNumeric()) {
-// // non-BigDecimal floating points
-// if (dte1 == DTE.XSDFloat || dte2 == DTE.XSDFloat)
-// return numericalMath(num1.floatValue(), num2.floatValue(), op);
-// else
-// return numericalMath(num1.doubleValue(), num2.doubleValue(), op);
-// } else {
-// // non-BigInteger integers
-// if (dte1 == DTE.XSDInt && dte2 == DTE.XSDInt)
-// return numericalMath(num1.intValue(), num2.intValue(), op);
-// else
-// return numericalMath(num1.longValue(), num2.longValue(), op);
-// }
-
- }
-
/**
- * The XPath numeric functions: abs, ceiling, floor, and round.
- *
- * @param iv1
- * The operand.
- * @param op
- * The operation.
-
- * @return The result.
- *
- * @see XPathMathFunctions
- */
- public static final IV numericalFunc(final IV iv1, final NumericOp op) {
-
- if (!iv1.isInline())
- throw new IllegalArgumentException(
- "left term is not inline: left=" + iv1 );
-
-
- if (!iv1.isLiteral())
- throw new IllegalArgumentException(
- "left term is not literal: left=" + iv1 );
-
-
- final DTE dte1 = iv1.getDTE();
-
- if (!dte1.isNumeric())
- throw new IllegalArgumentException(
- "left term is not numeric: left=" + iv1 );
-
- final AbstractLiteralIV num1 = (AbstractLiteralIV) iv1;
-
- /*
- * FIXME These xpath functions have very custom semantics. They need to
- * be lifted out of this class and put into their own static methods
- * with their own test suites.
- */
-// switch (op) {
-// case ABS:
-// return XPathMathFunctions.abs(iv1);
-// case CEIL:
-// return XPathMathFunctions.ceiling(iv1);
-// case FLOOR:
-// return XPathMathFunctions.floor(iv1);
-// case ROUND:
-// return XPathMathFunctions.round(iv1);
-// default:
-// throw new UnsupportedOperationException(op.toString());
-// }
-
- // if one's a BigDecimal we should use the BigDecimal comparator for
- // both
- if (dte1 == DTE.XSDDecimal) {
- return numericalFunc(num1.decimalValue(), op);
- } else if (dte1 == DTE.XSDInteger) {
- return numericalFunc(num1.integerValue(), op);
- } else if (dte1.isFloatingPointNumeric()) {
- return numericalFunc(num1.floatValue(), op);
- } else if (dte1.equals(DTE.XSDInt)) {
- return numericalFunc(num1.intValue(), op);
- } else if (dte1.equals(DTE.XSDDouble)) {
- return numericalFunc(num1.doubleValue(), op);
- } else {
- return numericalFunc(num1.longValue(), op);
- }
- }
-
- @Deprecated
- private static final IV numericalFunc(final BigDecimal left, final NumericOp op) {
- switch(op) {
- case ABS:
- return new XSDDecimalIV(left.abs());
- case CEIL:
- return new XSDDecimalIV(new BigDecimal(Math.round(Math.ceil(left.doubleValue()))));
- case FLOOR:
- return new XSDDecimalIV(new BigDecimal(Math.round(Math.floor(left.doubleValue()))));
- case ROUND:
- return new XSDDecimalIV(new BigDecimal(Math.round(left.doubleValue())));
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- @Deprecated
- private static final IV numericalFunc(final BigInteger left, final NumericOp op) {
- switch(op) {
- case ABS:
- return new XSDIntegerIV(left.abs());
- case CEIL:
- return new XSDNumericIV(Math.ceil(left.doubleValue()));
- case FLOOR:
- return new XSDNumericIV(Math.floor(left.doubleValue()));
- case ROUND:
- return new XSDNumericIV(Math.round(left.doubleValue()));
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- @Deprecated
- private static final IV numericalFunc(final float left, final NumericOp op) {
- switch(op) {
- case ABS:
- return new XSDNumericIV(Math.abs(left));
- case CEIL:
- return new XSDNumericIV(Math.ceil(left));
- case FLOOR:
- return new XSDNumericIV(Math.floor(left));
- case ROUND:
- return new XSDNumericIV(Math.round(left));
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- @Deprecated
- private static final IV numericalFunc(final int left, final NumericOp op) {
- switch(op) {
- case ABS:
- return new XSDNumericIV(Math.abs(left));
- case CEIL:
- return new XSDNumericIV(Math.ceil(left));
- case FLOOR:
- return new XSDNumericIV(Math.floor(left));
- case ROUND:
- return new XSDNumericIV(Math.round(left));
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- @Deprecated
- private static final IV numericalFunc(final long left, final NumericOp op) {
- switch(op) {
- case ABS:
- return new XSDNumericIV(Math.abs(left));
- case CEIL:
- return new XSDNumericIV(Math.ceil(left));
- case FLOOR:
- return new XSDNumericIV(Math.floor(left));
- case ROUND:
- return new XSDNumericIV(Math.round(left));
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- @Deprecated
- private static final IV numericalFunc(final double left, final NumericOp op) {
- switch(op) {
- case ABS:
- return new XSDNumericIV(Math.abs(left));
- case CEIL:
- return new XSDNumericIV(Math.ceil(left));
- case FLOOR:
- return new XSDNumericIV(Math.floor(left));
- case ROUND:
- return new XSDNumericIV(Math.round(left));
- default:
- throw new UnsupportedOperationException();
- }
- }
-
- private static final IV numericalMath(final BigDecimal left,
- final BigDecimal right, final MathOp op) {
-
- switch(op) {
- case PLUS:
- return new XSDDecimalIV(left.add(right));
- case MINUS:
- return new XSDDecimalIV(left.subtract(right));
- case MULTIPLY:
- return new XSDDecimalIV(left.multiply(right));
- case DIVIDE:
- /*
- * Note: Change per mroycsi. Reverts to a half-rounding mode iff
- * an exact quotient can not be represented.
- */
- try {
- // try for exact quotient
- return new XSDDecimalIV(left.divide(right));
- } catch (ArithmeticException ae) {
- // half-rounding mode.
- return new XSDDecimalIV(left.divide(right, 20,
- RoundingMode.HALF_UP));
- }
-// return new XSDDecimalIV(left.divide(right, RoundingMode.HALF_UP));
- case MIN:
- return new XSDDecimalIV(left.compareTo(right) < 0 ? left : right);
- case MAX:
- return new XSDDecimalIV(left.compareTo(right) > 0 ? left : right);
- default:
- throw new UnsupportedOperationException();
- }
-
- }
-
- private static final IV numericalMath(final BigInteger left,
- final BigInteger right, final MathOp op) {
-
- switch(op) {
- case PLUS:
- return new XSDIntegerIV(left.add(right));
- case MINUS:
- return new XSDIntegerIV(left.subtract(right));
- case MULTIPLY:
- return new XSDIntegerIV(left.multiply(right));
- case DIVIDE:
- return new XSDIntegerIV(left.divide(right));
- case MIN:
- return new XSDIntegerIV(left.compareTo(right) < 0 ? left : right);
- case MAX:
- return new XSDIntegerIV(left.compareTo(right) > 0 ? left : right);
- default:
- throw new UnsupportedOperationException();
- }
-
- }
-
- private static final IV numericalMath(final float left,
- final float right, final MathOp op) {
-
- switch(op) {
- case PLUS:
- return new XSDNumericIV(left+right);
- case MINUS:
- return new XSDNumericIV(left-right);
- case MULTIPLY:
- return new XSDNumericIV(left*right);
- case DIVIDE:
- return new XSDNumericIV(left/right);
- case MIN:
- return new XSDNumericIV(Math.min(left,right));
- case MAX:
- return new XSDNumericIV(Math.max(left,right));
- default:
- throw new UnsupportedOperationException();
- }
-
- }
-
- private static final IV numericalMath(final double left,
- final double right, final MathOp op) {
-
- switch(op) {
- case PLUS:
- return new XSDNumericIV(left+right);
- case MINUS:
- return new XSDNumericIV(left-right);
- case MULTIPLY:
- return new XSDNumericIV(left*right);
- case DIVIDE:
- return new XSDNumericIV(left/right);
- case MIN:
- return new XSDNumericIV(Math.min(left,right));
- case MAX:
- return new XSDNumericIV(Math.max(left,right));
- default:
- throw new UnsupportedOperationException();
- }
-
- }
-
- /**
* Encode an RDF value into a key for one of the statement indices. Handles
* null {@link IV} references gracefully.
*
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -157,22 +157,27 @@
}
+ private static boolean isRealTermId(final IV iv) {
+
+ return !iv.isInline() && !iv.isNullIV();
+
+ }
+
private static boolean compareLiterals(
final IV<BigdataValue, ?> left,
final IV<BigdataValue, ?> right,
final CompareOp op) {
/*
- * Handle the special case where we have exact termId equality.
- * Probably would never hit this because of SameTermBOp
+ * Handle the special case where we have exact termId equality. This
+ * only works if both are "real" termIds (i.e. not mock IVs).
*/
- if (!left.isInline() && !right.isInline() && op == CompareOp.EQ) {
+ if (op == CompareOp.EQ && isRealTermId(left) && isRealTermId(right)) {
- if (!left.isNullIV() && !right.isNullIV() && left.equals(right)) {
- /*
- * Neither may be a NullIV (or mock IV) and they are equals().
- */
+ if (left.equals(right)) {
+
return true;
+
}
}
@@ -207,55 +212,15 @@
}
}
-//
-// if (left.isInline() && left.isNumeric() && right.isInline() && right.isNumeric()) {
-//
-// final DTE dte1 = left.getDTE();
-// final DTE dte2 = right.getDTE();
-//
-// // we can use the natural ordering if they have the same DTE
-// // this will naturally take care of two booleans or two numerics of the
-// // same datatype
-// if (dte1 == dte2)
-// return _accept(left.compareTo(right), op);
-//
-// // otherwise we need to try to convert them into comparable numbers
-// final AbstractLiteralIV num1 = (AbstractLiteralIV) left;
-// final AbstractLiteralIV num2 = (AbstractLiteralIV) right;
-//
-// // if one's a BigDecimal we should use the BigDecimal comparator for both
-// if (dte1 == DTE.XSDDecimal || dte2 == DTE.XSDDecimal) {
-// return _accept(num1.decimalValue().compareTo(num2.decimalValue()), op);
-// }
-//
-// // same for BigInteger
-// if (dte1 == DTE.XSDInteger || dte2 == DTE.XSDInteger) {
-// return _accept(num1.integerValue().compareTo(num2.integerValue()), op);
-// }
-//
-// // fixed length numerics
-// if (dte1.isFloatingPointNumeric() || dte2.isFloatingPointNumeric()) {
-// // non-BigDecimal floating points - use doubles
-// return _accept(Double.compare(num1.doubleValue(), num2.doubleValue()), op);
-// } else {
-// // non-BigInteger integers - use longs
-// final long a = num1.longValue();
-// final long b = num2.longValue();
-// return _accept(a == b ? 0 : a < b ? -1 : 1, op);
-// }
-//
-// }
/*
* Now that the IVs implement the right openrdf interfaces,
* we should be able to mix and match inline with non-inline,
* using either the IV directly or its materialized value.
*/
-// final Literal l1 = left.isInline() ? (Literal) left : left.getValue();
- final Literal l1 = (Literal) left;
+ final Literal l1 = asLiteral(left);
-// final Literal l2 = right.isInline() ? (Literal) right : right.getValue();
- final Literal l2 = (Literal) right;
+ final Literal l2 = asLiteral(right);
if (log.isDebugEnabled()) {
log.debug(l1);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DatatypeBOp.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -87,7 +87,7 @@
final BigdataValueFactory vf = super.getValueFactory();
@SuppressWarnings("rawtypes")
- final IV iv = get(0).get(bs);
+ final IV iv = getAndCheckLiteral(0, bs);
// not yet bound
if (iv == null)
@@ -97,12 +97,17 @@
log.debug(iv);
}
- if (iv.isInline() && !iv.isExtension()) {
+ /*
+ * We don't need to do this anymore. asValue(IV) does the right thing,
+ * it will let us work with the IV directly in the right cases. The
+ * BOps should no longer be doing this kind of logic directly.
+ */
+// if (iv.isInline() && !iv.isExtension()) {
+//
+// return asIV(iv.getDTE().getDatatypeURI(), bs);
+//
+// }
- return asIV(iv.getDTE().getDatatypeURI(), bs);
-
- }
-
final Value val = asValue(iv);
if (val instanceof Literal) {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateTimeUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateTimeUtility.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/DateTimeUtility.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -10,12 +10,10 @@
import org.openrdf.model.vocabulary.XMLSchema;
import com.bigdata.rdf.error.SparqlTypeErrorException;
-import com.bigdata.rdf.internal.ILexiconConfiguration;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
import com.bigdata.rdf.internal.impl.literal.XSDNumericIV;
import com.bigdata.rdf.model.BigdataLiteral;
-import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.model.BigdataValueFactory;
import com.bigdata.rdf.sparql.ast.DummyConstantNode;
@@ -29,104 +27,102 @@
}
}
- static public IV dateTimeMath(Literal l1, IV iv1, Literal l2, IV iv2, MathOp op, BigdataValueFactory vf, ILexiconConfiguration lc) {
- final URI dt1 = l1.getDatatype();
- final URI dt2 = l2.getDatatype();
- if(dt1==null||dt2==null){
+ static public boolean checkDateTimeDatatype(final Literal... args) {
+ for (Literal lit : args) {
+ final URI dt = lit.getDatatype();
+ if (dt == null)
+ return false;
+ if (!(XMLDatatypeUtil.isCalendarDatatype(dt) || dt.equals(XMLSchema.DURATION)))
+ return false;
+ }
+ return true;
+ }
+
+ static public IV dateTimeMath(
+ final Literal l1, final IV iv1,
+ final Literal l2, final IV iv2,
+ final MathOp op,
+ final BigdataValueFactory vf) {
+
+ if (!checkDateTimeDatatype(l1, l2))
throw new SparqlTypeErrorException();
- }
- if ((XMLDatatypeUtil.isCalendarDatatype(dt1) || dt1.equals(XMLSchema.DURATION)) && (XMLDatatypeUtil.isCalendarDatatype(dt2) || dt2.equals(XMLSchema.DURATION))) {
- XMLGregorianCalendar c1 = XMLDatatypeUtil.isCalendarDatatype(dt1) ? l1.calendarValue() : null;
- XMLGregorianCalendar c2 = XMLDatatypeUtil.isCalendarDatatype(dt2) ? l2.calendarValue() : null;
- Duration d1 = dt1.equals(XMLSchema.DURATION) ? datatypeFactory.newDuration(l1.getLabel()) : null;
- Duration d2 = dt2.equals(XMLSchema.DURATION) ? datatypeFactory.newDuration(l2.getLabel()) : null;
- if (op == MathOp.PLUS) {
- if (c1 != null && c2 != null) {
- throw new IllegalArgumentException("Cannot add 2 calendar literals:" + l1 + ":" + l2);
- } else if (c1 != null && d2 != null) {
- c1.add(d2);
- final BigdataLiteral str = vf.createLiteral(c1);
- return createIV(str, lc);
- } else if (c2 != null && d1 != null) {
- c2.add(d1);
- final BigdataLiteral str = vf.createLiteral(c2);
- return createIV(str, lc);
- } else if (d1 != null && d2 != null) {
- Duration result = d1.add(d2);
- final BigdataLiteral str = vf.createLiteral(result.toString(), XMLSchema.DURATION);
- return createIV(str, lc);
+
+ final URI dt1 = l1.getDatatype();
+ final URI dt2 = l2.getDatatype();
+ XMLGregorianCalendar c1 = XMLDatatypeUtil.isCalendarDatatype(dt1) ? l1.calendarValue() : null;
+ XMLGregorianCalendar c2 = XMLDatatypeUtil.isCalendarDatatype(dt2) ? l2.calendarValue() : null;
+ Duration d1 = dt1.equals(XMLSchema.DURATION) ? datatypeFactory.newDuration(l1.getLabel()) : null;
+ Duration d2 = dt2.equals(XMLSchema.DURATION) ? datatypeFactory.newDuration(l2.getLabel()) : null;
+ if (op == MathOp.PLUS) {
+ if (c1 != null && c2 != null) {
+ throw new IllegalArgumentException("Cannot add 2 calendar literals:" + l1 + ":" + l2);
+ } else if (c1 != null && d2 != null) {
+ c1.add(d2);
+ final BigdataLiteral str = vf.createLiteral(c1);
+ return DummyConstantNode.toDummyIV(str);
+ } else if (c2 != null && d1 != null) {
+ c2.add(d1);
+ final BigdataLiteral str = vf.createLiteral(c2);
+ return DummyConstantNode.toDummyIV(str);
+ } else if (d1 != null && d2 != null) {
+ Duration result = d1.add(d2);
+ final BigdataLiteral str = vf.createLiteral(result.toString(), XMLSchema.DURATION);
+ return DummyConstantNode.toDummyIV(str);
+ } else {
+ throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
+ }
+ } else if (op == MathOp.MINUS) {
+ if (c1 != null && c2 != null) {
+ long milliseconds = c1.toGregorianCalendar().getTimeInMillis() - c2.toGregorianCalendar().getTimeInMillis();
+ double days = ((double) milliseconds) / ((double) (1000 * 60 * 60 * 24));
+ return new XSDNumericIV(days);
+ } else if (d1 != null && d2 != null) {
+ Duration result = d1.subtract(d2);
+ final BigdataLiteral str = vf.createLiteral(result.toString(), XMLSchema.DURATION);
+ return DummyConstantNode.toDummyIV(str);
+ } else {
+ throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
+ }
+
+ } else if (op == MathOp.MIN) {
+ if (c1 != null && c2 != null) {
+ int comp = c1.compare(c2);
+ if (comp <= 0) {
+ return iv1;
} else {
- throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
+ return iv2;
}
- } else if (op == MathOp.MINUS) {
- if (c1 != null && c2 != null) {
- long milliseconds = c1.toGregorianCalendar().getTimeInMillis() - c2.toGregorianCalendar().getTimeInMillis();
- double days = ((double) milliseconds) / ((double) (1000 * 60 * 60 * 24));
- return new XSDNumericIV(days);
- } else if (d1 != null && d2 != null) {
- Duration result = d1.subtract(d2);
- final BigdataLiteral str = vf.createLiteral(result.toString(), XMLSchema.DURATION);
- return createIV(str, lc);
- } else {
- throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
+ } else if (d1 != null && d2 != null) {
+ int comp= d1.compare(d2);
+ if(comp<=0){
+ return iv1;
+ }else{
+ return iv2;
}
-
- } else if (op == MathOp.MIN) {
- if (c1 != null && c2 != null) {
- int comp = c1.compare(c2);
- if (comp <= 0) {
- return iv1;
- } else {
- return iv2;
- }
- } else if (d1 != null && d2 != null) {
- int comp= d1.compare(d2);
- if(comp<=0){
- return iv1;
- }else{
- return iv2;
- }
+ } else {
+ throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
+ }
+ } else if (op == MathOp.MAX) {
+ if (c1 != null && c2 != null) {
+ int comp = c1.compare(c2);
+ if (comp >= 0) {
+ return iv1;
} else {
- throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
+ return iv2;
}
- } else if (op == MathOp.MAX) {
- if (c1 != null && c2 != null) {
- int comp = c1.compare(c2);
- if (comp >= 0) {
- return iv1;
- } else {
- return iv2;
- }
- } else if (d1 != null && d2 != null) {
- int comp= d1.compare(d2);
- if(comp>=0){
- return iv1;
- }else{
- return iv2;
- }
- } else {
- throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
- }
+ } else if (d1 != null && d2 != null) {
+ int comp= d1.compare(d2);
+ if(comp>=0){
+ return iv1;
+ }else{
+ return iv2;
+ }
} else {
- throw new SparqlTypeErrorException();
+ throw new IllegalArgumentException("Cannot add process datatype literals:" + l1 + ":" + l2);
}
+ } else {
+ throw new SparqlTypeErrorException();
}
- throw new SparqlTypeErrorException();
}
- private static IV createIV(final BigdataValue value, final ILexiconConfiguration lc) {
-
- // see if we happen to have the value in the vocab or can otherwise
- // create an inline IV for it
- IV iv = lc != null ? lc.createInlineIV(value) : null;
- if (iv != null) {
- iv.setValue(value);
- } else {
- iv = DummyConstantNode.toDummyIV(value);
- }
-
- return iv;
-
- }
-
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -362,7 +362,7 @@
* be turned into a {@link Literal} without an index read.
*/
@SuppressWarnings("rawtypes")
- final protected Literal asLiteral(final IV iv) {
+ final static public Literal asLiteral(final IV iv) {
if (iv == null)
throw new SparqlTypeErrorException();
@@ -370,23 +370,10 @@
if (!iv.isLiteral())
throw new SparqlTypeErrorException();
-// final BigdataValueFactory vf = getValueFactory();
-
if (iv.isInline() && !iv.needsMaterialization()) {
-// if (iv instanceof Literal) {
+ return (Literal) iv;
- return (Literal) iv;
-
-// } else {
-//
-// final BigdataURI datatype = vf
-// .asValue(iv.getDTE().getDatatypeURI());
-//
-// return vf.createLiteral(((Value) iv).stringValue(), datatype);
-//
-// }
-
} else if (iv.hasValue()) {
return (BigdataLiteral) iv.getValue();
@@ -414,7 +401,7 @@
* be turned into a {@link Literal} without an index read.
*/
@SuppressWarnings("rawtypes")
- final static protected Value asValue(final IV iv) {
+ final static public Value asValue(final IV iv) {
if (iv == null)
throw new SparqlTypeErrorException();
@@ -490,7 +477,7 @@
if (!iv.isLiteral())
throw new SparqlTypeErrorException();
- if (!iv.isInline() && !iv.hasValue())
+ if (iv.needsMaterialization() && !iv.hasValue())
throw new NotMaterializedException();
return iv;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2012-04-05 08:39:41 UTC (rev 6256)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -153,72 +153,46 @@
final public IV get(final IBindingSet bs) {
- final IV left = left().get(bs);
-
- // not yet bound?
- if (left == null)
- throw new SparqlTypeErrorException.UnboundVarException();
-
- final IV right = right().get(bs);
+ final IV iv1 = getAndCheckLiteral(0, bs);
+ final IV iv2 = getAndCheckLiteral(1, bs);
- // not yet bound?
- if (right == null)
- throw new SparqlTypeErrorException.UnboundVarException();
-
- try {
-
- if (log.isDebugEnabled()) {
- log.debug(toString(left.toString(), right.toString()));
- }
-
- if (left.isInline() && right.isInline()) {
-
- return IVUtility.numericalMath(left, right, op());
-
- } else {
-
- final BigdataValue val1 = left.getValue();
-
- final BigdataValue val2 = right.getValue();
-
- if (val1 == null || val2 == null)
- throw new NotMaterializedException();
-
- if (!(val1 instanceof Literal) || !(val2 instanceof Literal)) {
- throw new SparqlTypeErrorException();
- }
-
- try{
-
- return IVUtility.literalMath(
- (Literal) val1, (Literal) val2, op());
-
- } catch(IllegalArgumentException iae){
-
- ILexiconConfiguration lc = null;
- try {
- lc = getLexiconConfiguration(bs);
- } catch (ContextNotAvailableException ex) {
- // can't use the LC (e.g. test cases)
- }
-
- return DateTimeUtility.dateTimeMath((Literal)val1, left,
- (Literal)val2, right, op(), vf(), lc);
-
- }
-
- }
-
- } catch (IllegalArgumentException ex) {
-
- if (log.isDebugEnabled()) {
- log.debug("illegal argument, filtering solution");
- }
-
- throw new SparqlTypeErrorException();
-
- }
+ if (log.isDebugEnabled()) {
+ log.debug(toString(iv1.toString(), iv2.toString()));
+ }
+
+ final Literal lit1 = asLiteral(iv1);
+ final Literal lit2 = asLiteral(iv2);
+ if (MathUtility.checkNumericDatatype(lit1, lit2)) {
+
+ return MathUtility.literalMath(lit1, lit2, op());
+
+ } else if (DateTimeUtility.checkDateTimeDatatype(lit1, lit2)) {
+
+ final IV iv =
+ DateTimeUtility.dateTimeMath(lit1, iv1, lit2, iv2, op(), vf());
+
+ // try to create a real IV if possible
+ if (iv.isNullIV()) {
+
+ final BigdataValue val = iv.getValue();
+
+ return asIV(val, bs);
+
+ } else {
+
+ return iv;
+
+ }
+
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("illegal argument(s), filtering solution: " + iv1 + ", " + iv2);
+ }
+
+ throw new SparqlTypeErrorException();
+
}
public IValueExpression<? extends IV> left() {
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathUtility.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathUtility.java 2012-04-05 11:16:55 UTC (rev 6257)
@@ -0,0 +1,509 @@
+package com.bigdata.rdf.internal.constraints;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.datatypes.XMLDatatypeUtil;
+import org.openrdf.model.vocabulary.XMLSchema;
+
+import com.bigdata.rdf.error.SparqlTypeErrorException;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.XPathMathFunctions;
+import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
+import com.bigdata.rdf.internal.constraints.NumericBOp.NumericOp;
+import com.bigdata.rdf.internal.impl.literal.NumericIV;
+import com.bigdata.rdf.internal.impl.literal.XSDDecimalIV;
+import com.bigdata.rdf.internal.impl.literal.XSDIntegerIV;
+import com.bigdata.rdf.internal.impl.literal.XSDNumericIV;
+
+public class MathUtility {
+
+ public static boolean checkNumericDatatype(final Literal... args) {
+ for (Literal lit : args) {
+ final URI dt = lit.getDatatype();
+
+ if (dt == null || !XMLDatatypeUtil.isNumericDatatype(dt))
+ return false;
+ }
+ return true;
+ }
+
+ public static NumericIV literalMath(final Literal l1, final Literal l2,
+ final MathOp op)
+ {
+ if (!checkNumericDatatype(l1, l2))
+ throw new IllegalArgumentException("Not numbers: " + l1 + ", " + l2);
+
+ final URI dt1 = l1.getDatatype();
+ final URI dt2 = l2.getDatatype();
+
+ // Determine most specific datatype that the arguments have in common,
+ // choosing from xsd:integer, xsd:decimal, xsd:float and xsd:double as
+ // per the SPARQL/XPATH spec
+ URI commonDatatype;
+
+ if (dt1.equals(XMLSchema.DOUBLE) || dt2.equals(XMLSchema.DOUBLE)) {
+ commonDatatype = XMLSchema.DOUBLE;
+ } else if (dt1.equals(XMLSchema.FLOAT) || dt2.equals(XMLSchema.FLOAT)) {
+ commonDatatype = XMLSchema.FLOAT;
+ } else if (dt1.equals(XMLSchema.DECIMAL) || dt2.equals(XMLSchema.DECIMAL)) {
+ commonDatatype = XMLSchema.DECIMAL;
+ } else if (op == MathOp.DIVIDE) {
+ // Result of integer divide is decimal and requires the arguments to
+ // be handled as such, see for details:
+ // http://www.w3.org/TR/xpath-functions/#func-numeric-divide
+ commonDatatype = XMLSchema.DECIMAL;
+ } else {
+ commonDatatype = XMLSchema.INTEGER;
+ }
+
+ // Note: Java already handles cases like divide-by-zero appropriately
+ // for floats and doubles, see:
+ // http://www.particle.kth.se/~lindsey/JavaCourse/Book/Part1/Tech/
+ // Chapter02/floatingPt2.html
+
+ try {
+ if (commonDatatype.equals(XMLSchema.DOUBLE)) {
+ double left = l1.doubleValue();
+ double right = l2.doubleValue();
+ return numericalMath(left, right, op);
+ }
+ else if (commonDatatype.equals(XMLSchema.FLOAT)) {
+ float left = l1.floatValue();
+ float right = l2.floatValue();
+ return numericalMath(left, right, op);
+ }
+ else if (commonDatatype.equals(XMLSchema.DECIMAL)) {
+ BigDecimal left = l1.decimalValue();
+ BigDecimal right = l2.decimalValue();
+ return numericalMath(left, right, op);
+ }
+ else { // XMLSchema.INTEGER
+ BigInteger left = l1.integerValue();
+ BigInteger right = l2.integerValue();
+ return numericalMath(left, right, op);
+ }
+ } catch (NumberFormatException e) {
+ throw new SparqlTypeErrorException();
+ } catch (ArithmeticException e) {
+ throw new SparqlTypeErrorException();
+ }
+
+ }
+
+// public static final IV numericalMath(final Literal l1, final IV iv2,
+// final MathOp op) {
+//
+// final URI dt1 = l1.getDatatype();
+//
+// // Only numeric value can be used in math expressions
+// if (dt1 == null || !XMLDatatypeUtil.isNumericDatatype(dt1)) {
+// throw new IllegalArgumentException("Not a number: " + l1);
+// }
+//
+// if (!iv2.isInline())
+// throw new IllegalArgumentException(
+// "right term is not inline: left=" + l1 + ", right=" + iv2);
+//
+// if (!iv2.isLiteral())
+// throw new IllegalArgumentException(
+// "right term is not literal: left=" + l1 + ", right=" + iv2);
+//
+// final DTE dte2 = iv2.getDTE();
+//
+// if (!dte2.isNumeric())
+// throw new Il...
[truncated message content] |
|
From: <tho...@us...> - 2012-04-09 14:03:13
|
Revision: 6263
http://bigdata.svn.sourceforge.net/bigdata/?rev=6263&view=rev
Author: thompsonbry
Date: 2012-04-09 14:03:06 +0000 (Mon, 09 Apr 2012)
Log Message:
-----------
Added a unit test to verify that the KEY_ORDER query hint is also applied to the pipeline join.
Note: This test does NOT verify that the pipeline join actually obeys that query hint. You need to verify that with a unit test of the pipeline join evaluation and check to see that the right index is used for each join.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestQueryHints.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/query-hints-05a.rq
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2012-04-09 10:29:06 UTC (rev 6262)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2012-04-09 14:03:06 UTC (rev 6263)
@@ -405,14 +405,29 @@
* @param it
* The iterator.
*/
- @SuppressWarnings("rawtypes")
- public static List<IVariable> toList(final Iterator<IVariable<?>> it) {
+// @SuppressWarnings("rawtypes")
+// public static List<IVariable> toList(final Iterator<IVariable<?>> it) {
+//
+// final List<IVariable> c = new LinkedList<IVariable>();
+//
+// while (it.hasNext()) {
+//
+// final IVariable v = it.next();
+//
+// c.add(v);
+//
+// }
+//
+// return c;
+//
+// }
+ public static <T> List<T> toList(final Iterator<T> it) {
- final List<IVariable> c = new LinkedList<IVariable>();
+ final List<T> c = new LinkedList<T>();
while (it.hasNext()) {
- final IVariable v = it.next();
+ final T v = it.next();
c.add(v);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestQueryHints.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestQueryHints.java 2012-04-09 10:29:06 UTC (rev 6262)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestQueryHints.java 2012-04-09 14:03:06 UTC (rev 6263)
@@ -463,6 +463,119 @@
}
/**
+ * Unit test for the {@link IPredicate.Annotations#KEY_ORDER} query hint on
+ * a pipeline join (versus a hash join).
+ * <p>
+ * Note: This test does NOT verify that the pipeline join actually obeys
+ * that query hint. You need to verify that with a unit test of the pipeline
+ * join evaluation and check to see that the right index is used for each
+ * join.
+ */
+ public void test_query_hints_05a() throws Exception {
+
+ final ASTContainer astContainer = new TestHelper(
+ "query-hints-05a",
+ "query-hints-05a.rq",
+ "query-hints-05.trig",
+ "query-hints-05.srx"
+ )
+ .runTest();
+
+ /*
+ * Check the optimized AST. The magic predicates which correspond to
+ * query hints should have be removed and various annotations made to
+ * the AST which capture the semantics of those query hints.
+ */
+ {
+
+ final JoinGroupNode whereClause = (JoinGroupNode) astContainer
+ .getOptimizedAST().getWhereClause();
+
+ /*
+ * The hint to disable the join order optimizer should show up on
+ * the JoinGroupNodes. For the sample query, that means just the
+ * top-level WHERE clause.
+ */
+ assertEquals(QueryOptimizerEnum.None,
+ whereClause.getProperty(QueryHints.OPTIMIZER));
+
+ // There are two statement pattern nodes left (after removing the
+ // query hint SPs).
+ assertEquals(2, whereClause.arity());
+
+ {
+
+ final StatementPatternNode sp = (StatementPatternNode) whereClause
+ .get(0);
+
+ assertEquals(RDFS.LABEL, sp.p().getValue());
+
+ }
+
+ {
+
+ final StatementPatternNode sp = (StatementPatternNode) whereClause
+ .get(1);
+
+ assertEquals(RDF.TYPE, sp.p().getValue());
+
+ assertEquals(SPOKeyOrder.PCSO.toString(),
+ sp.getQueryHint(IPredicate.Annotations.KEY_ORDER));
+
+// assertEquals("true",
+// sp.getQueryHint(QueryHints.HASH_JOIN));
+
+ }
+
+ } // end check of the AST.
+
+ /*
+ * Check the query plan.
+ */
+ {
+
+ /*
+ * Should be two pipeline joins.
+ *
+ * The first pipeline join is (?x rdfs:label ?o)
+ *
+ * The second pipeline join is (?x rdf:type foaf:Person)
+ */
+ @SuppressWarnings("rawtypes")
+ final PipelineJoin[] joins = BOpUtility.toList(
+ BOpUtility.visitAll(astContainer.getQueryPlan(),
+ PipelineJoin.class)).toArray(new PipelineJoin[0]);
+
+ assertEquals("#joins", 2, joins.length);
+ {
+
+ @SuppressWarnings("rawtypes")
+ final PipelineJoin join = joins[1];
+
+ assertEquals(RDFS.LABEL, ((IV<?,?>)join.getPredicate().get(1/* p */)
+ .get()).getValue());
+
+ }
+
+ {
+
+ @SuppressWarnings("rawtypes")
+ final PipelineJoin join = joins[0];
+
+ assertEquals(RDF.TYPE, ((IV<?,?>) join.getPredicate().get(1/* p */)
+ .get()).getValue());
+
+ final IPredicate<?> pred = join.getPredicate();
+
+ assertEquals(SPOKeyOrder.PCSO, pred.getKeyOrder());
+
+ }
+
+ }
+
+ }
+
+ /**
* Unit test for {@link QueryHints#CHUNK_SIZE}.
*
* <pre>
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/query-hints-05a.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/query-hints-05a.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/query-hints-05a.rq 2012-04-09 14:03:06 UTC (rev 6263)
@@ -0,0 +1,20 @@
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+
+SELECT ?x ?o
+WHERE {
+
+ # disable join order optimizer
+ hint:Group hint:optimizer "None" .
+
+ # One-bound using P___ index.
+ ?x rdfs:label ?o .
+
+ # Two-bound. Uses POCS index by default, which is optimal.
+ ?x rdf:type foaf:Person .
+
+ # Override the statement index.
+ hint:Prior hint:com.bigdata.bop.IPredicate.keyOrder "POCS" .
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-04-09 19:38:04
|
Revision: 6266
http://bigdata.svn.sourceforge.net/bigdata/?rev=6266&view=rev
Author: thompsonbry
Date: 2012-04-09 19:37:57 +0000 (Mon, 09 Apr 2012)
Log Message:
-----------
This commit incorporates the memory manager journal mode (MemStrategy, which is enabled by BuffedMode.MemStore) [1].
It also incorporates some progress on the bigdata extension for SPARQL Update [2]. I am beginning to work through the integration to support this feature. I have written code for writing solution sets onto the memory manager and reading them back. That code has not yet been tested. This functionality is currently disabled in QueryHints and in the test suite.
[1] https://sourceforge.net/apps/trac/bigdata/ticket/541 (MemoryManager Journal mode)
[2] https://sourceforge.net/apps/trac/bigdata/ticket/531 (SPARQL UPDATE for Solution Sets).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/FileMetadata.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/RootBlockView.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStrategy.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestCommitHistory.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestTx.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/rwstore/sector/TestMemStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest2.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/FileMetadata.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/FileMetadata.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/FileMetadata.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -815,6 +815,9 @@
case Temporary:
buffer = null;
break;
+ case MemStore:
+ buffer = null;
+ break;
default:
throw new AssertionError();
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/RootBlockView.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/RootBlockView.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/RootBlockView.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -184,7 +184,7 @@
* readable). Finally, new root block images MUST be formed using the
* {@link #currentVersion}.
*/
- static final int currentVersion = VERSION2;
+ public static final int currentVersion = VERSION2;
/**
* The buffer holding the backing data.
@@ -355,7 +355,7 @@
* superseded by another journal).
* @param version
*/
- RootBlockView(//
+ public RootBlockView(//
final boolean rootBlock0, final int offsetBits,
final long nextOffset, final long firstCommitTime,
final long lastCommitTime, final long commitCounter,
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -353,6 +353,11 @@
return m_root.getSectorCount();
}
+ @Override
+ public void commit() {
+ m_root.commit();
+ }
+
// private SectorAllocation m_head = null;
//
// /**
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -276,5 +276,7 @@
public int getMaxSectors();
public void close();
+
+ public void commit();
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStore.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStore.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -165,7 +165,7 @@
root.addCounter("UUID", new OneShotInstrument<String>(getUUID()
.toString()));
- root.attach(m_strategy.m_mmgr.getCounters());
+ root.attach(m_strategy.getMemoryManager().getCounters());
return root;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStrategy.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStrategy.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemStrategy.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -12,20 +12,39 @@
import com.bigdata.journal.IBufferStrategy;
import com.bigdata.journal.IJournal;
import com.bigdata.journal.IRootBlockView;
+import com.bigdata.journal.RootBlockView;
+import com.bigdata.journal.StoreTypeEnum;
import com.bigdata.mdi.IResourceMetadata;
import com.bigdata.rawstore.IAddressManager;
+import com.bigdata.util.ChecksumUtility;
+/**
+ * A buffer implementation backed by an {@link IMemoryManager}.
+ *
+ * @author <a href="mailto:mat...@us...">Martyn Cutcher</a>
+ * @version $Id$
+ */
public class MemStrategy implements IBufferStrategy {
- final IMemoryManager m_mmgr;
- final IAddressManager m_am;
+ final private IMemoryManager m_mmgr;
+ final private IAddressManager m_am;
- boolean m_modifiable = true;
- boolean m_open = true;
+ private volatile boolean m_modifiable = true;
+ private volatile boolean m_open = true;
- public MemStrategy(IMemoryManager mmgr) {
- m_mmgr = mmgr;
- m_am = new IAddressManager() {
+ private volatile boolean m_dirty = true;
+
+ private volatile IRootBlockView m_rb0 = null;
+ private volatile IRootBlockView m_rb1 = null;
+
+ public MemStrategy(final IMemoryManager mmgr) {
+
+ if (mmgr == null)
+ throw new IllegalArgumentException();
+
+ m_mmgr = mmgr;
+
+ m_am = new IAddressManager() {
@Override
public int getByteCount(long addr) {
@@ -55,11 +74,28 @@
@Override
public String toString(final long addr) {
- // TODO Auto-generated method stub
return "PhysicalAddress: " + getPhysicalAddress(addr) + ", length: " + getByteCount(addr);
}
};
+
+ // initialise RootBlocks
+ final UUID uuid = UUID.randomUUID(); // Journal's UUID.
+ final long createTime = System.currentTimeMillis();
+ final ChecksumUtility checker = new ChecksumUtility();
+ m_rb0 = new RootBlockView(true, 0,
+ 0, 0, 0, 0, 0, 0,
+ uuid,
+ 0, 0, 0,
+ StoreTypeEnum.RW,
+ createTime, 0, RootBlockView.currentVersion, checker);
+
+ m_rb1 = new RootBlockView(false, 0,
+ 0, 0, 0, 0, 0, 0,
+ uuid,
+ 0, 0, 0,
+ StoreTypeEnum.RW,
+ createTime, 0, RootBlockView.currentVersion, checker);
}
public IMemoryManager getMemoryManager() {
@@ -68,7 +104,7 @@
@Override
public void abort() {
- throw new UnsupportedOperationException();
+ // NOP
}
@Override
@@ -78,7 +114,8 @@
@Override
public void commit(IJournal journal) {
- // NOP
+ m_mmgr.commit();
+ m_dirty = false;
}
@Override
@@ -92,11 +129,20 @@
}
@Override
- public CounterSet getCounters() {
- // TODO Auto-generated method stub
- return null;
+ synchronized public CounterSet getCounters() {
+
+ if (root == null) {
+
+ root = new CounterSet();
+
+ }
+
+ return root;
+
}
+ private CounterSet root;
+
@Override
public long getExtent() {
// return the amount of currently reserved memory
@@ -158,14 +204,12 @@
@Override
public ByteBuffer readRootBlock(boolean rootBlock0) {
- // NOP
- return null;
+ return (rootBlock0 ? m_rb0 : m_rb1).asReadOnlyBuffer();
}
@Override
public boolean requiresCommit(IRootBlockView block) {
- // NOP
- return false;
+ return m_dirty;
}
@Override
@@ -186,7 +230,11 @@
@Override
public void writeRootBlock(IRootBlockView rootBlock,
ForceEnum forceOnCommitEnum) {
- throw new UnsupportedOperationException();
+ if (rootBlock.isRootBlock0()) {
+ m_rb0 = rootBlock;
+ } else {
+ m_rb1 = rootBlock;
+ }
}
@Override
@@ -200,6 +248,8 @@
if (!m_modifiable) {
throw new IllegalStateException("The store is not modifiable");
}
+ m_mmgr.free(addr);
+ m_dirty = true;
}
@Override
@@ -264,16 +314,20 @@
@Override
public long write(ByteBuffer data) {
+ m_dirty = true;
+
return m_mmgr.allocate(data);
}
@Override
public long write(ByteBuffer data, long oldAddr) {
- // since there is no commit, we might as well
- // immediately recycle!
+ m_dirty = true;
+
+ final long ret = write(data);
+
m_mmgr.free(oldAddr);
-
- return write(data);
+
+ return ret;
}
// AddressManager delagates
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -27,6 +27,7 @@
import java.io.File;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
@@ -1001,6 +1002,14 @@
return new PSInputStream(this, addr);
}
+ @Override
+ public void commit() {
+ final Iterator<SectorAllocator> sectors = m_sectors.iterator();
+ while (sectors.hasNext()) {
+ sectors.next().commit();
+ }
+ }
+
/*
* DONE: The constructor must be able to accept nsectors :=
* Integer.MAX_VALUE in order to indicate that the #of backing buffers is
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -682,4 +682,12 @@
}
+ /**
+ * Called from MemoryManager to commit bits
+ */
+ public void commit() {
+ m_commitbits = m_bits.clone();
+ m_transientbits = m_bits.clone();
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestCommitHistory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestCommitHistory.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestCommitHistory.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -33,6 +33,7 @@
import com.bigdata.btree.BTree;
import com.bigdata.btree.BytesUtil;
+import com.bigdata.rwstore.sector.MemStrategy;
import com.bigdata.service.AbstractTransactionService;
/**
@@ -334,7 +335,8 @@
assertTrue(commitRecordIndexAddrs[i]!=0L);
- if ((!(journal.getBufferStrategy() instanceof RWStrategy)) && i > 0)
+ final IBufferStrategy strat = journal.getBufferStrategy();
+ if ((!(strat instanceof RWStrategy || strat instanceof MemStrategy)) && i > 0)
assertTrue(commitRecordIndexAddrs[i] > commitRecordIndexAddrs[i - 1]);
// get the current commit record.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestTx.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestTx.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestTx.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -41,6 +41,7 @@
import com.bigdata.btree.IIndex;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
+import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.btree.Tuple;
import com.bigdata.btree.isolation.IsolatedFusedView;
@@ -266,6 +267,9 @@
final long tx1 = journal.newTx(ITx.UNISOLATED);
+ if(log.isDebugEnabled())
+ log.debug("State A, tx1: " + tx1 + "\n" + showCRI(journal));
+
{
/*
@@ -291,8 +295,12 @@
IIndex index = journal.getIndex(name);
assertNull(index.insert(k2, v2));
+ assertTrue(index.contains(k2));
- assertNotSame(0L, journal.commit());
+ final long c2 = journal.commit();
+ if(log.isDebugEnabled())
+ log.debug("State B, c2: " + c2 + "\n" + showCRI(journal));
+ assertNotSame(0L, c2);
}
@@ -310,10 +318,26 @@
}
+ {
+
+ final IIndex index = journal.getIndex(name);
+
+ assertTrue(index.contains(k1));
+ assertTrue(index.contains(k2));
+
+ }
+
+ /*
+ * start another transaction and verify that the 2nd committed
+ * write is now visible to that transaction.
+ */
+
final long tx2 = journal.newTx(ITx.UNISOLATED);
+
+ if(log.isDebugEnabled())
+ log.debug("tx1: " + tx1 + ", tx2: " + tx2 + "\n" + showCRI(journal));
{
-
/*
* start another transaction and verify that the 2nd committed
* write is now visible to that transaction.
@@ -337,7 +361,46 @@
}
}
+
+ String showCRI(final Journal journal) {
+ final ITupleIterator<CommitRecordIndex.Entry> commitRecords;
+ /*
+ * Commit can be called prior to Journal initialisation, in which case
+ * the commitRecordIndex will not be set.
+ */
+ final IIndex commitRecordIndex = journal.getReadOnlyCommitRecordIndex();
+ if (commitRecordIndex == null) { // TODO Why is this here?
+ return "EMPTY";
+ }
+ final IndexMetadata metadata = commitRecordIndex.getIndexMetadata();
+
+ commitRecords = commitRecordIndex.rangeIterator();
+
+ StringBuilder out = new StringBuilder();
+ while (commitRecords.hasNext()) {
+
+ final ITuple<CommitRecordIndex.Entry> tuple = commitRecords.next();
+
+ final CommitRecordIndex.Entry entry = tuple.getObject();
+
+ try {
+
+ final ICommitRecord record = CommitRecordSerializer.INSTANCE
+ .deserialize(journal.read(entry.addr));
+
+ out.append(record.toString() + "\n");
+ } catch (RuntimeException re) {
+
+ throw new RuntimeException("Problem with entry at "
+ + entry.addr, re);
+
+ }
+ }
+ return out.toString();
+
+ }
+
/**
* Test verifies that an isolated write is visible inside of a transaction
* (tx1) but not in a concurrent transaction (tx2) and not in the unisolated
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/rwstore/sector/TestMemStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/rwstore/sector/TestMemStore.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/rwstore/sector/TestMemStore.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -5,20 +5,17 @@
import junit.extensions.proxy.ProxyTestSuite;
import junit.framework.Test;
-import junit.framework.TestCase2;
import com.bigdata.io.DirectBufferPool;
+import com.bigdata.journal.AbstractJournalTestCase;
import com.bigdata.journal.AbstractMRMWTestCase;
import com.bigdata.journal.AbstractMROWTestCase;
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.Journal;
+import com.bigdata.journal.Journal.Options;
+import com.bigdata.journal.TestJournalBasics;
import com.bigdata.rawstore.AbstractRawStoreTestCase;
-import com.bigdata.journal.AbstractJournalTestCase;
-
import com.bigdata.rawstore.IRawStore;
-import com.bigdata.rwstore.RWStore;
-import com.bigdata.journal.TestJournalBasics;
-import com.bigdata.journal.Journal.Options;
/**
* Test suite for {@link MemStore}.
@@ -101,7 +98,7 @@
* all the tests will run with the configuration specified in this test
* class and its optional .properties file.
*/
- // suite.addTest(TestJournalBasics.suite());
+ suite.addTest(TestJournalBasics.suite());
return suite;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -86,7 +86,7 @@
/**
* Return <code>true</code> if the target is a SOLUTION SET.
*/
- final boolean isTargetSolutionSet() {
+ public final boolean isTargetSolutionSet() {
return getProperty(Annotations.TARGET) instanceof VarNode;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -386,4 +386,16 @@
boolean DEFAULT_HASH_JOIN = false;
+ /**
+ * Option controls whether or not the bigdata extension to SPARQL Update for
+ * named solution sets is enabled.
+ *
+ * @see <a
+ * href="https://sourceforge.net/apps/mediawiki/bigdata/index.php?title=SPARQL_Update">
+ * SPARQL Update </a>
+ */
+ String SOLUTION_SET_CACHE = "solutionSetCache";
+
+ boolean DEFAULT_SOLUTION_SET_CACHE = false;
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -28,7 +28,6 @@
package com.bigdata.rdf.sparql.ast.cache;
import com.bigdata.bop.IBindingSet;
-import com.bigdata.rdf.sparql.ast.QueryBase;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
import com.bigdata.striterator.ICloseableIterator;
@@ -51,6 +50,48 @@
*/
void close();
+ /**
+ * Clear the specified named solution set.
+ *
+ * @param solutionSet
+ * The name of the solution set.
+ *
+ * @return <code>true</code> iff a solution set by that name existed and was
+ * cleared.
+ */
+ boolean clear(AST2BOpContext ctx,String solutionSet);
+
+ /**
+ * Clear all named solution sets.
+ */
+ void clearAll(AST2BOpContext ctx);
+
+ /**
+ * Save the solutions a named solution set.
+ *
+ * @param ctx
+ * @param solutionSet
+ * The name of the solution set.
+ * @param src
+ * The solutions.
+ */
+ void put(AST2BOpContext ctx, String solutionSet,
+ ICloseableIterator<IBindingSet[]> src);
+
+ /**
+ * Read the solutions from a named solution set.
+ *
+ * @param ctx
+ * @param solutionSet
+ * The name of the solution set.
+ *
+ * @return An iterator from which the solutions may be drained.
+ *
+ * @throws IllegalStateException
+ * if no solution set with that name exists.
+ */
+ ICloseableIterator<IBindingSet[]> get(AST2BOpContext ctx, String solutionSet);
+
// /**
// * Return the result from the cache -or- <code>null</code> if there is a
// * cache miss.
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -0,0 +1,333 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2012. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Apr 9, 2012
+ */
+
+package com.bigdata.rdf.sparql.ast.cache;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.NoSuchElementException;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.log4j.Logger;
+
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.btree.Checkpoint;
+import com.bigdata.io.DataInputBuffer;
+import com.bigdata.io.DataOutputBuffer;
+import com.bigdata.rawstore.Bytes;
+import com.bigdata.rawstore.IRawStore;
+import com.bigdata.rdf.internal.encoder.IVSolutionSetDecoder;
+import com.bigdata.rdf.internal.encoder.IVSolutionSetEncoder;
+import com.bigdata.rwstore.PSOutputStream;
+import com.bigdata.rwstore.sector.IMemoryManager;
+import com.bigdata.striterator.ICloseableIterator;
+
+/**
+ * Metadata for a solution set declaration (sort of like a checkpoint record).
+ *
+ * TODO Unit tests (this can be tested in isolation).
+ *
+ * TODO Work out the relationship to {@link Checkpoint} records.
+ */
+final class SolutionSetMetadata {
+
+ private static final Logger log = Logger
+ .getLogger(SolutionSetMetadata.class);
+
+ public final String name;
+
+ private final IMemoryManager allocationContext;
+
+ private long addr;
+
+ public SolutionSetMetadata(final String name,
+ final IMemoryManager allocationContext) {
+
+ if (name == null)
+ throw new IllegalArgumentException();
+
+ if (allocationContext == null)
+ throw new IllegalArgumentException();
+
+ this.name = name;
+
+ this.allocationContext = allocationContext;
+
+ }
+
+ public void clear() {
+
+ allocationContext.clear();
+ addr = IRawStore.NULL;
+
+ }
+
+ public ICloseableIterator<IBindingSet[]> get() {
+
+ final long addr = this.addr;
+
+ if (addr == IRawStore.NULL)
+ throw new IllegalStateException();
+
+ try {
+ return new SolutionSetStreamDecoder(addr);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ /**
+ * Stream decoder for solution sets.
+ */
+ private class SolutionSetStreamDecoder implements
+ ICloseableIterator<IBindingSet[]> {
+
+ private final DataInputStream in;
+ private final IVSolutionSetDecoder decoder;
+
+ private boolean open = false;
+
+ /** The next chunk of solutions to be visited. */
+ private IBindingSet[] bsets = null;
+
+ public SolutionSetStreamDecoder(final long addr)
+ throws IOException {
+
+ this.in = new DataInputStream(
+ wrapInputStream(allocationContext.getInputStream(addr)));
+
+ this.open = true;
+
+ this.decoder = new IVSolutionSetDecoder();
+
+ }
+
+ @Override
+ public void close() {
+ if(open) {
+ open = false;
+ try {
+ in.close();
+ } catch (IOException e) {
+ // Unexpected exception.
+ log.error(e, e);
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+
+ if (open && bsets == null) {
+
+ /*
+ * Read ahead and extract a chunk of solutions.
+ */
+
+ try {
+
+ bsets = decodeNextChunk();
+
+ } catch (IOException e) {
+
+ throw new RuntimeException(e);
+
+ }
+
+ }
+
+ return open && bsets != null;
+
+ }
+
+ /**
+ * Read ahead and decode the next chunk of solutions.
+ *
+ * @return The decoded solutions.
+ */
+ private IBindingSet[] decodeNextChunk() throws IOException {
+
+ // #of solutions in this chunk.
+ final int chunkSize = in.readInt();
+
+ // #of bytes in this chunk.
+ final int byteLength = in.readInt();
+
+ // Read in all the bytes in the next chunk.
+ final byte[] a = new byte[byteLength];
+
+ // read data
+ in.readFully(a);
+
+ // Wrap byte[] as stream.
+ final DataInputBuffer buf = new DataInputBuffer(a);
+
+ // Allocate array for the decoded solutions.
+ final IBindingSet[] t = new IBindingSet[chunkSize];
+
+ // Decode the solutions into the array.
+ for (int i = 0; i < chunkSize; i++) {
+
+ t[i] = decoder
+ .decodeSolution(buf, true/* resolveCachedValues */);
+
+ }
+
+ // Return the decoded solutions.
+ return t;
+
+ }
+
+ @Override
+ public IBindingSet[] next() {
+
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ final IBindingSet[] t = bsets;
+
+ bsets = null;
+
+ return t;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ public void put(final ICloseableIterator<IBindingSet[]> src) {
+
+ if (src == null)
+ throw new IllegalArgumentException();
+
+ final IVSolutionSetEncoder encoder = new IVSolutionSetEncoder();
+
+ // Stream writing onto the backing store.
+ final PSOutputStream out = allocationContext.getOutputStream();
+ // Address from which the solutions may be read.
+ final long newAddr;
+ // #of solutions written.
+ long nsolutions = 0;
+ // #of bytes for the encoded solutions (before compression).
+ long nbytes = 0;
+ try {
+
+ final DataOutputBuffer buf = new DataOutputBuffer();
+
+ final OutputStream os = wrapOutputStream(out);
+
+ while (src.hasNext()) {
+
+ // Discard the data in the buffer.
+ buf.reset();
+
+ // Chunk of solutions to be written.
+ final IBindingSet[] chunk = src.next();
+
+ // Write solutions.
+ for (int i = 0; i < chunk.length; i++) {
+
+ encoder.encodeSolution(buf, chunk[i]);
+
+ }
+
+ // #of bytes written onto the buffer.
+ final int bytesBuffered = buf.limit();
+
+ // Write header (#of solutions in this chunk).
+ os.write(chunk.length);
+
+ // Write header (#of bytes buffered).
+ os.write(bytesBuffered);
+
+ // transfer buffer data to output stream.
+ os.write(buf.array(), 0/* off */, bytesBuffered);
+
+ // += headerSize (chunkSize,bytesBuffered) + bytesBuffered.
+ nbytes += (Bytes.SIZEOF_INT + Bytes.SIZEOF_INT) + bytesBuffered;
+
+ nsolutions += chunk.length;
+
+ }
+
+ os.flush();
+
+ out.flush();
+
+ newAddr = out.getAddr();
+
+ if (log.isDebugEnabled())
+ log.debug("Wrote " + nsolutions + "; encodedBytes=" + nbytes
+ + " bytes, bytesWritten=" + out.getBytesWritten());
+
+ } catch (IOException e) {
+
+ throw new RuntimeException(e);
+
+ } finally {
+
+ try {
+ out.close();
+ } catch (IOException e) {
+ // Unexpected exception.
+ log.error(e, e);
+ }
+
+ }
+
+ if (addr != IRawStore.NULL) {
+
+ allocationContext.free(addr);
+
+ }
+
+ addr = newAddr;
+
+ }
+
+ private OutputStream wrapOutputStream(final OutputStream out)
+ throws IOException {
+
+ return new GZIPOutputStream(out);
+
+ }
+
+ private InputStream wrapInputStream(final InputStream in)
+ throws IOException {
+
+ return new GZIPInputStream(in);
+
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -27,7 +27,10 @@
package com.bigdata.rdf.sparql.ast.cache;
+import java.util.Iterator;
+import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@@ -35,6 +38,7 @@
import org.apache.log4j.Logger;
import com.bigdata.bfs.BigdataFileSystem;
+import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.btree.view.FusedView;
import com.bigdata.journal.AbstractJournal;
@@ -50,13 +54,16 @@
import com.bigdata.rdf.changesets.IChangeLog;
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.webapp.ConfigParams;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
import com.bigdata.relation.locator.DefaultResourceLocator;
import com.bigdata.resources.IndexManager;
import com.bigdata.rwstore.RWStore;
import com.bigdata.rwstore.sector.IMemoryManager;
+import com.bigdata.rwstore.sector.MemStrategy;
import com.bigdata.rwstore.sector.MemoryManager;
import com.bigdata.service.IDataService;
import com.bigdata.sparse.SparseRowStore;
+import com.bigdata.striterator.ICloseableIterator;
/**
* A SPARQL cache.
@@ -149,7 +156,29 @@
* stores, etc.
*/
private final CacheJournal cache;
-
+
+ /**
+ * TODO There are several problems here.
+ * <p>
+ * 1. We need a common semantics for visibility for the named solution sets
+ * and the query and update operations. This cache can not provide that
+ * without being somehow integrated with the MVCC architecture.
+ * <p>
+ * 2. We need to expire (at least some) cache objects. That expiration
+ * should have a default but should also be configurable for each cache
+ * object. The visibility issue also exists for expiration (we can not
+ * expire a result set while it is being used).
+ * <p>
+ * 3. If we allow updates against named solution sets, then the visibility
+ * of those updates must again be consistent with the MVCC architecture for
+ * the query and update operations.
+ * <p>
+ * 4. We need to have metadata about solution sets on hand for explicit
+ * CREATEs (e.g., supporting declared join variables).
+ */
+ private final ConcurrentHashMap<String/*name*/,SolutionSetMetadata> cacheMap;
+// private final ConcurrentWeakValueCacheWithTimeout<String/* name */, IMemoryManager /* allocationContext */> cacheMap;
+
// /**
// * The performance counters for the {@link IBufferStrategy} backing the
// * {@link #cache}.
@@ -215,21 +244,48 @@
this.queryEngine = queryEngine;
/*
- * FIXME Setup properties from Journal or Federation (mainly the maximum
+ * TODO Setup properties from Journal or Federation (mainly the maximum
* amount of RAM to use, but we can not limit that if we are using this
* for to store named solution sets rather than as a cache).
+ *
+ * TODO Setup an expire thread or a priority heap for expiring named
+ * solution sets from the cache.
*/
final Properties properties = new Properties();
-
- // FIXME Use MemStore to back this (new BufferMode).
+
+ /*
+ * Note: The cache will be backed by ByteBuffer objects allocated on the
+ * native process heap (Zero GC).
+ */
properties.setProperty(com.bigdata.journal.Options.BUFFER_MODE,
- BufferMode.Transient.name());
+ BufferMode.MemStore.name());
properties.setProperty(com.bigdata.journal.Options.INITIAL_EXTENT, ""
+ (10 * Bytes.megabyte));
+
+ properties.setProperty(com.bigdata.journal.Options.CREATE_TEMP_FILE,
+ "true");
+
+// properties.setProperty(Journal.Options.COLLECT_PLATFORM_STATISTICS,
+// "false");
+//
+// properties.setProperty(Journal.Options.COLLECT_QUEUE_STATISTICS,
+// "false");
+//
+// properties.setProperty(Journal.Options.HTTPD_PORT, "-1"/* none */);
+
+ this.cache = new CacheJournal(properties);
- this.cache = new CacheJournal( properties );
-
+// /*
+// * TODO The expire should be per cached object, not global. We would
+// * need a different cache map class for that.
+// */
+// final long timeoutNanos = TimeUnit.SECONDS.toNanos(20);
+
+// this.cacheMap = new ConcurrentWeakValueCacheWithTimeout<String, IMemoryManager>(
+// 0/* queueCapacity */, timeoutNanos);
+ this.cacheMap = new ConcurrentHashMap<String, SolutionSetMetadata>();
+
}
@Override
@@ -255,10 +311,101 @@
@Override
public void close() {
+ cacheMap.clear();
+
cache.destroy();
}
+ @Override
+ public void clearAll(final AST2BOpContext ctx) {
+
+ final Iterator<Map.Entry<String, SolutionSetMetadata>> itr = cacheMap
+ .entrySet().iterator();
+
+ while (itr.hasNext()) {
+
+ final Map.Entry<String, SolutionSetMetadata> e = itr.next();
+
+ final String solutionSet = e.getKey();
+
+ final SolutionSetMetadata sset = e.getValue();
+
+ if (log.isInfoEnabled())
+ log.info("solutionSet: " + solutionSet);
+
+ sset.clear();
+
+ itr.remove();
+
+ }
+
+ }
+
+ @Override
+ public boolean clear(final AST2BOpContext ctx, final String solutionSet) {
+
+ if (log.isInfoEnabled())
+ log.info("solutionSet: " + solutionSet);
+
+ final SolutionSetMetadata sset = cacheMap.remove(solutionSet);
+
+ if (sset != null) {
+ sset.clear();
+
+ return true;
+
+ }
+
+ return false;
+
+ }
+
+ public void put(final AST2BOpContext ctx, final String solutionSet,
+ final ICloseableIterator<IBindingSet[]> src) {
+
+ if (solutionSet == null)
+ throw new IllegalArgumentException();
+
+ if (src == null)
+ throw new IllegalArgumentException();
+
+ /*
+ * TODO Deal with visibility issues on update (when the modified
+ * solution set state becomes visible).
+ */
+
+ SolutionSetMetadata sset = cacheMap.get(solutionSet);
+
+ if (sset == null) {
+
+ final IMemoryManager mmrgr = ((MemStrategy) cache
+ .getBufferStrategy()).getMemoryManager();
+
+ sset = new SolutionSetMetadata(solutionSet,
+ mmrgr.createAllocationContext());
+
+ }
+
+ sset.put(src);
+
+ }
+
+ public ICloseableIterator<IBindingSet[]> get(final AST2BOpContext ctx,
+ final String solutionSet) {
+
+ if (solutionSet == null)
+ throw new IllegalArgumentException();
+
+ final SolutionSetMetadata sset = cacheMap.get(solutionSet);
+
+ if (sset == null)
+ throw new IllegalStateException("Not found: " + solutionSet);
+
+ return sset.get();
+
+ }
+
// @Override
// public ICacheHit get(final AST2BOpContext ctx,
// final QueryBase queryOrSubquery) {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -295,18 +295,6 @@
.getIndexManager());
/*
- * Cache for SPARQL solution sets.
- *
- * TODO Define a query hint for enabling or disabling the SPARQL cache
- * for an operation. However, the cache reference must still be set
- * since we must still issue invalidation notes to the cache even if it
- * is disabled during update operations (and it might not make sense to
- * use the cache during SPARQL UPDATEs at all).
- */
- this.sparqlCache = null;
-// this.sparqlCache = SparqlCacheFactory.getSparqlCache(queryEngine);
-
- /*
* Figure out the query UUID that will be used. This will be bound onto
* the query plan when it is generated. We figure out what it will be up
* front so we can refer to its UUID in parts of the query plan. For
@@ -331,6 +319,29 @@
this.queryId = queryId;
+ /*
+ * Cache for SPARQL named solution sets.
+ *
+ * TODO Define a query hint for enabling or disabling the SPARQL cache
+ * for an operation.
+ */
+ {
+ final boolean enable = queryHints == null ? QueryHints.DEFAULT_SOLUTION_SET_CACHE
+ : Boolean
+ .valueOf(queryHints
+ .getProperty(
+ QueryHints.SOLUTION_SET_CACHE,
+ QueryHints.DEFAULT_SOLUTION_SET_CACHE ? Boolean.TRUE
+ .toString() : Boolean.FALSE
+ .toString()));
+ if (enable) {
+ this.sparqlCache = SparqlCacheFactory
+ .getSparqlCache(queryEngine);
+ } else {
+ this.sparqlCache = null;
+ }
+ }
+
this.context = new BOpContextBase(queryEngine);
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -713,11 +713,11 @@
if (!sourceGraph.equals(targetGraph)) {
- clearGraph(targetGraph, null/* scope */, context);
+ clearOneGraph(targetGraph, context);
copyStatements(context, op.isSilent(), sourceGraph, targetGraph);
- clearGraph(sourceGraph, null/* scope */, context);
+ clearOneGraph(sourceGraph, context);
}
@@ -754,7 +754,7 @@
if (!sourceGraph.equals(targetGraph)) {
- clearGraph(targetGraph, null/* scope */, context);
+ clearOneGraph(targetGraph, context);
copyStatements(context, op.isSilent(), sourceGraph, targetGraph);
@@ -1054,36 +1054,82 @@
final DropGraph op, final AST2BOpUpdateContext context)
throws RepositoryException, SailException {
+ if (runOnQueryEngine)
+ throw new UnsupportedOperationException();
+
final TermNode targetGraphNode = op.getTargetGraph();
final BigdataURI targetGraph = targetGraphNode == null ? null
: (BigdataURI) targetGraphNode.getValue();
- final Scope scope = op.getScope();
+ clearGraph(op.isSilent(), op.getTargetSolutionSet(), targetGraph,
+ op.getScope(), op.isAllGraphs(), op.isAllSolutionSets(),
+ context);
- if (runOnQueryEngine)
- throw new UnsupportedOperationException();
-
- // FIXME This assumes that the target is a GRAPH (vs SOLUTION SET).
- clearGraph(targetGraph, scope, context);
-
return left;
}
/**
- * Remove all statements from the target graph or the specified
- * {@link Scope}.
+ * Clear one graph (SILENT).
*
* @param targetGraph
+ * The graph to be cleared -or- <code>null</code> if no target
+ * graph was named.
+ * @param context
+ * The {@link AST2BOpUpdateContext} used to perform the
+ * operation.
+ *
+ * @throws RepositoryException
+ * @throws SailException
+ */
+ private static final void clearOneGraph(final URI targetGraph, //
+ final AST2BOpUpdateContext context//
+ ) throws RepositoryException, SailException {
+
+ clearGraph(true/* silent */, null/* targetSolutionSet */, targetGraph,
+ null/* scope */, false/* allGraphs */,
+ false/* allSolutionSets */, context);
+
+ }
+
+ /**
+ * Clear one or more graphs and/or solution sets.
+ *
+ * @param silent
+ * When <code>true</code>, some kinds of problems will not be
+ * reported to the caller.
+ * @param targetSolutionSet
+ * The target solution set to be cleared -or- <code>null</code>
+ * if no target solution set was named.
+ * @param targetGraph
+ * The graph to be cleared -or- <code>null</code> if no target
+ * graph was named.
* @param scope
+ * The scope iff just the graphs in either the
+ * {@link Scope#DEFAULT_CONTEXTS} or {@link Scope#NAMED_CONTEXTS}
+ * should be cleared and otherwise <code>null</code>.
+ * @param allGraphs
+ * iff all graphs should be cleared.
+ * @param allSolutionSets
+ * iff all solution sets should be cleared.
* @param context
+ * The {@link AST2BOpUpdateContext} used to perform the
+ * operation.
+ *
* @throws RepositoryException
* @throws SailException
*/
- private static void clearGraph(final URI targetGraph, final Scope scope,
- final AST2BOpUpdateContext context) throws RepositoryException,
- SailException {
+ // CLEAR/DROP ( SILENT )? (GRAPH IRIref | DEFAULT | NAMED | ALL | GRAPHS | SOLUTIONS | SOLUTIONS %VARNAME)
+ private static void clearGraph(//
+ final boolean silent,//
+ final String solutionSet,//
+ final URI targetGraph, //
+ final Scope scope,//
+ final boolean allGraphs,//
+ final boolean allSolutionSets,//
+ final AST2BOpUpdateContext context//
+ ) throws RepositoryException, SailException {
if (log.isDebugEnabled())
log.debug("targetGraph=" + targetGraph + ", scope=" + scope);
@@ -1093,6 +1139,17 @@
*/
final BigdataSailConnection sailConn = context.conn.getSailConnection();
+ if (solutionSet != null) {
+
+ // Clear the named solution set.
+ if (!context.sparqlCache.clear(context, solutionSet) && !silent) {
+
+ throw new SailException("solutionSet=" + solutionSet);
+
+ }
+
+ }
+
if (targetGraph != null) {
/*
@@ -1102,7 +1159,9 @@
sailConn.removeStatements(null/* s */, null/* p */, null/* o */,
targetGraph);
- } else if (scope != null) {
+ }
+
+ if (scope != null) {
if (scope == Scope.DEFAULT_CONTEXTS) {
@@ -1141,7 +1200,9 @@
}
- } else {
+ }
+
+ if(allGraphs) {
/*
* Addressing ALL graphs.
@@ -1158,7 +1219,20 @@
}
+ /*
+ * Note: We need to verify that the backing data structure is enabled
+ * since the default semantics of CLEAR ALL and DROP ALL also imply all
+ * named solution sets.
+ */
+ if (allSolutionSets && context.sparqlCache != null) {
+
+ // Delete all solution sets.
+ context.sparqlCache.clearAll(context);
+
+ }
+
}
+
/**
* If the graph already exists (context has at least one statement), then
* this is an error (unless SILENT). Otherwise it is a NOP.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest2.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest2.java 2012-04-09 19:31:58 UTC (rev 6265)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest2.java 2012-04-09 19:37:57 UTC (rev 6266)
@@ -47,7 +47,6 @@
import junit.framework.TestCase2;
import org.apache.log4j.Logger;
-import org.openrdf.model.ValueFactory;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.QueryLanguage;
import org.openrdf.query.QueryResultUtil;
@@ -420,8 +419,8 @@
sb.append("}\n");
// sb.append("ORDER BY ?name");
- con.prepareUpdate(QueryLanguage.SPARQL,sb.toString()).execute();
-
+ con.prepareUpdate(QueryLanguage.SPARQL, sb.toString()).execute();
+
}
// Query it.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-04-10 15:03:01
|
Revision: 6269
http://bigdata.svn.sourceforge.net/bigdata/?rev=6269&view=rev
Author: thompsonbry
Date: 2012-04-10 15:02:48 +0000 (Tue, 10 Apr 2012)
Log Message:
-----------
Working through the integration of the bigdata extensions to SPARQL
UPDATE to support named solution sets [1,2]. You can now do INSERT
INTO but not yet DELETE FROM or DELETE/INSERT. CLEAR/DROP and CREATE
now work as well. Both DELETE FROM and DELETE/INSERT will require the
ability to resolve the named solution set when it appears outside of
the QueryRoot. We probably need to pass the ASTContext or the
SPARQLCache object into the StaticAnalysis constructor in order to
support those lookups.
I have added a method ASTEvalHelper which will run a SELECT query
without converting the results into openrdf binding sets. You can
specify whether or not the IVs will be materialized. This is progress
toward (a) not materializing IVs in SPARQL UPDATE [3]; and (b) not
de-chunking and re-chunking solutions for SPARQL query [4]. Howveer,
there is still a bunch more to do on both issues.
[1] https://sourceforge.net/apps/trac/bigdata/ticket/524 (SPARQL Cache)
[2] https://sourceforge.net/apps/mediawiki/bigdata/index.php?title=SPARQL_Update
[3] http://sourceforge.net/apps/trac/bigdata/ticket/522 (SPARQL UPDATE should not materialize RDF Values)
[4] http://sourceforge.net/apps/trac/bigdata/ticket/483 (Eliminate unnecessary dechunking and rechunking)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QuadsDataOrNamedSolutionSet.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/Update.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpFilters.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/update/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/sparql/TestUpdateExprBuilder2.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest2.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationIterator.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/cache/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/cache/TestSolutionSetCache.java
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationIterator.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationIterator.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationIterator.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -0,0 +1,146 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2012. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Apr 10, 2012
+ */
+
+package com.bigdata.bop.rdf.join;
+
+import java.util.NoSuchElementException;
+
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IVariable;
+import com.bigdata.rdf.lexicon.LexiconRelation;
+import com.bigdata.striterator.ICloseableIterator;
+
+/**
+ * Iterator pattern for chunked materialization.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ *
+ * @see ChunkedMaterializationOp
+ */
+public class ChunkedMaterializationIterator implements
+ ICloseableIterator<IBindingSet[]> {
+
+ private final IVariable<?>[] required;
+
+ private final LexiconRelation lex;
+
+ private final boolean materializeInlineIVs;
+
+ private final ICloseableIterator<IBindingSet[]> src;
+
+ private boolean open = false;
+
+ /**
+ *
+ * @param vars
+ * The variables to be materialized (required; must not be an
+ * empty array).
+ * @param lex
+ * The {@link LexiconRelation}.
+ * @param materializeInlineIVs
+ * When <code>true</code>, inline IVs will also be materialized.
+ * @param src
+ * The source iterator.
+ */
+ public ChunkedMaterializationIterator(//
+ final IVariable<?>[] vars,//
+ final LexiconRelation lex,//
+ final boolean materializeInlineIVs,//
+ final ICloseableIterator<IBindingSet[]> src) {
+
+ if (vars == null)
+ throw new IllegalArgumentException();
+
+ if (vars != null && vars.length == 0)
+ throw new IllegalArgumentException();
+
+ if (lex == null)
+ throw new IllegalArgumentException();
+
+ if (src == null)
+ throw new IllegalArgumentException();
+
+ this.required = vars;
+
+ this.lex = lex;
+
+ this.materializeInlineIVs = materializeInlineIVs;
+
+ this.src = src;
+
+ }
+
+ @Override
+ public void close() {
+
+ if (open) {
+
+ open = false;
+
+ src.close();
+
+ }
+
+ }
+
+ @Override
+ public boolean hasNext() {
+
+ if (open && !src.hasNext()) {
+
+ close();
+
+ return false;
+
+ }
+
+ return open;
+
+ }
+
+ @Override
+ public IBindingSet[] next() {
+
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ final IBindingSet[] chunk = src.next();
+
+ ChunkedMaterializationOp.resolveChunk(required, lex, chunk,
+ materializeInlineIVs);
+
+ return chunk;
+
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
\ No newline at end of file
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationIterator.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/ChunkedMaterializationOp.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -60,6 +60,9 @@
* {@link BigdataBindingSetResolverator} replaces the {@link IV} in the solution
* with the {@link BigdataValue}. Also, this class does not filter out variables
* which are not being materialized.
+ *
+ * @see ChunkedMaterializationIterator
+ * @see BigdataBindingSetResolverator
*/
public class ChunkedMaterializationOp extends PipelineOp {
@@ -71,10 +74,10 @@
public interface Annotations extends PipelineOp.Annotations {
/**
- * The variables to be materialized. When <code>null</code> or not
- * specified, ALL variables will be materialized. This may not be an
- * empty array as that would imply that there is no need to use this
- * operator.
+ * The {@link IVariable}[] identifying the variables to be materialized.
+ * When <code>null</code> or not specified, ALL variables will be
+ * materialized. This may not be an empty array as that would imply that
+ * there is no need to use this operator.
*/
String VARS = ChunkedMaterializationOp.class.getName()+".vars";
@@ -85,12 +88,12 @@
/**
* If true, materialize inline values in addition to term IDs.
*/
- String MATERIALIZE_ALL = ChunkedMaterializationOp.class.getName()+".materializeAll";
+ String MATERIALIZE_INLINE_IVS = ChunkedMaterializationOp.class.getName()+".materializeAll";
/**
* Default materialize all is false.
*/
- boolean DEFAULT_MATERIALIZE_ALL = false;
+ boolean DEFAULT_MATERIALIZE_INLINE_IVS = false;
}
@@ -149,10 +152,16 @@
);
}
- public boolean materializeAll() {
-
- return getProperty(Annotations.MATERIALIZE_ALL, Annotations.DEFAULT_MATERIALIZE_ALL);
-
+ /**
+ * When <code>true</code>, inline {@link IV}s are also materialized.
+ *
+ * @see Annotations#MATERIALIZE_INLINE_IVS
+ */
+ public boolean materializeInlineIVs() {
+
+ return getProperty(Annotations.MATERIALIZE_INLINE_IVS,
+ Annotations.DEFAULT_MATERIALIZE_INLINE_IVS);
+
}
public FutureTask<Void> eval(final BOpContext<IBindingSet> context) {
@@ -177,7 +186,7 @@
private final long timestamp;
- private final boolean materializeAll;
+ private final boolean materializeInlineIVs;
ChunkTask(final ChunkedMaterializationOp op,
final BOpContext<IBindingSet> context
@@ -194,7 +203,7 @@
timestamp = (Long) op.getProperty(Annotations.TIMESTAMP);
- materializeAll = op.materializeAll();
+ materializeInlineIVs = op.materializeInlineIVs();
}
@@ -219,7 +228,7 @@
stats.chunksIn.increment();
stats.unitsIn.add(a.length);
- resolveChunk(vars, lex, a, materializeAll);
+ resolveChunk(vars, lex, a, materializeInlineIVs);
sink.add(a);
@@ -252,10 +261,10 @@
* @param chunk
* The chunk of solutions whose variables will be materialized.
*/
- static private void resolveChunk(final IVariable<?>[] required,
+ static void resolveChunk(final IVariable<?>[] required,
final LexiconRelation lex,//
final IBindingSet[] chunk,//
- final boolean materializeAll
+ final boolean materializeInlineIVs
) {
if (log.isInfoEnabled())
@@ -304,7 +313,7 @@
}
- if (iv.needsMaterialization() || materializeAll) {
+ if (iv.needsMaterialization() || materializeInlineIVs) {
ids.add(iv);
@@ -331,7 +340,7 @@
}
- if (iv.needsMaterialization() || materializeAll) {
+ if (iv.needsMaterialization() || materializeInlineIVs) {
ids.add(iv);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/AbstractOneGraphManagement.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -88,7 +88,7 @@
*/
public final boolean isTargetSolutionSet() {
- return getProperty(Annotations.TARGET) instanceof VarNode;
+ return getProperty(Annotations.TARGET) instanceof String;
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QuadsDataOrNamedSolutionSet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QuadsDataOrNamedSolutionSet.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QuadsDataOrNamedSolutionSet.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -35,12 +35,19 @@
import com.bigdata.bop.NV;
/**
- * An AST node which models either {@link QuadData} or a named solution set.
- * This is used for the INSERT clause and DELETE clause of a
- * {@link DeleteInsertGraph} operation.
+ * An AST node which models either {@link QuadData} or a named solution set in
+ * support of the INSERT clause and DELETE clause of a {@link DeleteInsertGraph}
+ * operations. Use {@link #isQuads()} or {@link #isSolutions()} to identify how
+ * this AST node should be interpreted.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
+ * @version $Id: QuadsDataOrNamedSolutionSet.java 6196 2012-03-27 20:06:22Z
+ * thompsonbry $
+ *
+ * TODO Rather than overriding this for two very different things
+ * (quads data and the name of a solution set) it might be better to
+ * relayer the AST model or just handle this via polymorphism in the
+ * {@link DeleteInsertGraph} AST node.
*/
public class QuadsDataOrNamedSolutionSet extends QueryNodeBase implements
INamedSolutionSet, IProjectionDecl {
@@ -54,7 +61,8 @@
INamedSolutionSet.Annotations, IProjectionDecl.Annotations {
/**
- * The optional {@link QuadData}.
+ * The {@link QuadData} (optional; when present this is modeling QUADS
+ * data).
*/
String QUAD_DATA = "quadData";
@@ -107,6 +115,26 @@
}
+ /**
+ * Return <code>true</code> iff this models QUADS data (rather than named
+ * solutions)
+ */
+ public boolean isQuads() {
+
+ return getProperty(Annotations.QUAD_DATA)!=null;
+
+ }
+
+ /**
+ * Return <code>true</code> iff this models a reference to some named
+ * solutions (rather than QUADS data).
+ */
+ public boolean isSolutions() {
+
+ return getProperty(Annotations.NAMED_SET) != null;
+
+ }
+
public String getName() {
return (String) getProperty(Annotations.NAMED_SET);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -46,6 +46,7 @@
import com.bigdata.rdf.internal.constraints.INeedsMaterialization.Requirement;
import com.bigdata.rdf.internal.constraints.IPassesMaterialization;
import com.bigdata.rdf.internal.impl.literal.FullyInlineTypedLiteralIV;
+import com.bigdata.rdf.sparql.ast.cache.ISparqlCache;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
import com.bigdata.rdf.sparql.ast.optimizers.ASTBottomUpOptimizer;
import com.bigdata.rdf.sparql.ast.optimizers.ASTLiftPreFiltersOptimizer;
@@ -198,7 +199,16 @@
// private static final Logger log = Logger.getLogger(StaticAnalysis.class);
/**
+ * FIXME This will go away now once we have the ability to resolve named
+ * subqueries against the {@link ISparqlCache}.
*
+ * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/531">
+ * SPARQL Update for Named Solution Sets </a>
+ */
+ private static boolean requireDeclaredNamedSubquery = !QueryHints.DEFAULT_SOLUTION_SET_CACHE;
+
+ /**
+ *
* @param queryRoot
* The root of the query. We need to have this on hand in order
* to resolve {@link NamedSubqueryInclude}s during static
@@ -805,12 +815,12 @@
final NamedSubqueryRoot nsr = nsi.getNamedSubqueryRoot(queryRoot);
- if (nsr == null)
+ if (nsr != null)
+ vars.addAll(getDefinitelyProducedBindings(nsr));
+ else if(requireDeclaredNamedSubquery)
throw new RuntimeException("No named subquery declared: name="
+ nsi.getName());
- vars.addAll(getDefinitelyProducedBindings(nsr));
-
} else if(node instanceof ServiceNode) {
final ServiceNode service = (ServiceNode) node;
@@ -943,12 +953,12 @@
final NamedSubqueryRoot nsr = nsi.getNamedSubqueryRoot(queryRoot);
- if (nsr == null)
+ if (nsr != null)
+ vars.addAll(getMaybeProducedBindings(nsr));
+ else if(requireDeclaredNamedSubquery)
throw new RuntimeException("No named subquery declared: name="
+ nsi.getName());
- vars.addAll(getMaybeProducedBindings(nsr));
-
} else if(node instanceof ServiceNode) {
final ServiceNode service = (ServiceNode) node;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/Update.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/Update.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/Update.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -98,8 +98,8 @@
*/
public Update(final UpdateType updateType) {
- setProperty(Annotations.UPDATE_TYPE,updateType);
-
+ setProperty(Annotations.UPDATE_TYPE, updateType);
+
}
/**
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -29,6 +29,7 @@
import com.bigdata.bop.IBindingSet;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+import com.bigdata.rdf.spo.ISPO;
import com.bigdata.striterator.ICloseableIterator;
/**
@@ -59,14 +60,28 @@
* @return <code>true</code> iff a solution set by that name existed and was
* cleared.
*/
- boolean clear(AST2BOpContext ctx,String solutionSet);
+ boolean clearSolutions(AST2BOpContext ctx, String solutionSet);
/**
* Clear all named solution sets.
*/
- void clearAll(AST2BOpContext ctx);
+ void clearAllSolutions(AST2BOpContext ctx);
/**
+ * Create a named solution set.
+ *
+ * @param ctx
+ * @param solutionSet
+ * The name of the solution set.
+ * @param params
+ * The configuration parameters (optional).
+ *
+ * @throws RuntimeException
+ * if a solution set exists for that name.
+ */
+ void createSolutions(AST2BOpContext ctx, String solutionSet, ISPO[] params);
+
+ /**
* Save the solutions a named solution set.
*
* @param ctx
@@ -75,7 +90,7 @@
* @param src
* The solutions.
*/
- void put(AST2BOpContext ctx, String solutionSet,
+ void putSolutions(AST2BOpContext ctx, String solutionSet,
ICloseableIterator<IBindingSet[]> src);
/**
@@ -90,8 +105,20 @@
* @throws IllegalStateException
* if no solution set with that name exists.
*/
- ICloseableIterator<IBindingSet[]> get(AST2BOpContext ctx, String solutionSet);
+ ICloseableIterator<IBindingSet[]> getSolutions(AST2BOpContext ctx,
+ String solutionSet);
+ /**
+ * Return <code>true</code> iff a named solution set exists.
+ *
+ * @param ctx
+ * @param solutionSet
+ * The name of the solution set.
+ *
+ * @return <code>true</code> iff a solution set having that name exists.
+ */
+ boolean existsSolutions(AST2BOpContext ctx, String solutionSet);
+
// /**
// * Return the result from the cache -or- <code>null</code> if there is a
// * cache miss.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -28,12 +28,13 @@
package com.bigdata.rdf.sparql.ast.cache;
import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.NoSuchElementException;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.InflaterInputStream;
import org.apache.log4j.Logger;
@@ -45,6 +46,7 @@
import com.bigdata.rawstore.IRawStore;
import com.bigdata.rdf.internal.encoder.IVSolutionSetDecoder;
import com.bigdata.rdf.internal.encoder.IVSolutionSetEncoder;
+import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rwstore.PSOutputStream;
import com.bigdata.rwstore.sector.IMemoryManager;
import com.bigdata.striterator.ICloseableIterator;
@@ -52,23 +54,56 @@
/**
* Metadata for a solution set declaration (sort of like a checkpoint record).
*
- * TODO Unit tests (this can be tested in isolation).
- *
- * TODO Work out the relationship to {@link Checkpoint} records.
+ * TODO Work out the relationship to {@link Checkpoint} records. Everything
+ * about this solution set which is updatable should be organized into a
+ * checkpoint. The metadata declaration for the solution set should also be
+ * organized into the checkpoint, perhaps as an ISPO[], perhaps using a rigid
+ * schema. The size of the solution set should be visible when its metadata is
+ * looked at as a graph.
*/
final class SolutionSetMetadata {
private static final Logger log = Logger
.getLogger(SolutionSetMetadata.class);
+ /**
+ * The name of the solution set.
+ */
public final String name;
+ /**
+ * The metadata describing the solution set.
+ */
+ private final ISPO[] metadata;
+
+ /**
+ * The {@link IMemoryManager} on which the solution set will be written and
+ * from which it will be read.
+ */
private final IMemoryManager allocationContext;
- private long addr;
+ /**
+ * The #of solutions in this solutionset.
+ */
+ private long solutionCount;
+
+ /**
+ * The address from which the solution set may be read.
+ */
+ private long solutionSetAddr;
+ /**
+ *
+ * @param name
+ * The name of the solution set.
+ * @param allocationContext
+ * The {@link IMemoryManager} on which the solution set will be
+ * written and from which it will be read.
+ * @param metadata
+ * The metadata describing the solution set.
+ */
public SolutionSetMetadata(final String name,
- final IMemoryManager allocationContext) {
+ final IMemoryManager allocationContext, final ISPO[] metadata) {
if (name == null)
throw new IllegalArgumentException();
@@ -80,24 +115,26 @@
this.allocationContext = allocationContext;
+ this.metadata = metadata;
+
}
public void clear() {
allocationContext.clear();
- addr = IRawStore.NULL;
+ solutionSetAddr = IRawStore.NULL;
}
public ICloseableIterator<IBindingSet[]> get() {
- final long addr = this.addr;
+ final long solutionSetAddr = this.solutionSetAddr;
- if (addr == IRawStore.NULL)
+ if (solutionSetAddr == IRawStore.NULL)
throw new IllegalStateException();
try {
- return new SolutionSetStreamDecoder(addr);
+ return new SolutionSetStreamDecoder(solutionSetAddr, solutionCount);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -105,11 +142,22 @@
}
/**
+ * The per-chunk header is currently two int32 fields. The first field is
+ * the #of solutions in that chunk. The second field is the #of bytes to
+ * follow in the payload for that chunk.
+ *
+ * TODO Should there be an overall header for the solution set or are we
+ * going to handle that through the solution set metadata?
+ */
+ static private int CHUNK_HEADER_SIZE = Bytes.SIZEOF_INT + Bytes.SIZEOF_INT;
+
+ /**
* Stream decoder for solution sets.
*/
private class SolutionSetStreamDecoder implements
ICloseableIterator<IBindingSet[]> {
+ private final long solutionSetCount;
private final DataInputStream in;
private final IVSolutionSetDecoder decoder;
@@ -117,17 +165,23 @@
/** The next chunk of solutions to be visited. */
private IBindingSet[] bsets = null;
+
+ /** The #of solution sets which have been decoded so far. */
+ private long nsolutions = 0;
- public SolutionSetStreamDecoder(final long addr)
- throws IOException {
+ public SolutionSetStreamDecoder(final long solutionSetAddr,
+ final long solutionSetCount) throws IOException {
+ this.solutionSetCount = solutionSetCount;
+
this.in = new DataInputStream(
- wrapInputStream(allocationContext.getInputStream(addr)));
+ wrapInputStream(allocationContext
+ .getInputStream(solutionSetAddr)));
this.open = true;
-
+
this.decoder = new IVSolutionSetDecoder();
-
+
}
@Override
@@ -154,18 +208,23 @@
try {
- bsets = decodeNextChunk();
-
+ if ((bsets = decodeNextChunk()) == null) {
+
+ // Nothing more to be read.
+ close();
+
+ }
+
} catch (IOException e) {
-
+
throw new RuntimeException(e);
-
+
}
}
-
+
return open && bsets != null;
-
+
}
/**
@@ -175,6 +234,17 @@
*/
private IBindingSet[] decodeNextChunk() throws IOException {
+ if (nsolutions == solutionSetCount) {
+
+ // Nothing more to be read.
+
+ if (log.isDebugEnabled())
+ log.debug("Read solutionSet: solutionSetSize=" + nsolutions);
+
+ return null;
+
+ }
+
// #of solutions in this chunk.
final int chunkSize = in.readInt();
@@ -201,6 +271,14 @@
}
+ // Update the #of solution sets which have been decoded.
+ nsolutions += chunkSize;
+
+ if (log.isTraceEnabled())
+ log.trace("Read chunk: chunkSize=" + chunkSize + ", bytesRead="
+ + (CHUNK_HEADER_SIZE + byteLength)
+ + ", solutionSetSize=" + nsolutions);
+
// Return the decoded solutions.
return t;
@@ -221,7 +299,9 @@
@Override
public void remove() {
+
throw new UnsupportedOperationException();
+
}
}
@@ -241,55 +321,75 @@
long nsolutions = 0;
// #of bytes for the encoded solutions (before compression).
long nbytes = 0;
+ // The #of chunks written.
+ long chunkCount = 0;
try {
final DataOutputBuffer buf = new DataOutputBuffer();
- final OutputStream os = wrapOutputStream(out);
+ final DataOutputStream os = new DataOutputStream(
+ wrapOutputStream(out));
- while (src.hasNext()) {
+ try {
- // Discard the data in the buffer.
- buf.reset();
+ while (src.hasNext()) {
- // Chunk of solutions to be written.
- final IBindingSet[] chunk = src.next();
+ // Discard the data in the buffer.
+ buf.reset();
- // Write solutions.
- for (int i = 0; i < chunk.length; i++) {
+ // Chunk of solutions to be written.
+ final IBindingSet[] chunk = src.next();
- encoder.encodeSolution(buf, chunk[i]);
+ // Write solutions.
+ for (int i = 0; i < chunk.length; i++) {
- }
+ encoder.encodeSolution(buf, chunk[i]);
- // #of bytes written onto the buffer.
- final int bytesBuffered = buf.limit();
+ }
- // Write header (#of solutions in this chunk).
- os.write(chunk.length);
+ // #of bytes written onto the buffer.
+ final int bytesBuffered = buf.limit();
- // Write header (#of bytes buffered).
- os.write(bytesBuffered);
-
- // transfer buffer data to output stream.
- os.write(buf.array(), 0/* off */, bytesBuffered);
+ // Write header (#of solutions in this chunk).
+ os.writeInt(chunk.length);
- // += headerSize (chunkSize,bytesBuffered) + bytesBuffered.
- nbytes += (Bytes.SIZEOF_INT + Bytes.SIZEOF_INT) + bytesBuffered;
+ // Write header (#of bytes buffered).
+ os.writeInt(bytesBuffered);
- nsolutions += chunk.length;
+ // transfer buffer data to output stream.
+ os.write(buf.array(), 0/* off */, bytesBuffered);
+ // += headerSize + bytesBuffered.
+ nbytes += CHUNK_HEADER_SIZE + bytesBuffered;
+
+ nsolutions += chunk.length;
+
+ chunkCount++;
+
+ if (log.isDebugEnabled())
+ log.debug("Wrote chunk: chunkSize=" + chunk.length
+ + ", chunkCount=" + chunkCount
+ + ", bytesBuffered=" + bytesBuffered
+ + ", solutionSetSize=" + nsolutions);
+
+ }
+
+ os.flush();
+
+ } finally {
+
+ os.close();
+
}
- os.flush();
-
out.flush();
newAddr = out.getAddr();
if (log.isDebugEnabled())
- log.debug("Wrote " + nsolutions + "; encodedBytes=" + nbytes
- + " bytes, bytesWritten=" + out.getBytesWritten());
+ log.debug("Wrote solutionSet: solutionSetSize=" + nsolutions
+ + ", chunkCount=" + chunkCount + ", encodedBytes="
+ + nbytes + ", bytesWritten=" + out.getBytesWritten());
} catch (IOException e) {
@@ -306,28 +406,42 @@
}
- if (addr != IRawStore.NULL) {
+ if (solutionSetAddr != IRawStore.NULL) {
- allocationContext.free(addr);
+ allocationContext.free(solutionSetAddr);
}
- addr = newAddr;
+ // TODO This is not atomic (needs lock).
+ solutionSetAddr = newAddr;
+ solutionCount = nsolutions;
}
+ /**
+ * TODO Test performance with and without gzip. Extract into the CREATE
+ * schema.
+ */
+ private static final boolean zip = true;
+
private OutputStream wrapOutputStream(final OutputStream out)
throws IOException {
-
- return new GZIPOutputStream(out);
-
+
+ if (zip)
+ return new DeflaterOutputStream(out);
+
+ return out;
+
}
private InputStream wrapInputStream(final InputStream in)
throws IOException {
- return new GZIPInputStream(in);
+ if (zip)
+ return new InflaterInputStream(in);
+ return in;
+
}
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -28,6 +28,8 @@
package com.bigdata.rdf.sparql.ast.cache;
import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
@@ -55,6 +57,7 @@
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.webapp.ConfigParams;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+import com.bigdata.rdf.spo.ISPO;
import com.bigdata.relation.locator.DefaultResourceLocator;
import com.bigdata.resources.IndexManager;
import com.bigdata.rwstore.RWStore;
@@ -63,6 +66,7 @@
import com.bigdata.rwstore.sector.MemoryManager;
import com.bigdata.service.IDataService;
import com.bigdata.sparse.SparseRowStore;
+import com.bigdata.striterator.CloseableIteratorWrapper;
import com.bigdata.striterator.ICloseableIterator;
/**
@@ -232,9 +236,8 @@
* {@link IndexManager} inside the {@link IDataService} and provides direct
* access to {@link FusedView}s (aka shards).
*
- * @param indexManager
- * The <em>local</em> {@link IIndexManager}.
- *
+ * @param queryEngine
+ * The {@link QueryEngine}.
*/
public SparqlCache(final QueryEngine queryEngine) {
@@ -317,8 +320,22 @@
}
+ /**
+ * Return the {@link IMemoryManager} backing all transient named solution
+ * sets. The caller is responsible for creating a child allocation context
+ * when writing a named solution set onto the {@link IMemoryManager}.
+ *
+ * @return The shared {@link IMemoryManager}.
+ */
+ protected IMemoryManager getMemoryManager() {
+
+ return ((MemStrategy) cache
+ .getBufferStrategy()).getMemoryManager();
+
+ }
+
@Override
- public void clearAll(final AST2BOpContext ctx) {
+ public void clearAllSolutions(final AST2BOpContext ctx) {
final Iterator<Map.Entry<String, SolutionSetMetadata>> itr = cacheMap
.entrySet().iterator();
@@ -343,7 +360,7 @@
}
@Override
- public boolean clear(final AST2BOpContext ctx, final String solutionSet) {
+ public boolean clearSolutions(final AST2BOpContext ctx, final String solutionSet) {
if (log.isInfoEnabled())
log.info("solutionSet: " + solutionSet);
@@ -361,7 +378,7 @@
}
- public void put(final AST2BOpContext ctx, final String solutionSet,
+ public void putSolutions(final AST2BOpContext ctx, final String solutionSet,
final ICloseableIterator<IBindingSet[]> src) {
if (solutionSet == null)
@@ -379,19 +396,59 @@
if (sset == null) {
- final IMemoryManager mmrgr = ((MemStrategy) cache
- .getBufferStrategy()).getMemoryManager();
+ final IMemoryManager mmrgr = getMemoryManager()
+ .createAllocationContext();
- sset = new SolutionSetMetadata(solutionSet,
- mmrgr.createAllocationContext());
+ sset = new SolutionSetMetadata(solutionSet, mmrgr,
+ getDefaultMetadata());
+ cacheMap.put(solutionSet, sset);
+
}
+ // write the solutions onto the memory manager.
sset.put(src);
+
+ }
+
+ public void createSolutions(final AST2BOpContext ctx,
+ final String solutionSet, final ISPO[] params) {
+
+ if (solutionSet == null)
+ throw new IllegalArgumentException();
+ /*
+ * TODO Deal with visibility issues on update (when the modified
+ * solution set state becomes visible and race conditions on create).
+ */
+
+ SolutionSetMetadata sset = cacheMap.get(solutionSet);
+
+ if (sset != null)
+ throw new RuntimeException("Exists: " + solutionSet);
+
+ final IMemoryManager mmrgr = getMemoryManager()
+ .createAllocationContext();
+
+ sset = new SolutionSetMetadata(solutionSet, mmrgr,
+ params == null ? getDefaultMetadata() : params);
+
+ cacheMap.put(solutionSet, sset);
+
+ {
+
+ final List<IBindingSet[]> emptySolutionSet = new LinkedList<IBindingSet[]>();
+
+ final ICloseableIterator<IBindingSet[]> src = new CloseableIteratorWrapper<IBindingSet[]>(
+ emptySolutionSet.iterator());
+
+ // write the solutions onto the memory manager.
+ sset.put(src);
+ }
+
}
- public ICloseableIterator<IBindingSet[]> get(final AST2BOpContext ctx,
+ public ICloseableIterator<IBindingSet[]> getSolutions(final AST2BOpContext ctx,
final String solutionSet) {
if (solutionSet == null)
@@ -402,9 +459,34 @@
if (sset == null)
throw new IllegalStateException("Not found: " + solutionSet);
+ // Return iterator over the decoded solutions.
return sset.get();
}
+
+ public boolean existsSolutions(final AST2BOpContext ctx,
+ final String solutionSet) {
+
+ if (solutionSet == null)
+ throw new IllegalArgumentException();
+
+ final SolutionSetMetadata sset = cacheMap.get(solutionSet);
+
+ return sset != null;
+
+ }
+
+ /**
+ * Return the default metadata used when a named solution set is declared
+ * implicitly rather than explicitly.
+ *
+ * @return The metadata describing that solution set.
+ */
+ protected ISPO[] getDefaultMetadata() {
+
+ return new ISPO[]{};
+
+ }
// @Override
// public ICacheHit get(final AST2BOpContext ctx,
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpContext.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -13,6 +13,8 @@
import com.bigdata.bop.rdf.join.ChunkedMaterializationOp;
import com.bigdata.htree.HTree;
import com.bigdata.journal.IIndexManager;
+import com.bigdata.journal.ITx;
+import com.bigdata.journal.TimestampUtility;
import com.bigdata.rdf.lexicon.LexiconRelation;
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.EmptySolutionSetStats;
@@ -443,4 +445,26 @@
}
+ /**
+ * Return the timestamp which will be used to read on the lexicon.
+ * <p>
+ * Note: This uses the timestamp of the triple store view unless this is a
+ * read/write transaction, in which case we need to use the last commit
+ * point in order to see any writes which it may have performed (lexicon
+ * writes are always unisolated).
+ */
+ public long getLexiconReadTimestamp() {
+
+ long timestamp = db.getTimestamp();
+
+ if (TimestampUtility.isReadWriteTx(timestamp)) {
+
+ timestamp = ITx.UNISOLATED;
+
+ }
+
+ return timestamp;
+
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpFilters.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpFilters.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpFilters.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -51,8 +51,6 @@
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.bop.rdf.join.ChunkedMaterializationOp;
import com.bigdata.bop.rdf.join.InlineMaterializeOp;
-import com.bigdata.journal.ITx;
-import com.bigdata.journal.TimestampUtility;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.NotMaterializedException;
import com.bigdata.rdf.internal.constraints.CompareBOp;
@@ -240,9 +238,9 @@
if (nvars == 0)
return left;
- final long timestamp = getLexiconReadTimestamp(ctx);
+ final long timestamp = ctx.getLexiconReadTimestamp();
- final String ns = ctx.db.getLexiconRelation().getNamespace();
+ final String ns = ctx.getLexiconNamespace();
if (nvars >= 1) {
/*
@@ -413,28 +411,6 @@
return left;
}
-
- /**
- * Return the timestamp which will be used to read on the lexicon.
- * <p>
- * Note: This uses the timestamp of the triple store view unless this is a
- * read/write transaction, in which case we need to use the last commit
- * point in order to see any writes which it may have performed (lexicon
- * writes are always unisolated).
- */
- static protected long getLexiconReadTimestamp(final AST2BOpContext ctx) {
-
- long timestamp = ctx.db.getTimestamp();
-
- if (TimestampUtility.isReadWriteTx(timestamp)) {
-
- timestamp = ITx.UNISOLATED;
-
- }
-
- return timestamp;
-
- }
// /**
// * Wrapper for handling the {@link AST2BOpContext} / {@link BOpContextBase}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -110,6 +110,7 @@
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.store.BD;
+import com.bigdata.striterator.ICloseableIterator;
/**
* Class handles SPARQL update query plan generation.
@@ -484,7 +485,7 @@
* the WHERE clause results when both the DELETE clause and the
* INSERT clause are present.
*
- * TODO For large intermediate results, we would be much better
+ * FIXME For large intermediate results, we would be much better
* off putting the data onto an HTree and processing the
* bindings as IVs rather than materializing them as RDF Values
* (and even for small data sets, we would be better off
@@ -563,38 +564,117 @@
} else {
/*
- * FIXME Support INSERT INTO / DELETE FROM here.
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/524 (SPARQL Cache)
+ * DELETE/INSERT.
+ *
+ * Note: For this code path, only the INSERT clause -or- the
+ * DELETE clause was specified. We handle the case where BOTH
+ * clauses were specified above.
*/
- final QuadData quadData = (insertClause == null ? deleteClause
- : insertClause).getQuadData();
-
- final ConstructNode template = quadData.flatten();
- // Set the CONSTRUCT template (quads patterns).
- queryRoot.setConstruct(template);
+ // true iff this is an INSERT
+ final boolean isInsert = insertClause != null;
+// final boolean isDelete = deleteClause != null;
+
+ // The clause (either for INSERT or DELETE)
+ final QuadsDataOrNamedSolutionSet clause = isInsert ? insertClause
+ : deleteClause;
+
+ assert clause != null;
- // Run as a CONSTRUCT query.
- final GraphQueryResult result = ASTEvalHelper
- .evaluateGraphQuery(context.db, astContainer, null/* bindingSets */);
+ // Figure out whether we are insert or deleting solutions.
+ final boolean isSolutionSet = clause.isSolutions();
- try {
+ if(isSolutionSet) {
+
+ /*
+ * Target is solution set.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/524
+ * (SPARQL Cache)
+ */
+
+ // The named solution set on which we will write.
+ final String solutionSet = clause.getName();
- while (result.hasNext()) {
+ // Set the projection node.
+ queryRoot.setProjection(clause.getProjection());
- final BigdataStatement stmt = (BigdataStatement) result
- .next();
+ if (!isInsert) {
+ /*
+ * FIXME We need to find and remove the matching
+ * solutions. Probably the best way to do that is to
+ * transform the WHERE clause with a MINUS joining
+ * against the target solution set via an INCLUDE. The
+ * solutions which are produced by the query can then be
+ * written directly onto the named solution set. That
+ * way, both DELETE and INSERT will wind up as
+ * putSolutions().
+ */
+ throw new UnsupportedOperationException();
+ }
+
+ // Run as a SELECT query : Do NOT materialize IVs.
+ final ICloseableIterator<IBindingSet[]> result = ASTEvalHelper
+ .evaluateTupleQuery2(context.db, astContainer,
+ null/* bindingSets */, false/* materialize */);
- addOrRemoveStatement(context.conn.getSailConnection(),
- stmt, isInsertOnly);
-
+ try {
+
+ // Write the solutions onto the named solution set.
+ context.sparqlCache.putSolutions(context, solutionSet,
+ result);
+
+ } finally {
+
+ result.close();
+
}
+
+ } else {
- } finally {
+ /*
+ * Target is graph.
+ */
+
+ final QuadData quadData = (insertClause == null ? deleteClause
+ : insertClause).getQuadData();
- result.close();
+ final ConstructNode template = quadData.flatten();
+ // Set the CONSTRUCT template (quads patterns).
+ queryRoot.setConstruct(template);
+
+ /*
+ * Run as a CONSTRUCT query
+ *
+ * FIXME Can we avoid IV materialization for this code path?
+ * Note that we have to do Truth Maintenance. However, I
+ * suspect that we do not need to do IV materialization if
+ * we can tunnel into the Sail's assertion and retraction
+ * buffers.
+ */
+ final GraphQueryResult result = ASTEvalHelper
+ .evaluateGraphQuery(context.db, astContainer, null/* bindingSets */);
+
+ try {
+
+ while (result.hasNext()) {
+
+ final BigdataStatement stmt = (BigdataStatement) result
+ .next();
+
+ addOrRemoveStatement(
+ context.conn.getSailConnection(), stmt,
+ isInsertOnly);
+
+ }
+
+ } finally {
+
+ result.close();
+
+ }
+
}
}
@@ -1142,7 +1222,7 @@
if (solutionSet != null) {
// Clear the named solution set.
- if (!context.sparqlCache.clear(context, solutionSet) && !silent) {
+ if (!context.sparqlCache.clearSolutions(context, solutionSet) && !silent) {
throw new SailException("solutionSet=" + solutionSet);
@@ -1227,41 +1307,67 @@
if (allSolutionSets && context.sparqlCache != null) {
// Delete all solution sets.
- context.sparqlCache.clearAll(context);
+ context.sparqlCache.clearAllSolutions(context);
}
}
/**
- * If the graph already exists (context has at least one statement), then
- * this is an error (unless SILENT). Otherwise it is a NOP.
+ * GRAPHS : If the graph already exists (context has at least one
+ * statement), then this is an error (unless SILENT). Otherwise it is a NOP.
+ * <p>
+ * SOLUTIONS : If the named solution set already exists (is registered, but
+ * may be empty), then this is an error (unless SILENT). Otherwise, the
+ * named solution set is provisioned according to the optional parameters.
*
* @param left
* @param op
* @param context
* @return
- *
- * FIXME Support CREATE SOLUTIONS here.
*/
private static PipelineOp convertCreateGraph(final PipelineOp left,
final CreateGraph op, final AST2BOpUpdateContext context) {
- if (!op.isSilent()) {
+ if (op.isTargetSolutionSet()) {
+ final String solutionSet = op.getTargetSolutionSet();
+
+ final boolean exists = context.sparqlCache.existsSolutions(context,
+ solutionSet);
+
+ if (!op.isSilent() && exists) {
+
+ throw new RuntimeException("Solutions exists:" + solutionSet);
+
+ }
+
+ if (!exists) {
+
+ context.sparqlCache.createSolutions(context, solutionSet,
+ op.getParams());
+
+ }
+
+ } else {
+
final BigdataURI c = (BigdataURI) ((CreateGraph) op)
.getTargetGraph().getValue();
if (log.isDebugEnabled())
log.debug("targetGraph=" + c);
- if (context.db.getAccessPath(null/* s */, null/* p */, null/* o */,
- c.getIV()).rangeCount(false/* exact */) != 0) {
+ if (!op.isSilent()) {
- throw new RuntimeException("Graph exists: " + c);
+ if (context.db.getAccessPath(null/* s */, null/* p */,
+ null/* o */, c.getIV()).rangeCount(false/* exact */) != 0) {
+ throw new RuntimeException("Graph exists: " + c);
+
+ }
+
}
-
+
}
return left;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -47,7 +47,6 @@
import com.bigdata.bop.controller.ServiceCallJoin;
import com.bigdata.bop.controller.Steps;
import com.bigdata.bop.controller.Union;
-import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.join.HTreeHashIndexOp;
import com.bigdata.bop.join.HTreeMergeJoin;
@@ -513,10 +512,9 @@
if (!tmp.isEmpty()) {
- final long timestamp = getLexiconReadTimestamp(ctx);
+ final long timestamp = ctx.getLexiconReadTimestamp();
- final String ns = ctx.db.getLexiconRelation()
- .getNamespace();
+ final String ns = ctx.getLexiconNamespace();
final IVariable<?>[] vars = tmp.toArray(new IVariable[tmp
.size()]);
@@ -525,7 +523,7 @@
new NV(ChunkedMaterializationOp.Annotations.VARS, vars),//
new NV(ChunkedMaterializationOp.Annotations.RELATION_NAME, new String[] { ns }), //
new NV(ChunkedMaterializationOp.Annotations.TIMESTAMP, timestamp), //
- new NV(ChunkedMaterializationOp.Annotations.MATERIALIZE_ALL, true), //
+ new NV(ChunkedMaterializationOp.Annotations.MATERIALIZE_INLINE_IVS, true), //
new NV(PipelineOp.Annotations.SHARED_STATE, !ctx.isCluster()),// live stats, but not on the cluster.
new NV(BOp.Annotations.BOP_ID, ctx.nextId())//
);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java 2012-04-10 09:08:53 UTC (rev 6268)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTEvalHelper.java 2012-04-10 15:02:48 UTC (rev 6269)
@@ -57,8 +57,10 @@
import com.bigdata.bop.PipelineOp;
import com.bigdata.bop.bindingSet.ListBindingSet;
import com.bigdata.bop.engine.IRunningQuery;
+import com.bigdata.bop.rdf.join.ChunkedMaterializationIterator;
import com.bigdata.journal.TimestampUtility;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.IVCache;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.sail.Bigdata2Sesame2BindingSetIterator;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
@@ -236,8 +238,142 @@
));
}
-
+
/**
+ * Evaluate a SELECT query without converting the results into openrdf
+ * solutions.
+ *
+ * @param store
+ * The {@link AbstractTripleStore} having the data.
+ * @param queryPlan
+ * The {@link ASTContainer}.
+ * @param bs
+ * The initial solution to kick things off.
+ * @param materialize
+ * When <code>true</code>, {@link IV}s will be materialized
+ * (their {@link IVCache} association will be set to the
+ * corresponding RDF {@link Value}). When <code>false</code>,
+ * this materialization step will be skipped. However, it is
+ * possible that {@link IV}s in the query plan will be
+ * materialized anyway (for example, materialization might be
+ * required to support FILTERs in the query).
+ *
+ * @return An object from which the solutions may be drained.
+ *
+ * @throws QueryEvaluationException
+ */
+ static public ICloseableIterator<IBindingSet[]> evaluateTupleQuery2(
+ final AbstractTripleStore store, final ASTContainer astContainer,
+ final QueryBindingSet bs, final boolean materialize)
+ throws QueryEvaluationException {
+
+ final AST2BOpContext context = new AST2BOpContext(astContainer, store);
+
+ // Clear the optimized AST.
+ astContainer.clearOptimizedAST();
+
+ // Batch resolve Values to IVs and convert to bigdata binding set.
+ final IBindingSet[] bindingSets = mergeBindingSets(astContainer,
+ batchResolveIVs(store, bs));
+
+ // Convert the query (generates an optimized AST as a side-effect).
+ AST2BOpUtility.convert(context, bindingSets);
+
+ // The optimized AST.
+ final QueryRoot optimizedQuery = astContainer.getOptimizedAST();
+
+ // true iff we can materialize the projection inside of the query plan.
+ final boolean materializeProjectionInQuery = materialize && context.materializeProjectionInQuery
+ && !optimizedQuery.hasSlice();
+
+ final List<String> projectedSet;
+
+ if (materialize) {
+
+ /*
+ * Add a materialization step.
+ */
+
+ // Get the projection for the query.
+ final IVariable<?>[] projected = astContainer.getOptimizedAST()
+ .getProjection().getProjectionVars();
+
+ projectedSet = new LinkedList<String>();
+
+ for (IVariable<?> var : projected)
+ projectedSet.add(var.getName());
+
+ } else {
+
+ /*
+ * Do not add a materialization step.
+ */
+
+ projectedSet = null;
+
+ }
+
+ doSparqlLogging(context);
+
+ final PipelineOp queryPlan = astContainer.getQueryPlan();
+
+ IRunningQuery runningQuery = null;
+ try {
+
+ // Submit query for evaluation.
+ runningQuery = context.queryEngine.eval(queryPlan, bindingSets);
+
+ // The iterator draining the query solutions.
+ ...
[truncated message content] |
|
From: <tho...@us...> - 2012-04-12 11:06:47
|
Revision: 6278
http://bigdata.svn.sourceforge.net/bigdata/?rev=6278&view=rev
Author: thompsonbry
Date: 2012-04-12 11:06:36 +0000 (Thu, 12 Apr 2012)
Log Message:
-----------
javadoc on AbstractLocalTransactionManager (TODO)
IVValueExpression - generics.
LangMatchesBOp - slight reorganization to provide fast path for type errors and not materialized exceptions.
NeedsMaterializationBOp - removed debug statements to trim the class.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/ProxyTestCase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/LangMatchesBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NeedsMaterializationBOp.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2012-04-11 20:34:54 UTC (rev 6277)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2012-04-12 11:06:36 UTC (rev 6278)
@@ -41,7 +41,9 @@
* preparing will remain in this collection until it has either successfully
* prepared or aborted.
*
- * @todo config initial capacity and concurrency.
+ * TODO Configure the initial capacity and concurrency. For example, this
+ * should be sized to the #of client connections for both the
+ * initialCapacity and the concurrency level.
*/
final private ConcurrentHashMap<Long, Tx> activeTx = new ConcurrentHashMap<Long, Tx>();
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/ProxyTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/ProxyTestCase.java 2012-04-11 20:34:54 UTC (rev 6277)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/ProxyTestCase.java 2012-04-12 11:06:36 UTC (rev 6278)
@@ -163,6 +163,7 @@
final ThreadGroup grp = Thread.currentThread().getThreadGroup();
final int tearDownActiveThreads = grp.activeCount();
+ int nremaining = 0;
if (startupActiveThreads != tearDownActiveThreads) {
final Thread[] threads = new Thread[tearDownActiveThreads];
grp.enumerate(threads);
@@ -175,17 +176,18 @@
info.append(',');
info.append("[" + t.getName() + "]");
first = false;
+ nremaining++;
}
final String failMessage = "Threads left active after task"
+": test=" + getName()//
+ ", delegate="+getOurDelegate().getClass().getName()
+ ", startupCount=" + startupActiveThreads
- + ", teardownCount=" + tearDownActiveThreads
+ + ", teardownCount=" + nremaining
+ ", thisThread="+Thread.currentThread().getName()
+ ", threads: " + info;
- if (grp.activeCount() != startupActiveThreads)
+ if (nremaining > startupActiveThreads)
log.error(failMessage);
/*
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java 2012-04-11 20:34:54 UTC (rev 6277)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IVValueExpression.java 2012-04-12 11:06:36 UTC (rev 6278)
@@ -472,7 +472,7 @@
protected IV getAndCheckLiteral(final int i, final IBindingSet bs)
throws SparqlTypeErrorException, NotMaterializedException {
- final IV iv = getAndCheckBound(i, bs);
+ final IV<?, ?> iv = getAndCheckBound(i, bs);
if (!iv.isLiteral())
throw new SparqlTypeErrorException();
@@ -505,7 +505,7 @@
protected IV getAndCheckBound(final int i, final IBindingSet bs)
throws SparqlTypeErrorException, NotMaterializedException {
- final IV iv = get(i).get(bs);
+ final IV<?, ?> iv = get(i).get(bs);
if (iv == null)
throw new SparqlTypeErrorException.UnboundVarException();
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/LangMatchesBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/LangMatchesBOp.java 2012-04-11 20:34:54 UTC (rev 6277)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/LangMatchesBOp.java 2012-04-12 11:06:36 UTC (rev 6278)
@@ -77,53 +77,70 @@
protected boolean accept(final IBindingSet bs) {
- final IV tag = get(0).get(bs);
- final IV range = get(1).get(bs);
-
+ final IV<?, ?> tag = get(0).get(bs);
+
+ // not yet bound
+ if (tag == null)
+ throw new SparqlTypeErrorException();
+
+ final IV<?, ?> range = get(1).get(bs);
+
+ // not yet bound
+ if (range == null)
+ throw new SparqlTypeErrorException();
+
// if (log.isDebugEnabled()) {
// log.debug(tag);
// log.debug(range);
// }
- // not yet bound
- if (tag == null || range == null)
- throw new SparqlTypeErrorException();
final BigdataValue tagVal = tag.getValue();
+
+ // not yet materialized
+ if (tagVal == null)
+ throw new NotMaterializedException();
+
+ if (!QueryEvaluationUtil.isSimpleLiteral(tagVal))
+ throw new SparqlTypeErrorException();
+
final BigdataValue rangeVal = range.getValue();
-
+
+ // not yet materialized
+ if (rangeVal == null)
+ throw new NotMaterializedException();
+
+ if (!QueryEvaluationUtil.isSimpleLiteral(rangeVal))
+ throw new SparqlTypeErrorException();
+
// if (log.isDebugEnabled()) {
// log.debug(tagVal);
// log.debug(rangeVal);
// }
- // not yet materialized
- if (tagVal == null || rangeVal == null)
- throw new NotMaterializedException();
-
- if (QueryEvaluationUtil.isSimpleLiteral(tagVal)
- && QueryEvaluationUtil.isSimpleLiteral(rangeVal))
- {
- final String langTag = ((Literal)tagVal).getLabel();
- final String langRange = ((Literal)rangeVal).getLabel();
+// if (QueryEvaluationUtil.isSimpleLiteral(tagVal)
+// && QueryEvaluationUtil.isSimpleLiteral(rangeVal))
+// {
+ final String langTag = ((Literal) tagVal).getLabel();
+ final String langRange = ((Literal) rangeVal).getLabel();
- boolean result = false;
- if (langRange.equals("*")) {
- result = langTag.length() > 0;
- }
- else if (langTag.length() == langRange.length()) {
- result = langTag.equalsIgnoreCase(langRange);
- }
- else if (langTag.length() > langRange.length()) {
- // check if the range is a prefix of the tag
- final String prefix = langTag.substring(0, langRange.length());
- result = prefix.equalsIgnoreCase(langRange) && langTag.charAt(langRange.length()) == '-';
- }
-
- return result;
+ boolean result = false;
+ if (langRange.equals("*")) {
+ result = langTag.length() > 0;
}
+ else if (langTag.length() == langRange.length()) {
+ result = langTag.equalsIgnoreCase(langRange);
+ }
+ else if (langTag.length() > langRange.length()) {
+ // check if the range is a prefix of the tag
+ final String prefix = langTag.substring(0, langRange.length());
+ result = prefix.equalsIgnoreCase(langRange) && langTag.charAt(langRange.length()) == '-';
+ }
- throw new SparqlTypeErrorException();
+ return result;
+// }
+//
+// throw new SparqlTypeErrorException();
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NeedsMaterializationBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NeedsMaterializationBOp.java 2012-04-11 20:34:54 UTC (rev 6277)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/NeedsMaterializationBOp.java 2012-04-12 11:06:36 UTC (rev 6278)
@@ -26,12 +26,9 @@
import java.util.Map;
-import org.apache.log4j.Logger;
-
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
-import com.bigdata.bop.NV;
import com.bigdata.rdf.internal.NotMaterializedException;
import com.bigdata.util.InnerCause;
@@ -47,8 +44,8 @@
*/
private static final long serialVersionUID = 4767476516948560884L;
- private static final transient Logger log = Logger
- .getLogger(NeedsMaterializationBOp.class);
+// private static final transient Logger log = Logger
+// .getLogger(NeedsMaterializationBOp.class);
public NeedsMaterializationBOp(final IValueExpression<?> x) {
@@ -82,15 +79,15 @@
try {
- if (log.isDebugEnabled()) {
- log.debug("about to attempt evaluation prior to materialization");
- }
+// if (log.isDebugEnabled()) {
+// log.debug("about to attempt evaluation prior to materialization");
+// }
ve.get(bs);
- if (log.isDebugEnabled()) {
- log.debug("successfully evaluated constraint without materialization");
- }
+// if (log.isDebugEnabled()) {
+// log.debug("successfully evaluated constraint without materialization");
+// }
return false;
@@ -98,9 +95,9 @@
if (InnerCause.isInnerCause(t, NotMaterializedException.class)) {
- if (log.isDebugEnabled()) {
- log.debug("could not evaluate constraint without materialization");
- }
+// if (log.isDebugEnabled()) {
+// log.debug("could not evaluate constraint without materialization");
+// }
return true;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-04-12 15:27:42
|
Revision: 6279
http://bigdata.svn.sourceforge.net/bigdata/?rev=6279&view=rev
Author: thompsonbry
Date: 2012-04-12 15:27:30 +0000 (Thu, 12 Apr 2012)
Log Message:
-----------
Code changes to support packing TIDs on the Journal. This does not yet support packing longs on a cluster since we also would need to address the TermIdEncoder. Packed TIDs are disabled by default in IVUtility.PACK_TIDS since they are incompatible with existing data on the disk.
@see https://sourceforge.net/apps/trac/bigdata/ticket/548 (Pack TIDs)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/ByteArrayBuffer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/LongPacker.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/io/TestLongPacker.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/TermId.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestTermIV.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestId2TermTupleSerializer.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -39,6 +39,7 @@
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleSerializer;
+import com.bigdata.io.LongPacker;
/**
* A class that may be used to form multi-component keys but which does not
@@ -61,7 +62,7 @@
* len field to the mark. keys with multiple components could benefit from
* allowing multiple marks (the sparse row store is the main use case).
*/
-public class KeyBuilder implements IKeyBuilder {
+public class KeyBuilder implements IKeyBuilder, LongPacker.IByteBuffer {
private static final transient Logger log = Logger.getLogger(KeyBuilder.class);
@@ -737,6 +738,100 @@
}
/**
+ * Packs a non-negative long value into the minimum #of bytes in which the
+ * value can be represented and writes those bytes onto the buffer. The
+ * first byte determines whether or not the long value was packed and, if
+ * packed, how many bytes were required to represent the packed long value.
+ * When the high bit of the first byte is a one (1), then the long value
+ * could not be packed and the long value is found by clearing the high bit
+ * and interpreting the first byte plus the next seven (7) bytes as a long.
+ * Otherwise the next three (3) bits are interpreted as an unsigned integer
+ * giving the #of bytes (nbytes) required to represent the packed long
+ * value. To recover the long value the high nibble is cleared and the first
+ * byte together with the next nbytes are interpreted as an unsigned long
+ * value whose leading zero bytes were not written.
+ *
+ * <pre>
+ *
+ * [0|1|2|3|4|5|6|7]
+ * 1 - - - nbytes = 8, clear high bit and interpret this plus the next 7 bytes as a long.
+ * 0 1 1 1 nbytes = 7, clear high nibble and interpret this plus the next 6 bytes as a long.
+ * 0 1 1 0 nbytes = 6, clear high nibble and interpret this plus the next 5 bytes as a long.
+ * 0 1 0 1 nbytes = 5, clear high nibble and interpret this plus the next 4 bytes as a long.
+ * 0 1 0 0 nbytes = 4, clear high nibble and interpret this plus the next 3 bytes as a long.
+ * 0 0 1 1 nbytes = 3, clear high nibble and interpret this plus the next 3 bytes as a long.
+ * 0 0 1 0 nbytes = 2, clear high nibble and interpret this plus the next byte as a long.
+ * 0 0 0 1 nbytes = 1, clear high nibble. value is the low nibble.
+ *
+ * </pre>
+ *
+ * Note: These are decodable (no loss) but negative longs are not allowed.
+ * <p>
+ * Note: The order is NOT fully preserved. Any long which is encoded into
+ * less than 8 bytes has its order preserved. However, a long which is
+ * encoded into 8 bytes will wind up ordered before any longs which pack
+ * into fewer bytes.
+ *
+ * @param v
+ * The unsigned long value.
+ *
+ * @return The #of bytes onto which the unsigned long value was packed.
+ */
+ final public KeyBuilder pack(final long v) {
+
+ LongPacker.packLong(v, pbuf, this);
+
+ return this;
+
+ }
+
+ /**
+ * Relative <i>put</i> method for writing a byte[] on the buffer.
+ *
+ * @param b
+ * The byte[].
+ * @param off
+ * The offset of the first byte in <i>b</i> to be written on
+ * the buffer.
+ * @param len
+ * The #of bytes in <i>b</i> to be written on the buffer.
+ */
+ public void put(final byte[] b, final int off, final int len) {
+
+ ensureFree(len);
+
+ System.arraycopy(b/* src */, 0/* srcPos */, buf/* dest */,
+ this.len/* destPos */, len/* length */);
+
+ this.len += len;
+
+ }
+
+ /**
+ * Private buffer for packing long integers.
+ */
+ final private byte[] pbuf = new byte[8];
+
+// /**
+// * Unpack a long value from the current buffer position.
+// *
+// * @param buf
+// * The buffer containing the data to be decoded.
+// * @param off
+// * The offset of the first byte of the value to be decoded.
+// * @param limit
+// * The exclusive upper bound available for decoding
+// *
+// * @return The long value.
+// */
+// static final public long unpackLong(final byte[] buf, final int off,
+// final int limit) {
+//
+// return LongPacker.unpackLong(buf, off, limit);
+//
+// }
+
+ /**
* Return the value that will impose the lexiographic ordering as an
* unsigned long integer.
*
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/ByteArrayBuffer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/ByteArrayBuffer.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/ByteArrayBuffer.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -70,7 +70,7 @@
* @version $Id$
*/
public class ByteArrayBuffer extends OutputStream implements IByteArrayBuffer,
- RepositionableStream {
+ RepositionableStream, LongPacker.IByteBuffer {
private static final transient Logger log = Logger
.getLogger(ByteArrayBuffer.class);
@@ -1012,101 +1012,10 @@
*
* @return The #of bytes onto which the unsigned long value was packed.
*/
- final public int packLong( final long v ) {
+ final public int packLong(final long v) {
+
+ return LongPacker.packLong(v, pbuf, this);
- /*
- * You can only pack non-negative long values with this method.
- */
-
- if (v < 0) {
-
- throw new IllegalArgumentException("negative value: v=" + v);
-
- }
-
- /*
- * If the high byte is non-zero then we will write the value as a normal
- * long and return nbytes == 8. This case handles large positive long
- * values.
- */
- if( ( v >> 56 ) != 0 ) {
- pbuf[0] = ( (byte)((0xff & (v >> 56))|0x80) ); // note: set the high bit.
- pbuf[1] = ( (byte)(0xff & (v >> 48)) );
- pbuf[2] = ( (byte)(0xff & (v >> 40)) );
- pbuf[3] = ( (byte)(0xff & (v >> 32)) );
- pbuf[4] = ( (byte)(0xff & (v >> 24)) );
- pbuf[5] = ( (byte)(0xff & (v >> 16)) );
- pbuf[6] = ( (byte)(0xff & (v >> 8)) );
- pbuf[7] = ( (byte)(0xff & v) );
- put(pbuf, 0, 8);
- return 8;
- }
-
- // #of nibbles required to represent the long value.
- final int nnibbles = getNibbleLength( v );
-
- /*
- * Is [nnibbles] even? (If it is even then we need to pad out an extra
- * zero nibble in the first byte.)
- */
- final boolean evenNibbleCount = ( nnibbles == ( ( nnibbles >> 1 ) << 1 ) );
-
- // #of bytes required to represent the long value (plus the header nibble).
- final int nbytes = ( ( nnibbles +1 ) >> 1 ) + (evenNibbleCount?1:0);
-
- int nwritten = 0;
-
- if( evenNibbleCount ) {
-
- /*
- * An even nibble count requires that we pad the low nibble of the
- * first byte with zeros.
- */
-
- // header byte. low nibble is empty.
- byte b = (byte) ( nbytes << 4 );
-
- pbuf[nwritten++] = b;
-
- // remaining bytes containing the packed value.
- for( int i=(nnibbles-2)<<2; i>=0; i-=8 ) {
-
- b = (byte) (0xff & (v >> i));
-
- pbuf[nwritten++] = b;
-
- }
-
- } else {
-
- /*
- * An odd nibble count means that we pack the first nibble of the
- * long value into the low nibble of the header byte. In this case
- * the first nibble will always be the low nibble of the first
- * non-zero byte in the long value (the high nibble of that byte
- * must be zero since there is an odd nibble count).
- */
-
- byte highByte = (byte) (0xff & (v >> ((nbytes-1)*8) ));
-
- byte b = (byte) ( ( nbytes << 4 ) | highByte );
-
- pbuf[nwritten++] = b;
-
- for( int i=(nnibbles-3)<<2; i>=0; i-=8 ) {
-
- b = (byte) (0xff & (v >> i));
-
- pbuf[nwritten++] = b;
-
- }
-
- }
-
- put(pbuf,0,nwritten);
-
- return nwritten;
-
}
/**
@@ -1114,37 +1023,38 @@
*/
final private byte[] pbuf = new byte[8];
- /**
- * Return the #of non-zero nibbles, counting from the first non-zero nibble
- * in the long value. A value of <code>0L</code> is considered to be one
- * nibble for our purposes.
- *
- * @param v
- * The long value.
- *
- * @return The #of nibbles in [1:16].
- */
- static protected final int getNibbleLength( final long v )
- {
-
- for( int i=56, j=16; i>=0; i-=8, j-=2 ) {
-
- if( (0xf0 & (v >> i)) != 0 ) return j;
-
- if( (0x0f & (v >> i)) != 0 ) return j-1;
-
- }
-
- if (v != 0)
- throw new AssertionError("v=" + v);
-
- /*
- * value is zero, which is considered to be one nibble for our purposes.
- */
-
- return 1;
-
- }
+// /**
+// * Return the #of non-zero nibbles, counting from the first non-zero nibble
+// * in the long value. A value of <code>0L</code> is considered to be one
+// * nibble for our purposes.
+// *
+// * @param v
+// * The long value.
+// *
+// * @return The #of nibbles in [1:16].
+// */
+// static protected final int getNibbleLength( final long v )
+// {
+// return LongPacker.getNibbleLength(v);
+//
+//// for( int i=56, j=16; i>=0; i-=8, j-=2 ) {
+////
+//// if( (0xf0 & (v >> i)) != 0 ) return j;
+////
+//// if( (0x0f & (v >> i)) != 0 ) return j-1;
+////
+//// }
+////
+//// if (v != 0)
+//// throw new AssertionError("v=" + v);
+////
+//// /*
+//// * value is zero, which is considered to be one nibble for our purposes.
+//// */
+////
+//// return 1;
+////
+// }
/*
* Pack unsigned short integer.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/LongPacker.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/LongPacker.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/io/LongPacker.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -39,7 +39,6 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-
public class LongPacker
{
@@ -242,6 +241,157 @@
}
/**
+ * Narrow interface to support packing against different buffer classes.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ public interface IByteBuffer {
+ /**
+ * Relative <i>put</i> method for writing a byte[] on the buffer.
+ *
+ * @param b
+ * The byte[].
+ * @param off
+ * The offset of the first byte in <i>b</i> to be written on
+ * the buffer.
+ * @param len
+ * The #of bytes in <i>b</i> to be written on the buffer.
+ */
+ void put(final byte[] b, final int off, final int len);
+ }
+
+ /**
+ * Packs a non-negative long value into the minimum #of bytes in which the
+ * value can be represented and writes those bytes onto the buffer.
+ * The first byte determines whether or not the long value was packed and,
+ * if packed, how many bytes were required to represent the packed long
+ * value. When the high bit of the first byte is a one (1), then the long
+ * value could not be packed and the long value is found by clearing the
+ * high bit and interpreting the first byte plus the next seven (7) bytes as
+ * a long. Otherwise the next three (3) bits are interpreted as an unsigned
+ * integer giving the #of bytes (nbytes) required to represent the packed
+ * long value. To recover the long value the high nibble is cleared and the
+ * first byte together with the next nbytes are interpreted as an unsigned
+ * long value whose leading zero bytes were not written.
+ *
+ * <pre>
+ *
+ * [0|1|2|3|4|5|6|7]
+ * 1 - - - nbytes = 8, clear high bit and interpret this plus the next 7 bytes as a long.
+ * 0 1 1 1 nbytes = 7, clear high nibble and interpret this plus the next 6 bytes as a long.
+ * 0 1 1 0 nbytes = 6, clear high nibble and interpret this plus the next 5 bytes as a long.
+ * 0 1 0 1 nbytes = 5, clear high nibble and interpret this plus the next 4 bytes as a long.
+ * 0 1 0 0 nbytes = 4, clear high nibble and interpret this plus the next 3 bytes as a long.
+ * 0 0 1 1 nbytes = 3, clear high nibble and interpret this plus the next 3 bytes as a long.
+ * 0 0 1 0 nbytes = 2, clear high nibble and interpret this plus the next byte as a long.
+ * 0 0 0 1 nbytes = 1, clear high nibble. value is the low nibble.
+ *
+ * </pre>
+ *
+ * @param v The unsigned long value.
+ *
+ * @return The #of bytes onto which the unsigned long value was packed.
+ */
+ static final public int packLong(final long v, final byte[] pbuf,
+ final IByteBuffer buf) {
+
+ /*
+ * You can only pack non-negative long values with this method.
+ */
+
+ if (v < 0) {
+
+ throw new IllegalArgumentException("negative value: v=" + v);
+
+ }
+
+ /*
+ * If the high byte is non-zero then we will write the value as a normal
+ * long and return nbytes == 8. This case handles large positive long
+ * values.
+ */
+ if( ( v >> 56 ) != 0 ) {
+ pbuf[0] = ( (byte)((0xff & (v >> 56))|0x80) ); // note: set the high bit.
+ pbuf[1] = ( (byte)(0xff & (v >> 48)) );
+ pbuf[2] = ( (byte)(0xff & (v >> 40)) );
+ pbuf[3] = ( (byte)(0xff & (v >> 32)) );
+ pbuf[4] = ( (byte)(0xff & (v >> 24)) );
+ pbuf[5] = ( (byte)(0xff & (v >> 16)) );
+ pbuf[6] = ( (byte)(0xff & (v >> 8)) );
+ pbuf[7] = ( (byte)(0xff & v) );
+ buf.put(pbuf, 0, 8);
+ return 8;
+ }
+
+ // #of nibbles required to represent the long value.
+ final int nnibbles = LongPacker.getNibbleLength( v );
+
+ /*
+ * Is [nnibbles] even? (If it is even then we need to pad out an extra
+ * zero nibble in the first byte.)
+ */
+ final boolean evenNibbleCount = ( nnibbles == ( ( nnibbles >> 1 ) << 1 ) );
+
+ // #of bytes required to represent the long value (plus the header nibble).
+ final int nbytes = ( ( nnibbles +1 ) >> 1 ) + (evenNibbleCount?1:0);
+
+ int nwritten = 0;
+
+ if( evenNibbleCount ) {
+
+ /*
+ * An even nibble count requires that we pad the low nibble of the
+ * first byte with zeros.
+ */
+
+ // header byte. low nibble is empty.
+ byte b = (byte) ( nbytes << 4 );
+
+ pbuf[nwritten++] = b;
+
+ // remaining bytes containing the packed value.
+ for( int i=(nnibbles-2)<<2; i>=0; i-=8 ) {
+
+ b = (byte) (0xff & (v >> i));
+
+ pbuf[nwritten++] = b;
+
+ }
+
+ } else {
+
+ /*
+ * An odd nibble count means that we pack the first nibble of the
+ * long value into the low nibble of the header byte. In this case
+ * the first nibble will always be the low nibble of the first
+ * non-zero byte in the long value (the high nibble of that byte
+ * must be zero since there is an odd nibble count).
+ */
+
+ byte highByte = (byte) (0xff & (v >> ((nbytes-1)*8) ));
+
+ byte b = (byte) ( ( nbytes << 4 ) | highByte );
+
+ pbuf[nwritten++] = b;
+
+ for( int i=(nnibbles-3)<<2; i>=0; i-=8 ) {
+
+ b = (byte) (0xff & (v >> i));
+
+ pbuf[nwritten++] = b;
+
+ }
+
+ }
+
+ buf.put(pbuf,0,nwritten);
+
+ return nwritten;
+
+ }
+
+ /**
* Read a byte from an {@link InputStream} ala {@link DataInput#readByte()}
*
* @param is
@@ -369,6 +519,41 @@
}
/**
+ * Unpack a long value from the buffer position.
+ *
+ * @param buf
+ * The buffer
+ * @param off
+ * The offset from which the data will be unpacked.
+ * @return The long value.
+ */
+ static final public long unpackLong(final byte[] buf, int off) {
+ int b = buf[off++];
+ int nbytes;
+ long l;
+ if ((b & 0x80) != 0) {
+ // high bit is set.
+ nbytes = 8; // use 8 bytes (this one plus the next 7).
+ l = b & 0x7f; // clear the high bit - the rest of the byte is the
+ // start value.
+ } else {
+ // high bit is clear.
+ nbytes = b >> 4; // nbytes is the upper nibble. (right shift one
+ // nibble).
+ l = b & 0x0f; // starting value is lower nibble (clear the upper
+ // nibble).
+ }
+ for (int i = 1; i < nbytes; i++) {
+ // Read the next byte.
+ b = buf[off++];
+ // Shift the existing value one byte left and add into the low
+ // (unsigned) byte.
+ l = (l << 8) + (0xff & b);
+ }
+ return l;
+ }
+
+ /**
* Convenience method unpacks long and throws an exception if the value
* exceeds {@link Integer#MAX_VALUE}.
*
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -42,6 +42,7 @@
import junit.framework.TestCase2;
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.BytesUtil.UnsignedByteArrayComparator;
+import com.bigdata.io.LongPacker;
/**
* Test suite for high level operations that build variable length _unsigned_
@@ -1500,6 +1501,31 @@
}
+// /*
+// * Packed long integers.
+// *
+// * These are decodable (no loss) but negative longs are not allowed.
+// */
+// public void test_packLong() {
+//
+// final KeyBuilder keyBuilder = new KeyBuilder();
+//
+// /*
+// * TODO Do loop, appending into the buffer. Then do decode of each
+// * packed value in turn.
+// */
+// final long v = 1;
+// final int off = keyBuilder.off();
+// keyBuilder.pack(1);
+// final int nbytes = LongPacker.getByteLength(v);
+// assertEquals("nbytes", off + nbytes, keyBuilder.off());
+//
+// final long d = KeyBuilder.unpackLong(keyBuilder.array(), off, off
+// + nbytes);
+// assertEquals("decodedValue", v, d);
+//
+// }
+
/*
* BigInteger.
*
@@ -1508,10 +1534,10 @@
public void test_BigInteger_ctor() {
- Random r = new Random();
-
- for(int i=0; i<10000; i++) {
-
+ final Random r = new Random();
+
+ for (int i = 0; i < 10000; i++) {
+
final BigInteger v1 = BigInteger.valueOf(r.nextLong());
// Note: This DOES NOT work.
@@ -2568,7 +2594,8 @@
for (BigDecimal i : a) {
i = i.stripTrailingZeros();
- System.err.println("i="
+ if(log.isInfoEnabled())
+ log.info("i="
+ i
+ "\t(scale="
+ i.scale()
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/io/TestLongPacker.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/io/TestLongPacker.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/io/TestLongPacker.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -37,7 +37,6 @@
import junit.framework.TestCase;
-
/**
* Test suite for packing and unpacking unsigned long integers using the
* {@link DataInputBuffer} and the {@link ByteArrayBuffer}.
@@ -79,9 +78,9 @@
throws IOException
{
- DataInputBuffer dib = new DataInputBuffer(packed);
+ final DataInputBuffer dib = new DataInputBuffer(packed);
- long actual = dib.unpackLong();
+ final long actual = dib.unpackLong();
assertEquals( "value", expected, actual );
@@ -208,46 +207,46 @@
{
// Note: zero (0) is interpreted as being one nibble for our purposes.
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0x0 ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0x0 ) );
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0x1 ) );
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0x2 ) );
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0x7 ) );
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0x8 ) );
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0xe ) );
- assertEquals( "nibbles", 1, DataOutputBuffer.getNibbleLength( 0xf ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0x1 ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0x2 ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0x7 ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0x8 ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0xe ) );
+ assertEquals( "nibbles", 1, LongPacker.getNibbleLength( 0xf ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x10 ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x11 ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x12 ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x17 ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x18 ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x1e ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x1f ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x7f ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0x8f ) );
- assertEquals( "nibbles", 2, DataOutputBuffer.getNibbleLength( 0xff ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x10 ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x11 ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x12 ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x17 ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x18 ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x1e ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x1f ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x7f ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0x8f ) );
+ assertEquals( "nibbles", 2, LongPacker.getNibbleLength( 0xff ) );
- assertEquals( "nibbles", 3, DataOutputBuffer.getNibbleLength( 0x100 ) );
- assertEquals( "nibbles", 3, DataOutputBuffer.getNibbleLength( 0x101 ) );
- assertEquals( "nibbles", 3, DataOutputBuffer.getNibbleLength( 0x121 ) );
- assertEquals( "nibbles", 3, DataOutputBuffer.getNibbleLength( 0x1ee ) );
- assertEquals( "nibbles", 3, DataOutputBuffer.getNibbleLength( 0x1ff ) );
- assertEquals( "nibbles", 3, DataOutputBuffer.getNibbleLength( 0xfff ) );
+ assertEquals( "nibbles", 3, LongPacker.getNibbleLength( 0x100 ) );
+ assertEquals( "nibbles", 3, LongPacker.getNibbleLength( 0x101 ) );
+ assertEquals( "nibbles", 3, LongPacker.getNibbleLength( 0x121 ) );
+ assertEquals( "nibbles", 3, LongPacker.getNibbleLength( 0x1ee ) );
+ assertEquals( "nibbles", 3, LongPacker.getNibbleLength( 0x1ff ) );
+ assertEquals( "nibbles", 3, LongPacker.getNibbleLength( 0xfff ) );
- assertEquals( "nibbles", 4, DataOutputBuffer.getNibbleLength( 0x1ff0 ) );
- assertEquals( "nibbles", 4, DataOutputBuffer.getNibbleLength( 0x7ff0 ) );
- assertEquals( "nibbles", 4, DataOutputBuffer.getNibbleLength( 0xfff0 ) );
- assertEquals( "nibbles", 4, DataOutputBuffer.getNibbleLength( 0xfff1 ) );
+ assertEquals( "nibbles", 4, LongPacker.getNibbleLength( 0x1ff0 ) );
+ assertEquals( "nibbles", 4, LongPacker.getNibbleLength( 0x7ff0 ) );
+ assertEquals( "nibbles", 4, LongPacker.getNibbleLength( 0xfff0 ) );
+ assertEquals( "nibbles", 4, LongPacker.getNibbleLength( 0xfff1 ) );
- assertEquals( "nibbles", 5, DataOutputBuffer.getNibbleLength( 0x12345 ) );
- assertEquals( "nibbles", 5, DataOutputBuffer.getNibbleLength( 0x54321 ) );
+ assertEquals( "nibbles", 5, LongPacker.getNibbleLength( 0x12345 ) );
+ assertEquals( "nibbles", 5, LongPacker.getNibbleLength( 0x54321 ) );
- assertEquals( "nibbles", 6, DataOutputBuffer.getNibbleLength( 0x123456 ) );
- assertEquals( "nibbles", 6, DataOutputBuffer.getNibbleLength( 0x654321 ) );
+ assertEquals( "nibbles", 6, LongPacker.getNibbleLength( 0x123456 ) );
+ assertEquals( "nibbles", 6, LongPacker.getNibbleLength( 0x654321 ) );
- assertEquals( "nibbles", 7, DataOutputBuffer.getNibbleLength( 0x1234567 ) );
- assertEquals( "nibbles", 7, DataOutputBuffer.getNibbleLength( 0x7654321 ) );
+ assertEquals( "nibbles", 7, LongPacker.getNibbleLength( 0x1234567 ) );
+ assertEquals( "nibbles", 7, LongPacker.getNibbleLength( 0x7654321 ) );
/*
* Note: At 8 nibbles we have 32 bits. When the high bit is one, this
@@ -255,32 +254,32 @@
* will be interpreted as a negative integer and sign extended to a
* negative long.
*/
- assertEquals( "nibbles", 8, DataOutputBuffer.getNibbleLength( 0x12345678L ) );
- assertEquals( "nibbles", 8, DataOutputBuffer.getNibbleLength( 0x87654321L ) );
+ assertEquals( "nibbles", 8, LongPacker.getNibbleLength( 0x12345678L ) );
+ assertEquals( "nibbles", 8, LongPacker.getNibbleLength( 0x87654321L ) );
- assertEquals( "nibbles", 9, DataOutputBuffer.getNibbleLength( 0x123456789L ) );
- assertEquals( "nibbles", 9, DataOutputBuffer.getNibbleLength( 0x987654321L ) );
+ assertEquals( "nibbles", 9, LongPacker.getNibbleLength( 0x123456789L ) );
+ assertEquals( "nibbles", 9, LongPacker.getNibbleLength( 0x987654321L ) );
- assertEquals( "nibbles", 10, DataOutputBuffer.getNibbleLength( 0x123456789aL ) );
- assertEquals( "nibbles", 10, DataOutputBuffer.getNibbleLength( 0xa987654321L ) );
+ assertEquals( "nibbles", 10, LongPacker.getNibbleLength( 0x123456789aL ) );
+ assertEquals( "nibbles", 10, LongPacker.getNibbleLength( 0xa987654321L ) );
- assertEquals( "nibbles", 11, DataOutputBuffer.getNibbleLength( 0x123456789abL ) );
- assertEquals( "nibbles", 11, DataOutputBuffer.getNibbleLength( 0xba987654321L ) );
+ assertEquals( "nibbles", 11, LongPacker.getNibbleLength( 0x123456789abL ) );
+ assertEquals( "nibbles", 11, LongPacker.getNibbleLength( 0xba987654321L ) );
- assertEquals( "nibbles", 12, DataOutputBuffer.getNibbleLength( 0x123456789abcL ) );
- assertEquals( "nibbles", 12, DataOutputBuffer.getNibbleLength( 0xcba987654321L ) );
+ assertEquals( "nibbles", 12, LongPacker.getNibbleLength( 0x123456789abcL ) );
+ assertEquals( "nibbles", 12, LongPacker.getNibbleLength( 0xcba987654321L ) );
- assertEquals( "nibbles", 13, DataOutputBuffer.getNibbleLength( 0x123456789abcdL ) );
- assertEquals( "nibbles", 13, DataOutputBuffer.getNibbleLength( 0xdcba987654321L ) );
+ assertEquals( "nibbles", 13, LongPacker.getNibbleLength( 0x123456789abcdL ) );
+ assertEquals( "nibbles", 13, LongPacker.getNibbleLength( 0xdcba987654321L ) );
- assertEquals( "nibbles", 14, DataOutputBuffer.getNibbleLength( 0x123456789abcdeL ) );
- assertEquals( "nibbles", 14, DataOutputBuffer.getNibbleLength( 0xedcba987654321L ) );
+ assertEquals( "nibbles", 14, LongPacker.getNibbleLength( 0x123456789abcdeL ) );
+ assertEquals( "nibbles", 14, LongPacker.getNibbleLength( 0xedcba987654321L ) );
- assertEquals( "nibbles", 15, DataOutputBuffer.getNibbleLength( 0x123456789abcdefL ) );
- assertEquals( "nibbles", 15, DataOutputBuffer.getNibbleLength( 0xfedcba987654321L ) );
+ assertEquals( "nibbles", 15, LongPacker.getNibbleLength( 0x123456789abcdefL ) );
+ assertEquals( "nibbles", 15, LongPacker.getNibbleLength( 0xfedcba987654321L ) );
- assertEquals( "nibbles", 16, DataOutputBuffer.getNibbleLength( 0x1234567890abcdefL ) );
- assertEquals( "nibbles", 16, DataOutputBuffer.getNibbleLength( 0xfedcba0987654321L ) );
+ assertEquals( "nibbles", 16, LongPacker.getNibbleLength( 0x1234567890abcdefL ) );
+ assertEquals( "nibbles", 16, LongPacker.getNibbleLength( 0xfedcba0987654321L ) );
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -43,6 +43,7 @@
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.io.LongPacker;
import com.bigdata.rdf.internal.impl.AbstractIV;
import com.bigdata.rdf.internal.impl.AbstractInlineIV;
import com.bigdata.rdf.internal.impl.BlobIV;
@@ -71,6 +72,7 @@
import com.bigdata.rdf.internal.impl.uri.VocabURIShortIV;
import com.bigdata.rdf.lexicon.BlobsIndexHelper;
import com.bigdata.rdf.lexicon.ITermIndexCodes;
+import com.bigdata.rdf.lexicon.TermIdEncoder;
import com.bigdata.rdf.model.BigdataBNode;
import com.bigdata.rdf.model.BigdataLiteral;
import com.bigdata.rdf.model.BigdataURI;
@@ -91,6 +93,18 @@
// private static final transient Logger log = Logger.getLogger(IVUtility.class);
+ /**
+ * When <code>true</code>, we will pack term identifiers using
+ * {@link LongPacker}.
+ * <p>
+ * Note: This option requires that term identifiers are non-negative. That
+ * is not currently true for the cluster due to the {@link TermIdEncoder}.
+ *
+ * @see <a href="http://sourceforge.net/apps/trac/bigdata/ticket/529">
+ * Improve load performance </a>
+ */
+ public static final boolean PACK_TIDS = false;
+
public static boolean equals(final IV iv1, final IV iv2) {
// same IV or both null
@@ -386,7 +400,12 @@
*/
// decode the term identifier.
- final long termId = KeyBuilder.decodeLong(key, o);
+ final long termId;
+ if(PACK_TIDS) {
+ termId = LongPacker.unpackLong(key, o);
+ } else {
+ termId = KeyBuilder.decodeLong(key, o);
+ }
if (termId == TermId.NULL) {
if(nullIsNullRef) {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/TermId.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/TermId.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/TermId.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -29,9 +29,12 @@
import org.openrdf.model.Value;
import com.bigdata.btree.keys.IKeyBuilder;
+import com.bigdata.btree.keys.KeyBuilder;
+import com.bigdata.io.LongPacker;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.internal.DTE;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.IVUtility;
import com.bigdata.rdf.internal.VTE;
import com.bigdata.rdf.model.BigdataValue;
@@ -228,15 +231,23 @@
return (int) (termId ^ (termId >>> 32));
}
-
+
public int byteLength() {
- return 1 + Bytes.SIZEOF_LONG;
+ if (IVUtility.PACK_TIDS) {
+
+ return 1 + LongPacker.getByteLength(termId);
+
+ } else {
+
+ return 1 + Bytes.SIZEOF_LONG;
+
+ }
}
@Override
- public int _compareTo(IV o) {
+ public int _compareTo(final IV o) {
final long termId2 = ((TermId<?>) o).termId;
@@ -250,7 +261,17 @@
// First emit the flags byte.
keyBuilder.appendSigned(flags());
- keyBuilder.append(getTermId());
+ if (IVUtility.PACK_TIDS) {
+
+ // variable length encoding
+ ((KeyBuilder) keyBuilder).pack(termId);
+
+ } else {
+
+ // fixed length encoding.
+ keyBuilder.append(termId);
+
+ }
return keyBuilder;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -135,7 +135,7 @@
public class LexiconRelation extends AbstractRelation<BigdataValue>
implements IDatatypeURIResolver {
- final static Logger log = Logger.getLogger(LexiconRelation.class);
+ private final static Logger log = Logger.getLogger(LexiconRelation.class);
private final Set<String> indexNames;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdTupleSerializer.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -97,7 +97,10 @@
*/
// super(keyBuilderFactory);
super(keyBuilderFactory, getDefaultLeafKeysCoder(),
- new com.bigdata.btree.raba.codec.FixedLengthValueRabaCoder(9));
+ IVUtility.PACK_TIDS //
+ ? new com.bigdata.btree.raba.codec.SimpleRabaCoder()//
+ : new com.bigdata.btree.raba.codec.FixedLengthValueRabaCoder(9)//
+ );
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestTermIV.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestTermIV.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestTermIV.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -164,16 +164,28 @@
public void test_TermId_URI_Counter_MINUS_ONE() {
- doTermIVTest(VTE.URI,-1L);
+ if (!IVUtility.PACK_TIDS)
+ doTermIVTest(VTE.URI, -1L);
}
public void test_TermId_URI_Counter_MIN_VALUE() {
- doTermIVTest(VTE.URI, Long.MIN_VALUE);
+ if (!IVUtility.PACK_TIDS)
+ doTermIVTest(VTE.URI, Long.MIN_VALUE);
}
+ /*
+ * Note: This is hitting odd fence posts having to do with equality and
+ * mock IVs.
+ */
+// public void test_TermId_URI_Counter_ZERO() {
+//
+// doTermIVTest(VTE.URI, 0);
+//
+// }
+
public void test_TermId_URI_Counter_MAX_VALUE() {
doTermIVTest(VTE.URI, Long.MAX_VALUE);
@@ -190,7 +202,8 @@
for (VTE vte : VTE.values()) {
// 64 bit random term identifier.
- final long termId = r.nextLong();
+ final long termId = IVUtility.PACK_TIDS ? Math.abs(r.nextLong())
+ : r.nextLong();
final TermId<?> v = new TermId<BigdataValue>(vte, termId);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestId2TermTupleSerializer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestId2TermTupleSerializer.java 2012-04-12 11:06:36 UTC (rev 6278)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestId2TermTupleSerializer.java 2012-04-12 15:27:30 UTC (rev 6279)
@@ -34,6 +34,7 @@
import junit.framework.TestCase2;
import com.bigdata.btree.BytesUtil;
import com.bigdata.io.SerializerUtil;
+import com.bigdata.rdf.internal.IVUtility;
import com.bigdata.rdf.internal.VTE;
import com.bigdata.rdf.internal.impl.TermId;
import com.bigdata.rdf.model.BigdataURI;
@@ -67,9 +68,16 @@
final Id2TermTupleSerializer fixture = new Id2TermTupleSerializer(
namespace, BigdataValueFactoryImpl.getInstance(namespace));
- final TermId<?> id1 = new TermId<BigdataURI>(VTE.URI, -1);
- final TermId<?> id2 = new TermId<BigdataURI>(VTE.URI, 0);
- final TermId<?> id3 = new TermId<BigdataURI>(VTE.URI, 1);
+ final TermId<?> id1, id2, id3;
+ if (IVUtility.PACK_TIDS) {
+ id1 = new TermId<BigdataURI>(VTE.URI, 0);
+ id2 = new TermId<BigdataURI>(VTE.URI, 1);
+ id3 = new TermId<BigdataURI>(VTE.URI, 2);
+ } else {
+ id1 = new TermId<BigdataURI>(VTE.URI, -1);
+ id2 = new TermId<BigdataURI>(VTE.URI, 0);
+ id3 = new TermId<BigdataURI>(VTE.URI, 1);
+ }
final byte[] k1 = fixture.id2key(id1);
final byte[] k2 = fixture.id2key(id2);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2012-04-13 15:12:27
|
Revision: 6285
http://bigdata.svn.sourceforge.net/bigdata/?rev=6285&view=rev
Author: mrpersonick
Date: 2012-04-13 15:12:13 +0000 (Fri, 13 Apr 2012)
Log Message:
-----------
added an exact match capability to the value-centric text index
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/search/FullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestPrefixSearch.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearch.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearchRestartSafe.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataValueCentricFullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITextIndexer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestFullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/search/FullTextIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/search/FullTextIndex.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/search/FullTextIndex.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -885,34 +885,6 @@
/**
* Performs a full text search against indexed documents returning a hit
- * list using the configured default timeout.
- *
- * @param query
- * The query (it will be parsed into tokens).
- * @param languageCode
- * The language code that should be used when tokenizing the
- * query -or- <code>null</code> to use the default
- * {@link Locale}).
- * @param minCosine
- * The minimum cosine that will be returned.
- * @param maxRank
- * The upper bound on the #of hits in the result set.
- *
- * @return The hit list.
- *
- * @see Options#INDEXER_TIMEOUT
- */
- public Hiterator<Hit<V>> search(final String query, final String languageCode,
- final double minCosine, final int maxRank) {
-
- return search(query, languageCode, false/* prefixMatch */, minCosine,
- 1.0d/* maxCosine */, 1/* minRank */, maxRank,
- false/* matchAllTerms */, this.timeout, TimeUnit.MILLISECONDS);
-
- }
-
- /**
- * Performs a full text search against indexed documents returning a hit
* list.
* <p>
* The basic algorithm computes cosine between the term-frequency vector of
@@ -976,11 +948,12 @@
public Hiterator<Hit<V>> search(final String query, final String languageCode,
final boolean prefixMatch,
final double minCosine, final double maxCosine,
- final int minRank, final int maxRank, final boolean matchAllTerms,
+ final int minRank, final int maxRank,
+ final boolean matchAllTerms, final boolean matchExact,
long timeout, final TimeUnit unit) {
final Hit<V>[] a = _search(query, languageCode, prefixMatch, minCosine,
- maxCosine, minRank, maxRank, matchAllTerms, timeout, unit);
+ maxCosine, minRank, maxRank, matchAllTerms, matchExact, timeout, unit);
return new Hiterator<Hit<V>>(//
Arrays.asList(a)//
@@ -997,7 +970,7 @@
final boolean prefixMatch) {
return count(query, languageCode, prefixMatch, 0.0d, 1.0d, 1, 10000,
- false, this.timeout,//
+ false, false, this.timeout,//
TimeUnit.MILLISECONDS);
}
@@ -1006,11 +979,12 @@
public int count(final String query, final String languageCode,
final boolean prefixMatch,
final double minCosine, final double maxCosine,
- final int minRank, final int maxRank, final boolean matchAllTerms,
+ final int minRank, final int maxRank,
+ final boolean matchAllTerms, final boolean matchExact,
long timeout, final TimeUnit unit) {
final Hit[] a = _search(query, languageCode, prefixMatch, minCosine,
- maxCosine, minRank, maxRank, matchAllTerms, timeout, unit);
+ maxCosine, minRank, maxRank, matchAllTerms, matchExact, timeout, unit);
return a.length;
@@ -1021,7 +995,8 @@
final boolean prefixMatch,
final double minCosine, final double maxCosine,
final int minRank, final int maxRank,
- final boolean matchAllTerms, long timeout, final TimeUnit unit) {
+ final boolean matchAllTerms, final boolean matchExact,
+ long timeout, final TimeUnit unit) {
final long begin = System.currentTimeMillis();
@@ -1063,7 +1038,7 @@
}
final FullTextSearchQuery cacheKey = new FullTextSearchQuery(
- query, matchAllTerms, prefixMatch, timeout, unit
+ query, matchAllTerms, matchExact, prefixMatch, timeout, unit
);
Hit<V>[] a;
@@ -1218,9 +1193,10 @@
/*
* If match all is specified, remove any hits with a term count less
- * than the number of search tokens.
+ * than the number of search tokens. It's also an optimization to
+ * run the pruning if we're going to do matchExact.
*/
- if (matchAllTerms && qdata.distinctTermCount() > 1) {
+ if ((matchAllTerms || matchExact) && qdata.distinctTermCount() > 1) {
final int nterms = qdata.terms.size();
@@ -1247,6 +1223,15 @@
}
+ /*
+ * Delegate match exact to subclasses.
+ */
+ if (matchExact) {
+
+ a = matchExact(a, query);
+
+ }
+
if (a.length == 0) {
log.warn("No hits after matchAllTerms pruning: languageCode=[" + languageCode + "], query=["
@@ -1421,6 +1406,19 @@
}
+ /**
+ * Subclasses can override this method to do exact match processing. This
+ * involves materializing the hits into their original text values and
+ * checking for the query string in the materialized value. Not possible
+ * from the base class. The value-centric RDF version can use the
+ * lexicon to materialize the hits and check them for exact match.
+ */
+ protected Hit<V>[] matchExact(final Hit<V>[] hits, final String query) {
+
+ throw new UnsupportedOperationException();
+
+ }
+
/*
* @todo implement the relevant methods.
*/
@@ -1465,6 +1463,7 @@
private final String search;
private final boolean matchAllTerms;
+ private final boolean matchExact;
private final boolean prefixMatch;
private final long timeout;
private final TimeUnit unit;
@@ -1472,12 +1471,14 @@
public FullTextSearchQuery(
final String search,
final boolean matchAllTerms,
+ final boolean matchExact,
final boolean prefixMatch,
final long timeout,
final TimeUnit unit) {
this.search = search;
this.matchAllTerms = matchAllTerms;
+ this.matchExact = matchExact;
this.prefixMatch = prefixMatch;
this.timeout = timeout;
this.unit = unit;
@@ -1492,6 +1493,7 @@
final int prime = 31;
int result = 1;
result = prime * result + (matchAllTerms ? 1231 : 1237);
+ result = prime * result + (matchExact ? 1231 : 1237);
result = prime * result + (prefixMatch ? 1231 : 1237);
result = prime * result
+ ((search == null) ? 0 : search.hashCode());
@@ -1514,6 +1516,8 @@
FullTextSearchQuery other = (FullTextSearchQuery) obj;
if (matchAllTerms != other.matchAllTerms)
return false;
+ if (matchExact != other.matchExact)
+ return false;
if (prefixMatch != other.prefixMatch)
return false;
if (search == null) {
@@ -1527,7 +1531,7 @@
return false;
return true;
}
-
+
}
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestPrefixSearch.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestPrefixSearch.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestPrefixSearch.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -113,7 +113,7 @@
final Hiterator<?> itr = ndx.search("The quick brown dog",
languageCode, false/* prefixMatch */
, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if (log.isInfoEnabled())
log.info("hits:" + itr);
@@ -143,7 +143,7 @@
final Hiterator<?> itr = ndx.search("The qui bro do",
languageCode, true/*prefixMatch*/, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if(log.isInfoEnabled()) log.info("hits:" + itr);
@@ -172,14 +172,14 @@
final Hiterator<?> itr = ndx
.search("brown", languageCode, false/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if(log.isInfoEnabled())
log.info("hits:" + itr);
assertEquals(2, ndx
.count("brown", languageCode, false/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit));
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit));
}
@@ -190,13 +190,13 @@
final Hiterator<?> itr = ndx
.search("brown", languageCode, true/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if(log.isInfoEnabled()) log.info("hits:" + itr);
assertEquals(2, ndx
.count("brown", languageCode, true/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit));
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit));
}
@@ -207,13 +207,13 @@
final Hiterator<?> itr = ndx
.search("bro", languageCode, true/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if(log.isInfoEnabled()) log.info("hits:" + itr);
assertEquals(2, ndx
.count("bro", languageCode, true/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit));
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit));
}
@@ -224,7 +224,7 @@
final Hiterator<?> itr = ndx
.search("bro", languageCode, false/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if(log.isInfoEnabled())
log.info("hits:" + itr);
@@ -240,7 +240,7 @@
final Hiterator<?> itr = ndx
.search("qui", languageCode, true/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if(log.isInfoEnabled())
log.info("hits:" + itr);
@@ -256,7 +256,7 @@
final Hiterator<?> itr = ndx
.search("qui", languageCode, false/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if (log.isInfoEnabled())
log.info("hits:" + itr);
@@ -272,7 +272,7 @@
final Hiterator<?> itr = ndx
.search("quick", languageCode, false/* prefixMatch */, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
if (log.isInfoEnabled())
log.info("hits:" + itr);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearch.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearch.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearch.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -154,7 +154,7 @@
final Hiterator<Hit<Long>> itr = ndx.search(query,
languageCode, prefixMatch, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
// query, languageCode, 0d/* minCosine */,
// Integer.MAX_VALUE/* maxRank */);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearchRestartSafe.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearchRestartSafe.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/search/TestSearchRestartSafe.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -144,7 +144,7 @@
// );
ndx.search(text,
languageCode, prefixMatch, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
assertEquals(1, itr.size()); // Note: 2nd result pruned by cosine.
@@ -177,7 +177,7 @@
final Hiterator<?> itr = // ndx.search(text, languageCode);
ndx.search(text,
languageCode, prefixMatch, minCosine, maxCosine,
- minRank, maxRank, matchAllTerms, timeout, unit);
+ minRank, maxRank, matchAllTerms, false/* matchExact*/, timeout, unit);
assertEquals(1, itr.size()); // Note: 2nd result pruned by cosine.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataValueCentricFullTextIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataValueCentricFullTextIndex.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataValueCentricFullTextIndex.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -28,7 +28,10 @@
package com.bigdata.rdf.lexicon;
import java.io.StringReader;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.Properties;
import java.util.UUID;
@@ -42,6 +45,8 @@
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.btree.raba.codec.SimpleRabaCoder;
import com.bigdata.journal.IIndexManager;
+import com.bigdata.journal.ITx;
+import com.bigdata.journal.TimestampUtility;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.store.AbstractTripleStore;
@@ -121,7 +126,7 @@
assertWritable();
- final String name = getNamespace() + "."+NAME_SEARCH;
+ final String name = getNamespace() + "." + NAME_SEARCH;
final IIndexManager indexManager = getIndexManager();
@@ -303,4 +308,146 @@
}
+ final synchronized public LexiconRelation getLexiconRelation() {
+
+ if (lexiconRelation == null) {
+
+ long t = getTimestamp();
+
+ if (TimestampUtility.isReadWriteTx(t)) {
+
+ /*
+ * A read-write tx must use the unisolated view of the lexicon.
+ */
+ t = ITx.UNISOLATED;
+
+ }
+
+ // lexicon namespace, since this index is inside the lexicon
+ final String ns = getNamespace();
+
+ if (log.isDebugEnabled())
+ log.debug(ns);
+
+ lexiconRelation = (LexiconRelation) getIndexManager()
+ .getResourceLocator().locate(ns, t);
+
+ }
+
+ return lexiconRelation;
+
+ }
+ private LexiconRelation lexiconRelation;
+
+// protected Hit[] matchExact2(final Hit[] hits, final String query) {
+//
+// final Map<IV<?,?>, Hit> iv2Hit = new HashMap<IV<?,?>, Hit>(hits.length);
+//
+// for (Hit h : hits) {
+//
+// iv2Hit.put((IV<?,?>) h.getDocId(), h);
+//
+// }
+//
+// final LexiconRelation lex = getLexiconRelation();
+//
+// final Map<IV<?,?>, BigdataValue> terms = lex.getTerms(iv2Hit.keySet());
+//
+// final Hit[] tmp = new Hit[hits.length];
+//
+// int i = 0;
+// for (Map.Entry<IV<?,?>, BigdataValue> e : terms.entrySet()) {
+//
+// final IV<?,?> iv = e.getKey();
+//
+// final BigdataValue term = e.getValue();
+//
+// if (term.stringValue().contains(query)) {
+//
+// tmp[i++] = iv2Hit.get(iv);
+//
+// }
+//
+// }
+//
+// if (i < hits.length) {
+//
+// final Hit[] a = new Hit[i];
+// System.arraycopy(tmp, 0, a, 0, i);
+// return a;
+//
+// } else {
+//
+// return hits;
+//
+// }
+//
+// }
+
+ @Override
+ protected Hit[] matchExact(final Hit[] hits, final String query) {
+
+// /*
+// * Too big to do efficient exact matching.
+// */
+// if (hits.length > 10000) {
+//
+// return hits;
+//
+// }
+
+ final int chunkSize = 1000;
+
+ final Hit[] tmp = new Hit[hits.length];
+
+ final Map<IV<?,?>, Hit> iv2Hit = new HashMap<IV<?,?>, Hit>(chunkSize);
+
+ final LexiconRelation lex = getLexiconRelation();
+
+ int i = 0, k = 0;
+ while (i < hits.length) {
+
+ iv2Hit.clear();
+
+ for (int j = 0; j < chunkSize && i < hits.length; j++) {
+
+ final Hit h = hits[i++];
+
+ iv2Hit.put((IV<?,?>) h.getDocId(), h);
+
+ }
+
+ final Map<IV<?,?>, BigdataValue> terms = lex.getTerms(iv2Hit.keySet());
+
+ for (Map.Entry<IV<?,?>, BigdataValue> e : terms.entrySet()) {
+
+ final IV<?,?> iv = e.getKey();
+
+ final BigdataValue term = e.getValue();
+
+ if (term.stringValue().contains(query)) {
+
+ tmp[k++] = iv2Hit.get(iv);
+
+ }
+
+ }
+
+ }
+
+ if (k < hits.length) {
+
+ final Hit[] a = new Hit[k];
+ System.arraycopy(tmp, 0, a, 0, k);
+ return a;
+
+ } else {
+
+ return hits;
+
+ }
+
+ }
+
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITextIndexer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITextIndexer.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/ITextIndexer.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -110,6 +110,8 @@
* The max rank of the search result.
* @param matchAllTerms
* if true, return only hits that match all search terms
+ * @param matchExact
+ * if true, return only hits that have an exact match of the search string
* @param timeout
* The timeout -or- ZERO (0) for NO timeout (this is equivalent
* to using {@link Long#MAX_VALUE}).
@@ -122,7 +124,8 @@
final boolean prefixMatch,
final double minCosine, final double maxCosine,
final int minRank, final int maxRank,
- final boolean matchAllTerms, long timeout, final TimeUnit unit);
+ final boolean matchAllTerms, final boolean matchExact,
+ long timeout, final TimeUnit unit);
/**
* Count free text search results.
@@ -154,6 +157,8 @@
* The max rank of the search result.
* @param matchAllTerms
* if true, return only hits that match all search terms
+ * @param matchExact
+ * if true, return only hits that have an exact match of the search string
* @param timeout
* The timeout -or- ZERO (0) for NO timeout (this is equivalent
* to using {@link Long#MAX_VALUE}).
@@ -166,6 +171,7 @@
final boolean prefixMatch,
final double minCosine, final double maxCosine,
final int minRank, final int maxRank,
- final boolean matchAllTerms, long timeout, final TimeUnit unit);
+ final boolean matchAllTerms, final boolean matchExact,
+ long timeout, final TimeUnit unit);
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/ASTSearchOptimizer.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -104,6 +104,7 @@
set.add(BD.MAX_RELEVANCE);
set.add(BD.MIN_RELEVANCE);
set.add(BD.MATCH_ALL_TERMS);
+ set.add(BD.MATCH_EXACT);
set.add(BD.SUBJECT_SEARCH);
set.add(BD.SEARCH_TIMEOUT);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/SearchServiceFactory.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -283,6 +283,10 @@
assertObjectIsLiteral(sp);
+ } else if(uri.equals(BD.MATCH_EXACT)) {
+
+ assertObjectIsLiteral(sp);
+
} else if(uri.equals(BD.SUBJECT_SEARCH)) {
assertObjectIsLiteral(sp);
@@ -348,6 +352,7 @@
private final Literal minRelevance;
private final Literal maxRelevance;
private final boolean matchAllTerms;
+ private final boolean matchExact;
private final boolean subjectSearch;
private final Literal searchTimeout;
@@ -393,6 +398,7 @@
Literal minRelevance = null;
Literal maxRelevance = null;
boolean matchAllTerms = false;
+ boolean matchExact = false;
boolean subjectSearch = false;
Literal searchTimeout = null;
@@ -420,6 +426,8 @@
maxRelevance = (Literal) oVal;
} else if (BD.MATCH_ALL_TERMS.equals(p)) {
matchAllTerms = ((Literal) oVal).booleanValue();
+ } else if (BD.MATCH_EXACT.equals(p)) {
+ matchExact = ((Literal) oVal).booleanValue();
} else if (BD.SUBJECT_SEARCH.equals(p)) {
subjectSearch = ((Literal) oVal).booleanValue();
} else if (BD.SEARCH_TIMEOUT.equals(p)) {
@@ -438,6 +446,7 @@
this.minRelevance = minRelevance;
this.maxRelevance = maxRelevance;
this.matchAllTerms = matchAllTerms;
+ this.matchExact = matchExact;
this.subjectSearch = subjectSearch;
this.searchTimeout = searchTimeout;
@@ -474,6 +483,7 @@
minRank == null ? BD.DEFAULT_MIN_RANK/*1*/ : minRank.intValue()/* minRank */,
maxRank == null ? BD.DEFAULT_MAX_RANK/*Integer.MAX_VALUE*/ : maxRank.intValue()/* maxRank */,
matchAllTerms,
+ matchExact,
searchTimeout == null ? BD.DEFAULT_TIMEOUT/*0L*/ : searchTimeout.longValue()/* timeout */,
TimeUnit.MILLISECONDS);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BD.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -281,6 +281,27 @@
final boolean DEFAULT_MATCH_ALL_TERMS = false;
+ /**
+ * Magic predicate used to query for free text search metadata. Use
+ * in conjunction with {@link #SEARCH} as follows:
+ * <p>
+ * <pre>
+ *
+ * select ?s
+ * where {
+ * ?s bd:search "scale-out RDF triplestore" .
+ * ?s bd:matchExact "true" .
+ * }
+ *
+ * </pre>
+ * <p>
+ * This operation will be rather expensive as it will require materializing
+ * all the hits to check their values.
+ */
+ final URI MATCH_EXACT = new URIImpl(SEARCH_NAMESPACE+"matchExact");
+
+ final boolean DEFAULT_MATCH_EXACT = false;
+
final boolean DEFAULT_PREFIX_MATCH = false;
/**
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestFullTextIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestFullTextIndex.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestFullTextIndex.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -132,6 +132,7 @@
minCosine, 1.0d/* maxCosine */,
1/* minRank */, Integer.MAX_VALUE/* maxRank */,
false/* matchAllTerms */,
+ false, // matchExact
Long.MAX_VALUE,//2L/* timeout */,
TimeUnit.MILLISECONDS// TimeUnit.SECONDS
);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/lexicon/TestSubjectCentricFullTextIndex.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -137,6 +137,7 @@
minCosine, 1.0d/* maxCosine */,
1/* minRank */, Integer.MAX_VALUE/* maxRank */,
false/* matchAllTerms */,
+ false, // matchExact
Long.MAX_VALUE,//2L/* timeout */,
TimeUnit.MILLISECONDS// TimeUnit.SECONDS
);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java 2012-04-13 13:36:27 UTC (rev 6284)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java 2012-04-13 15:12:13 UTC (rev 6285)
@@ -799,6 +799,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -866,6 +867,7 @@
BD.DEFAULT_MIN_RANK,//1
maxHits, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -936,6 +938,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1010,6 +1013,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1086,6 +1090,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1160,6 +1165,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1230,6 +1236,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
true, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1299,6 +1306,7 @@
minRank,//1
maxRank,//10000, // maxRank (=maxResults + 1)
false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1338,6 +1346,7 @@
BD.DEFAULT_MIN_RANK, // minRank
BD.DEFAULT_MAX_RANK, // maxRank
false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1477,6 +1486,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1542,6 +1552,7 @@
BD.DEFAULT_MIN_RANK,//1
maxHits, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1613,6 +1624,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1689,6 +1701,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1766,6 +1779,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1841,6 +1855,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1912,6 +1927,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
true, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -1983,6 +1999,7 @@
minRank,//1
maxRank,//10000, // maxRank (=maxResults + 1)
false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -2022,6 +2039,7 @@
BD.DEFAULT_MIN_RANK, // minRank
BD.DEFAULT_MAX_RANK, // maxRank
false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -2167,6 +2185,7 @@
BD.DEFAULT_MIN_RANK,//1
BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ false, // matchExact
BD.DEFAULT_TIMEOUT,//1000L, // timeout
TimeUnit.MILLISECONDS // unit
);
@@ -2426,5 +2445,161 @@
FILTER (?category1 = <http://www.ms2w.com/ontologies/autocad/AutoCADBlock> || ?category1 = <http://www.ms2w.com/ontologies/autocad/AutoCADBlockAttribute> || ?category1 = <http://www.ms2w.com/ontologies/autocad/AutoCADBlockReference> || ?category1 = <http://www.ms2w.com/ontologies/autocad/AutoCADFile> || ?category1 = <http://www.ms2w.com/ontologies/autocad/AutoCADTable> || ?category1 = <http://www.ms2w.com/ontologies/autocad/AutoCADTitleBlock> || ?category1 = <http://www.ms2w.com/ontologies/file/Directory> || ?category1 = <http://www.ms2w.com/ontologies/file/File> || ?category1 = <http://www.ms2w.com/ontologies/pdffile/PdfAnnotation> || ?category1 = <http://www.ms2w.com/ontologies/pdffile/PdfFile> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmDocumentAssembly> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmTableTypeBOM> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/Component> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/Configuration> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmDocumentDrawing> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmBOMTableTypeIndented> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmDocumentPart> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmBOMTableTypePartsOnly> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmTableTypeRevision> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmToolboxCopiedPart> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmToolboxStandardPart> || ?category1 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmBOMTableTypeTopLevelOnly> ) .
FILTER (?category2 = <http://www.ms2w.com/ontologies/autocad/AutoCADBlock> || ?category2 = <http://www.ms2w.com/ontologies/autocad/AutoCADBlockAttribute> || ?category2 = <http://www.ms2w.com/ontologies/autocad/AutoCADBlockReference> || ?category2 = <http://www.ms2w.com/ontologies/autocad/AutoCADFile> || ?category2 = <http://www.ms2w.com/ontologies/autocad/AutoCADTable> || ?category2 = <http://www.ms2w.com/ontologies/autocad/AutoCADTitleBlock> || ?category2 = <http://www.ms2w.com/ontologies/file/Directory> || ?category2 = <http://www.ms2w.com/ontologies/file/File> || ?category2 = <http://www.ms2w.com/ontologies/pdffile/PdfAnnotation> || ?category2 = <http://www.ms2w.com/ontologies/pdffile/PdfFile> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmDocumentAssembly> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmTableTypeBOM> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/Component> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/Configuration> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmDocumentDrawing> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmBOMTableTypeIndented> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmDocumentPart> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmBOMTableTypePartsOnly> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmTableTypeRevision> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmToolboxCopiedPart> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmToolboxStandardPart> || ?category2 = <http://www.ms2w.com/ontologies/solidworks/20091023/swDmBOMTableTypeTopLevelOnly> ) .
*/
+
+ /**
+ * FIXME This should be migrated to the data-driven test suite in
+ * {@link TestSearch}.
+ *
+ * @see TestSearch
+ */
+ public void testExactMatch() throws Exception {
+
+ final BigdataSail sail = getSail();
+ try {
+
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+ final BigdataSailRepositoryConnection cxn =
+ (BigdataSailRepositoryConnection) repo.getConnection();
+
+ try {
+
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ final URI s1 = vf.createURI(BD.NAMESPACE+"s1");
+ final URI s2 = vf.createURI(BD.NAMESPACE+"s2");
+ final URI s3 = vf.createURI(BD.NAMESPACE+"s3");
+ final URI s4 = vf.createURI(BD.NAMESPACE+"s4");
+ final URI s5 = vf.createURI(BD.NAMESPACE+"s5");
+ final URI s6 = vf.createURI(BD.NAMESPACE+"s6");
+ final URI s7 = vf.createURI(BD.NAMESPACE+"s7");
+ final URI s8 = vf.createURI(BD.NAMESPACE+"s8");
+ final Literal l1 = vf.createLiteral("0");
+ final Literal l2 = vf.createLiteral("0 0");
+ final Literal l3 = vf.createLiteral("0 1 0");
+ final Literal l4 = vf.createLiteral("0,0");
+ final Literal l5 = vf.createLiteral("1 2 0 0 3");
+ final Literal l6 = vf.createLiteral("brown cow");
+ final Literal l7 = vf.createLiteral("how now brown cow");
+ final Literal l8 = vf.createLiteral("toilet");
+
+ cxn.add(s1, RDFS.LABEL, l1);
+ cxn.add(s2, RDFS.LABEL, l2);
+ cxn.add(s3, RDFS.LABEL, l3);
+ cxn.add(s4, RDFS.LABEL, l4);
+ cxn.add(s5, RDFS.LABEL, l5);
+ cxn.add(s6, RDFS.LABEL, l6);
+ cxn.add(s7, RDFS.LABEL, l7);
+ cxn.add(s8, RDFS.LABEL, l8);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ final Map<IV, Literal> literals = new LinkedHashMap<IV, Literal>();
+ literals.put(((BigdataValue)l1).getIV(), l1);
+ literals.put(((BigdataValue)l2).getIV(), l2);
+ literals.put(((BigdataValue)l3).getIV(), l3);
+ literals.put(((BigdataValue)l4).getIV(), l4);
+ literals.put(((BigdataValue)l5).getIV(), l5);
+ literals.put(((BigdataValue)l6).getIV(), l6);
+ literals.put(((BigdataValue)l7).getIV(), l7);
+ literals.put(((BigdataValue)l8).getIV(), l8);
+
+ final Map<IV, URI> uris = new LinkedHashMap<IV, URI>();
+ uris.put(((BigdataValue)l1).getIV(), s1);
+ uris.put(((BigdataValue)l2).getIV(), s2);
+ uris.put(((BigdataValue)l3).getIV(), s3);
+ uris.put(((BigdataValue)l4).getIV(), s4);
+ uris.put(((BigdataValue)l5).getIV(), s5);
+ uris.put(((BigdataValue)l6).getIV(), s6);
+ uris.put(((BigdataValue)l7).getIV(), s7);
+ uris.put(((BigdataValue)l8).getIV(), s8);
+
+/**/
+ if (log.isInfoEnabled()) {
+ log.info("\n" + sail.getDatabase().dumpStore());
+ }
+
+ {
+ final String searchQuery = "0 0";
+
+ final String query =
+ "select ?s ?o ?score " +
+ "where " +
+ "{ " +
+ " ?s <"+RDFS.LABEL+"> ?o . " +
+ " ?o <"+BD.SEARCH+"> \""+searchQuery+"\" . " +
+ " ?o <"+BD.MATCH_EXACT+"> \"true\" . " +
+ " ?o <"+BD.RELEVANCE+"> ?score . " +
+ "} " +
+ "order by desc(?score)";
+
+ final TupleQuery tupleQuery =
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+ TupleQueryResult result = tupleQuery.evaluate();
+
+ int i = 0;
+ while (result.hasNext()) {
+ final BindingSet tmp = result.next();
+ if (log.isInfoEnabled())
+ log.info(i + ": " + tmp.toString());
+ i++;
+ }
+ assertEquals("wrong # of results", 2, i);
+
+ Collection<BindingSet> answer = new LinkedList<BindingSet>();
+
+ final ITextIndexer search =
+ sail.getDatabase().getLexiconRelation().getSearchEngine();
+ final Hiterator<IHit> hits =
+ search.search(searchQuery,
+ null, // languageCode
+ BD.DEFAULT_PREFIX_MATCH,//false, // prefixMatch
+ BD.DEFAULT_MIN_RELEVANCE,//0d, // minCosine
+ BD.DEFAULT_MAX_RELEVANCE,//1.0d, // maxCosine
+ BD.DEFAULT_MIN_RANK,//1
+ BD.DEFAULT_MAX_RANK,//10000, // maxRank (=maxResults + 1)
+ BD.DEFAULT_MATCH_ALL_TERMS,//false, // matchAllTerms
+ true, // matchExact
+ BD.DEFAULT_TIMEOUT,//1000L, // timeout
+ TimeUnit.MILLISECONDS // unit
+ );
+
+ while (hits.hasNext()) {
+ final IHit hit = hits.next();
+ final IV id = (IV)hit.getDocId();
+ final Literal score = vf.createLiteral(hit.getCosine());
+ final URI s = uris.get(id);
+ final Literal o = literals.get(id);
+ final BindingSet bs = createBindingSet(
+ new BindingImpl("s", s),
+ new BindingImpl("o", o),
+ new BindingImpl("score", score));
+ if(log.isInfoEnabled())
+ log.info(bs);
+ answer.add(bs);
+ }
+
+ result = tupleQuery.evaluate();
+ compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ }
+ } finally {
+ sail.__tearDownUnitTest();
+ }
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2012-04-13 15:22:01
|
Revision: 6286
http://bigdata.svn.sourceforge.net/bigdata/?rev=6286&view=rev
Author: mrpersonick
Date: 2012-04-13 15:21:49 +0000 (Fri, 13 Apr 2012)
Log Message:
-----------
range bop integration part I - single datatype ranges
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionNode.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTSetValueExpressionsOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTStaticJoinOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/RangeNode.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/RangeHint.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTRangeOptimizer.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpBase.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpBase.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -213,6 +213,15 @@
}
/**
+ * A copy of the annotations.
+ */
+ final protected Map<String, Object> annotationsCopy() {
+
+ return new LinkedHashMap<String, Object>(annotations);
+
+ }
+
+ /**
* A reference to the actual annotations map object. This is used in some
* hot spots to avoid creating a new annotations map when we know that the
* annotations will not be modified (annotations are always set within the
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -330,7 +330,39 @@
*/
public Predicate<E> asBound(final IBindingSet bindingSet) {
- return new Predicate<E>(argsCopy(), annotationsRef())
+ Map<String, Object> anns = annotationsRef();
+
+ final RangeBOp rangeBOp = range();
+
+ if (rangeBOp != null) {
+
+ /*
+ * Attempt to evaluate the RangeBOp.
+ */
+ final RangeBOp asBound = rangeBOp.asBound(bindingSet);
+
+ // reference test is ok here
+ if (asBound != rangeBOp) {
+
+ // make a copy of the anns
+ // set the asBound range on it
+ // use those for the pred below
+
+ anns = annotationsCopy();
+ anns.put(Annotations.RANGE, asBound);
+
+ }
+
+ }
+
+// /*
+// * If we have a range we need to modify the annotations with the asBound
+// * range.
+// */
+// final Map<String, Object> anns =
+// range() != null ? annotationsCopy() : annotationsRef();
+
+ return new Predicate<E>(argsCopy(), anns)
._asBound(bindingSet);
}
@@ -376,30 +408,29 @@
}
- /*
- * FIXME When putting the RangeBOp back into use, be very careful of the
- * optimization in asBound(). Predicate#asBound() is NOT making a copy
- * of the annotations map. The code below will therefore cause a
- * modification to the source predicate's annotations, not the copy's.
- * This violates the "effectively immutable" contract.
- *
- * What the code should probably do is check in asBound() and only use
- * the code path which avoids the annotations map copy when the RANGE is
- * not to be set on the new Predicate instance.
- */
-
+// /*
+// * FIXME When putting the RangeBOp back into use, be very careful of the
+// * optimization in asBound(). Predicate#asBound() is NOT making a copy
+// * of the annotations map. The code below will therefore cause a
+// * modification to the source predicate's annotations, not the copy's.
+// * This violates the "effectively immutable" contract.
+// *
+// * What the code should probably do is check in asBound() and only use
+// * the code path which avoids the annotations map copy when the RANGE is
+// * not to be set on the new Predicate instance.
+// */
+//
// final RangeBOp rangeBOp = range();
//
-// // we don't have a range bop for ?o
-// if (rangeBOp == null)
-// return tmp;
+// if (rangeBOp != null) {
//
-// /*
-// * Attempt to evaluate the RangeBOp.
-// */
-// final RangeBOp asBound = rangeBOp.asBound(bindingSet);
-//
-// tmp._setProperty(Annotations.RANGE, asBound);
+// /*
+// * Attempt to evaluate the RangeBOp.
+// */
+// final RangeBOp asBound = rangeBOp.asBound(bindingSet);
+//
+// _setProperty(Annotations.RANGE, asBound);
+// }
return this;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -34,6 +34,7 @@
import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.ImmutableBOp;
+import com.bigdata.bop.ModifiableBOpBase;
import com.bigdata.bop.NV;
import com.bigdata.bop.ap.Predicate;
import com.bigdata.rdf.error.SparqlTypeErrorException;
@@ -48,7 +49,7 @@
* @author mrpersonick
*/
@SuppressWarnings("rawtypes")
-final public class RangeBOp extends BOpBase implements IVariable<Range> {
+final public class RangeBOp extends ModifiableBOpBase { // implements IVariable<Range> {
/**
*
@@ -73,17 +74,23 @@
}
- /** Cached to/from lookups. */
- private transient volatile IValueExpression<IV> to, from;
+// /** Cached to/from lookups. */
+// private transient volatile IValueExpression<IV> to, from;
- public RangeBOp(final IVariable<IV> var,
- final IValueExpression<IV> from,
- final IValueExpression<IV> to) {
+ public RangeBOp(final IVariable<? extends IV> var) {
+
+ this(BOp.NOARGS, NV.asMap(new NV(Annotations.VAR, var)));
+
+ }
+
+ public RangeBOp(final IVariable<? extends IV> var,
+ final IValueExpression<? extends IV> from,
+ final IValueExpression<? extends IV> to) {
- this(BOp.NOARGS,
- NV.asMap(new NV(Annotations.VAR, var),
- new NV(Annotations.FROM, from),
- new NV(Annotations.TO, to)));
+ this(BOp.NOARGS, NV.asMap(
+ new NV(Annotations.VAR, var),
+ new NV(Annotations.FROM, from),
+ new NV(Annotations.TO, to)));
}
@@ -91,17 +98,7 @@
* Required shallow copy constructor.
*/
public RangeBOp(final BOp[] args, final Map<String,Object> anns) {
-
super(args,anns);
-
- if (getProperty(Annotations.VAR) == null
- || getProperty(Annotations.FROM) == null
- || getProperty(Annotations.TO) == null) {
-
- throw new IllegalArgumentException();
-
- }
-
}
/**
@@ -112,195 +109,217 @@
}
@SuppressWarnings("unchecked")
- public IVariable<IV> var() {
- return (IVariable<IV>) getRequiredProperty(Annotations.VAR);
+ public IVariable<? extends IV> var() {
+ return (IVariable<? extends IV>) getProperty(Annotations.VAR);
}
@SuppressWarnings("unchecked")
- public IValueExpression<IV> from() {
- if (from == null) {
- from = (IValueExpression<IV>) getRequiredProperty(Annotations.FROM);
- }
- return from;
+ public IValueExpression<? extends IV> from() {
+ return (IValueExpression<? extends IV>) getProperty(Annotations.FROM);
}
@SuppressWarnings("unchecked")
- public IValueExpression<IV> to() {
- if (to == null) {
- to = (IValueExpression<IV>) getRequiredProperty(Annotations.TO);
- }
- return to;
+ public IValueExpression<? extends IV> to() {
+ return (IValueExpression<? extends IV>) getProperty(Annotations.TO);
}
+
+ public void setFrom(final IValueExpression<? extends IV> from) {
+ setProperty(Annotations.FROM, from);
+ }
- final public Range get(final IBindingSet bs) {
-
-// log.debug("getting the asBound value");
-
- final IV from = from().get(bs);
-
-// log.debug("from: " + from);
-
- // sort of like Var.get(), which returns null when the variable
- // is not yet bound
- if (from == null)
- return null;
-
- final IV to = to().get(bs);
-
-// log.debug("to: " + to);
-
- // sort of like Var.get(), which returns null when the variable
- // is not yet bound
- if (to == null)
- return null;
+ public void setTo(final IValueExpression<? extends IV> to) {
+ setProperty(Annotations.TO, to);
+ }
- try {
- /*
- * FIXME Should handle from/to (non-)exclusive boundaries using a
- * successor pattern.
- */
- // let Range ctor() do the type checks and valid range checks
- return new Range(from, to);
- } catch (IllegalArgumentException ex) {
- // log the reason the range is invalid
-// if (log.isInfoEnabled())
-// log.info("dropping solution: " + ex.getMessage());
- // drop the solution
- throw new SparqlTypeErrorException();
- }
-
- }
-
- /**
- * FIXME This needs to be hooked from {@link Predicate#asBound(IBindingSet)}.
- */
+// final public Range get(final IBindingSet bs) {
+//
+//// log.debug("getting the asBound value");
+//
+// final IV from = from().get(bs);
+//
+//// log.debug("from: " + from);
+//
+// // sort of like Var.get(), which returns null when the variable
+// // is not yet bound
+// if (from == null)
+// return null;
+//
+// final IV to = to().get(bs);
+//
+//// log.debug("to: " + to);
+//
+// // sort of like Var.get(), which returns null when the variable
+// // is not yet bound
+// if (to == null)
+// return null;
+//
+// try {
+// /*
+// * FIXME Should handle from/to (non-)exclusive boundaries using a
+// * successor pattern.
+// */
+// // let Range ctor() do the type checks and valid range checks
+// return new Range(from, to);
+// } catch (IllegalArgumentException ex) {
+// // log the reason the range is invalid
+//// if (log.isInfoEnabled())
+//// log.info("dropping solution: " + ex.getMessage());
+// // drop the solution
+// throw new SparqlTypeErrorException();
+// }
+//
+// }
+//
final public RangeBOp asBound(final IBindingSet bs) {
- final IV from, to;
- try {
- // log.debug("getting the asBound value");
-
- from = from().get(bs);
-
- // log.debug("from: " + from);
-
- // sort of like Var.get(), which returns null when the variable
- // is not yet bound
- if (from == null)
- return this;
-
- to = to().get(bs);
-
- // log.debug("to: " + to);
-
- // sort of like Var.get(), which returns null when the variable
- // is not yet bound
- if (to == null)
- return this;
-
- } catch (SparqlTypeErrorException ex) {
-
- /*
- * Ignore. If the variables in the RangeBOp value expressions are
- * not fully bound or has the wrong dynamic type then the range bop
- * can not be evaluated yet.
- */
-
+ /*
+ * Only care if we change from a non-null, non-Constant to a ground IV.
+ */
+ final IValueExpression<? extends IV> origFrom = from();
+ final IValueExpression<? extends IV> origTo = to();
+
+ IValueExpression<? extends IV> asBoundFrom;
+ {
+ if (origFrom == null) {
+ asBoundFrom = null;
+ } else if (origFrom instanceof IConstant) {
+ asBoundFrom = origFrom;
+ } else {
+ try {
+ final IV iv = origFrom.get(bs);
+ asBoundFrom = new Constant<IV>(iv);
+ } catch (SparqlTypeErrorException ex) {
+ asBoundFrom = origFrom;
+ }
+ }
+ }
+
+ IValueExpression<? extends IV> asBoundTo;
+ {
+ if (origTo == null) {
+ asBoundTo = null;
+ } else if (origTo instanceof IConstant) {
+ asBoundTo = origTo;
+ } else {
+ try {
+ final IV iv = origTo.get(bs);
+ asBoundTo = new Constant<IV>(iv);
+ } catch (SparqlTypeErrorException ex) {
+ asBoundTo = origTo;
+ }
+ }
+ }
+
+ /*
+ * Null means no value expression, constant value expression, or not
+ * able to evaluate at this time. Non-null means the asBound is
+ * different from the original.
+ */
+ if (asBoundFrom == origFrom && asBoundTo == origTo) {
return this;
-
}
-
- // Note: defer clone() until everything is bound.
- final RangeBOp asBound = (RangeBOp) this.clone();
-
- asBound._setProperty(Annotations.FROM, new Constant<IV>(from));
- asBound._setProperty(Annotations.TO, new Constant<IV>(to));
-
+
+ final RangeBOp asBound = new RangeBOp(var());
+ if (asBoundFrom != null)
+ asBound.setFrom(asBoundFrom);
+ if (asBoundTo != null)
+ asBound.setTo(asBoundTo);
+
return asBound;
-
+
}
- final public boolean isFullyBound() {
-
- return from() instanceof IConstant && to() instanceof IConstant;
-
+// final public boolean isFullyBound() {
+//
+// return (from() == null || from() instanceof IConstant) &&
+// (to() == null || to() instanceof IConstant);
+//
+// }
+
+ final public boolean isFromBound() {
+ return from() instanceof IConstant;
}
-
-// @Override
- public boolean isVar() {
- return true;
- }
-
-// @Override
- public boolean isConstant() {
- return false;
- }
-
-// @Override
- public Range get() {
-// log.debug("somebody tried to get me");
-
- return null;
- }
-
-// @Override
- public String getName() {
- return var().getName();
- }
-
-// @Override
- public boolean isWildcard() {
- return false;
- }
-
-
- /*
- * TODO The default BOp equals() and hashCode() should be fine.
- */
-
- // TODO This looks dangerous. It is only considering the variable!
- final public boolean equals(final IVariableOrConstant op) {
-
- if (op == null)
- return false;
-
- if (this == op)
- return true;
-
- if (op instanceof IVariable<?>) {
-
- return var().getName().equals(((IVariable<?>) op).getName());
-
- }
-
- return false;
-
+
+ final public boolean isToBound() {
+ return to() instanceof IConstant;
}
-// final private boolean _equals(final RangeBOp op) {
+//
+//// @Override
+// public boolean isVar() {
+// return true;
+// }
+//
+//// @Override
+// public boolean isConstant() {
+// return false;
+// }
+//
+//// @Override
+// public Range get() {
+//// log.debug("somebody tried to get me");
+//
+// return null;
+// }
+//
+//// @Override
+// public String getName() {
+// return var().getName();
+// }
+//
+//// @Override
+// public boolean isWildcard() {
+// return false;
+// }
+//
+//
+// /*
+// * TODO The default BOp equals() and hashCode() should be fine.
+// */
+//
+// // TODO This looks dangerous. It is only considering the variable!
+// final public boolean equals(final IVariableOrConstant op) {
+//
+// if (op == null)
+// return false;
//
-// return var().equals(op.var())
-// && from().equals(op.from())
-// && to().equals(op.to());
+// if (this == op)
+// return true;
//
+// if (op instanceof IVariable<?>) {
+//
+// return var().getName().equals(((IVariable<?>) op).getName());
+//
+// }
+//
+// return false;
+//
// }
-
- /**
- * Caches the hash code.
- */
-// private int hash = 0;
- public int hashCode() {
-//
-// int h = hash;
-// if (h == 0) {
-// h = 31 * h + var().hashCode();
-// h = 31 * h + from().hashCode();
-// h = 31 * h + to().hashCode();
-// hash = h;
-// }
-// return h;
-//
- return var().hashCode();
- }
+//
+//// final private boolean _equals(final RangeBOp op) {
+////
+//// return var().equals(op.var())
+//// && from().equals(op.from())
+//// && to().equals(op.to());
+////
+//// }
+//
+// /**
+// * Caches the hash code.
+// */
+//// private int hash = 0;
+// public int hashCode() {
+////
+//// int h = hash;
+//// if (h == 0) {
+//// h = 31 * h + var().hashCode();
+//// h = 31 * h + from().hashCode();
+//// h = 31 * h + to().hashCode();
+//// hash = h;
+//// }
+//// return h;
+////
+// return var().hashCode();
+// }
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/BindingsClause.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -194,14 +194,26 @@
sb.append("{");
- for(IBindingSet bset : bindingSets) {
-
+ if (bindingSets.size() <= 10) {
+
+ for(IBindingSet bset : bindingSets) {
+
+ sb.append("\n");
+
+ sb.append(s1);
+
+ sb.append(bset.toString());
+
+ }
+
+ } else {
+
sb.append("\n");
sb.append(s1);
- sb.append(bset.toString());
-
+ sb.append("[ count=" + bindingSets.size() + " ]");
+
}
sb.append("\n");
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionNode.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/FunctionNode.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -113,7 +113,7 @@
super.clearProperty(Annotations.VALUE_EXPR);
super.invalidate();
-
+
}
/**
@@ -188,6 +188,16 @@
}
/**
+ * Return <code>t1 - t2</code> (aka SUBTRACT).
+ */
+ static public FunctionNode subtract(final TermNode t1, final TermNode t2) {
+
+ return new FunctionNode(FunctionRegistry.SUBTRACT, null/* scalarValues */,
+ new ValueExpressionNode[] { t1, t2 });
+
+ }
+
+ /**
* Return <code>sameTerm(t1,t2)</code> (aka EQ).
*/
static public FunctionNode sameTerm(final TermNode t1, final TermNode t2) {
@@ -238,6 +248,28 @@
}
/**
+ * Return <code>min(v1,v2)</code>
+ */
+ static public FunctionNode MIN(
+ final ValueExpressionNode v1, final ValueExpressionNode v2) {
+
+ return new FunctionNode(FunctionRegistry.MIN, null/* scalarValues */,
+ new ValueExpressionNode[] { v1, v2 });
+
+ }
+
+ /**
+ * Return <code>max(v1,v2)</code>
+ */
+ static public FunctionNode MAX(
+ final ValueExpressionNode v1, final ValueExpressionNode v2) {
+
+ return new FunctionNode(FunctionRegistry.MAX, null/* scalarValues */,
+ new ValueExpressionNode[] { v1, v2 });
+
+ }
+
+ /**
* Return a binary function <code>op(t1,t2)</code>
*/
static public FunctionNode binary(final URI uri, final TermNode t1,
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -398,4 +398,11 @@
boolean DEFAULT_SOLUTION_SET_CACHE = false;
+ /**
+ * Used to mark a predicate as "range safe" - that is, we can safely
+ * apply the range bop to constrain the predicate. This can only be
+ * used currently when there is a single datatype for attribute values.
+ */
+ String RANGE_SAFE = "rangeSafe";
+
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/RangeNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/RangeNode.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/RangeNode.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -0,0 +1,127 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.sparql.ast;
+
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.NV;
+import com.bigdata.rdf.internal.constraints.Range;
+import com.bigdata.rdf.internal.constraints.RangeBOp;
+
+/**
+ * It's a value expression because it does eventually evaluate to a value -
+ * a {@link Range} value.
+ */
+public class RangeNode extends ASTBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1954230684539505857L;
+
+ public interface Annotations extends ASTBase.Annotations {
+
+ /**
+ * The variable whose range is restricted by the associated
+ * {@link #FROM} and/or {@link #TO} filters.
+ */
+ String VAR = RangeNode.class.getName() + ".var";
+
+ /**
+ * The inclusive lower bound.
+ */
+ String FROM = RangeNode.class.getName() + ".from";
+
+ /**
+ * The exclusive upper bound.
+ */
+ String TO = RangeNode.class.getName() + ".to";
+
+ /**
+ * The physical bop to be executed.
+ */
+ String RANGE_BOP = RangeNode.class.getName() + ".bop";
+
+ }
+
+ /**
+ * Set the bounds later.
+ */
+ public RangeNode(final VarNode var) {
+ this(BOp.NOARGS, NV.asMap(
+ new NV(Annotations.VAR, var)
+ ));
+ }
+
+ /**
+ * Set the bounds now.
+ */
+ public RangeNode(final VarNode var, final IValueExpressionNode from,
+ final IValueExpressionNode to) {
+ this(BOp.NOARGS, NV.asMap(
+ new NV(Annotations.VAR, var),
+ new NV(Annotations.FROM, from),
+ new NV(Annotations.TO, to)
+ ));
+ }
+
+ public RangeNode(final BOp[] args, final Map<String, Object> anns) {
+ super(args, anns);
+ }
+
+ public RangeNode(final RangeNode bop) {
+ super(bop);
+ }
+
+ public void setFrom(final ValueExpressionNode ve) {
+ setProperty(Annotations.FROM, ve);
+ }
+
+ public void setTo(final ValueExpressionNode ve) {
+ setProperty(Annotations.TO, ve);
+ }
+
+ public ValueExpressionNode from() {
+ return (ValueExpressionNode) getProperty(Annotations.FROM);
+ }
+
+ public ValueExpressionNode to() {
+ return (ValueExpressionNode) getProperty(Annotations.TO);
+ }
+
+ public VarNode var() {
+ return (VarNode) getProperty(Annotations.VAR);
+ }
+
+ public RangeBOp getRangeBOp() {
+ return (RangeBOp) getProperty(Annotations.RANGE_BOP);
+ }
+
+ public void setRangeBOp(final RangeBOp bop) {
+ setProperty(Annotations.RANGE_BOP, bop);
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -345,6 +345,23 @@
setProperty(Annotations.OPTIONAL, optional);
}
+
+ /**
+ * Attach a {@link RangeNode} that describes a range for the statement
+ * pattern's O value.
+ * @param range
+ */
+ final public void setRange(final RangeNode range) {
+
+ setProperty(Annotations.RANGE, range);
+
+ }
+
+ final public RangeNode getRange() {
+
+ return (RangeNode) getProperty(Annotations.RANGE);
+
+ }
final public List<FilterNode> getAttachedJoinFilters() {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -117,6 +117,7 @@
import com.bigdata.rdf.sparql.ast.QueryBase;
import com.bigdata.rdf.sparql.ast.QueryHints;
import com.bigdata.rdf.sparql.ast.QueryRoot;
+import com.bigdata.rdf.sparql.ast.RangeNode;
import com.bigdata.rdf.sparql.ast.SliceNode;
import com.bigdata.rdf.sparql.ast.SolutionSetStats;
import com.bigdata.rdf.sparql.ast.StatementPatternNode;
@@ -3431,6 +3432,12 @@
// Mark the join as optional.
anns.add(new NV(IPredicate.Annotations.OPTIONAL, Boolean.TRUE));
}
+
+ final RangeNode range = sp.getRange();
+ if (range != null) {
+ // Add the RangeBOp
+ anns.add(new NV(IPredicate.Annotations.RANGE, range.getRangeBOp()));
+ }
final Properties queryHints = sp.getQueryHints();
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -101,6 +101,7 @@
add(new RunOnceHint());
add(new OptimizerQueryHint());
add(new OptimisticQueryHint());
+ add(new RangeHint());
add(new AnalyticQueryHint());
add(new NativeDistinctQueryHint());
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/RangeHint.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/RangeHint.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/RangeHint.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -0,0 +1,63 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Nov 27, 2011
+ */
+
+package com.bigdata.rdf.sparql.ast.hints;
+
+import com.bigdata.rdf.sparql.ast.ASTBase;
+import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+
+final class RangeHint extends AbstractQueryHint<Boolean> {
+
+ protected RangeHint() {
+ super(QueryHints.RANGE_SAFE, false);
+ }
+
+ @Override
+ public void handle(final AST2BOpContext context,
+ final QueryHintScope scope, final ASTBase op,
+ final Boolean value) {
+
+ if (op instanceof StatementPatternNode) {
+
+ _setQueryHint(context, scope, op, getName(), value);
+
+ return;
+
+ }
+
+ }
+
+ @Override
+ public Boolean validate(String value) {
+
+ return Boolean.valueOf(value);
+
+ }
+
+}
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -0,0 +1,285 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 10, 2011
+ */
+
+package com.bigdata.rdf.sparql.ast.optimizers;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.URI;
+
+import com.bigdata.bop.IValueExpression;
+import com.bigdata.bop.IVariable;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.constraints.RangeBOp;
+import com.bigdata.rdf.sparql.ast.FilterNode;
+import com.bigdata.rdf.sparql.ast.FunctionNode;
+import com.bigdata.rdf.sparql.ast.FunctionRegistry;
+import com.bigdata.rdf.sparql.ast.GlobalAnnotations;
+import com.bigdata.rdf.sparql.ast.IQueryNode;
+import com.bigdata.rdf.sparql.ast.JoinGroupNode;
+import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.sparql.ast.RangeNode;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode;
+import com.bigdata.rdf.sparql.ast.StaticAnalysis;
+import com.bigdata.rdf.sparql.ast.ValueExpressionNode;
+import com.bigdata.rdf.sparql.ast.VarNode;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpUtility;
+
+/**
+ * Attach range bops.
+ */
+public class ASTRangeOptimizer extends AbstractJoinGroupOptimizer
+ implements IASTOptimizer {
+
+ private static final transient Logger log = Logger.getLogger(ASTRangeOptimizer.class);
+
+ /**
+ * Optimize the join group, attach range nodes.
+ */
+ @SuppressWarnings("rawtypes")
+ protected void optimizeJoinGroup(final AST2BOpContext ctx,
+ final StaticAnalysis sa, final JoinGroupNode group) {
+
+ /*
+ * First see if we have anything "rangeable".
+ */
+ boolean rangeSafe = false;
+ for (IQueryNode node : group) {
+
+ if (!(node instanceof StatementPatternNode))
+ continue;
+
+ final StatementPatternNode sp = (StatementPatternNode) node;
+
+ rangeSafe |= sp.getQueryHintAsBoolean(QueryHints.RANGE_SAFE, false);
+
+ }
+ if (!rangeSafe)
+ return;
+
+ final Map<VarNode, RangeNode> ranges =
+ new LinkedHashMap<VarNode, RangeNode>();
+
+ for (IQueryNode node : group) {
+
+ if (!(node instanceof FilterNode))
+ continue;
+
+ final FilterNode filter = (FilterNode) node;
+
+ if (log.isDebugEnabled())
+ log.debug(filter);
+
+ if (!(filter.getValueExpressionNode() instanceof FunctionNode))
+ continue;
+
+ final FunctionNode function = (FunctionNode)
+ filter.getValueExpressionNode();
+
+ if (log.isDebugEnabled())
+ log.debug(function);
+
+ processFunction(function, ranges);
+
+ }
+
+ // didn't find any
+ if (ranges.isEmpty())
+ return;
+
+ final GlobalAnnotations globals = new GlobalAnnotations(
+ ctx.getLexiconNamespace(), ctx.getTimestamp());
+
+ for (IQueryNode node : group) {
+
+ if (!(node instanceof StatementPatternNode))
+ continue;
+
+ final StatementPatternNode sp = (StatementPatternNode) node;
+
+ if (!sp.getQueryHintAsBoolean(QueryHints.RANGE_SAFE, false))
+ continue;
+
+ if (!sp.o().isVariable())
+ continue;
+
+ final VarNode v = (VarNode) sp.o();
+
+ if (!ranges.containsKey(v))
+ continue;
+
+ final RangeNode range = ranges.get(v);
+
+ final RangeBOp bop = toRangeBOp(range, globals);
+
+ if (log.isDebugEnabled()) {
+ log.debug("attaching a range:\n" + range + "\n to statement pattern: " + sp);
+ }
+
+ range.setRangeBOp(bop);
+
+ sp.setRange(range);
+
+ }
+
+ }
+
+ /**
+ * Public static faciliatates the test cases.
+ */
+ public static RangeBOp toRangeBOp(
+ final RangeNode range, final GlobalAnnotations globals) {
+
+ final IVariable<? extends IV> var = range.var().getValueExpression();
+
+ final RangeBOp bop = new RangeBOp(var);
+
+ final ValueExpressionNode from = range.from();
+
+ if (from != null) {
+
+ final IValueExpression<? extends IV> ve =
+ AST2BOpUtility.toVE(globals, from);
+
+ bop.setFrom(ve);
+
+ }
+
+ final ValueExpressionNode to = range.to();
+
+ if (to != null) {
+
+ final IValueExpression<? extends IV> ve =
+ AST2BOpUtility.toVE(globals, to);
+
+ bop.setTo(ve);
+
+ }
+
+ return bop;
+
+ }
+
+ private void processFunction(
+ final FunctionNode function, final Map<VarNode, RangeNode> ranges) {
+
+ final URI uri = function.getFunctionURI();
+
+ if (uri.equals(FunctionRegistry.AND)) {
+
+ final ValueExpressionNode left = (ValueExpressionNode) function.get(0);
+
+ if (left instanceof FunctionNode) {
+
+ processFunction((FunctionNode) left, ranges);
+
+ }
+
+ final ValueExpressionNode right = (ValueExpressionNode) function.get(1);
+
+ if (right instanceof FunctionNode) {
+
+ processFunction((FunctionNode) right, ranges);
+
+ }
+
+ } else if (uri.equals(FunctionRegistry.GT) || uri.equals(FunctionRegistry.GE)) {
+
+ final ValueExpressionNode left = (ValueExpressionNode) function.get(0);
+ final ValueExpressionNode right = (ValueExpressionNode) function.get(1);
+
+ // ?left > ?right
+ if (left instanceof VarNode) {
+ addLowerBound((VarNode) left, right, ranges);
+ }
+
+ if (right instanceof VarNode) {
+ addUpperBound((VarNode) right, left, ranges);
+ }
+
+ } else if (uri.equals(FunctionRegistry.LT) || uri.equals(FunctionRegistry.LE)) {
+
+ final ValueExpressionNode left = (ValueExpressionNode) function.get(0);
+ final ValueExpressionNode right = (ValueExpressionNode) function.get(1);
+
+ // ?left < ?right
+ if (left instanceof VarNode) {
+ addUpperBound((VarNode) left, right, ranges);
+ }
+
+ if (right instanceof VarNode) {
+ addLowerBound((VarNode) right, left, ranges);
+ }
+
+ }
+
+ }
+
+ private void addUpperBound(final VarNode var, final ValueExpressionNode ve,
+ final Map<VarNode, RangeNode> ranges) {
+
+ RangeNode range = ranges.get(var);
+ if (range == null) {
+ range = new RangeNode(var);
+ ranges.put(var, range);
+ }
+
+ ValueExpressionNode to = range.to();
+ if (to == null) {
+ to = (ValueExpressionNode) ve.clone();
+ } else {
+ to = FunctionNode.MIN(to, ve);
+ }
+
+ range.setTo(to);
+
+ }
+
+ private void addLowerBound(final VarNode var, final ValueExpressionNode ve,
+ final Map<VarNode, RangeNode> ranges) {
+
+ RangeNode range = ranges.get(var);
+ if (range == null) {
+ range = new RangeNode(var);
+ ranges.put(var, range);
+ }
+
+ ValueExpressionNode from = range.from();
+ if (from == null) {
+ from = (ValueExpressionNode) ve.clone();
+ } else {
+ from = FunctionNode.MAX(from, ve);
+ }
+
+ range.setFrom(from);
+
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTSetValueExpressionsOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTSetValueExpressionsOptimizer.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTSetValueExpressionsOptimizer.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -40,6 +40,9 @@
import com.bigdata.rdf.sparql.ast.IValueExpressionNodeContainer;
import com.bigdata.rdf.sparql.ast.OrderByExpr;
import com.bigdata.rdf.sparql.ast.QueryRoot;
+import com.bigdata.rdf.sparql.ast.RangeNode;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode;
+import com.bigdata.rdf.sparql.ast.ValueExpressionNode;
import com.bigdata.rdf.sparql.ast.VarNode;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
import com.bigdata.rdf.sparql.ast.eval.AST2BOpUtility;
@@ -155,6 +158,8 @@
return true;
if (obj instanceof HavingNode)
return true;
+ if (obj instanceof StatementPatternNode)
+ return true;
return false;
}
});
@@ -183,8 +188,8 @@
if (op instanceof IValueExpressionNodeContainer) {
// AssignmentNode, FilterNode, OrderByExpr
- AST2BOpUtility.toVE(globals, ((IValueExpressionNodeContainer) op)
- .getValueExpressionNode());
+ AST2BOpUtility.toVE(globals,
+ ((IValueExpressionNodeContainer) op).getValueExpressionNode());
} else if (op instanceof HavingNode) {
@@ -196,6 +201,22 @@
}
+ } else if (op instanceof StatementPatternNode) {
+
+ final StatementPatternNode sp = (StatementPatternNode) op;
+
+ final RangeNode range = sp.getRange();
+
+ if (range != null) {
+
+ if (range.from() != null)
+ AST2BOpUtility.toVE(globals, range.from());
+
+ if (range.to() != null)
+ AST2BOpUtility.toVE(globals, range.to());
+
+ }
+
}
// if(op instanceof AssignmentNode) {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTStaticJoinOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTStaticJoinOptimizer.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTStaticJoinOptimizer.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -45,6 +45,7 @@
import com.bigdata.bop.joinGraph.fast.DefaultEvaluationPlan2;
import com.bigdata.journal.ITx;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.constraints.RangeBOp;
import com.bigdata.rdf.sparql.ast.GraphPatternGroup;
import com.bigdata.rdf.sparql.ast.IGroupMemberNode;
import com.bigdata.rdf.sparql.ast.IJoinNode;
@@ -57,6 +58,7 @@
import com.bigdata.rdf.sparql.ast.QueryHints;
import com.bigdata.rdf.sparql.ast.QueryOptimizerEnum;
import com.bigdata.rdf.sparql.ast.QueryRoot;
+import com.bigdata.rdf.sparql.ast.RangeNode;
import com.bigdata.rdf.sparql.ast.StatementPatternNode;
import com.bigdata.rdf.sparql.ast.StaticAnalysis;
import com.bigdata.rdf.sparql.ast.TermNode;
@@ -174,6 +176,10 @@
if (!(queryNode instanceof QueryRoot))
return queryNode;
+ if (log.isDebugEnabled()) {
+ log.debug("before:\n"+queryNode);
+ }
+
final QueryRoot queryRoot = (QueryRoot) queryNode;
final IBindingSet exogenousBindings = getExogenousBindings(bindingSets);
@@ -221,6 +227,10 @@
// log.error("\nafter rewrite:\n" + queryNode);
+ if (log.isDebugEnabled()) {
+ log.debug("after:\n"+queryNode);
+ }
+
return queryNode;
}
@@ -543,8 +553,11 @@
final IV<?, ?> p = getIV(sp.p(), exogenousBindings);
final IV<?, ?> o = getIV(sp.o(), exogenousBindings);
final IV<?, ?> c = getIV(sp.c(), exogenousBindings);
+
+ final RangeNode rangeNode = sp.getRange();
+ final RangeBOp range = rangeNode != null ? rangeNode.getRangeBOp() : null;
- final IAccessPath<?> ap = db.getAccessPath(s, p, o, c);
+ final IAccessPath<?> ap = db.getAccessPath(s, p, o, c, range);
final long cardinality = ap.rangeCount(false/* exact */);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/DefaultOptimizerList.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -421,6 +421,17 @@
*/
add(new ASTRunFirstRunLastOptimizer());
+ /*
+ * FIXME Datatype and value range constraints. Per the notes immediately
+ * above, incorporate an optimizer which leverages information about
+ * ground and non-ground datatype constraints and value-range
+ * constraints within the allowable ground datatypes for a variable when
+ * it is first bound by an SP.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/238
+ */
+ add(new ASTRangeOptimizer());
+
/**
* Run the static join order optimizer. This attaches the estimated
* cardinality data (fast range counts) and uses fast algorithm to
@@ -460,17 +471,6 @@
add(new ASTStaticJoinOptimizer());
/*
- * FIXME Datatype and value range constraints. Per the notes immediately
- * above, incorporate an optimizer which leverages information about
- * ground and non-ground datatype constraints and value-range
- * constraints within the allowable ground datatypes for a variable when
- * it is first bound by an SP.
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/238
- */
-// add(new ASTRangeConstraintOptimizer());
-
- /*
* The joins are now ordered. Everything from here down MUST NOT change
* the join order when making changes to the join groups and MAY rely on
* the join order to make decisions about filter attachment, whether a
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -517,8 +517,7 @@
boolean noneBound = true;
- final RangeBOp range = (RangeBOp)
- predicate.getProperty(IPredicate.Annotations.RANGE);
+ final RangeBOp range = getRange(predicate);
for (int i = 0; i < keyArity; i++) {
@@ -527,10 +526,20 @@
final IVariableOrConstant<?> term = predicate.get(index);
if (term == null || term.isVar()) {
- if (index == 2 && range != null && range.isFullyBound()) {
+ if (index == 2 && range != null && range.isFromBound()) {
+ /*
+ * We are on the O term, it's a variable, and we have a
+ * lower bound for it.
+ */
final IConstant<IV> c = (IConstant<IV>) range.from();
appendKeyComponent(keyBuilder, i, c.get());
noneBound = false;
+
+ if (log.isInfoEnabled()) {
+ log.info("used range to build to key: " + c);
+ log.info(predicate);
+ }
+
} else {
break;
}
@@ -547,6 +556,36 @@
}
+
+ /*
+ * Ranges are only useful when O is not bound and either SP bound or just P
+ * SP?: SPO index
+ * ?P?: POS index
+ * S??: SPO index, range is not useful without an SOP index
+ * ???: SPO index, range is not useful unless we switch to OSP
+ *
+ * This reduces to when O == null && P != null
+ */
+ private RangeBOp getRange(final IPredicate<ISPO> predicate) {
+
+ final RangeBOp range = (RangeBOp)
+ predicate.getProperty(IPredicate.Annotations.RANGE);
+
+ if (range == null)
+ return null;
+
+// final IVariableOrConstant<?> s = predicate.get(0);
+ final IVariableOrConstant<?> p = predicate.get(1);
+ final IVariableOrConstant<?> o = predicate.get(2);
+
+ if ((o == null || o.isVar()) && (p != null && p.isConstant())) {
+ return range;
+ } else {
+ return null;
+ }
+
+ }
+
/**
* {@inheritDoc}
*
@@ -570,8 +609,7 @@
boolean noneBound = true;
- final RangeBOp range = (RangeBOp)
- predicate.getProperty(IPredicate.Annotations.RANGE);
+ final RangeBOp range = getRange(predicate);
for (int i = 0; i < keyArity; i++) {
@@ -581,10 +619,20 @@
// Note: term MAY be null for the context position.
if (term == null || term.isVar()) {
- if (index == 2 && range != null && range.isFullyBound()) {
+ if (index == 2 && range != null && range.isToBound()) {
+ /*
+ * We are on the O term, it's a variable, and we have an
+ * upper bound for it.
+ */
final IConstant<IV> c = (IConstant<IV>) range.to();
appendKeyComponent(keyBuilder, i, c.get());
noneBound = false;
+
+ if (log.isInfoEnabled()) {
+ log.info("used range to build to key: " + c);
+ log.info(predicate);
+ }
+
} else {
break;
}
@@ -842,16 +890,16 @@
return keyOrder;
}
- final RangeBOp range = (RangeBOp)
- predicate.getProperty(IPredicate.Annotations.RANGE);
+// final RangeBOp range = (RangeBOp)
+// predicate.getProperty(IPredicate.Annotations.RANGE);
+//
+// final boolean rangeIsBound = range != null && range.isFullyBound();
- final boolean rangeIsBound = range != null && range.isFullyBound();
-
final boolean s = !predicate.get(0).isVar();
final boolean p = !predicate.get(1).isVar();
- final boolean o = !predicate.get(2).isVar() || rangeIsBound;
+ final boolean o = !predicate.get(2).isVar(); // || rangeIsBound;
if (keyArity == 3) {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -33,7 +33,9 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
@@ -77,6 +79,7 @@
import com.bigdata.rdf.inf.Justification;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.IVUtility;
+import com.bigdata.rdf.internal.constraints.RangeBOp;
import com.bigdata.rdf.internal.impl.bnode.SidIV;
import com.bigdata.rdf.lexicon.ITermIVFilter;
import com.bigdata.rdf.lexicon.LexiconRelation;
@@ -931,22 +934,39 @@
if (keyArity != 3)
throw new UnsupportedOperationException();
- return getAccessPath(s, p, o, null/* c */, null/* filter */);
+ return getAccessPath(s, p, o, null/* c */, null/* filter */, null/* range */);
}
/**
- * Return the access path for a triple or quad pattern with an optional
- * filter.
+ * Return the access path for a triple or quad pattern.
*/
public IAccessPath<ISPO> getAccessPath(final IV s, final IV p,
final IV o, final IV c) {
- return getAccessPath(s, p, o, c, null/*filter*/);
+ return getAccessPath(s, p, o, c, null/*filter*/, null/*range*/);
}
/**
+ * Return the access path for a triple or quad pattern with a range.
+ */
+ public IAccessPath<ISPO> getAccessPath(final IV s, final IV p,
+ final IV o, final IV c, final RangeBOp range) {
+
+ return getAccessPath(s, p, o, c, null/*filter*/, range);
+
+ }
+
+ /**
+ * Return the access path for a triple or quad pattern with a filter.
+ */
+ public IAccessPath<ISPO> getAccessPath(final IV s, final IV p,
+ final IV o, final IV c, IElementFilter<ISPO> filter) {
+ return getAccessPath(s, p, o, c, filter, null);
+ }
+
+ /**
* Return the access path for a triple or quad pattern with an optional
* filter (core implementation). All arguments are optional. Any bound
* argument will restrict the returned access path. For a triple pattern,
@@ -965,16 +985,18 @@
* store).
* @param filter
* The filter (optional).
- *
+ * @param range
+ * The range (optional).
+ *
* @return The best access path for that triple or quad pattern.
*
* @throws UnsupportedOperationException
* for a triple store without statement identifiers if the
* <i>c</i> is non-{@link #NULL}.
*/
- @SuppressWarnings("unchecked")
public IAccessPath<ISPO> getAccessPath(final IV s, final IV p,
- final IV o, final IV c, IElementFilter<ISPO> filter) {
+ final IV o, final IV c, IElementFilter<ISPO> filter,
+ final RangeBOp range) {
final IVariableOrConstant<IV> S = (s == null ? Var.var("s")
: new Constant<IV>(s));
@@ -1008,12 +1030,18 @@
}
+ final Map<String, Object> anns = new LinkedHashMap<String, Object>();
+ anns.put(IPredicate.Annotations.RELATION_NAME,
+ new String[] { getNamespace() });
+ if (range != null) {
+ anns.put(IPredicate.Annotations.RANGE, range);
+ }
+
Predicate<ISPO> pred = new SPOPredicate(
// keyArity == 4 ?
(keyArity == 4 || statementIdentifiers) ?
new BOp[] { S, P, O, C } : new BOp[] { S, P, O },
- new NV(IPredicate.Annotations.RELATION_NAME,
- new String[] { getNamespace() }));
+ anns);
if (filter != null) {
// Layer on an optional filter.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2012-04-13 15:12:13 UTC (rev 6285)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2012-04-13 15:21:49 UTC (rev 6286)
@@ -97,6 +97,7 @@
import com.bigdata.rdf.internal.IExtensionFactory;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.VTE;
+import com.bigdata.rdf.internal.constraints.RangeBOp;
import com.bigdata.rdf.internal.impl.BlobIV;
import com.bigdata.rdf.internal.impl.extensions.XSDStringExtension;
import com.bigdata.rdf.lexicon.BigdataSubjectCentricFullTextIndex;
@@ -1953,12 +1954,12 @@
if (exact) {
- return getAccessPath(null/* s */, null/* p */, null/* o */, c, null/* filter */)
+ return getAccessPath(null/* s */, null/* p */, null/* o */, c)
.rangeCount(exact);
} else {
- return getAccessPath(null/* s */, null/* p */, null/* o */, c, null/* filter */)
+ return getAccessPath(null/* s */, null/* p */, null/* o */, c)
.rangeCount(exact);
}
@@ -1980,7 +1981,7 @@
public long getExplicitStatementCount(final Resource c) {
return getAccessPath(null/* s */, null/* p */, null/* o */, c,
- ExplicitSPOFilter.INSTANCE).rangeCount(true/* exact */);
+ ExplicitSPOFilter.INSTANCE, null).rangeCount(true/* exact */);
}
@@ -2531,7 +2532,7 @@
final public long removeStatements(final Resource s, final URI p,
final Value o, final Resource c) {
- return getAccessPath(s, p, o, c, null/* filter */).removeAll();
+ return getAccessPath(s, p, o, c).removeAll();
}
@@ -2647,26 +2648,27 @@
public IAccessPath<ISPO> getAccessPath(final Resource s, final URI p,
final Value o) {
- return getAccessPath(s, p, o, null/*c*/, null/* filter */);
+ return getAccessPath(s, p, o, null/*c*/, null/* filter */, null/* range */);
}
public IAccessPath<ISPO> getAccessPath(final Resource s, final URI p,
final Value o, final IElementFilter<ISPO> filter) {
- return getAccessPath(s, p, o, null/* c */, filter);
+ return getAccessPath(s, p, o, null/* c */, filter, null/* range */);
}
final public IAccessPath<ISPO> getAccessPath(final Resource s, final URI p,
final Value o, Resource c) {
- return getAccessPath(s, p, o, c, null/* filter */);
+ return getAccessPath(s, p, o, c, null/* filter */, null/* range */);
}
final public IAccessPath<ISPO> getAccessPath(final Resource s, final URI p,
- final Value o, final Resource c, final IElementFilter<ISPO> filter) {
+ final Value o, final Resource c, final IElementFilter<ISPO> filter,
+ final RangeBOp range) {
/*
* Convert other Value object types to our object types.
@@ -2776,7 +2778,7 @@
final IV o) {
return getSPORelation()
- .getAccessPath(s, p, o, null/* c */, null/* filter */);
+ .getAccessPath(s, p, o, null/* c */);
}
@@ -2790,10 +2792,18 @@
final IV o,final IV c) {
return getSPORelation()
- .getAccessPath(s, p, o, c, null/* filter */);
+ .getAccessPath(s, p, o, c);
}
+ final public IAccessPath<ISPO> getAccessPath(final IV s, final IV p,
+ final IV o,final IV c, final RangeBOp range) {
+
+ return getSPORelation()
+ .getAccessPath(s, p, o, c, range);
+
+ }
+
final public IAccessPath<ISPO> getAccessPath(final IV s, final IV p,
final IV o, final IV c, final IElementFilter<ISPO> filter) {
@@ -3079,7 +3089,7 @@
final IV p = term.getIV();
final long n = getSPORelation().getAccessPath(null, p, null,
- null, null/* filter */).rangeCount(false/* exact */);
+ null).rangeCount(false/* exact */);
/*
* FIXME do efficient term resolution for scale-out. This will
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTRangeOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTRangeOptimizer.java (rev 0)
...
[truncated message content] |
|
From: <tho...@us...> - 2012-04-17 20:11:43
|
Revision: 6294
http://bigdata.svn.sourceforge.net/bigdata/?rev=6294&view=rev
Author: thompsonbry
Date: 2012-04-17 20:11:31 +0000 (Tue, 17 Apr 2012)
Log Message:
-----------
We are special caseing the situation where the INCLUDE appears as the
first operator in a WHERE clause (of either the QueryRoot or a
NamedSubqueryRoot) and cardinality of the exogenous solutions is low.
In that case we are performing a scan over the named solutions,
joining against each exogenous solution in an inner loop. This is
efficient. It eliminates a problem where we are not assigning good
join variables, and it preserves the ORDER of the solutions in the
named solution set.
Otherwise, we build a hash index from the named solution set on the
shared definitely bound variables and then peform a hash join against
the source solutions.
- done. Write tests for SLICE of a named solution set which verify
that the order of the solutions in the named solution set is
preserved when it is the sole operator in a WHERE clause. Write
tests for both the top-level query and a sub-select. This uses a
new operator which performs a SCAN on the named solution set and
does an inner loop join against the solutions from the pipeline
(which must have low cardinality for this to be efficent).
- insert into solutions is now running.
- delete from solutions is transformed into a MINUS operation and is
now running.
- Modified the (otherwise unused)
BigdataOpenRDFBindingSetsResolverator to expose the chunk wise
resolution of BindingSet[] to IBindingSet[].
I am currently working on DELETE+INSERT for solution sets. This is a
different pattern and requires new code paths for handling the DELETE
of the solutions.
@see https://sourceforge.net/apps/trac/bigdata/ticket/531 (SPARQL UPDATE for SOLUTION SETS)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/engine/AbstractQueryEngineTestCase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/SolutionSetStatserator.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysisBase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTNamedSubqueryOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataOpenRDFBindingSetsResolverator.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/AbstractDataDrivenSPARQLTestCase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/cache/TestSolutionSetCache.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTNamedSubqueryOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest2.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/NamedSolutionSetScanOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestNamedSolutionSetScanOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestInclude.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/include_01.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/include_01.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/include_01.trig
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/include_02.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/include_02.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/include_02.trig
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test_deleteFromSolutions_01.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test_deleteFromSolutions_02.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test_deleteFromSolutions_03.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test_deleteInsertSolutions_01a.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test_deleteInsertSolutions_01b.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test_insertIntoSolutions_01.srx
Removed Paths:
-------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test01.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/data/test02.srx
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -47,7 +47,9 @@
import com.bigdata.bop.solutions.IGroupByRewriteState;
import com.bigdata.rdf.sparql.ast.GroupNodeBase;
import com.bigdata.rdf.sparql.ast.IGroupMemberNode;
+import com.bigdata.rdf.sparql.ast.SolutionSetStatserator;
import com.bigdata.relation.accesspath.IBlockingBuffer;
+import com.bigdata.striterator.CloseableIteratorWrapper;
import com.bigdata.striterator.ICloseableIterator;
import cutthecrap.utils.striterators.EmptyIterator;
@@ -999,6 +1001,22 @@
* An empty {@link IBindingSet}[].
*/
public static final IBindingSet[] EMPTY_CHUNK = new IBindingSet[0];
+
+ /**
+ * Wrap the solutions with an {@link ICloseableIterator}.
+ *
+ * @param bindingSets
+ * The solutions.
+ *
+ * @return The {@link ICloseableIterator}.
+ */
+ public static ICloseableIterator<IBindingSet[]> asIterator(
+ final IBindingSet[] bindingSets) {
+
+ return new CloseableIteratorWrapper<IBindingSet[]>(
+ new SingleValueIterator<IBindingSet[]>(bindingSets));
+
+ }
/**
* Pretty print a bop.
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/NamedSolutionSetScanOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/NamedSolutionSetScanOp.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/NamedSolutionSetScanOp.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -0,0 +1,270 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 25, 2010
+ */
+
+package com.bigdata.bop.join;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.BOpContext;
+import com.bigdata.bop.BOpUtility;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IConstraint;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.engine.BOpStats;
+import com.bigdata.rdf.sparql.ast.cache.ISparqlCache;
+import com.bigdata.rdf.sparql.ast.cache.SparqlCacheFactory;
+import com.bigdata.relation.accesspath.IBlockingBuffer;
+import com.bigdata.relation.accesspath.UnsyncLocalOutputBuffer;
+import com.bigdata.striterator.ICloseableIterator;
+
+/**
+ * This operator performs a nested loop join for solutions read from a named
+ * solution set <em>lacking an index</em> against solutions read from the
+ * pipeline (left). The operator is useful when the cardinality of the source
+ * solutions in the pipeline is low (typically one empty source solution which
+ * is exogenous to the query, but it is also cost efficient when there is a
+ * small set of source solutions to be tested for each solution drained from the
+ * named solution set). As the number of source solutions to be drained from the
+ * pipeline grows, it eventually becomes cheaper to build a hash index over the
+ * named solution set and perform a hash join with the source solutions.
+ * <p>
+ * Note: This operator MUST NOT reorder the solutions which are being scanned
+ * from the named solution set. The query planner relies on that behavior to
+ * optimize a SLICE from a pre-computed named solution set.
+ *
+ * @see <a
+ * href="https://sourceforge.net/apps/mediawiki/bigdata/index.php?title=SPARQL_Update">
+ * SPARQL Update </a>
+ *
+ * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/531"> SPARQL
+ * UPDATE for SOLUTION SETS </a>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: CopyOp.java 6010 2012-02-10 20:11:20Z thompsonbry $
+ */
+public class NamedSolutionSetScanOp extends PipelineOp {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends JoinAnnotations {
+
+ /**
+ * The name of the pre-existing named solution set to be scanned.
+ */
+ String NAME = NamedSolutionSetScanOp.class.getName() + ".name";
+
+ }
+
+ /**
+ * Deep copy constructor.
+ *
+ * @param op
+ */
+ public NamedSolutionSetScanOp(final NamedSolutionSetScanOp op) {
+
+ super(op);
+
+ }
+
+ /**
+ * Shallow copy constructor.
+ *
+ * @param args
+ * @param annotations
+ */
+ public NamedSolutionSetScanOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ super(args, annotations);
+
+ // MUST be given.
+ getRequiredProperty(Annotations.NAME);
+
+ }
+
+ public NamedSolutionSetScanOp(final BOp[] args, final NV... annotations) {
+
+ this(args, NV.asMap(annotations));
+
+ }
+
+ /**
+ * @see Annotations#NAME
+ */
+ protected String getName() {
+
+ return (String) getRequiredProperty(Annotations.NAME);
+
+ }
+
+ /**
+ * @see Annotations#SELECT
+ */
+ protected IVariable<?>[] getSelect() {
+
+ return getProperty(Annotations.SELECT, null/* defaultValue */);
+
+ }
+
+ /**
+ * @see Annotations#CONSTRAINTS
+ */
+ protected IConstraint[] constraints() {
+
+ return getProperty(Annotations.CONSTRAINTS, null/* defaultValue */);
+
+ }
+
+ public FutureTask<Void> eval(final BOpContext<IBindingSet> context) {
+
+ return new FutureTask<Void>(new ChunkTask(this, context));
+
+ }
+
+ /**
+ * Copy the source to the sink.
+ */
+ static private class ChunkTask implements Callable<Void> {
+
+ private final NamedSolutionSetScanOp op;
+
+ private final BOpContext<IBindingSet> context;
+
+ private final ISparqlCache sparqlCache;
+
+ ChunkTask(final NamedSolutionSetScanOp op,
+ final BOpContext<IBindingSet> context) {
+
+ this.op = op;
+
+ this.context = context;
+
+ // Resolve the object which will give us access to the named
+ // solution set.
+ this.sparqlCache = SparqlCacheFactory
+ .getExistingSparqlCache(context.getRunningQuery()
+ .getQueryEngine());
+
+ if (sparqlCache == null) {
+
+ /*
+ * Not enabled?
+ */
+
+ throw new UnsupportedOperationException();
+
+ }
+
+ }
+
+ public Void call() throws Exception {
+
+ final BOpStats stats = context.getStats();
+
+ // Convert source solutions to array (low cardinality).
+ final IBindingSet[] leftSolutions = BOpUtility.toArray(
+ context.getSource(), stats);
+
+ // default sink
+ final IBlockingBuffer<IBindingSet[]> sink = context.getSink();
+
+ final UnsyncLocalOutputBuffer<IBindingSet> unsyncBuffer = new UnsyncLocalOutputBuffer<IBindingSet>(
+ op.getChunkCapacity(), sink);
+
+ final String name = op.getName();
+
+ final IVariable<?>[] selectVars = op.getSelect();
+
+ final IConstraint[] constraints = op.constraints();
+
+ final ICloseableIterator<IBindingSet[]> ritr = sparqlCache
+ .getSolutions(name);
+
+ try {
+
+ while (ritr.hasNext()) {
+
+ final IBindingSet[] a = ritr.next();
+
+ for (IBindingSet right : a) {
+
+ for (IBindingSet left : leftSolutions) {
+
+ // See if the solutions join.
+ final IBindingSet outSolution = BOpContext.bind(//
+ right,//
+ left,//
+ constraints,//
+ selectVars//
+ );
+
+ if (outSolution != null) {
+
+ // Output the solution.
+ unsyncBuffer.add(outSolution);
+
+ }
+
+ }
+
+ }
+
+ }
+
+ // flush the unsync buffer.
+ unsyncBuffer.flush();
+
+ // flush the sink.
+ sink.flush();
+
+ // Done.
+ return null;
+
+ } finally {
+
+ sink.close();
+
+ context.getSource().close();
+
+ if (ritr != null)
+ ritr.close();
+
+ }
+
+ }
+
+ } // class CopyTask
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/engine/AbstractQueryEngineTestCase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/engine/AbstractQueryEngineTestCase.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/engine/AbstractQueryEngineTestCase.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -22,11 +22,46 @@
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
/*
+Portions of this code are:
+
+Copyright Aduna (http://www.aduna-software.com/) � 2001-2007
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the copyright holder nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+*/
+/*
* Created on Aug 30, 2011
*/
package com.bigdata.bop.engine;
+import info.aduna.iteration.Iterations;
+import info.aduna.text.StringUtil;
+
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashSet;
@@ -37,13 +72,23 @@
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
+import junit.framework.AssertionFailedError;
import junit.framework.TestCase2;
import org.apache.log4j.Logger;
+import org.openrdf.model.Statement;
+import org.openrdf.model.util.ModelUtil;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryResultUtil;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.impl.MutableTupleQueryResult;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.rdf.sparql.ast.ASTContainer;
+import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.striterator.Dechunkerator;
import com.bigdata.striterator.ICloseableIterator;
@@ -649,4 +694,285 @@
}
+ /**
+ * Utility method compares expected and actual solutions and reports on any
+ * discrepancies.
+ *
+ * @param name
+ * The name of the test.
+ * @param testURI
+ * The URI for the test.
+ * @param store
+ * The {@link AbstractTripleStore} (optional).
+ * @param astContainer
+ * The {@link ASTContainer} (optional).
+ * @param queryResult
+ * The actual result.
+ * @param expectedResult
+ * The expected result.
+ * @param laxCardinality
+ * When <code>true</code>, strict cardinality will be enforced.
+ * @param checkOrder
+ * When <code>true</code>, the order must be the same.
+ * @throws AssertionFailedError
+ * if the results are not the same.
+ * @throws QueryEvaluationException
+ */
+ static public void compareTupleQueryResults(
+ final String name,//
+ final String testURI, //
+ final AbstractTripleStore store,//
+ final ASTContainer astContainer,//
+ final TupleQueryResult queryResult,//
+ final TupleQueryResult expectedResult,//
+ final boolean laxCardinality,//
+ final boolean checkOrder//
+ ) throws QueryEvaluationException {
+
+ /*
+ * Create MutableTupleQueryResult to be able to re-iterate over the
+ * results.
+ */
+
+ final MutableTupleQueryResult queryResultTable = new MutableTupleQueryResult(queryResult);
+
+ final MutableTupleQueryResult expectedResultTable = new MutableTupleQueryResult(expectedResult);
+
+ boolean resultsEqual;
+ if (laxCardinality) {
+ resultsEqual = QueryResultUtil.isSubset(queryResultTable, expectedResultTable);
+ }
+ else {
+ resultsEqual = QueryResultUtil.equals(queryResultTable, expectedResultTable);
+
+ if (checkOrder) {
+ // also check the order in which solutions occur.
+ queryResultTable.beforeFirst();
+ expectedResultTable.beforeFirst();
+
+ while (queryResultTable.hasNext()) {
+ final BindingSet bs = queryResultTable.next();
+ final BindingSet expectedBs = expectedResultTable.next();
+
+ if (! bs.equals(expectedBs)) {
+ resultsEqual = false;
+ break;
+ }
+ }
+ }
+ }
+
+ // Note: code block shows the expected and actual results.
+ StringBuilder expectedAndActualResults = null;
+ if (!resultsEqual && true) {
+ queryResultTable.beforeFirst();
+ expectedResultTable.beforeFirst();
+ final StringBuilder message = new StringBuilder(2048);
+ message.append("\n============ ");
+ message.append(name);
+ message.append(" =======================\n");
+ message.append("Expected result: \n");
+ while (expectedResultTable.hasNext()) {
+ message.append(expectedResultTable.next());
+ message.append("\n");
+ }
+ message.append("=============");
+ StringUtil.appendN('=', name.length(), message);
+ message.append("========================\n");
+ message.append("Query result: \n");
+ while (queryResultTable.hasNext()) {
+ message.append(queryResultTable.next());
+ message.append("\n");
+ }
+ message.append("=============");
+ StringUtil.appendN('=', name.length(), message);
+ message.append("========================\n");
+ expectedAndActualResults = message;
+// log.error(message);
+ }
+
+ if (!resultsEqual) {
+
+ queryResultTable.beforeFirst();
+ expectedResultTable.beforeFirst();
+
+ final List<BindingSet> queryBindings = Iterations.asList(queryResultTable);
+ final List<BindingSet> expectedBindings = Iterations.asList(expectedResultTable);
+
+ final List<BindingSet> missingBindings = new ArrayList<BindingSet>(expectedBindings);
+ missingBindings.removeAll(queryBindings);
+
+ final List<BindingSet> unexpectedBindings = new ArrayList<BindingSet>(queryBindings);
+ unexpectedBindings.removeAll(expectedBindings);
+
+ final StringBuilder message = new StringBuilder(2048);
+ message.append("\n");
+ message.append(testURI);
+ message.append("\n");
+ message.append(name);
+ message.append("\n===================================\n");
+
+ if (!missingBindings.isEmpty()) {
+
+ message.append("Missing bindings: \n");
+ for (BindingSet bs : missingBindings) {
+ message.append(bs);
+ message.append("\n");
+ }
+
+ message.append("=============");
+ StringUtil.appendN('=', name.length(), message);
+ message.append("========================\n");
+ }
+
+ if (!unexpectedBindings.isEmpty()) {
+ message.append("Unexpected bindings: \n");
+ for (BindingSet bs : unexpectedBindings) {
+ message.append(bs);
+ message.append("\n");
+ }
+
+ message.append("=============");
+ StringUtil.appendN('=', name.length(), message);
+ message.append("========================\n");
+ }
+
+ if (checkOrder && missingBindings.isEmpty() && unexpectedBindings.isEmpty()) {
+ message.append("Results are not in expected order.\n");
+ message.append(" =======================\n");
+ message.append("query result: \n");
+ for (BindingSet bs: queryBindings) {
+ message.append(bs);
+ message.append("\n");
+ }
+ message.append(" =======================\n");
+ message.append("expected result: \n");
+ for (BindingSet bs: expectedBindings) {
+ message.append(bs);
+ message.append("\n");
+ }
+ message.append(" =======================\n");
+
+ log.error(message.toString());
+ }
+
+ if (expectedAndActualResults != null) {
+ message.append(expectedAndActualResults);
+ }
+
+// RepositoryConnection con = ((DatasetRepository)dataRep).getDelegate().getConnection();
+// System.err.println(con.getClass());
+// try {
+ if(astContainer!=null) {
+ message.append("\n===================================\n");
+ message.append(astContainer.toString());
+ }
+ if(store!=null&&
+ store.getStatementCount()<100) {
+ message.append("\n===================================\n");
+ message.append("database dump:\n");
+ message.append(store.dumpStore());
+ }
+// RepositoryResult<Statement> stmts = con.getStatements(null, null, null, false);
+// while (stmts.hasNext()) {
+// message.append(stmts.next());
+// message.append("\n");
+// }
+// } finally {
+// con.close();
+// }
+
+ log.error(message.toString());
+ fail(message.toString());
+ }
+ /* debugging only: print out result when test succeeds
+ else {
+ queryResultTable.beforeFirst();
+
+ List<BindingSet> queryBindings = Iterations.asList(queryResultTable);
+ StringBuilder message = new StringBuilder(128);
+
+ message.append("\n============ ");
+ message.append(getName());
+ message.append(" =======================\n");
+
+ message.append(" =======================\n");
+ message.append("query result: \n");
+ for (BindingSet bs: queryBindings) {
+ message.append(bs);
+ message.append("\n");
+ }
+
+ System.out.print(message.toString());
+ }
+ */
+ }
+
+ /**
+ * Compare the expected and actual query results for a graph query.
+ *
+ * @param name
+ * The name of the test.
+ * @param queryResult
+ * The actual result.
+ * @param expectedResult
+ * The expected result.
+ *
+ * @throws AssertionFailedError
+ * if the results are not the same.
+ */
+ static public void compareGraphs(final String name,
+ final Set<Statement> queryResult,
+ final Set<Statement> expectedResult) {
+
+ if (!ModelUtil.equals(expectedResult, queryResult)) {
+ // Don't use RepositoryUtil.difference, it reports incorrect diffs
+ /*
+ * Collection<? extends Statement> unexpectedStatements =
+ * RepositoryUtil.difference(queryResult, expectedResult);
+ * Collection<? extends Statement> missingStatements =
+ * RepositoryUtil.difference(expectedResult, queryResult);
+ * StringBuilder message = new StringBuilder(128);
+ * message.append("\n=======Diff: "); message.append(getName());
+ * message.append("========================\n"); if
+ * (!unexpectedStatements.isEmpty()) { message.append("Unexpected
+ * statements in result: \n"); for (Statement st :
+ * unexpectedStatements) { message.append(st.toString());
+ * message.append("\n"); } message.append("============="); for (int
+ * i = 0; i < getName().length(); i++) { message.append("="); }
+ * message.append("========================\n"); } if
+ * (!missingStatements.isEmpty()) { message.append("Statements
+ * missing in result: \n"); for (Statement st : missingStatements) {
+ * message.append(st.toString()); message.append("\n"); }
+ * message.append("============="); for (int i = 0; i <
+ * getName().length(); i++) { message.append("="); }
+ * message.append("========================\n"); }
+ */
+ StringBuilder message = new StringBuilder(128);
+ message.append("\n============ ");
+ message.append(name);
+ message.append(" =======================\n");
+ message.append("Expected result: \n");
+ for (Statement st : expectedResult) {
+ message.append(st.toString());
+ message.append("\n");
+ }
+ message.append("=============");
+ StringUtil.appendN('=', name.length(), message);
+ message.append("========================\n");
+
+ message.append("Query result: \n");
+ for (Statement st : queryResult) {
+ message.append(st.toString());
+ message.append("\n");
+ }
+ message.append("=============");
+ StringUtil.appendN('=', name.length(), message);
+ message.append("========================\n");
+
+ log.error(message.toString());
+ fail(message.toString());
+ }
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestAll.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestAll.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -78,6 +78,12 @@
suite.addTestSuite(TestHTreeHashIndexOp.class);
suite.addTestSuite(TestHTreeSolutionSetHashJoin.class);
+ /*
+ * Test suite for join against a named solution set when the left
+ * cardinality is low.
+ */
+ suite.addTestSuite(TestNamedSolutionSetScanOp.class);
+
return suite;
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestNamedSolutionSetScanOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestNamedSolutionSetScanOp.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/bop/join/TestNamedSolutionSetScanOp.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -0,0 +1,65 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Apr 16, 2012
+ */
+package com.bigdata.bop.join;
+
+import com.bigdata.rdf.sparql.ast.eval.TestInclude;
+
+import junit.framework.TestCase2;
+
+/**
+ * Test suite for low to no cardinality joins against a named solution set.
+ *
+ * @author thompsonbry
+ *
+ * FIXME Verify that we are handling SELECT and CONSTRAINTS as well as
+ * the operator specific annotations (NAME, SPARQL_CACHE). Look at the
+ * existing test suites for hash joins for examples that we can setup
+ * here.
+ *
+ * FIXME Verify that the output of this join operator is order
+ * preserving (that could be done in a data driven unit test at the
+ * SPARQL layer for INCLUDE).
+ */
+public class TestNamedSolutionSetScanOp extends TestCase2 {
+
+ public TestNamedSolutionSetScanOp() {
+ }
+
+ public TestNamedSolutionSetScanOp(String name) {
+ super(name);
+ }
+
+ /**
+ * Note: There are some tests at the data-driven level.
+ *
+ * @see TestInclude
+ */
+ public void test_something() {
+ fail("write tests");
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/CompiledSolutionSetStats.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -148,4 +148,21 @@
}
+ public String toString() {
+
+ final StringBuilder sb = new StringBuilder();
+
+ sb.append(super.toString());
+ sb.append("{size=" + nsolutions);
+ sb.append(",usedVars=" + usedVars);
+ sb.append(",alwaysBound=" + alwaysBound);
+ sb.append(",notAlwaysBound=" + notAlwaysBound);
+ sb.append(",materialized=" + materialized);
+ sb.append(",constants=" + constants);
+ sb.append("}");
+
+ return sb.toString();
+
+ }
+
}
\ No newline at end of file
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -404,5 +404,5 @@
* used currently when there is a single datatype for attribute values.
*/
String RANGE_SAFE = "rangeSafe";
-
-}
+
+}
\ No newline at end of file
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/SolutionSetStatserator.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/SolutionSetStatserator.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/SolutionSetStatserator.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -33,6 +33,7 @@
import java.util.NoSuchElementException;
import java.util.Set;
+import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstant;
import com.bigdata.bop.IVariable;
@@ -123,8 +124,7 @@
static public ISolutionSetStats get(final IBindingSet[] bindingSets) {
final SolutionSetStatserator itr = new SolutionSetStatserator(
- new CloseableIteratorWrapper<IBindingSet[]>(
- new SingleValueIterator(bindingSets)));
+ BOpUtility.asIterator(bindingSets));
try {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysis.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -201,16 +201,7 @@
// private static final Logger log = Logger.getLogger(StaticAnalysis.class);
/**
- * FIXME This will go away now once we have the ability to resolve named
- * subqueries against the {@link ISparqlCache}.
*
- * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/531">
- * SPARQL Update for Named Solution Sets </a>
- */
- private static boolean requireDeclaredNamedSubquery = !QueryHints.DEFAULT_SOLUTION_SET_CACHE;
-
- /**
- *
* @param queryRoot
* The root of the query. We need to have this on hand in order
* to resolve {@link NamedSubqueryInclude}s during static
@@ -2025,7 +2016,66 @@
}
+
/**
+ * Return the join variables for an INCLUDE of a pre-existing named solution
+ * set.
+ *
+ * @param nsi
+ * The {@link NamedSubqueryInclude}
+ * @param solutionSet
+ * The name of a pre-existing solution set.
+ * @param vars
+ * The caller's collection.
+ *
+ * @return The caller's collection.
+ */
+ public Set<IVariable<?>> getJoinVars(final NamedSubqueryInclude nsi,
+ final String solutionSet, final Set<IVariable<?>> vars) {
+
+ final String name = solutionSet;
+
+ /*
+ * The variables which will be definitely bound based on the statistics
+ * collected for that solution set.
+ */
+ final ISolutionSetStats stats = getSolutionSetStats(name);
+
+ if(stats == null)
+ throw new RuntimeException("Not found: "+name);
+
+ /*
+ * All variables which are bound in each solution of this solution set.
+ *
+ * Note: The summary data for a named solution set is typically
+ * immutable, so we insert the variables into a mutable collection in
+ * order to make changes to that collection below.
+ */
+ final Set<IVariable<?>> boundInSolutionSet = new LinkedHashSet<IVariable<?>>(
+ stats.getAlwaysBound());
+
+ /*
+ * The variables which are definitely bound on entry to the INCLUDE
+ * operator based on the static analysis of the query, including where
+ * it appears in the join order of the query.
+ */
+ final Set<IVariable<?>> incomingBindings = getDefinitelyIncomingBindings(
+ nsi, new LinkedHashSet<IVariable<?>>());
+
+ /*
+ * This is only those variables which are bound on entry into the
+ * INCLUDE *and* which are "must" bound variables projected by the
+ * pre-existing named solution set.
+ */
+ boundInSolutionSet.retainAll(incomingBindings);
+
+ vars.addAll(boundInSolutionSet);
+
+ return vars;
+
+ }
+
+ /**
* Return any variables which are used after the given node in the current
* ordering of its parent {@link JoinGroupNode} but DOES NOT consider the
* parent or the PROJECTION for the query in which this group appears.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysisBase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysisBase.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StaticAnalysisBase.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -38,7 +38,6 @@
import com.bigdata.bop.IConstant;
import com.bigdata.bop.IVariable;
import com.bigdata.rdf.sparql.ast.cache.ISparqlCache;
-import com.bigdata.rdf.sparql.ast.cache.SolutionSetMetadata;
import com.bigdata.rdf.sparql.ast.eval.IEvaluationContext;
/**
@@ -127,7 +126,7 @@
* @return The caller's {@link Set}.
*
* TODO Unit tests for different kinds of AST nodes to make sure
- * that we always get/ignore the variables in filters as approriate.
+ * that we always get/ignore the variables in filters as appropriate.
* For example, an optional {@link StatementPatternNode} can have
* filter nodes attached.
*/
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/ISparqlCache.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -61,7 +61,7 @@
* @return <code>true</code> iff a solution set by that name existed and was
* cleared.
*/
- boolean clearSolutions(IEvaluationContext ctx, String solutionSet);
+ boolean clearSolutions(String solutionSet);
/**
* Clear all named solution sets.
@@ -71,7 +71,6 @@
/**
* Create a named solution set.
*
- * @param ctx
* @param solutionSet
* The name of the solution set.
* @param params
@@ -80,24 +79,21 @@
* @throws RuntimeException
* if a solution set exists for that name.
*/
- void createSolutions(IEvaluationContext ctx, String solutionSet, ISPO[] params);
+ void createSolutions(String solutionSet, ISPO[] params);
/**
* Save the solutions a named solution set.
*
- * @param ctx
* @param solutionSet
* The name of the solution set.
* @param src
* The solutions.
*/
- void putSolutions(IEvaluationContext ctx, String solutionSet,
- ICloseableIterator<IBindingSet[]> src);
+ void putSolutions(String solutionSet, ICloseableIterator<IBindingSet[]> src);
/**
* Read the solutions from a named solution set.
*
- * @param ctx
* @param solutionSet
* The name of the solution set.
*
@@ -106,8 +102,7 @@
* @throws IllegalStateException
* if no solution set with that name exists.
*/
- ICloseableIterator<IBindingSet[]> getSolutions(IEvaluationContext ctx,
- String solutionSet);
+ ICloseableIterator<IBindingSet[]> getSolutions(String solutionSet);
/**
* Return computed statistics for a named solution set.
@@ -123,13 +118,12 @@
/**
* Return <code>true</code> iff a named solution set exists.
*
- * @param ctx
* @param solutionSet
* The name of the solution set.
*
* @return <code>true</code> iff a solution set having that name exists.
*/
- boolean existsSolutions(IEvaluationContext ctx, String solutionSet);
+ boolean existsSolutions(String solutionSet);
// /**
// * Return the result from the cache -or- <code>null</code> if there is a
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SolutionSetMetadata.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -47,7 +47,6 @@
import com.bigdata.rdf.internal.encoder.IVSolutionSetDecoder;
import com.bigdata.rdf.internal.encoder.IVSolutionSetEncoder;
import com.bigdata.rdf.sparql.ast.ISolutionSetStats;
-import com.bigdata.rdf.sparql.ast.SolutionSetStats;
import com.bigdata.rdf.sparql.ast.SolutionSetStatserator;
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rwstore.PSOutputStream;
@@ -149,17 +148,21 @@
public ICloseableIterator<IBindingSet[]> get() {
- final long solutionSetAddr = this.solutionSetAddr;
+ final long solutionSetAddr = this.solutionSetAddr;
- if (solutionSetAddr == IRawStore.NULL)
- throw new IllegalStateException();
+ if (solutionSetAddr == IRawStore.NULL)
+ throw new IllegalStateException();
- try {
- return new SolutionSetStreamDecoder(solutionSetAddr, solutionCount);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ try {
+ return new SolutionSetStreamDecoder(solutionSetAddr, solutionCount);
+
+ } catch (IOException e) {
+
+ throw new RuntimeException(e);
+
+ }
+
}
/**
@@ -259,9 +262,10 @@
// Nothing more to be read.
- if (log.isDebugEnabled())
- log.debug("Read solutionSet: solutionSetSize=" + nsolutions);
-
+ if (log.isDebugEnabled())
+ log.debug("Read solutionSet: name=" + name
+ + ", solutionSetSize=" + nsolutions);
+
return null;
}
@@ -290,15 +294,19 @@
t[i] = decoder
.decodeSolution(buf, true/* resolveCachedValues */);
+ if (log.isTraceEnabled())
+ log.trace("Read: name=" + name + ", solution=" + t[i]);
+
}
// Update the #of solution sets which have been decoded.
nsolutions += chunkSize;
if (log.isTraceEnabled())
- log.trace("Read chunk: chunkSize=" + chunkSize + ", bytesRead="
- + (CHUNK_HEADER_SIZE + byteLength)
- + ", solutionSetSize=" + nsolutions);
+ log.trace("Read chunk: name=" + name + ", chunkSize="
+ + chunkSize + ", bytesRead="
+ + (CHUNK_HEADER_SIZE + byteLength)
+ + ", solutionSetSize=" + nsolutions);
// Return the decoded solutions.
return t;
@@ -369,6 +377,10 @@
encoder.encodeSolution(buf, chunk[i]);
+ if (log.isTraceEnabled())
+ log.trace("Wrote name=" + name + ", solution="
+ + chunk[i]);
+
}
// #of bytes written onto the buffer.
@@ -391,10 +403,10 @@
chunkCount++;
if (log.isDebugEnabled())
- log.debug("Wrote chunk: chunkSize=" + chunk.length
- + ", chunkCount=" + chunkCount
- + ", bytesBuffered=" + bytesBuffered
- + ", solutionSetSize=" + nsolutions);
+ log.debug("Wrote chunk: name=" + name + ", chunkSize="
+ + chunk.length + ", chunkCount=" + chunkCount
+ + ", bytesBuffered=" + bytesBuffered
+ + ", solutionSetSize=" + nsolutions);
}
@@ -411,9 +423,10 @@
newAddr = out.getAddr();
if (log.isDebugEnabled())
- log.debug("Wrote solutionSet: solutionSetSize=" + nsolutions
- + ", chunkCount=" + chunkCount + ", encodedBytes="
- + nbytes + ", bytesWritten=" + out.getBytesWritten());
+ log.debug("Wrote solutionSet: name=" + name
+ + ", solutionSetSize=" + nsolutions + ", chunkCount="
+ + chunkCount + ", encodedBytes=" + nbytes
+ + ", bytesWritten=" + out.getBytesWritten());
} catch (IOException e) {
@@ -448,9 +461,9 @@
}
/**
- * TODO Test performance with and without gzip. Extract into the CREATE
- * schema.
- */
+ * TODO Test performance with and without gzip. Extract into the CREATE
+ * schema so we can do this declaratively.
+ */
private static final boolean zip = true;
private OutputStream wrapOutputStream(final OutputStream out)
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/cache/SparqlCache.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -116,9 +116,6 @@
* makes the cache somewhat more difficult to integration since the same
* query is not always the same (e.g., include the hash of the exogenous
* solutions in the query hash code and we will get less reuse).
- *
- * TODO If the {@link IEvaluationContext} is not required, then back it
- * out of the method signatures.
*/
public class SparqlCache implements ISparqlCache {
@@ -364,7 +361,7 @@
}
@Override
- public boolean clearSolutions(final IEvaluationContext ctx, final String solutionSet) {
+ public boolean clearSolutions(final String solutionSet) {
if (log.isInfoEnabled())
log.info("solutionSet: " + solutionSet);
@@ -382,8 +379,8 @@
}
- public void putSolutions(final IEvaluationContext ctx, final String solutionSet,
- final ICloseableIterator<IBindingSet[]> src) {
+ public void putSolutions(final String solutionSet,
+ final ICloseableIterator<IBindingSet[]> src) {
if (solutionSet == null)
throw new IllegalArgumentException();
@@ -415,8 +412,7 @@
}
- public void createSolutions(final IEvaluationContext ctx,
- final String solutionSet, final ISPO[] params) {
+ public void createSolutions(final String solutionSet, final ISPO[] params) {
if (solutionSet == null)
throw new IllegalArgumentException();
@@ -469,8 +465,8 @@
}
- public ICloseableIterator<IBindingSet[]> getSolutions(final IEvaluationContext ctx,
- final String solutionSet) {
+ public ICloseableIterator<IBindingSet[]> getSolutions(
+ final String solutionSet) {
if (solutionSet == null)
throw new IllegalArgumentException();
@@ -485,8 +481,7 @@
}
- public boolean existsSolutions(final IEvaluationContext ctx,
- final String solutionSet) {
+ public boolean existsSolutions(final String solutionSet) {
if (solutionSet == null)
throw new IllegalArgumentException();
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-16 15:07:46 UTC (rev 6293)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-04-17 20:11:31 UTC (rev 6294)
@@ -27,6 +27,8 @@
package com.bigdata.rdf.sparql.ast.eval;
+import info.aduna.iteration.CloseableIteration;
+
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -37,6 +39,7 @@
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.log4j.Logger;
@@ -46,6 +49,7 @@
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
import org.openrdf.query.GraphQueryResult;
import org.openrdf.query.QueryEvaluationException;
import org.openrdf.query.UpdateExecutionException;
@@ -78,13 +82,16 @@
import com.bigdata.bop.rdf.update.ParseOp;
import com.bigdata.bop.rdf.update.RemoveStatementsOp;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.lexicon.LexiconRelation;
import com.bigdata.rdf.model.BigdataStatement;
import com.bigdata.rdf.model.BigdataURI;
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
+import com.bigdata.rdf.sail.Sesame2BigdataIterator;
import com.bigdata.rdf.sparql.ast.ASTContainer;
import com.bigdata.rdf.sparql.ast.AbstractGraphDataUpdate;
import com.bigdata.rdf.sparql.ast.AddGraph;
+import com.bigdata.rdf.sparql.ast.BindingsClause;
import com.bigdata.rdf.sparql.ast.ConstantNode;
import com.bigdata.rdf.sparql.ast.ConstructNode;
import com.bigdata.rdf.sparql.ast.CopyGraph;
@@ -95,6 +102,7 @@
import com.bigdata.rdf.sparql.ast.JoinGroupNode;
import com.bigdata.rdf.sparql.ast.LoadGraph;
import com.bigdata.rdf.sparql.ast.MoveGraph;
+import com.bigdata.rdf.sparql.ast.NamedSubqueryInclude;
import com.bigdata.rdf.sparql.ast.ProjectionNode;
import com.bigdata.rdf.sparql.ast.QuadData;
import com.bigdata.rdf.sparql.ast.QuadsDataOrNamedSolutionSet;
@@ -110,8 +118,13 @@
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.store.BD;
+import com.bigdata.rdf.store.BigdataOpenRDFBindingSetsResolverator;
+import com.bigdata.striterator.Chunkerator;
import com.bigdata.striterator.ICloseableIterator;
+import cutthecrap.utils.striterators.Resolver;
+import cutthecrap.utils.striterators.Striterator;
+
/**
* Class handles SPARQL update query plan generation.
*
@@ -475,31 +488,39 @@
if (isDeleteInsert) {
/*
- * DELETE + INSERT.
- *
- * Note: The semantics of DELETE + INSERT are that the WHERE
- * clause is executed once. The solutions to that need to be fed
- * once through the DELETE clause. After the DELETE clauss has
- * been processed for all solutions to the WHERE clause, the
- * INSERT clause is then processed. So, we need to materalize
- * the WHERE clause results when both the DELETE clause and the
- * INSERT clause are present.
- *
- * FIXME For large intermediate results, we would be much better
- * off putting the data onto an HTree and processing the
- * bindings as IVs rather than materializing them as RDF Values
- * (and even for small data sets, we would be better off
- * avoiding materialization of the RDF Values and using an
- * ASTConstructIterator which builds ISPOs using IVs rather than
- * Values).
- *
- * FIXME Support INSERT INTO / DELETE FROM here.
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/524
- * (SPARQL Cache)
- */
+ * DELETE + INSERT.
+ *
+ * Note: The semantics of DELETE + INSERT are that the WHERE
+ * clause is executed once. The solutions to that need to be fed
+ * once through the DELETE clause. After the DELETE clauss has
+ * been processed for all solutions to the WHERE clause, the
+ * INSERT clause is then processed. So, we need to materalize
+ * the WHERE clause results when both the DELETE clause and the
+ * INSERT clause are present.
+ *
+ * FIXME For large intermediate results, we would be much better
+ * off putting the data onto an HTree (or, better yet, a chain
+ * of blocks) and processing the bindings as IVs rather than
+ * materializing them as RDF Values (and even for small data
+ * sets, we would be better off avoiding materialization of the
+ * RDF Values and using an ASTConstructIterator which builds
+ * ISPOs using IVs rather than Values).
+ *
+ * Note: Unlike operations against a graph, we do NOT perform
+ * truth maintenance for updates against solution sets,
+ * therefore we could get by nicely with operations on
+ * IBindingSet[]s without RDF Value materialization.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/524
+ * (SPARQL Cache)
+ */
- // Run as a SELECT query.
+ final LexiconRelation lexicon = context
+ .getAbstractTripleStore().getLexiconRelation();
+
+ final int chunkSize = 100; // TODO configure.
+
+ // Run as a SELECT query.
final MutableTupleQueryResult result = new MutableTupleQueryResult(
ASTEvalHelper.evaluateTupleQuery(context.db,
astContainer, null/* bindingSets */));
@@ -512,48 +533,239 @@
// rewind.
result.beforeFirst();
- final ConstructNode template = op.getDeleteClause()
- .getQuadData().flatten();
+ // Figure out if operating on solutions or graphs.
+ final boolean isSolutionSet = deleteClause.isSolutions();
- final ASTConstructIterator itr = new ASTConstructIterator(
- context.conn.getTripleStore(), template, result);
+ if (isSolutionSet) {
- while (itr.hasNext()) {
+ /*
+ * Target is solution set.
+ *
+ * @see
+ * https://sourceforge.net/apps/trac/bigdata/ticket
+ * /524 (SPARQL Cache)
+ *
+ * FIXME The DELETE+INSERT code path is failing
+ * because it is based on the DELETE FROM SELECT
+ * code path below and attempts to rewrite the query
+ * to use a MINUS operator. However, the setup is
+ * different in this case since we have already run
+ * the original WHERE clause into a rewindable tuple
+ * result set.
+ *
+ * The best way to fix this would be to stay within
+ * the native IBindingSet[] model and write the
+ * solutions from the WHERE clause onto a chained
+ * list of blocks, just as we do when writing on a
+ * named solution set. That chained list could then
+ * be joined into the query with an INCLUDE. Since
+ * we do not want this "temporary" solution set to
+ * be visible, we could prefix it with a UUID and
+ * make sure that it is written onto a memory
+ * manager, and also make sure that we eventually
+ * delete the named solution set since it should be
+ * temporary.
+ *
+ * Another way to handle this is to flow the
+ * solutions from a solution set into a new query
+ * which as the appropriate MINUS logic [This does
+ * not work because the solutions wind up on the
+ * LEFT side of the MINUS but they need to be on the
+ * RIGHT]. Here, the solutions would appear on the
+ * pipeline rather than being joined in. The result
+ * is the same. This would make it easier to manage
+ * the visibility of the solutions. Note that there
+ * is no API to flow an
+ * ICloseableIterator<IBindingSet[]> into an
+ * IRunningQuery. Only an IBindingSet[]. However, we
+ * might be able to wrap up the named solution set
+ * as an IChunkMessage.
+ */
- final BigdataStatement stmt = itr.next();
+ // The named solution set on which we will write.
+ final String solutionSet = deleteClause.getName();
- addOrRemoveStatement(
- context.conn.getSailConnection(), stmt,
- false/* insert */);
+ // A unique named solution set used to INCLUDE the
+ // solutions to be deleted.
+ final String tempSolutionSet = "-" + solutionSet
+ + "-" + UUID.randomUUID();
- }
+ // Write solutions to be deleted onto temp set.
+ context.sparqlCache.putSolutions(
+ tempSolutionSet,
+ asBigdataIterator(lexicon, chunkSize,
+ result));
- }
+ try {
+ /*
+ * Replace WHERE clause with an join group
+ * containing an INCLUDE for the solutions to be
+ * removed.
+ *
+ * WHERE := { INCLUDE %namedSet MINUS {INCLUDE %temp} }
+ *
+ * FIXME This is failing (I think) because
+ * the named solution set is not visible.
+ * It looks like it might be only visible
+ * when using a SCAN?!?
+ */
+// final JoinGroupNode oldWhereClause = (JoinGroupNode) queryRoot
+// .getWhereClause();
+
+ final JoinGroupNode newWhereClause = new JoinGroupNode();
+ queryRoot.setWhereClause(newWhereClause);
+
+ // Include the source solutions.
+ newWhereClause.addArg(new NamedSubqueryInclude(
+ solutionSet));
+
+ // MINUS solutions to be removed.
+ final JoinGroupNode minusOp = new JoinGroupNode(
+ new NamedSubqueryInclude(
+ tempSolutionSet));
+ newWhereClause.addArg(minusOp);
+ minusOp.setMinus(true);
+
+// log.error("oldWhereClause="+oldWhereClause);
+// log.error("newWhereClause="+newWhereClause);
+
+// /*
+// * Re-write the AST to handle DELETE solutions.
+// */
+// convertQueryForDeleteSolutions(queryRoot,
+// solutionSet);
+
+ // Set the projection node.
+ ...
[truncated message content] |
|
From: <mrp...@us...> - 2012-05-03 09:06:49
|
Revision: 6312
http://bigdata.svn.sourceforge.net/bigdata/?rev=6312&view=rev
Author: mrpersonick
Date: 2012-05-03 09:06:37 +0000 (Thu, 03 May 2012)
Log Message:
-----------
added support for query hint hint:cutoffLimit - will limit the input read into a join from a predicate's access path
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/CutoffLimitHint.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/IPredicate.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/IPredicate.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -34,6 +34,9 @@
import com.bigdata.bop.ap.filter.BOpFilterBase;
import com.bigdata.bop.ap.filter.BOpTupleFilter;
import com.bigdata.bop.ap.filter.DistinctFilter;
+import com.bigdata.bop.join.HTreeHashJoinOp;
+import com.bigdata.bop.join.JVMHashJoinOp;
+import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.bop.joinGraph.IEvaluationPlan;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
@@ -338,6 +341,20 @@
* an allowable value range for some slot in that predicate.
*/
String RANGE = IPredicate.class.getName() + ".range";
+
+ /**
+ * Limits the number of elements read from the access path for this
+ * predicate when used in a join ({@link HTreeHashJoinOp},
+ * {@link JVMHashJoinOp}, {@link PipelineJoin}). Note that this is
+ * limiting INPUT to a join, it says nothing about the output from
+ * a join.
+ */
+ String CUTOFF_LIMIT = IPredicate.class.getName() + ".cutoffLimit";
+
+ /**
+ * Deault is to not cut off the join.
+ */
+ final long DEFAULT_CUTOFF_LIMIT = Long.MAX_VALUE;
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -49,6 +49,7 @@
import com.bigdata.relation.accesspath.IBindingSetAccessPath;
import com.bigdata.relation.accesspath.IBlockingBuffer;
import com.bigdata.relation.accesspath.UnsyncLocalOutputBuffer;
+import com.bigdata.striterator.ICloseableIterator;
/**
* A hash join against an {@link IAccessPath} based on the {@link HTree} and
@@ -389,8 +390,19 @@
final UnsyncLocalOutputBuffer<IBindingSet> unsyncBuffer = new UnsyncLocalOutputBuffer<IBindingSet>(
op.getChunkCapacity(), sink);
+ final long cutoffLimit = pred.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
+ // Obtain the iterator for the current join dimension.
+ final ICloseableIterator<IBindingSet> itr;
+ if (cutoffLimit == Long.MAX_VALUE)
+ itr = ((IBindingSetAccessPath<?>) accessPath).solutions(stats);
+ else
+ itr = ((IBindingSetAccessPath<?>) accessPath).solutions(cutoffLimit, stats);
+
state.hashJoin(
- ((IBindingSetAccessPath<?>) accessPath).solutions(stats),// left
+ itr,// left
unsyncBuffer// out
);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -49,6 +49,7 @@
import com.bigdata.relation.accesspath.IBindingSetAccessPath;
import com.bigdata.relation.accesspath.IBlockingBuffer;
import com.bigdata.relation.accesspath.UnsyncLocalOutputBuffer;
+import com.bigdata.striterator.ICloseableIterator;
/**
* A hash join against an {@link IAccessPath} based on the Java collections
@@ -325,8 +326,19 @@
final UnsyncLocalOutputBuffer<IBindingSet> unsyncBuffer = new UnsyncLocalOutputBuffer<IBindingSet>(
op.getChunkCapacity(), sink);
+ final long cutoffLimit = pred.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
+ // Obtain the iterator for the current join dimension.
+ final ICloseableIterator<IBindingSet> itr;
+ if (cutoffLimit == Long.MAX_VALUE)
+ itr = ((IBindingSetAccessPath<?>) accessPath).solutions(stats);
+ else
+ itr = ((IBindingSetAccessPath<?>) accessPath).solutions(cutoffLimit, stats);
+
state.hashJoin(
- ((IBindingSetAccessPath<?>) accessPath).solutions(stats),// left
+ itr,// left
unsyncBuffer // where to write the solutions which join.
);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -1527,8 +1527,16 @@
*/
protected void handleJoin() {
+ final long cutoffLimit = predicate.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
// Obtain the iterator for the current join dimension.
- final IChunkedOrderedIterator<?> itr = accessPath.iterator();
+ final IChunkedOrderedIterator<?> itr;
+ if (cutoffLimit == Long.MAX_VALUE)
+ itr = accessPath.iterator();
+ else
+ itr = accessPath.iterator(0, limit, (int) limit);
try {
@@ -1638,9 +1646,16 @@
*/
protected void handleJoin2() {
+ final long cutoffLimit = predicate.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
// Obtain the iterator for the current join dimension.
- final ICloseableIterator<IBindingSet> itr = ((IBindingSetAccessPath<?>) accessPath)
- .solutions(stats);
+ final ICloseableIterator<IBindingSet> itr;
+ if (cutoffLimit == Long.MAX_VALUE)
+ itr = ((IBindingSetAccessPath<?>) accessPath).solutions(stats);
+ else
+ itr = ((IBindingSetAccessPath<?>) accessPath).solutions(cutoffLimit, stats);
try {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -732,6 +732,22 @@
}
+ /**
+ * {@inheritDoc}
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/209 (Access path
+ * should visit solutions for high level query).
+ */
+ public ICloseableIterator<IBindingSet> solutions(final long limit,
+ final BaseJoinStats stats) {
+
+// final IVariable<?>[] vars = BOpUtility
+// .getDistinctArgumentVariables(predicate);
+
+ return BOpContext.solutions(iterator(0, limit, (int) limit), predicate, /*vars,*/ stats);
+
+ }
+
final public IChunkedOrderedIterator<R> iterator() {
return iterator(0L/* offset */, 0L/* limit */, 0);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -56,5 +56,16 @@
* should visit solutions for high level query).
*/
ICloseableIterator<IBindingSet> solutions(BaseJoinStats stats);
-
+
+ /**
+ * Return an iterator which will visit the solutions drawn from the access
+ * path with a limit on the number of solutions drawn.
+ *
+ * TODO Should this be visiting IBindingSet[]s?
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/209 (Access path
+ * should visit solutions for high level query).
+ */
+ ICloseableIterator<IBindingSet> solutions(long limit, BaseJoinStats stats);
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/RangeBOp.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -23,20 +23,17 @@
*/
package com.bigdata.rdf.internal.constraints;
+import java.util.LinkedHashMap;
import java.util.Map;
import com.bigdata.bop.BOp;
-import com.bigdata.bop.BOpBase;
import com.bigdata.bop.Constant;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstant;
import com.bigdata.bop.IValueExpression;
-import com.bigdata.bop.IVariable;
-import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.ImmutableBOp;
import com.bigdata.bop.ModifiableBOpBase;
import com.bigdata.bop.NV;
-import com.bigdata.bop.ap.Predicate;
import com.bigdata.rdf.error.SparqlTypeErrorException;
import com.bigdata.rdf.internal.IV;
@@ -60,11 +57,11 @@
public interface Annotations extends ImmutableBOp.Annotations {
- /**
- * The variable whose range is restricted by the associated
- * {@link #FROM} and/or {@link #TO} filters.
- */
- String VAR = RangeBOp.class.getName() + ".var";
+// /**
+// * The variable whose range is restricted by the associated
+// * {@link #FROM} and/or {@link #TO} filters.
+// */
+// String VAR = RangeBOp.class.getName() + ".var";
/** The inclusive lower bound. */
String FROM = RangeBOp.class.getName() + ".from";
@@ -77,18 +74,24 @@
// /** Cached to/from lookups. */
// private transient volatile IValueExpression<IV> to, from;
- public RangeBOp(final IVariable<? extends IV> var) {
+ public RangeBOp() {
- this(BOp.NOARGS, NV.asMap(new NV(Annotations.VAR, var)));
+ this(BOp.NOARGS, new LinkedHashMap<String,Object>());
}
- public RangeBOp(final IVariable<? extends IV> var,
+// public RangeBOp(final IVariable<? extends IV> var) {
+//
+// this(BOp.NOARGS, NV.asMap(new NV(Annotations.VAR, var)));
+//
+// }
+
+ public RangeBOp(//final IVariable<? extends IV> var,
final IValueExpression<? extends IV> from,
final IValueExpression<? extends IV> to) {
this(BOp.NOARGS, NV.asMap(
- new NV(Annotations.VAR, var),
+// new NV(Annotations.VAR, var),
new NV(Annotations.FROM, from),
new NV(Annotations.TO, to)));
@@ -108,10 +111,10 @@
super(op);
}
- @SuppressWarnings("unchecked")
- public IVariable<? extends IV> var() {
- return (IVariable<? extends IV>) getProperty(Annotations.VAR);
- }
+// @SuppressWarnings("unchecked")
+// public IVariable<? extends IV> var() {
+// return (IVariable<? extends IV>) getProperty(Annotations.VAR);
+// }
@SuppressWarnings("unchecked")
public IValueExpression<? extends IV> from() {
@@ -219,7 +222,7 @@
return this;
}
- final RangeBOp asBound = new RangeBOp(var());
+ final RangeBOp asBound = new RangeBOp();
if (asBoundFrom != null)
asBound.setFrom(asBoundFrom);
if (asBoundTo != null)
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -31,6 +31,7 @@
import com.bigdata.bop.BufferAnnotations;
import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.fed.QueryEngineFactory;
@@ -405,4 +406,13 @@
*/
String RANGE_SAFE = "rangeSafe";
+ /**
+ * Used to mark a statement pattern with a cutoff limit for how many
+ * elements (maximum) should be read from its access path. This
+ * effectively limits the input into the join.
+ *
+ * @see {@link Annotations#CUTOFF_LIMIT}.
+ */
+ String CUTOFF_LIMIT = "cutoffLimit";
+
}
\ No newline at end of file
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -3808,6 +3808,12 @@
// Add the RangeBOp
anns.add(new NV(IPredicate.Annotations.RANGE, range.getRangeBOp()));
}
+
+ final String cutoffLimit = sp.getQueryHint(QueryHints.CUTOFF_LIMIT);
+ if (cutoffLimit != null) {
+ // Add the cutoff limit
+ anns.add(new NV(IPredicate.Annotations.CUTOFF_LIMIT, Long.valueOf(cutoffLimit)));
+ }
final Properties queryHints = sp.getQueryHints();
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/CutoffLimitHint.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/CutoffLimitHint.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/CutoffLimitHint.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -0,0 +1,63 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Nov 27, 2011
+ */
+
+package com.bigdata.rdf.sparql.ast.hints;
+
+import com.bigdata.rdf.sparql.ast.ASTBase;
+import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+
+final class CutoffLimitHint extends AbstractQueryHint<Long> {
+
+ protected CutoffLimitHint() {
+ super(QueryHints.CUTOFF_LIMIT, Long.MAX_VALUE);
+ }
+
+ @Override
+ public void handle(final AST2BOpContext context,
+ final QueryHintScope scope, final ASTBase op,
+ final Long value) {
+
+ if (op instanceof StatementPatternNode) {
+
+ _setQueryHint(context, scope, op, getName(), value);
+
+ return;
+
+ }
+
+ }
+
+ @Override
+ public Long validate(String value) {
+
+ return Long.valueOf(value);
+
+ }
+
+}
\ No newline at end of file
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/hints/QueryHintRegistry.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -101,7 +101,6 @@
add(new RunOnceHint());
add(new OptimizerQueryHint());
add(new OptimisticQueryHint());
- add(new RangeHint());
add(new AnalyticQueryHint());
add(new NativeDistinctQueryHint());
@@ -142,7 +141,21 @@
add(new PipelineMaxParallelHint());
add(new PipelineMaxMessagesPerTaskHint());
add(new PipelineQueueCapacityHint());
+
+ /*
+ * Mark a statement pattern as "range safe", which in effect means it
+ * uses only one datatype in it value space (for bindings for O) and
+ * that the filters in the query are respecting that datatype.
+ */
+ add(new RangeHint());
+ /*
+ * Limit the input into joins by limiting the number of elements read
+ * from an access path. Not exactly a cutoff join, which limits output
+ * from the join rather than input into it.
+ */
+ add(new CutoffLimitHint());
+
}
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTRangeOptimizer.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -157,9 +157,9 @@
public static RangeBOp toRangeBOp(
final RangeNode range, final GlobalAnnotations globals) {
- final IVariable<? extends IV> var = range.var().getValueExpression();
+// final IVariable<? extends IV> var = range.var().getValueExpression();
- final RangeBOp bop = new RangeBOp(var);
+ final RangeBOp bop = new RangeBOp();
final ValueExpressionNode from = range.from();
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -2769,7 +2769,7 @@
p == null ? null : _p.getIV(),//
o == null ? null : _o.getIV(),//
(c == null || !quads) ? null : _c.getIV(),//
- filter//
+ filter, range//
);
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2012-05-02 17:49:50 UTC (rev 6311)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2012-05-03 09:06:37 UTC (rev 6312)
@@ -409,7 +409,7 @@
new NV(BOp.Annotations.BOP_ID, nextId++),//
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RANGE, new RangeBOp(//
- origProperty1,// FIXME verify correct var w/ MikeP
+// origProperty1,// FIXME verify correct var w/ MikeP
new MathBOp(origProperty1, new Constant(
new XSDIntegerIV(BigInteger.valueOf(120))),
MathOp.MINUS,globals),//
@@ -440,7 +440,7 @@
new NV(BOp.Annotations.BOP_ID, nextId++),//
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RANGE, new RangeBOp(//
- origProperty2,// FIXME verify correct var with MikeP
+// origProperty2,// FIXME verify correct var with MikeP
new MathBOp(origProperty2, new Constant(
new XSDIntegerIV(BigInteger.valueOf(170))),
MathOp.MINUS,globals),//
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-05-09 15:31:46
|
Revision: 6313
http://bigdata.svn.sourceforge.net/bigdata/?rev=6313&view=rev
Author: thompsonbry
Date: 2012-05-09 15:31:31 +0000 (Wed, 09 May 2012)
Log Message:
-----------
This is a checkpoint on reification done right. The SPARQL grammar has been extended per the draft specification, a test suite has been written for that grammar extension, and the parser is passing that test suite. In addition, this commit includes a test suite for evaluation of the new refication syntax, but I have not yet started on query evaluation for that syntax. The code paths for reification done right are currently conditionally disabled by QueryHints#DEFAULT_REFICIATION_DONE_RIGHT. That property will go away once this feature is finished.
https://sourceforge.net/apps/trac/bigdata/ticket/526 (Reification done right)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/VarNode.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ASTVisitorBase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BigdataASTContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BlankNodeVarProcessor.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPattern.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPatternBuilder.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/TriplePatternExprBuilder.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/JJTSyntaxTreeBuilderState.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilder.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderConstants.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderTokenManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderTreeConstants.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/SyntaxTreeBuilderVisitor.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/sparql.jjt
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/sparql/AbstractBigdataExprBuilderTestCase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/sparql/TestAll_AST.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.ttl
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.ttl
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.ttl
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.rq
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.ttl
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTTRefPattern.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/sparql.jj
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/sparql/TestReificationDoneRightParser.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/QueryHints.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -400,6 +400,17 @@
boolean DEFAULT_SOLUTION_SET_CACHE = false;
/**
+ * Option controls whether or not the proposed SPARQL extension for
+ * reification done right is enabled.
+ *
+ * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ */
+ String REIFICATION_DONE_RIGHT = "reificationDoneRight";
+
+ boolean DEFAULT_REIFICATION_DONE_RIGHT = false;
+
+ /**
* Used to mark a predicate as "range safe" - that is, we can safely
* apply the range bop to constrain the predicate. This can only be
* used currently when there is a single datatype for attribute values.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/StatementPatternNode.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -159,6 +159,19 @@
*/
String RANGE = "range";
+ /**
+ * An optional annotation whose value is the variable which will be
+ * bound to the statement identifier for the matched statement patterns.
+ * The statement identifier is always formed from the subject, predicate
+ * and object (the triple). The context is NOT represented in the
+ * statement identifier. This keeps the semantics consistent with RDF
+ * reification.
+ *
+ * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ */
+ String SID = "sid";
+
}
/**
@@ -263,30 +276,71 @@
}
+ /**
+ * The variable or constant for the subject position (required).
+ */
final public TermNode s() {
return (TermNode) get(0);
}
+ /**
+ * The variable or constant for the predicate position (required).
+ */
final public TermNode p() {
return (TermNode) get(1);
}
+ /**
+ * The variable or constant for the object position (required).
+ */
final public TermNode o() {
return (TermNode) get(2);
}
+ /**
+ * The variable or constant for the context position (required iff in quads
+ * mode).
+ */
final public TermNode c() {
return (TermNode) get(3);
}
-
+
+ final public void setC(final TermNode c) {
+
+ this.setArg(3, c);
+
+ }
+
+ /**
+ * The statement identifier variable for triples which match this statement
+ * pattern (optional). The statement identifier is the composition of the
+ * (subject, predicate, and object) positions of the matched statements.
+ *
+ * @see Annotations#SID
+ */
+ final public VarNode sid() {
+
+ return (VarNode) getProperty(Annotations.SID);
+
+ }
+
+ /**
+ * Set the SID variable.
+ */
+ final public void setSid(final VarNode sid) {
+
+ setProperty(Annotations.SID, sid);
+
+ }
+
/**
* The scope for this statement pattern (either named graphs or default
* graphs).
@@ -300,6 +354,15 @@
}
+ final public void setScope(final Scope scope) {
+
+ if (scope == null)
+ throw new IllegalArgumentException();
+
+ setProperty(Annotations.SCOPE, scope);
+
+ }
+
/**
* {@inheritDoc}
* <p>
@@ -503,13 +566,18 @@
sb.append(", ").append(c);
}
- final Scope scope = getScope();
- if (scope != null) {
- sb.append(", ").append(scope);
- }
-
sb.append(")");
+ final VarNode sid = sid();
+ if (sid != null) {
+ sb.append(" [sid=" + sid + "]");
+ }
+
+ final Scope scope = getScope();
+ if (scope != null) {
+ sb.append(" [scope=" + scope + "]");
+ }
+
if(isOptional()) {
sb.append(" [optional]");
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/VarNode.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/VarNode.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/VarNode.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -31,15 +31,33 @@
* interpretation of the parse tree, AST, etc rather than one given
* directly in the original query).
*/
- String ANONYMOUS = VarNode.class.getName() + ".anonymous";
+ String ANONYMOUS = "anon";
boolean DEFAULT_ANONYMOUS = false;
+ /*
+ * Note: Annotating variables which are "SIDS" does not work out because
+ * we lack sufficient context in the parser to make this determination
+ * when the variable appears outside of the BIND( <<...>> as ?sid )
+ * context. In order for the parser to do this it would have to reason
+ * about the visibility of variables as well, which is just too much
+ * effort.
+ */
+// /**
+// * Annotation marks a variable which is a statement identifier.
+// *
+// * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+// * Reification Done Right</a>
+// */
+// String SID = "sid";
+//
+// boolean DEFAULT_SID = false;
+
/**
* Annotation marks a variable which is actually the name of a solution
* set.
*/
- String SOLUTION_SET = VarNode.class.getName() + ".solutionSet";
+ String SOLUTION_SET = "solutionSet";
boolean DEFAULT_SOLUTION_SET = false;
@@ -102,7 +120,31 @@
}
- /**
+// /**
+// * Return <code>true</code> if this is an SID variable (a variable which
+// * binds statement identifiers composed from triples). This marker is useful
+// * mainly when reading the AST tree as an aid to understanding where a given
+// * variable came from in the original query.
+// */
+// public boolean isSID() {
+//
+// return getProperty(Annotations.SID, Annotations.DEFAULT_SID);
+//
+// }
+//
+// /**
+// * Mark this as an SID variable (one which represents a triple reference
+// * pattern).
+// *
+// * @param val
+// */
+// public void setSID(final boolean val) {
+//
+// setProperty(Annotations.SID, val);
+//
+// }
+
+ /**
* Return <code>true</code> if the variable represents a solution set name.
*/
public boolean isSolutionSet() {
@@ -145,15 +187,14 @@
}
/**
- * Overridden to mark anonymous variables.
- *
- * @see #isAnonymous()
- */
+ * Overridden to mark metadata about variables.
+ */
@Override
public String toString() {
return "VarNode(" + getValueExpression() + ")"
+ (isAnonymous() ? "[anonymous]" : "")
+// + (isSID() ? "[sid]" : "")
+ (isSolutionSet() ? "[solutionSet]" : "")
;
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestAll.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/TestAll.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -24,6 +24,7 @@
package com.bigdata.rdf.sparql.ast.eval;
import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.sparql.ast.eval.reif.TestReificationDoneRightEval;
import junit.framework.Test;
import junit.framework.TestCase;
@@ -134,6 +135,15 @@
*/
suite.addTestSuite(TestTCK.class);
+ if (QueryHints.DEFAULT_REIFICATION_DONE_RIGHT) {
+
+ /*
+ * Test suite for the SPARQL extension for "reification done right".
+ */
+ suite.addTestSuite(TestReificationDoneRightEval.class);
+
+ }
+
// Test suite for embedded bigdata query hints.
suite.addTestSuite(TestQueryHints.class);
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,210 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 7, 2011
+ */
+
+package com.bigdata.rdf.sparql.ast.eval.reif;
+
+import com.bigdata.bop.ap.Predicate;
+import com.bigdata.rdf.sparql.ast.eval.AbstractDataDrivenSPARQLTestCase;
+
+/**
+ * Reification Done Right test suite.
+ * <p>
+ * The basic extension is:
+ *
+ * <pre>
+ * BIND(<<?s,?p,?o>> as ?sid)
+ * </pre>
+ *
+ * This triple reference pattern associates the bindings on the subject,
+ * predicate, and object positions of some triple pattern with the binding on a
+ * variable for the triple. Except in the case where ?sid is already bound, the
+ * triple pattern itself is processed normally and the bindings are concatenated
+ * to form a representation of a triple, which is then bound on ?sid. When ?sid
+ * is bound, it is decomposed into its subject, predicate, and object components
+ * and those values are bound on the triple pattern.
+ * <p>
+ * When there are nested triple reference patterns, then they are just unwound
+ * into simple triple reference patterns. A variables is created to provide the
+ * association between each nested triple reference pattern and role played by
+ * that triple reference pattern in the outer triple reference pattern.
+ * <p>
+ * We can handle this internally by allowing an optional named/positional role
+ * for a {@link Predicate} which (when defined) becomes bound to the composition
+ * of the (subject, predicate, and object) bindings for the predicate. It might
+ * be easiest to handle this if we allowed the [c] position to be optional as
+ * well and ignored it when in a triples only mode. The sid/triple binding would
+ * always be "in scope" but it would only be interpreted when non-null (either a
+ * constant or a variable).
+ *
+ * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: TestTCK.java 6261 2012-04-09 10:28:48Z thompsonbry $
+ */
+public class TestReificationDoneRightEval extends AbstractDataDrivenSPARQLTestCase {
+
+// private static final Logger log = Logger.getLogger(TestReificationDoneRight.class);
+
+ /**
+ *
+ */
+ public TestReificationDoneRightEval() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestReificationDoneRightEval(String name) {
+ super(name);
+ }
+
+ /**
+ * Simple query involving alice, bob, and an information extractor.
+ */
+ public void test_reificationDoneRight_01() throws Exception {
+
+ new TestHelper("reif/rdr-01", // testURI,
+ "reif/rdr-01.rq",// queryFileURL
+ "reif/rdr-01.ttl",// dataFileURL
+ "reif/rdr-01.srx"// resultFileURL
+ ).runTest();
+
+ }
+
+ /**
+ * Same data, but the query uses the BIND() syntax and pulls out some
+ * more information.
+ */
+ public void test_reificationDoneRight_01a() throws Exception {
+
+ new TestHelper("reif/rdr-01a", // testURI,
+ "reif/rdr-01a.rq",// queryFileURL
+ "reif/rdr-01.ttl",// dataFileURL
+ "reif/rdr-01.srx"// resultFileURL
+ ).runTest();
+
+ }
+
+ /**
+ * Simple query ("who bought sybase").
+ */
+ public void test_reificationDoneRight_02() throws Exception {
+
+ new TestHelper("reif/rdr-02", // testURI,
+ "reif/rdr-02.rq",// queryFileURL
+ "reif/rdr-02.ttl",// dataFileURL
+ "reif/rdr-02.srx"// resultFileURL
+ ).runTest();
+
+ }
+
+ /**
+ * Same data, but the query uses the BIND() syntax and pulls out some
+ * more information.
+ */
+ public void test_reificationDoneRight_02a() throws Exception {
+
+ new TestHelper("reif/rdr-02a", // testURI,
+ "reif/rdr-02a.rq",// queryFileURL
+ "reif/rdr-02a.ttl",// dataFileURL
+ "reif/rdr-02a.srx"// resultFileURL
+ ).runTest();
+
+ }
+
+ /**
+ * <pre>
+ * prefix : <http://example.com/>
+ * SELECT ?a {
+ * BIND( <<?a :b :c>> AS ?ignored )
+ * }
+ * </pre>
+ *
+ * <pre>
+ * prefix : <http://example.com/> .
+ * :a1 :b :c .
+ * :a2 :b :c .
+ * <<:a2 :b :c>> :d :e .
+ * <<:a3 :b :c>> :d :e .
+ * </pre>
+ *
+ * <pre>
+ * ?a
+ * ---
+ * :a1
+ * :a2
+ * </pre>
+ *
+ * @throws Exception
+ */
+ public void test_reificationDoneRight_03() throws Exception {
+
+ new TestHelper("reif/rdr-03", // testURI,
+ "reif/rdr-03.rq",// queryFileURL
+ "reif/rdr-03.ttl",// dataFileURL
+ "reif/rdr-03.srx"// resultFileURL
+ ).runTest();
+
+ }
+
+ /**
+ * <pre>
+ * prefix : <http://example.com/>
+ * SELECT ?a ?e {
+ * BIND( <<?a :b :c>> AS ?sid ) .
+ * ?sid :d ?e .
+ * }
+ * </pre>
+ *
+ * <pre>
+ * prefix : <http://example.com/> .
+ * :a1 :b :c .
+ * :a2 :b :c .
+ * <<:a2 :b :c>> :d :e .
+ * <<:a3 :b :c>> :d :e .
+ * </pre>
+ *
+ * <pre>
+ * ?a ?e
+ * ------
+ * :a2 :e
+ * </pre>
+ *
+ * @throws Exception
+ */
+ public void test_reificationDoneRight_03a() throws Exception {
+
+ new TestHelper("reif/rdr-03a", // testURI,
+ "reif/rdr-03a.rq",// queryFileURL
+ "reif/rdr-03a.ttl",// dataFileURL
+ "reif/rdr-03a.srx"// resultFileURL
+ ).runTest();
+
+ }
+
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.rq 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,10 @@
+# Who says Bob knows Alice?
+
+prefix foaf: <http://xmlns.com/foaf/0.1/>
+prefix dc: <http://purl.org/dc/terms/>
+
+select ?src where {
+ ?x foaf:name "Alice" .
+ ?y foaf:name "Bob" .
+ <<?x foaf:knows ?y>> dc:source ?src .
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.srx 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="src"/>
+ </head>
+ <results>
+ <result>
+ <binding name="src">
+ <uri>http://reasoner.example.com/engines#engine_1</uri>
+ </binding>
+ </result>
+ </results>
+</sparql>
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.ttl
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.ttl (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01.ttl 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,79 @@
+@prefix foaf: <http://xmlns.com/foaf/0.1/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix dc: <http://purl.org/dc/terms/> .
+@prefix re: <http://reasoner.example.com/engines#> .
+@prefix rr: <http://reasoner.example.com/rules#> .
+@prefix rv: <http://reasoner.example.com/vocabulary#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+_:alice
+ rdf:type foaf:Person ;
+ foaf:name "Alice" ;
+ foaf:mbox <mailto:alice@work> ;
+ foaf:knows _:bob.
+
+# The terse syntax:
+#<<_:alice foaf:mbox <mailto:alice@work>>>
+# dc:source <http://hr.example.com/employees#bob> ;
+# dc:created "2012-02-05T12:34:00Z"^^xsd:dateTime .
+#
+# The expanded syntax.
+_:s1 rdf:subject _:alice .
+_:s1 rdf:predicate foaf:mbox .
+_:s1 rdf:object <mailto:alice@work> .
+_:s1 rdf:type rdf:Statement .
+_:s1 dc:source <http://hr.example.com/employees#bob> ;
+ dc:created "2012-02-05T12:34:00Z"^^xsd:dateTime .
+
+_:s1 rdf:subject _:alice .
+_:s1 rdf:predicate foaf:mbox .
+_:s1 rdf:object <mailto:alice@work> .
+_:s1 rdf:type rdf:Statement .
+_:s1 dc:source <http://hr.example.com/employees#bob> ;
+ dc:created "2012-02-05T12:34:00Z"^^xsd:dateTime .
+
+# Terse
+#<<_:alice foaf:knows _:bob>>
+# dc:source re:engine_1;
+# rv:rule rr:rule524 ;
+# rv:confidence 0.9835 .
+# Expanded
+_:s2 rdf:subject _:alice .
+_:s2 rdf:predicate foaf:knows .
+_:s2 rdf:object _:bob .
+_:s2 rdf:type rdf:Statement .
+_:s2
+ dc:source re:engine_1;
+ rv:rule rr:rule524 ;
+ rv:confidence 0.9835 .
+
+_:bob
+ rdf:type foaf:Person ;
+ foaf:name "Bob" ;
+ foaf:knows _:alice ;
+ foaf:mbox <mailto:bob@work> ;
+ foaf:mbox <mailto:bob@home> .
+
+# Terse
+# <<_:bob foaf:mbox <mailto:bob@home>>>
+# Expanded
+_:s3 rdf:subject _:bob .
+_:s3 rdf:predicate foaf:mbox .
+_:s3 rdf:object <mailto:bob@home> .
+_:s3 rdf:type rdf:Statement .
+_:s3
+ dc:creator <http://hr.example.com/infra/crawlers#we1> ;
+ dc:created "2011-04-05T12:00:00Z"^^xsd:dateTime ;
+ dc:source <http://whatever.nu/profile/bob1975> .
+
+# Terse
+# <<_:bob foaf:mbox <mailto:bob@home>>>
+# Expanded
+_:s4 rdf:subject _:bob .
+_:s4 rdf:predicate foaf:mbox .
+_:s4 rdf:object <mailto:bob@home> .
+_:s4 rdf:type rdf:Statement .
+_:s4
+ dc:source <http://hr.example.com/employees/bob> ;
+ dc:created "2012-02-05T12:34:00Z"^^xsd:dateTime .
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.rq 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,14 @@
+# Who knows Alice, what is our source for that fact, and what is our confidence
+# in that fact?
+
+prefix foaf: <http://xmlns.com/foaf/0.1/>
+prefix dc: <http://purl.org/dc/terms/>
+prefix rv: <http://reasoner.example.com/vocabulary#>
+
+select ?who ?src ?conf where {
+ ?x foaf:name "Alice" .
+ ?y foaf:name ?who .
+ BIND( <<?x foaf:knows ?y>> as ?sid ) .
+ ?sid dc:source ?src .
+ ?sid rv:confidence ?src .
+}
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="who"/>
+ <variable name="src"/>
+ <variable name="conf"/>
+ </head>
+ <results>
+ <result>
+ <binding name="who">
+ <literal>Bob</literal>
+ </binding>
+ <binding name="src">
+ <uri>http://reasoner.example.com/engines#engine_1</uri>
+ </binding>
+ <binding name="conf">
+ <literal datatype="xsd:decimal">0.9835</literal>
+ </binding>
+ </result>
+ </results>
+</sparql>
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.rq 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,8 @@
+# Who bought Sybase and where did we get that fact?
+
+prefix : <http://example.com/>
+prefix dc: <http://purl.org/dc/terms/>
+
+SELECT ?src ?who {
+ <<?who :bought :sybase>> dc:source ?src
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.srx 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="who"/>
+ <variable name="src"/>
+ </head>
+ <results>
+ <result>
+ <binding name="who">
+ <uri>http://example.com/SAP</uri>
+ </binding>
+ <binding name="src">
+ <uri>http://example.com/news/us-sybase</uri>
+ </binding>
+ </result>
+ </results>
+</sparql>
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,16 @@
+# <<:SAP :bought :sybase>> dc:source news:us-sybase ;
+# dc:created "2011-04-05T12:00:00Z"^^xsd:dateTime .
+
+@prefix : <http://example.com/> .
+@prefix news: <http://example.com/news/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix dc: <http://purl.org/dc/terms/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+_:s1 rdf:subject :SAP .
+_:s1 rdf:predicate :bought .
+_:s1 rdf:object :sybase .
+_:s1 rdf:type rdf:Statement .
+_:s1 dc:source news:us-sybase .
+_:s1 dc:created "2011-04-05T12:00:00Z"^^xsd:dateTime .
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.rq 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,10 @@
+# Who bought Sybase and where did we get that fact?
+
+prefix : <http://example.com/>
+prefix dc: <http://purl.org/dc/terms/>
+
+SELECT ?src ?who ?created {
+ BIND( <<?who :bought :sybase>> as ?sid ) .
+ ?sid dc:source ?src .
+ OPTIONAL {?sid dc:created ?created}
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="who"/>
+ <variable name="src"/>
+ <variable name="created"/>
+ </head>
+ <results>
+ <result>
+ <binding name="who">
+ <uri>http://example.com/SAP</uri>
+ </binding>
+ <binding name="src">
+ <uri>http://example.com/news/us-sybase</uri>
+ </binding>
+ <binding name="created">
+ <literal datatype="xsd:dateTime">2011-04-05T12:00:00Z</literal>
+ </binding>
+ </result>
+ </results>
+</sparql>
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.ttl
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.ttl (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.ttl 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,16 @@
+# <<:SAP :bought :sybase>> dc:source news:us-sybase ;
+# dc:created "2011-04-05T12:00:00Z"^^xsd:dateTime .
+
+@prefix : <http://example.com/> .
+@prefix news: <http://example.com/news/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix dc: <http://purl.org/dc/terms/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+_:s1 rdf:subject :SAP .
+_:s1 rdf:predicate :bought .
+_:s1 rdf:object :sybase .
+_:s1 rdf:type rdf:Statement .
+_:s1 dc:source news:us-sybase .
+_:s1 dc:created "2011-04-05T12:00:00Z"^^xsd:dateTime .
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.rq 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,4 @@
+prefix : <http://example.com/>
+SELECT ?a {
+ BIND( <<?a :b :c>> AS ?ignored )
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.srx 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="a"/>
+ </head>
+ <results>
+ <result>
+ <binding name="a">
+ <uri>http://example.com/a1</uri>
+ </binding>
+ </result>
+ <result>
+ <binding name="a">
+ <uri>http://example.com/a2</uri>
+ </binding>
+ </result>
+ </results>
+</sparql>
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.ttl
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.ttl (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03.ttl 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,22 @@
+# prefix : <http://example.com/> .
+# :a1 :b :c .
+# :a2 :b :c .
+# <<:a2 :b :c>> :d :e .
+# <<:a3 :b :c>> :d :e .
+
+@prefix : <http://example.com/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+
+_:s1 rdf:subject :a1 .
+_:s1 rdf:predicate :b .
+_:s1 rdf:object :c .
+_:s1 rdf:type rdf:Statement .
+
+_:s2 rdf:subject :a2 .
+_:s2 rdf:predicate :b .
+_:s2 rdf:object :c .
+_:s2 rdf:type rdf:Statement .
+
+_:s1 :d :e .
+
+_:s2 :d :e .
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.rq
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.rq (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.rq 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,5 @@
+prefix : <http://example.com/>
+SELECT ?a {
+ BIND( <<?a :b :c>> AS ?sid ) .
+ ?sid :d ?e .
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.srx (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.srx 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<sparql
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:xs="http://www.w3.org/2001/XMLSchema#"
+ xmlns="http://www.w3.org/2005/sparql-results#" >
+ <head>
+ <variable name="a"/>
+ <variable name="e"/>
+ </head>
+ <results>
+ <result>
+ <binding name="a">
+ <uri>http://example.com/a2</uri>
+ </binding>
+ <binding name="e">
+ <uri>http://example.com/e</uri>
+ </binding>
+ </result>
+ </results>
+</sparql>
\ No newline at end of file
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.ttl
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.ttl (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-03a.ttl 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,22 @@
+# prefix : <http://example.com/> .
+# :a1 :b :c .
+# :a2 :b :c .
+# <<:a2 :b :c>> :d :e .
+# <<:a3 :b :c>> :d :e .
+
+@prefix : <http://example.com/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+
+_:s1 rdf:subject :a1 .
+_:s1 rdf:predicate :b .
+_:s1 rdf:object :c .
+_:s1 rdf:type rdf:Statement .
+
+_:s2 rdf:subject :a2 .
+_:s2 rdf:predicate :b .
+_:s2 rdf:object :c .
+_:s2 rdf:type rdf:Statement .
+
+_:s1 :d :e .
+
+_:s2 :d :e .
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ASTVisitorBase.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ASTVisitorBase.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ASTVisitorBase.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -887,4 +887,9 @@
return node.childrenAccept(this, data);
}
+ public Object visit(ASTTRefPattern node, Object data)
+ throws VisitorException {
+ return node.childrenAccept(this, data);
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BigdataASTContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BigdataASTContext.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BigdataASTContext.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -141,4 +141,18 @@
}
+ /**
+ * Return a new variable for a SID (Statement Identifier)
+ *
+ * @see <a href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ */
+ protected VarNode createSidVar() {
+
+ final VarNode var = new VarNode("-sid-" + constantVarID++);
+
+ return var;
+
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BlankNodeVarProcessor.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BlankNodeVarProcessor.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/BlankNodeVarProcessor.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -72,7 +72,7 @@
}
@Override
- public Object visit(ASTBasicGraphPattern node, Object data)
+ public Object visit(final ASTBasicGraphPattern node, final Object data)
throws VisitorException
{
// The same Blank node ID cannot be used across Graph Patterns
@@ -85,10 +85,10 @@
}
@Override
- public Object visit(ASTBlankNode node, Object data)
+ public Object visit(final ASTBlankNode node, final Object data)
throws VisitorException
{
- String bnodeID = node.getID();
+ final String bnodeID = node.getID();
String varName = findVarName(bnodeID);
if (varName == null) {
@@ -99,7 +99,7 @@
}
}
- ASTVar varNode = new ASTVar(SyntaxTreeBuilderTreeConstants.JJTVAR);
+ final ASTVar varNode = new ASTVar(SyntaxTreeBuilderTreeConstants.JJTVAR);
varNode.setName(varName);
varNode.setAnonymous(true);
@@ -108,10 +108,10 @@
return super.visit(node, data);
}
- private String findVarName(String bnodeID) throws VisitorException {
+ private String findVarName(final String bnodeID) throws VisitorException {
if (bnodeID == null)
return null;
- String varName = conversionMap.get(bnodeID);
+ final String varName = conversionMap.get(bnodeID);
if (varName == null && usedBNodeIDs.contains(bnodeID))
throw new VisitorException(
"BNodeID already used in another scope: " + bnodeID);
@@ -119,7 +119,7 @@
}
@Override
- public Object visit(ASTBlankNodePropertyList node, Object data)
+ public Object visit(final ASTBlankNodePropertyList node, final Object data)
throws VisitorException
{
node.setVarName(createAnonVarName());
@@ -127,7 +127,7 @@
}
@Override
- public Object visit(ASTCollection node, Object data)
+ public Object visit(final ASTCollection node, final Object data)
throws VisitorException
{
node.setVarName(createAnonVarName());
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPattern.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPattern.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPattern.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -183,10 +183,18 @@
log.info("pattern=" + sp);
// Fill in the inherited context and scope.
- final StatementPatternNode t = new StatementPatternNode(sp.s(), sp.p(),
- sp.o(), context, spScope);
+// final StatementPatternNode t = new StatementPatternNode(sp.s(), sp.p(),
+// sp.o(), context, spScope);
+
+ if (context != null) {
+
+ sp.setC(context);
+
+ }
+
+ sp.setScope(spScope);
- children.add(t);
+ children.add(sp);
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPatternBuilder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPatternBuilder.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/GroupGraphPatternBuilder.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -50,6 +50,7 @@
import com.bigdata.rdf.sail.sparql.ast.ASTNamedSubqueryInclude;
import com.bigdata.rdf.sail.sparql.ast.ASTOptionalGraphPattern;
import com.bigdata.rdf.sail.sparql.ast.ASTServiceGraphPattern;
+import com.bigdata.rdf.sail.sparql.ast.ASTTRefPattern;
import com.bigdata.rdf.sail.sparql.ast.ASTUnionGraphPattern;
import com.bigdata.rdf.sail.sparql.ast.ASTVar;
import com.bigdata.rdf.sail.sparql.ast.ASTWhereClause;
@@ -472,30 +473,54 @@
}
/**
- * A BIND (or FILTER) can appear in an {@link ASTBasicGraphPattern}.
- *
- * @return The {@link AssignmentNode} for the BIND.
- */
+ * A BIND (or FILTER) can appear in an {@link ASTBasicGraphPattern}.
+ *
+ * @return The {@link AssignmentNode} for the BIND -or- the
+ * <code>null</code> if this is <code>BIND(tripleRef AS var)<code>.
+ */
@Override
- final public AssignmentNode visit(final ASTBind node, Object data)
+ final public Object visit(final ASTBind node, final Object data)
throws VisitorException {
if (node.jjtGetNumChildren() != 2)
throw new AssertionError("Expecting two children, not "
+ node.jjtGetNumChildren() + ", node=" + node.dump(">>>"));
- final ValueExpressionNode ve = (ValueExpressionNode) node
- .jjtGetChild(0).jjtAccept(this, data);
+ if (node.jjtGetChild(0) instanceof ASTTRefPattern) {
+ /*
+ * Note: This case is handled by visit(ASTTRefPattern,Object) when
+ * we invoke jjtAccept() for the expression node. That method has
+ * already added the appropriate SP to the group. We do not want to
+ * do anything more here.
+ *
+ * Note: The caller does not actually do anything with the return
+ * value from this method, so returning a [null] here is Ok.
+ *
+ * @see <a
+ * href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ */
+
+ node.jjtGetChild(0).jjtAccept(this, data);
+
+ return null;
+
+ }
+
+ final Object expr = node.jjtGetChild(0).jjtAccept(this, data);
+
+ final ValueExpressionNode ve = (ValueExpressionNode) expr;
+
final Node aliasNode = node.jjtGetChild(1);
final String alias = ((ASTVar) aliasNode).getName();
- final AssignmentNode bind = new AssignmentNode(new VarNode(alias), ve);
+ final AssignmentNode bind = new AssignmentNode(new VarNode(alias), ve);
- graphPattern.add(bind);
+ graphPattern.add(bind);
- return bind;
+ return bind;
}
@@ -505,7 +530,7 @@
* @return The {@link AssignmentNode}.
*/
@Override
- final public AssignmentNode visit(final ASTLet node, Object data)
+ final public AssignmentNode visit(final ASTLet node, final Object data)
throws VisitorException {
if (node.jjtGetNumChildren() != 2)
@@ -535,7 +560,7 @@
* @return The constraint.
*/
@Override
- final public ValueExpressionNode visit(final ASTConstraint node, Object data)
+ final public ValueExpressionNode visit(final ASTConstraint node, final Object data)
throws VisitorException {
final ValueExpressionNode valueExpr = (ValueExpressionNode) node
@@ -551,7 +576,7 @@
* INCLUDE for a named subquery result set.
*/
@Override
- final public Void visit(final ASTNamedSubqueryInclude node, Object data)
+ final public Void visit(final ASTNamedSubqueryInclude node, final Object data)
throws VisitorException {
final NamedSubqueryInclude includeNode = new NamedSubqueryInclude(
@@ -629,7 +654,7 @@
* declarations for that.)
*/
@Override
- final public Void visit(final ASTServiceGraphPattern node, Object data)
+ final public Void visit(final ASTServiceGraphPattern node, final Object data)
throws VisitorException {
// left arg is the service reference (a value expression).
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/TriplePatternExprBuilder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/TriplePatternExprBuilder.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/TriplePatternExprBuilder.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -30,14 +30,24 @@
import java.util.ArrayList;
import java.util.List;
+import org.openrdf.model.BNode;
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
import org.openrdf.model.vocabulary.RDF;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.sail.sparql.ast.ASTBind;
import com.bigdata.rdf.sail.sparql.ast.ASTBlankNodePropertyList;
import com.bigdata.rdf.sail.sparql.ast.ASTCollection;
import com.bigdata.rdf.sail.sparql.ast.ASTObjectList;
import com.bigdata.rdf.sail.sparql.ast.ASTPropertyList;
import com.bigdata.rdf.sail.sparql.ast.ASTPropertyListPath;
+import com.bigdata.rdf.sail.sparql.ast.ASTTRefPattern;
+import com.bigdata.rdf.sail.sparql.ast.ASTVar;
+import com.bigdata.rdf.sail.sparql.ast.Node;
import com.bigdata.rdf.sail.sparql.ast.VisitorException;
+import com.bigdata.rdf.sparql.ast.ConstantNode;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode;
import com.bigdata.rdf.sparql.ast.TermNode;
import com.bigdata.rdf.sparql.ast.VarNode;
@@ -52,12 +62,12 @@
/**
* @param context
*/
- public TriplePatternExprBuilder(BigdataASTContext context) {
+ public TriplePatternExprBuilder(final BigdataASTContext context) {
super(context);
}
@Override
- final public Object visit(final ASTPropertyList propListNode, Object data)
+ final public Object visit(final ASTPropertyList propListNode, final Object data)
throws VisitorException {
final TermNode subject = (TermNode) data;
@@ -89,7 +99,7 @@
@Override
final public Object visit(final ASTPropertyListPath propListNode,
- Object data) throws VisitorException {
+ final Object data) throws VisitorException {
final TermNode subject = (TermNode) data;
@@ -150,7 +160,7 @@
}
@Override
- final public List<TermNode> visit(final ASTObjectList node, Object data)
+ final public List<TermNode> visit(final ASTObjectList node, final Object data)
throws VisitorException {
final int childCount = node.jjtGetNumChildren();
@@ -167,7 +177,7 @@
}
@Override
- final public VarNode visit(final ASTBlankNodePropertyList node, Object data)
+ final public VarNode visit(final ASTBlankNodePropertyList node, final Object data)
throws VisitorException {
final VarNode bnodeVar = context.createAnonVar(node.getVarName());
@@ -182,10 +192,11 @@
* Handle the RDF Collection syntax.
*/
@Override
- public VarNode visit(final ASTCollection node, Object data)
+ public VarNode visit(final ASTCollection node, final Object data)
throws VisitorException {
final String listVarName = node.getVarName();
+
final VarNode rootListVar = context.createAnonVar(listVarName);
TermNode listVar = rootListVar;
@@ -669,4 +680,172 @@
// }
// }
+ /*
+ * Reification done right.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/526 (Reification
+ * done right)
+ */
+
+ /**
+ * This is invoked in two different contexts. One for a bare triple
+ * reference pattern:
+ *
+ * <pre>
+ * <<a b c>> d e
+ * </pre>
+ *
+ * and the other for a BIND() of a triple reference pattern onto a statement
+ * identifier variable:
+ *
+ * <pre>
+ * BIND(<<a b c>> as ?sidVar)
+ * </pre>
+ *
+ * In both cases we translate the triple reference pattern into a new
+ * {@link StatementPatternNode}. We reach back to the parent to decide which
+ * of the two invocation contexts applies and either generate a new SID
+ * variable or use the one from the BIND(). Then we set the SID variable on
+ * the {@link StatementPatternNode} and add it to the current graph pattern
+ * group.
+ *
+ * <p>
+ * If either the subject or object position in the triple reference pattern
+ * is a triple reference pattern, then it will have been turned into a
+ * {@link StatementPatternNode} by recursion through this method and we
+ * replace it with the SID variable which was assigned to that
+ * {@link StatementPatternNode}.
+ *
+ * @return The SID variable. This allows the existing code paths to
+ * handle nested triple reference patterns without change.
+ */
+ public VarNode visit(final ASTTRefPattern node,
+ final Object data) throws VisitorException {
+
+ /*
+ * Accept and convert.
+ */
+
+ final TermNode s = (TermNode) node.jjtGetChild(0).jjtAccept(this, data);
+
+ final TermNode p = (TermNode) node.jjtGetChild(1).jjtAccept(this, data);
+
+ final TermNode o = (TermNode) node.jjtGetChild(2).jjtAccept(this, data);
+
+ /*
+ * Check constraints.
+ */
+
+ if(s instanceof ConstantNode) {
+
+ final BigdataValue v = ((ConstantNode)s).getValue();
+
+ if (v instanceof Literal) {
+
+ throw new VisitorException(
+ "Subject in triple reference pattern may not be literal.");
+
+ }
+
+ } else {
+ if (((VarNode) s).isAnonymous()) {
+ /*
+ * Blank nodes have already been translated into variables.
+ *
+ * TODO We could run into trouble here if there anonymous
+ * variables are introduced for anything other than a blank node
+ * (and there are). We need to explicitly mark anonymous
+ * variables which correspond to blank nodes in the original
+ * query and then test for isBlankNode() here.
+ */
+ throw new VisitorException(
+ "Subject in triple reference pattern may not be blank node.");
+ }
+ }
+
+ if (p instanceof ConstantNode) {
+
+ final BigdataValue v = ((ConstantNode) p).getValue();
+
+ if (!(v instanceof URI)) {
+
+ throw new VisitorException(
+ "Predicate in triple reference pattern must be IRI.");
+
+ }
+
+ }
+
+ if (o instanceof ConstantNode) {
+
+ final BigdataValue v = ((ConstantNode) o).getValue();
+
+ if (v instanceof BNode) {
+
+ throw new VisitorException(
+ "Object in triple reference pattern may not be blank node.");
+
+ }
+
+ } else {
+
+ if (((VarNode) o).isAnonymous()) {
+ /*
+ * Blank nodes have already been translated into variables.
+ */
+ throw new VisitorException(
+ "Object in triple reference pattern may not be blank node.");
+ }
+
+ }
+
+ /*
+ * Note: The caller will associate the SID variable with this
+ * StatementPatternNode if it appears in a BIND(<<...>> AS ?sid)
+ * construction. That will overwrite the variable which we assign here.
+ *
+ * @see GroupGraphPatternBuilder#visit(ASTBind,Object)
+ */
+
+ final StatementPatternNode sp = new StatementPatternNode(s, p, o,
+ graphPattern.getContext(),
+ graphPattern.getStatementPatternScope());
+
+ final Node parent = node.jjtGetParent();
+
+ final VarNode sidVar;
+
+ if(parent instanceof ASTBind) {
+
+ final ASTBind bind = (ASTBind) parent;
+
+ final Node aliasNode = bind.jjtGetChild(1);
+
+ final String alias = ((ASTVar) aliasNode).getName();
+
+ // Use the specified variable.
+ sidVar = new VarNode(alias);
+
+// sidVar.setSID(true);
+
+ } else {
+
+ // Create a new variable.
+ sidVar = context.createSidVar();
+
+ }
+
+ // Set the SID variable on the SP.
+ sp.setSid(sidVar);
+
+ // add to the current join group.
+ graphPattern.addSP(sp);
+
+// log.error("node:=" + node + " => " + sp);
+
+ // Return the SID variable.
+ return sidVar;
+
+ }
+
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTTRefPattern.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTTRefPattern.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/ASTTRefPattern.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -0,0 +1,21 @@
+/* Generated By:JJTree: Do not edit this line. ASTTRefPattern.java Version 4.3 */
+/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=false,NODE_PREFIX=AST,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_CLASS_VISIBILITY_PUBLIC=true */
+package com.bigdata.rdf.sail.sparql.ast;
+
+public
+class ASTTRefPattern extends SimpleNode {
+ public ASTTRefPattern(int id) {
+ super(id);
+ }
+
+ public ASTTRefPattern(SyntaxTreeBuilder p, int id) {
+ super(p, id);
+ }
+
+
+ /** Accept the visitor. **/
+ public Object jjtAccept(SyntaxTreeBuilderVisitor visitor, Object data) throws VisitorException {
+ return visitor.visit(this, data);
+ }
+}
+/* JavaCC - OriginalChecksum=ee63fb12e5388e4bb47e28aac24c0801 (do not edit this line) */
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/JJTSyntaxTreeBuilderState.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/JJTSyntaxTreeBuilderState.java 2012-05-03 09:06:37 UTC (rev 6312)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/ast/JJTSyntaxTreeBuilderState.java 2012-05-09 15:31:31 UTC (rev 6313)
@@ -1,123 +1,123 @@
-/* Generated By:JavaCC: Do not edit this line. JJTSyntaxTreeBuilderState.java Version 5.0 */
-package com.bigdata.rdf.sail.sparql.ast;
-
-public class JJTSyntaxTreeBuilderState {
- private java.util.List<Node> nodes;
- private java.util.List<Integer> marks;
-
- private int sp; // number of nodes on stack
- private int mk; // current mark
- private boolean node_created;
-
- public JJTSyntaxTreeBuilderState() {
- nodes = new java.util.ArrayList<Node>();
- marks = new java.util.ArrayList<Integer>();
- sp = 0;
- mk = 0;
- }
-
- /* Determines whether the current node was actually closed and
- pushed. This should only be called in the final user action of a
- node scope. */
- public boolean nodeCreated() {
- return node_created;
- }
-
- /* Call this to reinitialize the node stack. It is called
- automatically by the parser's ReInit() method. */
- public void reset() {
- nodes.clear();
- marks.clear();
- sp = 0;
- mk = 0;
- }
-
- /* Returns the root node of the AST. It only makes sense to call
- this after a successful parse. */
- public Node rootNode() {
- return nodes.get(0);
- }
-
- /* Pushes a node on to the stack. */
- public void pushNode(Node n) {
- nodes.add(n);
- ++sp;
- }
-
- /* Returns the node on the top of the stack, and remove it from the
- stack. */
- public Node popNode() {
- if (--sp < mk) {
- mk = marks.remove(marks.size()-1);
- }
- return nodes.remove(nodes.size()-1);
- }
-
- /* Returns the node currently on the top of the stack. */
- public Node peekNode() {
- return nodes.get(nodes.size()-1);
- }
-
- /* Returns the number of children on the stack in the current node
- scope. */
- public int nodeArity() {
- return sp - mk;
- }
-
-
- public void clearNodeScope(Node n) {
- while (sp > mk) {
- popNode();
- }
- mk = marks.remove(marks.size()-1);
- }
-
-
- public void openNodeScope(Node n) {
- marks.add(mk);
- mk = sp;
- n.jjtOpen();
- }
-
-
- /* A definite node is constructed from a specified number of
- children. That number of nodes are popped from the stack and
- made the children of the definite node. Then the definite node
- is pushed on to the stack. */
- public void closeNodeScope(Node n, int num) {
- mk = marks.remove(marks.size()-1);
- while (num-- > 0) {
- Node c = popNode();
- c.jjtSetParent(n);
- n.jjtAddChild(c, num);
- }
- n.jjtClose();
- pushNode(n);
- node_created = true;
- }
-
-
- /* A conditional node is constructed if its condition is true. All
...
[truncated message content] |
|
From: <tho...@us...> - 2012-05-10 15:28:31
|
Revision: 6314
http://bigdata.svn.sourceforge.net/bigdata/?rev=6314&view=rev
Author: thompsonbry
Date: 2012-05-10 15:28:21 +0000 (Thu, 10 May 2012)
Log Message:
-----------
Partial commit of query support for "reification done right." This does not include the small code block in AST2BOpUtility#toPredicate() due to a code conflict which MikeP will be resolving soon. Two unit tests for the new query syntax now pass. Those tests run against data which is inserted "by hand" in the correct form into the indices. This gives a good indication that the integration with the overall query plan will work once we modify the RDF parsers to load data into the indices using the appropriate (inline) representation.
@see https://sourceforge.net/apps/trac/bigdata/ticket/526 (Reification done right)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpContext.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/bnode/SidIV.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/empty.ttl
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpContext.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpContext.java 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/BOpContext.java 2012-05-10 15:28:21 UTC (rev 6314)
@@ -43,6 +43,12 @@
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.join.BaseJoinStats;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.impl.bnode.SidIV;
+import com.bigdata.rdf.model.BigdataBNode;
+import com.bigdata.rdf.sparql.ast.QueryHints;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.spo.SPO;
+import com.bigdata.rdf.spo.SPOPredicate;
import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IBlockingBuffer;
@@ -549,7 +555,7 @@
static public void copyValues(final IElement e, final IPredicate<?> pred,
final IBindingSet bindingSet) {
- final int arity = pred.arity();
+ final int arity = pred.arity();
for (int i = 0; i < arity; i++) {
@@ -596,6 +602,39 @@
}
+ if (QueryHints.DEFAULT_REIFICATION_DONE_RIGHT
+ && pred instanceof SPOPredicate) {
+
+ final SPOPredicate tmp = (SPOPredicate) pred;
+
+ final IVariable<?> sidVar = tmp.sid();
+
+ if (sidVar != null) {
+
+ /*
+ * Build a SidIV for the (s,p,o) and binding it on the sid
+ * variable.
+ *
+ * @see <a
+ * href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ *
+ * TODO This is RDF specific code. It would be nice if we
+ * did not have to put it into BOpContext.
+ */
+
+ final IV s = (IV) e.get(0);
+ final IV p = (IV) e.get(1);
+ final IV o = (IV) e.get(2);
+ final ISPO spo = new SPO(s, p, o);
+ final SidIV sidIV = new SidIV<BigdataBNode>(spo);
+
+ bindingSet.set(sidVar, new Constant(sidIV));
+
+ }
+
+ }
+
}
// /**
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/bnode/SidIV.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/bnode/SidIV.java 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/internal/impl/bnode/SidIV.java 2012-05-10 15:28:21 UTC (rev 6314)
@@ -200,12 +200,20 @@
return bi.toString();
}
- public boolean equals(Object o) {
+ /**
+ * Two {@link SidIV} are equal if their (s,p,o) IVs are equal.
+ */
+ public boolean equals(final Object o) {
if (this == o)
return true;
if (o instanceof SidIV) {
- final ISPO spo2 = ((SidIV) o).spo;
- return spo.equals(spo2);
+ final ISPO stmt2 = ((SidIV<?>) o).spo;
+ return
+ IVUtility.equals(spo.s(), stmt2.s()) && //
+ IVUtility.equals(spo.p(), stmt2.p()) && //
+ IVUtility.equals(spo.o(), stmt2.o())//
+ ;
+// return spo.equals(spo2);
}
return false;
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java 2012-05-10 15:28:21 UTC (rev 6314)
@@ -27,10 +27,12 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.NV;
import com.bigdata.bop.ap.Predicate;
import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.impl.bnode.SidIV;
import com.bigdata.relation.rule.IAccessPathExpander;
/**
@@ -51,6 +53,15 @@
*/
private static final long serialVersionUID = 3517916629931687107L;
+ public static interface Annotations extends Predicate.Annotations {
+
+ /**
+ * The SID {@link IVariable} (optional).
+ */
+ String SID = SPOPredicate.class.getName() + ".sid";
+
+ }
+
/**
* Variable argument version of the shallow copy constructor.
*/
@@ -267,6 +278,9 @@
}
+ /**
+ * The variable or constant for the subject position (required).
+ */
@SuppressWarnings({ "unchecked", "rawtypes" })
final public IVariableOrConstant<IV> s() {
@@ -274,6 +288,9 @@
}
+ /**
+ * The variable or constant for the predicate position (required).
+ */
@SuppressWarnings({ "unchecked", "rawtypes" })
final public IVariableOrConstant<IV> p() {
@@ -281,6 +298,9 @@
}
+ /**
+ * The variable or constant for the object position (required).
+ */
@SuppressWarnings({ "unchecked", "rawtypes" })
final public IVariableOrConstant<IV> o() {
@@ -288,12 +308,31 @@
}
+ /**
+ * The variable or constant for the context position (required iff in quads
+ * mode).
+ */
@SuppressWarnings({ "unchecked", "rawtypes" })
final public IVariableOrConstant<IV> c() {
return (IVariableOrConstant<IV>) get(3/* c */);
}
+
+ /**
+ * The variable for the statement identifier (optional). The statement
+ * identifier is the composition of the (subject, predicate, and object)
+ * positions of the fully predicate. When this variable is declared, it will
+ * be bound to a {@link SidIV} for matched statements.
+ *
+ * @see Annotations#SID
+ */
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ final public IVariable<IV> sid() {
+
+ return (IVariable<IV>) getProperty(Annotations.SID);
+
+ }
/*
* Note: Moved to Predicate. See notes there before putting back into
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/TestReificationDoneRightEval.java 2012-05-10 15:28:21 UTC (rev 6314)
@@ -28,7 +28,20 @@
package com.bigdata.rdf.sparql.ast.eval.reif;
import com.bigdata.bop.ap.Predicate;
+import com.bigdata.rdf.internal.XSD;
+import com.bigdata.rdf.internal.impl.bnode.SidIV;
+import com.bigdata.rdf.model.BigdataBNode;
+import com.bigdata.rdf.model.BigdataLiteral;
+import com.bigdata.rdf.model.BigdataStatement;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.model.StatementEnum;
import com.bigdata.rdf.sparql.ast.eval.AbstractDataDrivenSPARQLTestCase;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.spo.SPO;
+import com.bigdata.rdf.store.AbstractTripleStore;
+import com.bigdata.rdf.vocab.decls.DCTermsVocabularyDecl;
/**
* Reification Done Right test suite.
@@ -83,9 +96,147 @@
super(name);
}
+ /**
+ * Bootstrap test. The data are explicitly entered into the KB by hand. This
+ * makes it possible to test evaluation without having to fix the RDF data
+ * loader. The query is based on <code>rdf-02</code>.
+ */
+ public void test_reificationDoneRight_00() throws Exception {
+
+ final BigdataValueFactory vf = store.getValueFactory();
+
+ final BigdataURI SAP = vf.createURI("http://example.com/SAP");
+ final BigdataURI bought = vf.createURI("http://example.com/bought");
+ final BigdataURI sybase = vf.createURI("http://example.com/sybase");
+ final BigdataURI dcSource = vf.asValue(DCTermsVocabularyDecl.source);
+ final BigdataURI dcCreated = vf.asValue(DCTermsVocabularyDecl.created);
+ final BigdataURI newsSybase = vf.createURI("http://example.com/news/us-sybase");
+ final BigdataLiteral createdDate = vf.createLiteral("2011-04-05T12:00:00Z",XSD.DATETIME);
+ final BigdataURI g1 = vf.createURI("http://example.com/g1");
+
+ // Add/resolve the terms against the lexicon.
+ final BigdataValue[] terms = new BigdataValue[] { SAP, bought, sybase,
+ dcSource, dcCreated, newsSybase, createdDate, g1 };
+
+ final BigdataURI context = store.isQuads() ? g1 : null;
+
+ store.addTerms(terms);
+
+ // ground statement.
+ final BigdataStatement s0 = vf.createStatement(SAP, bought, sybase,
+ context, StatementEnum.Explicit);
+
+ // Setup blank node with SidIV for that Statement.
+ final BigdataBNode s1 = vf.createBNode("s1");
+ s1.setStatementIdentifier(true);
+ final ISPO spo = new SPO(SAP.getIV(), bought.getIV(), sybase.getIV(),
+ null/* NO CONTEXT */, StatementEnum.Explicit);
+ s1.setIV(new SidIV<BigdataBNode>(spo));
+
+ // metadata statements.
+
+ final BigdataStatement mds1 = vf.createStatement(s1, dcSource,
+ newsSybase, context, StatementEnum.Explicit);
+
+ final BigdataStatement mds2 = vf.createStatement(s1, dcCreated,
+ createdDate, context, StatementEnum.Explicit);
+
+ final ISPO[] stmts = new ISPO[] { s0, mds1, mds2 };
+
+ store.addStatements(stmts, stmts.length);
+
+ /*
+ * Now that we have populated the database, we can go ahead and compile
+ * the query. (If we compile the query first then it will not find any
+ * matching lexical items.)
+ */
+
+ final TestHelper h = new TestHelper("reif/rdr-00", // testURI,
+ "reif/rdr-02.rq",// queryFileURL
+ "reif/empty.ttl",// dataFileURL
+ "reif/rdr-02.srx"// resultFileURL
+ );
+
+ h.runTest();
+
+ }
+
+ /**
+ * Bootstrap test. The data are explicitly entered into the KB by hand. This
+ * makes it possible to test evaluation without having to fix the RDF data
+ * loader. The query is based on <code>rdf-02a</code>.
+ */
+ public void test_reificationDoneRight_00a() throws Exception {
+
+ final BigdataValueFactory vf = store.getValueFactory();
+
+ final BigdataURI SAP = vf.createURI("http://example.com/SAP");
+ final BigdataURI bought = vf.createURI("http://example.com/bought");
+ final BigdataURI sybase = vf.createURI("http://example.com/sybase");
+ final BigdataURI dcSource = vf.asValue(DCTermsVocabularyDecl.source);
+ final BigdataURI dcCreated = vf.asValue(DCTermsVocabularyDecl.created);
+ final BigdataURI newsSybase = vf.createURI("http://example.com/news/us-sybase");
+ final BigdataLiteral createdDate = vf.createLiteral("2011-04-05T12:00:00Z",XSD.DATETIME);
+ final BigdataURI g1 = vf.createURI("http://example.com/g1");
+
+ // Add/resolve the terms against the lexicon.
+ final BigdataValue[] terms = new BigdataValue[] { SAP, bought, sybase,
+ dcSource, dcCreated, newsSybase, createdDate, g1 };
+
+ final BigdataURI context = store.isQuads() ? g1 : null;
+
+ store.addTerms(terms);
+
+ // ground statement.
+ final BigdataStatement s0 = vf.createStatement(SAP, bought, sybase,
+ context, StatementEnum.Explicit);
+
+ // Setup blank node with SidIV for that Statement.
+ final BigdataBNode s1 = vf.createBNode("s1");
+ s1.setStatementIdentifier(true);
+ final ISPO spo = new SPO(SAP.getIV(), bought.getIV(), sybase.getIV(),
+ null/* NO CONTEXT */, StatementEnum.Explicit);
+ s1.setIV(new SidIV<BigdataBNode>(spo));
+
+ // metadata statements.
+
+ final BigdataStatement mds1 = vf.createStatement(s1, dcSource,
+ newsSybase, context, StatementEnum.Explicit);
+
+ final BigdataStatement mds2 = vf.createStatement(s1, dcCreated,
+ createdDate, context, StatementEnum.Explicit);
+
+ final ISPO[] stmts = new ISPO[] { s0, mds1, mds2 };
+
+ store.addStatements(stmts, stmts.length);
+
+ /*
+ * Now that we have populated the database, we can go ahead and compile
+ * the query. (If we compile the query first then it will not find any
+ * matching lexical items.)
+ */
+
+ final TestHelper h = new TestHelper("reif/rdr-00a", // testURI,
+ "reif/rdr-02a.rq",// queryFileURL
+ "reif/empty.ttl",// dataFileURL
+ "reif/rdr-02a.srx"// resultFileURL
+ );
+
+ h.runTest();
+
+ }
+
/**
- * Simple query involving alice, bob, and an information extractor.
- */
+ * Simple query involving alice, bob, and an information extractor.
+ *
+ * <pre>
+ * select ?src where {
+ * ?x foaf:name "Alice" .
+ * ?y foaf:name "Bob" .
+ * <<?x foaf:knows ?y>> dc:source ?src .
+ * }
+ * </pre>
+ */
public void test_reificationDoneRight_01() throws Exception {
new TestHelper("reif/rdr-01", // testURI,
@@ -97,8 +248,18 @@
}
/**
- * Same data, but the query uses the BIND() syntax and pulls out some
- * more information.
+ * Same data, but the query uses the BIND() syntax and pulls out some more
+ * information.
+ *
+ * <pre>
+ * select ?who ?src ?conf where {
+ * ?x foaf:name "Alice" .
+ * ?y foaf:name ?who .
+ * BIND( <<?x foaf:knows ?y>> as ?sid ) .
+ * ?sid dc:source ?src .
+ * ?sid rv:confidence ?src .
+ * }
+ * </pre>
*/
public void test_reificationDoneRight_01a() throws Exception {
@@ -112,6 +273,12 @@
/**
* Simple query ("who bought sybase").
+ *
+ * <pre>
+ * SELECT ?src ?who {
+ * <<?who :bought :sybase>> dc:source ?src
+ * }
+ * </pre>
*/
public void test_reificationDoneRight_02() throws Exception {
@@ -124,8 +291,16 @@
}
/**
- * Same data, but the query uses the BIND() syntax and pulls out some
- * more information.
+ * Same data, but the query uses the BIND() syntax and pulls out some more
+ * information.
+ *
+ * <pre>
+ * SELECT ?src ?who ?created {
+ * BIND( <<?who :bought :sybase>> as ?sid ) .
+ * ?sid dc:source ?src .
+ * OPTIONAL {?sid dc:created ?created}
+ * }
+ * </pre>
*/
public void test_reificationDoneRight_02a() throws Exception {
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/empty.ttl
===================================================================
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-01a.srx 2012-05-10 15:28:21 UTC (rev 6314)
@@ -18,7 +18,7 @@
<uri>http://reasoner.example.com/engines#engine_1</uri>
</binding>
<binding name="conf">
- <literal datatype="xsd:decimal">0.9835</literal>
+ <literal datatype="http://www.w3.org/2001/XMLSchema#decimal">0.9835</literal>
</binding>
</result>
</results>
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02.ttl 2012-05-10 15:28:21 UTC (rev 6314)
@@ -8,6 +8,7 @@
@prefix dc: <http://purl.org/dc/terms/> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+:SAP :bought :sybase .
_:s1 rdf:subject :SAP .
_:s1 rdf:predicate :bought .
_:s1 rdf:object :sybase .
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx 2012-05-09 15:31:31 UTC (rev 6313)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/eval/reif/rdr-02a.srx 2012-05-10 15:28:21 UTC (rev 6314)
@@ -18,7 +18,7 @@
<uri>http://example.com/news/us-sybase</uri>
</binding>
<binding name="created">
- <literal datatype="xsd:dateTime">2011-04-05T12:00:00Z</literal>
+ <literal datatype="http://www.w3.org/2001/XMLSchema#dateTime">2011-04-05T12:00:00Z</literal>
</binding>
</result>
</results>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2012-05-31 07:49:02
|
Revision: 6322
http://bigdata.svn.sourceforge.net/bigdata/?rev=6322&view=rev
Author: martyncutcher
Date: 2012-05-31 07:48:48 +0000 (Thu, 31 May 2012)
Log Message:
-----------
synchronization commit for initial GOM development. The two object managers - local and remote - are broadly functional. See TestGOM and TestRemoteGOM for example usage.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/IGPO.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/ILinkSet.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/IObjectManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/skin/GenericSkinRegistry.java
branches/BIGDATA_RELEASE_1_2_0/ctc-striterators/src/java/cutthecrap/utils/striterators/EmptyIterator.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/Alchemist.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/owl/
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/BasicSkin.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/GPO.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/LinkSet.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/NanoSparqlObjectManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectMgrModel.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/TestGOM.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/TestOwlGOM.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/TestRemoteGOM.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/testgom.n3
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/testowl.xml
branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/testowl2.xml
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/Alchemist.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/Alchemist.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/Alchemist.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,15 @@
+package com.bigdata.gom.alchemy;
+
+/**
+ * The RDF Alchemist will generate interface and implementation files to provide a
+ * typed interface to the underlying GPO representations.
+ *
+ * The pattern generated will exploit the Skin pattern to enable multiple typed
+ * interfaces to the same object.
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class Alchemist {
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/alchemy/Alchemist.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/BasicSkin.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/BasicSkin.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/BasicSkin.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,136 @@
+package com.bigdata.gom.gpo;
+
+import java.util.Calendar;
+import java.util.Date;
+
+import javax.xml.datatype.XMLGregorianCalendar;
+
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.CalendarLiteralImpl;
+import org.openrdf.model.impl.IntegerLiteralImpl;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.impl.NumericLiteralImpl;
+import org.semanticweb.yars.nx.NumericLiteral;
+
+import com.sun.org.apache.xerces.internal.jaxp.datatype.XMLGregorianCalendarImpl;
+
+/**
+ * The BasicSkin provides standard type wrappers to help with conversions
+ * @author Martyn Cutcher
+ *
+ */
+public class BasicSkin implements IGenericSkin {
+
+ final IGPO m_gpo;
+ final ValueFactory m_vf;
+
+ public BasicSkin(final IGPO gpo) {
+ m_gpo = gpo;
+ m_vf = gpo.getObjectManager().getValueFactory();
+ }
+
+ public void setValue(final URI property, final Value value) {
+ m_gpo.setValue(property, value);
+ }
+
+ public void setValue(final URI property, final String value) {
+ m_gpo.setValue(property, m_vf.createLiteral(value));
+ }
+
+ public void setValue(final URI property, final int value) {
+ m_gpo.setValue(property, m_vf.createLiteral(value));
+ }
+
+ public void setValue(final URI property, final double value) {
+ m_gpo.setValue(property, m_vf.createLiteral(value));
+ }
+
+ public void setValue(final URI property, final XMLGregorianCalendar value) {
+ m_gpo.setValue(property, m_vf.createLiteral(value));
+ }
+
+ public void setValue(final String property, final Value value) {
+ m_gpo.setValue(m_vf.createURI(property), value);
+ }
+
+ public void setValue(final String property, final String value) {
+ m_gpo.setValue(m_vf.createURI(property), m_vf.createLiteral(value));
+ }
+
+ public void setValue(final String property, final int value) {
+ m_gpo.setValue(m_vf.createURI(property), m_vf.createLiteral(value));
+ }
+
+ public void setValue(final String property, final double value) {
+ m_gpo.setValue(m_vf.createURI(property), m_vf.createLiteral(value));
+ }
+
+ public void setValue(final String property, final XMLGregorianCalendar value) {
+ m_gpo.setValue(m_vf.createURI(property), m_vf.createLiteral(value));
+ }
+
+ @Override
+ public IGPO asGeneric() {
+ return m_gpo;
+ }
+
+ public int getIntValue(final URI key) {
+ final Value v = m_gpo.getValue(key);
+
+ if (v instanceof Literal) {
+ return ((Literal) v).intValue();
+ } else {
+ return 0;
+ }
+ }
+
+ public double getDoubleValue(final URI key) {
+ final Value v = m_gpo.getValue(key);
+
+ if (v instanceof Literal) {
+ return ((Literal) v).doubleValue();
+ } else {
+ return 0;
+ }
+ }
+
+ public boolean getBooleanValue(final URI key) {
+ final Value v = m_gpo.getValue(key);
+
+ if (v instanceof Literal) {
+ return ((Literal) v).booleanValue();
+ } else {
+ return false;
+ }
+ }
+
+ public String getStringValue(final URI key) {
+ final Value v = m_gpo.getValue(key);
+
+ if (v instanceof Literal) {
+ return ((Literal) v).stringValue();
+ } else {
+ return null;
+ }
+ }
+
+ public int getInt(final String key) {
+ return getIntValue(m_vf.createURI(key));
+ }
+
+ public double getDouble(final String key) {
+ return getDoubleValue(m_vf.createURI(key));
+ }
+
+ public boolean getBoolean(final String key) {
+ return getBooleanValue(m_vf.createURI(key));
+ }
+
+ public String getString(final String key) {
+ return getStringValue(m_vf.createURI(key));
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/BasicSkin.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/GPO.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/GPO.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/GPO.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,651 @@
+package com.bigdata.gom.gpo;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.repository.RepositoryException;
+
+import com.bigdata.gom.om.IObjectManager;
+import com.bigdata.gom.om.ObjectMgrModel;
+import com.bigdata.gom.skin.GenericSkinRegistry;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.model.BigdataResource;
+import com.bigdata.rdf.model.BigdataStatement;
+import com.bigdata.rdf.model.BigdataStatementImpl;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactoryImpl;
+import com.bigdata.rdf.model.StatementEnum;
+import com.bigdata.striterator.ICloseableIterator;
+
+/**
+ * A TripleStore backed GPO yields a number of challenges.
+ *
+ * <ol>
+ * <li> We need a property mechanism to track the statement assertion
+ * and retractions required for efficient update. The GPOEntry class
+ * provides support for this with its linked lists of GPOValues, also
+ * supporting multiple values per predicate.</li>
+ *
+ * <li>A strategy for lazy materialization. This turns out to be fairly
+ * straightforward, with the GPO created unmaterialized and requesting
+ * as necessary</li>
+ *
+ * <li>Most importantly we have a consistency/referential integrity
+ * issue since the GPO state is not kept in sync with the underlying
+ * triple store. The simple solution is to ensure that any update
+ * is immediately pushed through to the underlying store, but this
+ * loses the advantage of the localised cache. Although it could be
+ * argued that a general query is made against committed data the
+ * same case cannot be made when building GPO models. It seems we
+ * need a mechanism to lazily flush updates relevant to a specific
+ * property.</li>
+ *
+ * <li>An alternate solution is to view the ObjectManager transactions
+ * as transactional updates to the GPO structures and that when these
+ * are committed then the model is consistent. Separating the
+ * GPO commit from any underlying TripleStore commit. So the GPO processing
+ * inside the GPO transaction is restricted and referential integrity
+ * is not maintained. This is similar to the read-committed query
+ * semantics of most databases, which is not to commend it in any way. I am
+ * keen to find a solution to this.</li>
+ *
+ * <li>One option is to force GPO related
+ * updates through to the underlying store, not necessarily a problem since
+ * a rollback already performs a TripleStore abort anyhow.</li>
+ * </ol>
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class GPO implements IGPO {
+
+ private static final Logger log = Logger.getLogger(GPO.class);
+
+ final IObjectManager m_om;
+ final Resource m_id;
+
+ boolean m_materialized = false;
+
+ boolean m_clean = true;
+
+ ArrayList<IGenericSkin> m_skins = null;
+
+ static class LinkValue {
+ final Value m_value;
+ LinkValue m_next;
+
+ LinkValue(Value value) {
+ m_value = value;
+ }
+ }
+ /**
+ * The GPOEntry retains the state necessary for providing delta updates to the underlying
+ * triple data. It supports multi-values against the same property and records values
+ * removed and added.
+ */
+ static class GPOEntry {
+ final URI m_key;
+ GPOEntry m_next;
+
+ LinkValue m_values;
+ LinkValue m_addedValues;
+ LinkValue m_removedValues;
+
+ GPOEntry(final URI key) {
+ assert key != null;
+
+ m_key = key;
+ }
+
+
+ /**
+ * initValue is called by ObjectManager materialize and resets the GPO
+ * to its read state. Therefore the value is added to the m_values
+ * list and not m_addedValues.
+ */
+ public void initValue(Value value) {
+ final LinkValue newValue = new LinkValue(value);
+ newValue.m_next = m_values;
+ m_values = newValue;
+
+ }
+
+ public void set(final Value value) {
+ m_addedValues = new LinkValue(value);
+
+ // move m_values to m_removedValues
+ LinkValue nxt = m_values;
+ while (nxt != null) {
+ LinkValue rem = nxt;
+ nxt = nxt.m_next;
+
+ rem.m_next = m_removedValues;
+ m_removedValues = rem;
+ }
+ // and clear any current values
+ m_values = null;
+ }
+
+ public Iterator<Value> values() {
+ return new Iterator<Value>() {
+ LinkValue m_cur = m_values;
+ LinkValue m_added = m_addedValues;
+ @Override
+ public boolean hasNext() {
+ return m_cur != null || m_added != null;
+ }
+
+ @Override
+ public Value next() {
+ final LinkValue ret = m_cur != null ? m_cur : m_added;
+ if (ret == null) {
+ throw new NoSuchElementException();
+ }
+ if (m_cur != null) {
+ m_cur = m_cur.m_next;
+ } else {
+ m_added = m_added.m_next;
+ }
+
+ return ret.m_value;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ static class ValueIterator implements Iterator<Value> {
+ LinkValue m_cur;
+
+ ValueIterator(final LinkValue cur) {
+ m_cur = cur;
+ }
+ @Override
+ public boolean hasNext() {
+ return m_cur != null;
+ }
+
+ @Override
+ public Value next() {
+ final LinkValue ret = m_cur;
+ m_cur = m_cur.m_next;
+
+ return ret.m_value;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ public Iterator<Value> removes() {
+ Iterator<Value> ret = new ValueIterator(m_removedValues);
+ return ret;
+ }
+
+ public Iterator<Value> additions() {
+ Iterator<Value> ret = new ValueIterator(m_addedValues);
+ return ret;
+ }
+
+ public Value getValue() {
+ if (m_values != null) {
+ return m_values.m_value;
+ } else if (m_addedValues != null) {
+ return m_addedValues.m_value;
+ } else {
+ return null;
+ }
+ }
+
+
+ /**
+ * Committing the entry checks for any added values and
+ * moves this to the values chain.
+ */
+ public void commit() {
+ if (m_addedValues != null) {
+ if (m_values == null) {
+ m_values = m_addedValues;
+ } else {
+ LinkValue tail = m_values;
+ while (tail.m_next != null)
+ tail = tail.m_next;
+ tail.m_next = m_addedValues;
+ }
+ m_addedValues = null;
+ }
+ m_removedValues = null;
+ }
+
+ /**
+ * A new value is only added if it does not already exist, ensuring the GPO maintains
+ * semantics with the underlying TripleStore.
+ *
+ * @return true if value was added
+ */
+ public boolean add(final Value value) {
+ final Iterator<Value> values = values();
+ while (values.hasNext()) {
+ if (values.next().equals(value)) {
+ return false;
+ }
+ }
+
+ final LinkValue nv = new LinkValue(value);
+ nv.m_next = m_addedValues;
+ m_addedValues = nv;
+
+ return true;
+ }
+ }
+ GPOEntry m_headEntry = null;
+ GPOEntry m_tailEntry = null;
+
+ GPOEntry establishEntry(final URI key) {
+ final URI fkey = m_om.internKey(key);
+ if (m_headEntry == null) {
+ m_headEntry = m_tailEntry = new GPOEntry(fkey);
+ } else {
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ if (entry.m_key == fkey) {
+ return entry;
+ }
+ entry = entry.m_next;
+ }
+ m_tailEntry = m_tailEntry.m_next = new GPOEntry(fkey);
+ }
+
+ return m_tailEntry;
+ }
+
+ /**
+ * The entry is interned to provide a unique Object and allow
+ * '==' testing.
+ *
+ * @param key
+ * @return the found entry, if any
+ */
+ GPOEntry getEntry(final URI key) {
+ final URI fkey = m_om.internKey(key);
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ if (entry.m_key == fkey) {
+ return entry;
+ }
+ entry = entry.m_next;
+ }
+ return null;
+ }
+
+ public void reset() {
+ m_headEntry = m_tailEntry = null;
+ }
+
+ public GPO(IObjectManager om, Resource id) {
+ m_om = om;
+ m_id = id;
+ }
+
+ @Override
+ public IGenericSkin asClass(Class theClassOrInterface) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Resource getId() {
+ return m_id;
+ }
+
+ /**
+ * getLinksIn simply filters the values for Resources and returns a
+ * wrapper.
+ */
+ @Override
+ public Set<IGPO> getLinksIn() {
+ // Does not require full materialization!
+
+ final String query = "SELECT ?x WHERE {?x ?p <" + getId().toString() + ">}";
+ System.out.println("Query: " + query);
+ final ICloseableIterator<BindingSet> res = m_om.evaluate(query);
+
+ final HashSet<IGPO> ret = new HashSet<IGPO>();
+ while (res.hasNext()) {
+ final BindingSet bs = res.next();
+ ret.add(m_om.getGPO((Resource) bs.getBinding("x").getValue()));
+ }
+
+ return ret;
+ }
+
+ /** All ?y where (?y,p,self). */
+ @Override
+ public ILinkSet getLinksIn(URI property) {
+ return new LinkSet(this, property, true);
+ }
+
+ /** All ?y where (self,?,?y). */
+ @Override
+ public Set<IGPO> getLinksOut() {
+ materialize();
+
+ final HashSet<IGPO> ret = new HashSet<IGPO>();
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ Iterator<Value> values = entry.values();
+ while (values.hasNext()) {
+ final Value value = values.next();
+ if (value instanceof Resource) {
+ ret.add(m_om.getGPO((Resource) value));
+ }
+ }
+ entry = entry.m_next;
+ }
+
+ return ret;
+ }
+
+ @Override
+ public ILinkSet getLinksOut(URI property) {
+ materialize();
+
+ return new LinkSet(this, property, false);
+ }
+
+ @Override
+ public IObjectManager getObjectManager() {
+ return m_om;
+ }
+
+ @Override
+ public Map<URI, Long> getReverseLinkProperties() {
+ materialize();
+
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Set<Statement> getStatements() {
+ materialize();
+
+ final HashSet<Statement> out = new HashSet<Statement>();
+
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ Iterator<Value> values = entry.values();
+ while (values.hasNext())
+ out.add(makeStatement(m_id, entry.m_key, values.next()));
+
+ entry = entry.m_next;
+ }
+
+ return out;
+ }
+
+ private Statement makeStatement(final Resource id, final URI key, final Value value) {
+ return m_om.getValueFactory().createStatement(id, key, value);
+ }
+
+ @Override
+ public Value getValue(final URI property) {
+ materialize();
+
+ final GPOEntry entry = getEntry(property);
+ return entry != null ? entry.getValue() : null;
+ }
+
+ @Override
+ public Set<Value> getValues(URI property) {
+ materialize();
+
+ return null;
+ }
+
+ @Override
+ public boolean isBound(URI property) {
+ materialize();
+
+ return getValue(property) != null;
+ }
+
+ @Override
+ public boolean isMemberOf(ILinkSet linkSet) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public void remove() {
+ m_om.remove(this);
+ }
+
+ public void dematerialize() {
+ m_materialized = false;
+ m_clean = true;
+ m_headEntry = m_tailEntry = null;
+ }
+
+ public void initValue(URI predicate, Value object) {
+ assert !m_materialized;
+
+ final GPOEntry entry = establishEntry(predicate);
+ entry.initValue(object);
+ }
+
+ @Override
+ public void setValue( final URI property, final Value newValue) {
+ materialize();
+
+ final GPOEntry entry = establishEntry(property);
+ entry.set(newValue);
+
+ if (false && newValue instanceof Resource) {
+ try {
+ update(entry);
+ } catch (RepositoryException e) {
+ throw new RuntimeException("Unable to update", e);
+ }
+ } else {
+ dirty();
+ }
+ }
+
+ private void dirty() {
+ if (m_clean) {
+ m_clean = false;
+ ((ObjectMgrModel) m_om).addToDirtyList(this);
+ }
+ }
+
+ @Override
+ public IGPO asGeneric() {
+ return this;
+ }
+
+ @Override
+ public String pp() {
+ materialize();
+
+ final StringBuilder out = new StringBuilder("ID: " + m_id.stringValue() + "\n");
+
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ Iterator<Value> values = entry.values();
+ while (values.hasNext())
+ out.append(entry.m_key.toString() + ": " + values.next().toString() + "\n");
+
+ entry = entry.m_next;
+ }
+
+ return out.toString();
+ }
+
+ @Override
+ public IGPO getType() {
+ materialize();
+
+ final URI tid = (URI) getValue(new URIImpl("attr:/type"));
+ if (tid != null) {
+ return m_om.getGPO(tid);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Called by the ObjectManager when flushing dirty objects. This can occur
+ * incrementally or on ObjectManager commit.
+ *
+ * The object is marked as clean once written.
+ *
+ * @throws RepositoryException
+ */
+ public void update() throws RepositoryException {
+ assert m_materialized;
+
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ update(entry);
+
+ entry = entry.m_next;
+ }
+
+ m_clean = true;
+ }
+
+ private void update(final GPOEntry entry) throws RepositoryException {
+ assert m_materialized;
+
+ final Iterator<Value> removes = entry.removes();
+ while (removes.hasNext()) {
+ m_om.retract(m_id, entry.m_key, removes.next());
+ }
+
+ final Iterator<Value> inserts = entry.additions();
+ while (inserts.hasNext()) {
+ m_om.insert(m_id, entry.m_key, inserts.next());
+ }
+
+ entry.commit();
+ }
+
+ /**
+ * Basis for lazy materialization, checks materialize state and if false
+ * requests matierialization from the ObjectManager
+ */
+ public void materialize() {
+ if (!m_materialized) {
+ synchronized (this) {
+ if (!m_materialized) {
+ m_om.materialize(this);
+ m_materialized = true;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void addValue(final URI property, final Value value) {
+ materialize();
+
+ final GPOEntry entry = establishEntry(property);
+ if (entry.add(value)) {
+ dirty();
+ }
+ }
+
+ public void setMaterialized(boolean b) {
+ m_materialized = true;
+ }
+
+ public void prepareBatchTerms() {
+ final ObjectMgrModel oom = (ObjectMgrModel) m_om;
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ final Iterator<Value> inserts = entry.additions();
+ while (inserts.hasNext()) {
+ final Value v = inserts.next();
+ oom.checkValue(v);
+ }
+
+ entry = entry.m_next;
+ }
+ }
+
+ /**
+ * adds statements for batch update
+ */
+ public void prepareBatchUpdate() {
+ final ObjectMgrModel oom = (ObjectMgrModel) m_om;
+ GPOEntry entry = m_headEntry;
+ while (entry != null) {
+ final Iterator<Value> inserts = entry.additions();
+ while (inserts.hasNext()) {
+ final Value v = inserts.next();
+ oom.insertBatch(m_id, entry.m_key, v);
+ }
+ final Iterator<Value> removes = entry.removes();
+ while (removes.hasNext()) {
+ final Value v = removes.next();
+ oom.removeBatch(m_id, entry.m_key, v);
+ }
+
+ entry = entry.m_next;
+ }
+ }
+
+ /**
+ * The getSkin method is inspired somewhat by the Microsoft Win32 getInterface
+ * that allowed an object to return multiple interfaces. The difference with
+ * the GPO skin is that the skin should be able to interact with any underlying
+ * GPO object.
+ *
+ * <p>It may be worthwhile performance wise to cache a skin. I believe of more
+ * importance is to preserve identity - even of the interface/skin object.
+ *
+ * @param skin interface required
+ * @return a skin if available
+ */
+ public IGenericSkin getSkin(final Class intf) {
+ IGenericSkin ret = null;
+ if (m_skins != null) {
+ for (int i = 0; i < m_skins.size(); i++) {
+ if (intf.isInstance(m_skins.get(i))) {
+ return m_skins.get(i);
+ }
+ }
+ } else {
+ m_skins = new ArrayList<IGenericSkin>(2);
+ }
+
+ ret = GenericSkinRegistry.asClass(this, intf);
+ m_skins.add(ret);
+
+ return ret;
+ }
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/GPO.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/IGPO.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/IGPO.java 2012-05-16 08:38:12 UTC (rev 6321)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/IGPO.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -111,6 +111,9 @@
/** Replace (self,p,?x) with (self,p,newValue). */
void setValue(URI property, Value newValue);
+ /** Assert (self,p,newValue). */
+ void addValue(URI property, Value newValue);
+
/** All (self,?p,?o). */
Set<Statement> getStatements();
@@ -189,4 +192,15 @@
*/
IGenericSkin asClass(Class theClassOrInterface);
+ /**
+ * @return a pretty printed representation of the GPO
+ */
+ String pp();
+
+ /**
+ * FIXME: this method will be moved to an as yet unnamed derived class
+ * that will become the superclass for alchemist generated subclasses.
+ */
+ IGPO getType();
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/ILinkSet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/ILinkSet.java 2012-05-16 08:38:12 UTC (rev 6321)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/ILinkSet.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -32,6 +32,15 @@
/**
* A collection of links into (edges in) or links out of (edges out) of an
* {@link IGPO}.
+ *
+ * <p>Note that the links out are only intended to be used to represent
+ * many-many associations. Standard one-many associations should be represented
+ * by linksIn (many links pointing to one resource rather than many links from
+ * one resource pointing to many other resources).
+ *
+ * <p>The LinksOut as the low-cardinality part of a many-many association reflect
+ * common asymmetry of such associations. Experience suggests that if the
+ * association is more symmetric the cardinalities tend not to be large.
*/
public interface ILinkSet extends Set<IGPO> {
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/LinkSet.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/LinkSet.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/LinkSet.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,229 @@
+package com.bigdata.gom.gpo;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.openrdf.model.Resource;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+
+import com.bigdata.gom.om.IObjectManager;
+import com.bigdata.striterator.ICloseableIterator;
+
+import cutthecrap.utils.striterators.EmptyIterator;
+
+public class LinkSet implements ILinkSet {
+ final IGPO m_owner;
+ final URI m_linkProperty;
+ final boolean m_linksIn;
+
+ public LinkSet(final IGPO owner, final URI linkProperty, final boolean linksIn) {
+ m_owner = owner;
+ m_linkProperty = linkProperty;
+ m_linksIn = linksIn;
+ }
+
+ @Override
+ public URI getLinkProperty() {
+ return m_linkProperty;
+ }
+
+ @Override
+ public IGPO getOwner() {
+ return m_owner;
+ }
+
+ @Override
+ public boolean isLinkSetIn() {
+ return m_linksIn;
+ }
+
+ @Override
+ public <C> Iterator<C> iterator(Class<C> theClassOrInterface) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public int size() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public long sizeLong() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public boolean add(IGPO arg0) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean addAll(Collection<? extends IGPO> arg0) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public void clear() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean contains(Object arg) {
+ if (!(arg instanceof IGPO)) {
+ throw new IllegalArgumentException("IGPO required");
+ }
+ IGPO gpo = (IGPO) arg;
+ return gpo.getValue(m_linkProperty) == m_owner;
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> arg0) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public Iterator<IGPO> iterator() {
+ if (m_linksIn) {
+ final IObjectManager om = m_owner.getObjectManager();
+
+ final String query = "SELECT ?x WHERE {?x <" + m_linkProperty.toString() + "> <" + m_owner.getId().toString() + ">}";
+ System.out.println("Query: " + query);
+ final ICloseableIterator<BindingSet> res = om.evaluate(query);
+
+ return new Iterator<IGPO>() {
+
+ @Override
+ public boolean hasNext() {
+ return res != null && res.hasNext();
+ }
+
+ @Override
+ public IGPO next() {
+ final BindingSet bs = res.next();
+
+ return om.getGPO((Resource) bs.getBinding("x").getValue());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ } else {
+ final GPO.GPOEntry entry = ((GPO) m_owner).getEntry(m_linkProperty);
+ if (entry == null) {
+ return new EmptyIterator<IGPO>();
+ }
+
+ return new Iterator<IGPO>() {
+ Iterator<Value> m_values = entry.values();
+ IGPO nextGPO = nextGPO();
+
+ private IGPO nextGPO() {
+ while (m_values.hasNext()) {
+ final Value val = m_values.next();
+ if (val instanceof Resource) {
+ return m_owner.getObjectManager().getGPO((Resource) val);
+ }
+ }
+ return null;
+ }
+ @Override
+ public boolean hasNext() {
+ return nextGPO != null;
+ }
+
+ @Override
+ public IGPO next() {
+ if (nextGPO == null)
+ throw new NoSuchElementException();
+
+ final IGPO ret = nextGPO;
+ nextGPO = nextGPO();
+
+ return ret;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+ }
+
+ @Override
+ public boolean remove(Object obj) {
+ if (!(obj instanceof IGPO)) {
+ throw new IllegalArgumentException("Expected an instance of IGPO");
+ }
+
+ final IGPO gpo = (IGPO) obj;
+
+ final boolean ret;
+ if (m_linksIn) {
+ ret = gpo.getValue(m_linkProperty) == m_owner;
+ if (ret) {
+ gpo.setValue(m_linkProperty, null);
+ }
+ } else { // FIXME: implement linksOut
+ throw new UnsupportedOperationException();
+ }
+
+ return ret;
+ }
+
+ @Override
+ public boolean removeAll(Collection<?> arg0) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean retainAll(Collection<?> arg0) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ /**
+ * Eagerly streams materialized objects into an array
+ */
+ @Override
+ public Object[] toArray() {
+ final ArrayList<Object> out = new ArrayList<Object>();
+
+ final Iterator<IGPO> gpos = iterator();
+ while (gpos.hasNext()) {
+ out.add(gpos.next());
+ }
+
+ return out.toArray();
+ }
+
+ @Override
+ public <T> T[] toArray(T[] arg0) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/gpo/LinkSet.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/IObjectManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/IObjectManager.java 2012-05-16 08:38:12 UTC (rev 6321)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/IObjectManager.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -28,7 +28,12 @@
package com.bigdata.gom.om;
import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
import org.openrdf.query.BindingSet;
+import org.openrdf.repository.RepositoryException;
import com.bigdata.gom.gpo.IGPO;
import com.bigdata.striterator.ICloseableIterator;
@@ -92,4 +97,39 @@
*/
void execute(String updateStr);
+ URI internKey(URI key);
+
+ void retract(Resource id, URI key, Value value) throws RepositoryException;
+
+ void insert(Resource id, URI key, Value value) throws RepositoryException;
+
+ /**
+ * The ObjectManager is able to assign automatic ids for a new object. These
+ * will be of the form "gpo:#[genid]"
+ *
+ * @return a new GPO
+ */
+ IGPO createGPO();
+
+ /**
+ * Simple save/recall interface that the ObjectManager provides to simplify
+ * other pattern implementations. Internally it uses a NameManager GPO
+ */
+ void save(URI key, Value value);
+
+ /**
+ * Simple save/recall interface that the ObjectManager provides to simplify
+ * other pattern implementations. Internally it uses a NameManager GPO
+ */
+ Value recall(URI key);
+
+ IGPO recallAsGPO(URI key);
+
+ void remove(IGPO gpo);
+
+ ICloseableIterator<Statement> evaluateGraph(String query);
+
+ ValueFactory getValueFactory();
+
+ void checkValue(Value newValue);
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/NanoSparqlObjectManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/NanoSparqlObjectManager.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/NanoSparqlObjectManager.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,248 @@
+package com.bigdata.gom.om;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.GraphQuery;
+import org.openrdf.query.GraphQueryResult;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.repository.RepositoryException;
+
+import com.bigdata.gom.gpo.GPO;
+import com.bigdata.gom.gpo.IGPO;
+import com.bigdata.rdf.model.BigdataResource;
+import com.bigdata.rdf.model.BigdataStatementImpl;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactoryImpl;
+import com.bigdata.rdf.model.StatementEnum;
+import com.bigdata.rdf.sail.webapp.client.IPreparedGraphQuery;
+import com.bigdata.rdf.sail.webapp.client.IPreparedTupleQuery;
+import com.bigdata.rdf.sail.webapp.client.IRemoteRepository;
+import com.bigdata.rdf.sail.webapp.client.RemoteRepository;
+import com.bigdata.rdf.sail.webapp.client.RemoteRepository.AddOp;
+import com.bigdata.rdf.sail.webapp.client.RemoteRepository.RemoveOp;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.spo.SPO;
+import com.bigdata.striterator.CloseableIteratorWrapper;
+import com.bigdata.striterator.ICloseableIterator;
+
+public class NanoSparqlObjectManager extends ObjectMgrModel {
+ final RemoteRepository m_repo;
+
+ public NanoSparqlObjectManager(final RemoteRepository repo, final String namespace) {
+ super(BigdataValueFactoryImpl.getInstance(namespace));
+
+ m_repo = repo;
+ }
+
+ @Override
+ public void close() {
+ // m_repo.close();
+ }
+
+ @Override
+ public ICloseableIterator<BindingSet> evaluate(String query) {
+ try {
+ final IPreparedTupleQuery q = m_repo.prepareTupleQuery(query);
+ final TupleQueryResult res = q.evaluate();
+ return new CloseableIteratorWrapper<BindingSet>(new Iterator<BindingSet>() {
+
+ @Override
+ public boolean hasNext() {
+ try {
+ return res.hasNext();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public BindingSet next() {
+ try {
+ return res.next();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ });
+ } catch (RepositoryException e1) {
+ e1.printStackTrace();
+ } catch (MalformedQueryException e1) {
+ e1.printStackTrace();
+ } catch (QueryEvaluationException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ @Override
+ public void execute(String updateStr) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean isPersistent() {
+ return true;
+ }
+
+ @Override
+ public void materialize(IGPO gpo) {
+ if (gpo == null || gpo.getId() == null)
+ throw new IllegalArgumentException("Materialization requires an identity");
+
+ if (log.isTraceEnabled())
+ log.trace("Materializing: " + gpo.getId());
+
+ ((GPO) gpo).reset();
+
+ // At present the DESCRIBE query will simply return a set of
+ // statements equivalent to a TupleQuery <id, ?, ?>
+ final String query = "DESCRIBE <" + gpo.getId().toString() + ">";
+ final ICloseableIterator<Statement> stmts = evaluateGraph(query);
+
+ while (stmts.hasNext()) {
+ final Statement stmt = stmts.next();
+ ((GPO) gpo).initValue(stmt.getPredicate(), stmt.getObject());
+ }
+ }
+
+ @Override
+ public void insert(final Resource id, final URI key, final Value val) {
+ if (log.isTraceEnabled())
+ log.trace("Inserting statement: " + id.stringValue() + " " + key.stringValue() + " " + val.stringValue());
+
+ final Statement statement = m_valueFactory.createStatement(id, key, val);
+ final ArrayList<Statement> batch = new ArrayList<Statement>(1);
+ batch.add(statement);
+ try {
+ m_repo.add(new AddOp(batch));
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to insert statement", e);
+ }
+ }
+
+ @Override
+ public void retract(final Resource id, final URI key, final Value val) {
+ if (false && log.isTraceEnabled())
+ log.trace("Removing statement: " + id.stringValue() + " " + key.stringValue() + " " + val.stringValue());
+
+ try {
+ m_repo.remove(new RemoveOp((URI) id, key, val, null));
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to remove statement", e);
+ }
+ }
+
+ @Override
+ void doCommit() {
+ // FIXME: The current NanoSparqlServer commits each update. This
+ // needs to change to associate with an IsolatedTransaction with
+ // an additional commit/rollback protocol
+ }
+
+ @Override
+ void doRollback() {
+ // FIXME: see comment above for doCommit()
+ }
+
+ @Override
+ public ICloseableIterator<Statement> evaluateGraph(String query) {
+ try {
+ final IPreparedGraphQuery q = m_repo.prepareGraphQuery(query);
+ final GraphQueryResult res = q.evaluate();
+ return new CloseableIteratorWrapper<Statement>(new Iterator<Statement>() {
+
+ @Override
+ public boolean hasNext() {
+ try {
+ return res.hasNext();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Statement next() {
+ try {
+ return res.next();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ });
+ } catch (RepositoryException e1) {
+ e1.printStackTrace();
+ } catch (MalformedQueryException e1) {
+ e1.printStackTrace();
+ } catch (QueryEvaluationException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ @Override
+ public void remove(IGPO gpo) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ void flushTerms() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ void flushStatements() {
+ // handle batch removes
+ try {
+ final RemoveOp rop = m_removes.size() > 0 ? new RemoveOp(m_removes) : null;
+ final AddOp iop = m_inserts.size() > 0 ? new AddOp(m_inserts) : null;
+
+ if (rop != null && iop != null) {
+ m_repo.update(rop, iop);
+ } else if (iop != null) {
+ m_repo.add(iop);
+ } else if (rop != null) {
+ m_repo.remove(rop);
+ }
+
+ m_inserts.clear();
+ m_removes.clear();
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to flush statements", e);
+ }
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/NanoSparqlObjectManager.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectManager.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectManager.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectManager.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,268 @@
+package com.bigdata.gom.om;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.GraphQuery;
+import org.openrdf.query.GraphQueryResult;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.repository.RepositoryException;
+
+import com.bigdata.gom.gpo.GPO;
+import com.bigdata.gom.gpo.IGPO;
+import com.bigdata.rdf.model.BigdataResource;
+import com.bigdata.rdf.model.BigdataStatement;
+import com.bigdata.rdf.model.BigdataStatementImpl;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.StatementEnum;
+import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.spo.SPO;
+import com.bigdata.striterator.CloseableIteratorWrapper;
+import com.bigdata.striterator.ICloseableIterator;
+
+public class ObjectManager extends ObjectMgrModel {
+ private static final Logger log = Logger.getLogger(IObjectManager.class);
+
+ final BigdataSailRepositoryConnection m_cxn;
+
+ public ObjectManager(final BigdataSailRepositoryConnection cxn) {
+ super(cxn.getTripleStore().getValueFactory());
+ m_cxn = cxn;
+ }
+
+ @Override
+ public void close() {
+ try {
+ m_cxn.close();
+ } catch (RepositoryException e) {
+ log.warn("Problem with close", e);
+ }
+ m_dict.clear();
+ }
+
+ @Override
+ public ICloseableIterator<BindingSet> evaluate(final String query) {
+ try {
+ final TupleQuery q = m_cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ final TupleQueryResult res = q.evaluate();
+ return new CloseableIteratorWrapper<BindingSet>(new Iterator<BindingSet>() {
+
+ @Override
+ public boolean hasNext() {
+ try {
+ return res.hasNext();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public BindingSet next() {
+ try {
+ return res.next();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ });
+ } catch (RepositoryException e1) {
+ e1.printStackTrace();
+ } catch (MalformedQueryException e1) {
+ e1.printStackTrace();
+ } catch (QueryEvaluationException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ public ICloseableIterator<Statement> evaluateGraph(final String query) {
+ try {
+ final GraphQuery q = m_cxn.prepareGraphQuery(QueryLanguage.SPARQL, query);
+ final GraphQueryResult res = q.evaluate();
+ return new CloseableIteratorWrapper<Statement>(new Iterator<Statement>() {
+
+ @Override
+ public boolean hasNext() {
+ try {
+ return res.hasNext();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Statement next() {
+ try {
+ return res.next();
+ } catch (QueryEvaluationException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ });
+ } catch (RepositoryException e1) {
+ e1.printStackTrace();
+ } catch (MalformedQueryException e1) {
+ e1.printStackTrace();
+ } catch (QueryEvaluationException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+ @Override
+ public void execute(String updateStr) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean isPersistent() {
+ return true; //
+ }
+
+ @Override
+ public void materialize(IGPO gpo) {
+ if (false && log.isTraceEnabled())
+ log.trace("Materializing: " + gpo.getId());
+
+ ((GPO) gpo).reset();
+
+ // At present the DESCRIBE query will simply return a set of
+ // statements equivalent to a TupleQuery <id, ?, ?>
+ final String query = "DESCRIBE <" + gpo.getId().toString() + ">";
+ final ICloseableIterator<Statement> stmts = evaluateGraph(query);
+
+ while (stmts.hasNext()) {
+ final Statement stmt = stmts.next();
+ ((GPO) gpo).initValue(stmt.getPredicate(), stmt.getObject());
+ }
+ }
+
+ @Override
+ public void insert(Resource id, URI key, Value val) throws RepositoryException {
+ if (false && log.isTraceEnabled())
+ log.trace("Inserting statement: " + id.stringValue() + " " + key.stringValue() + " " + val.stringValue());
+
+ // experiment with adding using batch syntax
+ if (false) {
+ // m_cxn.getTripleStore().addStatement(id, key, val);
+ m_cxn.add(id, key, val);
+ } else {
+ final ISPO spo = new BigdataStatementImpl((BigdataResource) id,
+ (BigdataURI) key,
+ (BigdataValue) val, null, StatementEnum.Explicit, false);
+
+ m_cxn.getTripleStore().addStatements(new ISPO[] {spo}, 1);
+ }
+ }
+
+ @Override
+ public void retract(Resource id, URI key, Value val) throws RepositoryException {
+ if (log.isTraceEnabled())
+ log.trace("Removing statement: " + id.stringValue() + " " + key.stringValue() + " " + val.stringValue());
+ m_cxn.remove(id, key, val);
+ }
+
+ @Override
+ void doCommit() {
+ m_cxn.getTripleStore().commit();
+ }
+
+ /**
+ * doRollback handles the "partial" updates written to maintain referential integrity and also
+ * incremental updates of "dirty" objects.
+ */
+ @Override
+ void doRollback() {
+ m_cxn.getTripleStore().abort();
+ }
+
+ @Override
+ public void remove(IGPO gpo) {
+ try {
+ // Removes all references
+ m_cxn.remove(gpo.getId(), null, null);
+ m_cxn.remove((Resource) null, null, gpo.getId());
+ } catch (RepositoryException e) {
+ throw new RuntimeException("Unable to remove object", e);
+ }
+ }
+
+ @Override
+ public ValueFactory getValueFactory() {
+ return m_valueFactory;
+ }
+
+ @Override
+ void flushTerms() {
+ if (m_terms.size() > 0) {
+ final BigdataValue[] terms = new BigdataValue[m_terms.size()];
+ m_terms.toArray(terms);
+ m_terms.clear();
+ final long start = System.currentTimeMillis();
+ m_cxn.getTripleStore().addTerms(terms);
+ if (log.isTraceEnabled())
+ log.trace("Added " + terms.length + " terms: " + (System.currentTimeMillis()-start) + "ms");
+ }
+ }
+
+ @Override
+ void flushStatements() {
+ // handle batch removes
+ if (m_removes.size() > 0) {
+ final ISPO[] spos = statementsToSPO(m_removes);
+ m_removes.clear();
+ m_cxn.getTripleStore().removeStatements(spos, spos.length);
+ }
+
+ // handle batch inserts
+ if (m_inserts.size() > 0) {
+ final ISPO[] spos = statementsToSPO(m_inserts);
+ m_inserts.clear();
+ m_cxn.getTripleStore().addStatements(spos, spos.length);
+
+ }
+ }
+
+ SPO[] statementsToSPO(List<Statement> statements) {
+ final int size = statements.size();
+ SPO[] ret = new SPO[size];
+
+ for (int i = 0; i < size; i++) {
+ final BigdataStatement s = (BigdataStatement) statements.get(i);
+ ret[i] = new SPO(s.getSubject().getIV(), s.getPredicate().getIV(), s.getObject().getIV(), StatementEnum.Explicit);
+ if (ret[i].s == null || ret[i].p == null || ret[i].o == null) {
+ throw new IllegalStateException("Values must be bound");
+ }
+ }
+
+ return ret;
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectManager.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectMgrModel.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectMgrModel.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectMgrModel.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,278 @@
+package com.bigdata.gom.om;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.repository.RepositoryException;
+
+import com.bigdata.gom.gpo.BasicSkin;
+import com.bigdata.gom.gpo.GPO;
+import com.bigdata.gom.gpo.IGPO;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.model.BigdataResource;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.model.StatementEnum;
+import com.bigdata.rdf.spo.SPO;
+
+public abstract class ObjectMgrModel implements IObjectManager {
+
+ protected static final Logger log = Logger.getLogger(IObjectManager.class);
+
+ final WeakHashMap<Resource, IGPO> m_dict = new WeakHashMap<Resource, IGPO>();
+
+ final ConcurrentHashMap<URI, URI> m_internedKeys = new ConcurrentHashMap<URI, URI>();
+
+ final ValueFactory m_valueFactory;
+
+ // new terms cache to enable batch term registration on update/flush
+ final ArrayList<BigdataValue> m_terms = new ArrayList<BigdataValue>();
+ final ArrayList<Statement> m_inserts = new ArrayList<Statement>();
+ final ArrayList<Statement> m_removes = new ArrayList<Statement>();
+
+ // Object Creation and ID Management patterns
+ final URI s_idMgr;
+ final URI s_idMgrNextId;
+ final URI s_nmeMgr;
+
+ int m_transactionCounter = 0;
+
+ ObjectMgrModel(final ValueFactory valueFactory) {
+ m_valueFactory = valueFactory;
+ s_idMgr = m_valueFactory.createURI("gpo:idMgr");
+ s_idMgrNextId = m_valueFactory.createURI("gpo:idMgr#nextId");
+ s_nmeMgr = m_valueFactory.createURI("gpo:nmeMgr");
+
+ addNewTerm((BigdataValue) s_idMgr);
+ addNewTerm((BigdataValue) s_idMgrNextId );
+ addNewTerm((BigdataValue) s_nmeMgr);
+ }
+
+ @Override
+ public URI internKey(final URI key) {
+ final URI old = m_internedKeys.putIfAbsent(key, key);
+
+
+ final URI uri = old != null ? old : key;
+
+ if (old == null && (uri instanceof BigdataURI) && ((BigdataURI) uri).getIV() == null)
+ addNewTerm((BigdataURI) uri);
+
+ return uri;
+ }
+
+ final ArrayList<GPO> m_dirtyGPOs = new ArrayList<GPO>();
+
+ final int m_maxDirtyListSize = 1000; // 5000; // FIXME: Init from property file
+
+ /**
+ * GPOs are added to the dirty list when initially modified.
+ *
+ * <p>The list cannot be allowed to grow unbounded since it retains a
+ * concrete reference to the GPO and OutOfMemory will occur. The
+ * solution is to incrementally flush the dirty list.</p>
+ */
+ public void addToDirtyList(GPO gpo) {
+ m_dirtyGPOs.add(gpo);
+ if (m_dirtyGPOs.size() > m_maxDirtyListSize) {
+ if (log.isTraceEnabled())
+ log.trace("Incremental flush of dirty objects");
+
+ flushDirtyObjects();
+ }
+ }
+
+ abstract void flushTerms();
+
+ private void flushDirtyObjects() {
+ // prepare values
+ Iterator<GPO> newValues = m_dirtyGPOs.iterator();
+ while (newValues.hasNext()) {
+ final GPO gpo = newValues.next();
+ gpo.prepareBatchTerms();
+ }
+
+ // flush terms
+ flushTerms();
+
+ final long start = System.currentTimeMillis();
+ final long count = m_dirtyGPOs.size();
+
+ if (true) {
+ Iterator<GPO> updates = m_dirtyGPOs.iterator();
+ while (updates.hasNext()) {
+ updates.next().prepareBatchUpdate();
+ }
+
+ flushStatements();
+ } else {
+ // update dirty objects - is it worth while batching SPO[]?
+ Iterator<GPO> updates = m_dirtyGPOs.iterator();
+ while (updates.hasNext()) {
+ try {
+ updates.next().update();
+ } catch (RepositoryException e) {
+ throw new RuntimeException("Unexpected update exception", e);
+ }
+ }
+ }
+ m_dirtyGPOs.clear();
+ if (log.isTraceEnabled())
+ log.trace("Flush took " + (System.currentTimeMillis()-start) + "ms for " + count + " objects");
+ }
+
+ abstract void flushStatements();
+
+ @Override
+ public IGPO getGPO(final Resource id) {
+ IGPO ret = m_dict.get(id);
+
+ if (ret == null) {
+ ret = new GPO(this, id);
+ // materialize(ret); // JFDI?
+ m_dict.put(id, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public synchronized int beginNativeTransaction() {
+ return m_transactionCounter++;
+ }
+
+ @Override
+ public int commitNativeTransaction(final int expectedCounter) {
+ final int ret = --m_transactionCounter;
+ if (ret != expectedCounter) {
+ throw new IllegalArgumentException("Unexpected transaction counter");
+ }
+
+ if (ret == 0) {
+ flushDirtyObjects();
+ }
+
+ doCommit();
+
+ return ret;
+ }
+
+ abstract void doCommit();
+
+ @Override
+ public int getNativeTransactionCounter() {
+ return m_transactionCounter;
+ }
+
+ @Override
+ public void rollbackNativeTransaction() {
+ // just clear the cache for now
+ m_dict.clear();
+ m_dirtyGPOs.clear();
+ m_transactionCounter = 0;
+ m_idMgr = null;
+
+ doRollback();
+ }
+
+ abstract void doRollback();
+
+ @Override
+ public IGPO createGPO() {
+ BasicSkin idMgr = getIdMgr();
+
+ int nxtId = idMgr.getIntValue(s_idMgrNextId)+1;
+ idMgr.setValue(s_idMgrNextId, nxtId);
+
+ final Resource uri = getValueFactory().createURI("gpo:#" + nxtId);
+ addNewTerm((BigdataValue) uri);
+ final GPO ret = (GPO) getGPO(uri);
+
+ ret.setMaterialized(true);
+
+ return ret;
+ }
+
+ protected void addNewTerm(final BigdataValue uri) {
+ if (uri.isRealIV())
+ throw new IllegalArgumentException("IV already available: " + uri.stringValue());
+
+ m_terms.add(uri);
+ }
+
+ BasicSkin m_idMgr = null;
+ protected BasicSkin getIdMgr() {
+ if (m_idMgr == null) {
+ if (log.isTraceEnabled())
+ log.trace("retrieving ID Manager");
+
+ IGPO idMgr = getGPO(s_idMgr);
+
+ m_idMgr = new BasicSkin(idMgr);
+ }
+
+ return m_idMgr;
+ }
+
+ /**
+ * Simple save/recall interface that the ObjectManager provides to simplify
+ * other pattern implementations. Internally it uses a NameManager GPO
+ */
+ public void save(final URI key, Value value) {
+ getGPO(s_nmeMgr).setValue(key, value);
+ }
+
+ /**
+ * Simple save/recall interface that the ObjectManager provides to simplify
+ * other pattern implementations. Internally it uses a NameManager GPO
+ */
+ public Value recall(final URI key) {
+ return getGPO(s_nmeMgr).getValue(key);
+ }
+
+ public IGPO recallAsGPO(final URI key) {
+ Value val = recall(key);
+
+ if (val instanceof Resource) {
+ return getGPO((Resource) val);
+ } else {
+ return null;
+ }
+ }
+
+ public void checkValue(Value newValue) {
+ final BigdataValue v = (BigdataValue) newValue;
+ if (!v.isRealIV()) {
+ addNewTerm(v);
+ }
+ }
+
+ public void clearCache() {
+ m_dict.clear();
+ m_dirtyGPOs.clear();
+ }
+
+ public void insertBatch(final Resource m_id, final URI bigdataURI, final Value v) {
+ m_inserts.add(m_valueFactory.createStatement(m_id, bigdataURI, v));
+ }
+
+ public void removeBatch(final Resource m_id, final URI bigdataURI, final Value v) {
+ m_removes.add(m_valueFactory.createStatement(m_id, bigdataURI, v));
+ }
+
+ @Override
+ public ValueFactory getValueFactory() {
+ return m_valueFactory;
+ }
+
+}
Property changes on: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/om/ObjectMgrModel.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/skin/GenericSkinRegistry.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/skin/GenericSkinRegistry.java 2012-05-16 08:38:12 UTC (rev 6321)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/java/com/bigdata/gom/skin/GenericSkinRegistry.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -436,7 +436,7 @@
{
Constructor c = implClass.getDeclaredConstructor
- ( new Class[] { IGenericSkin.class }
+ ( new Class[] { IGPO.class }
);
return (IGenericSkin) c.newInstance
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/TestGOM.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/TestGOM.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-gom/src/test/com/bigdata/gom/TestGOM.java 2012-05-31 07:48:48 UTC (rev 6322)
@@ -0,0 +1,382 @@
+package com.bigdata.gom;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.openrdf.model.Resource;
+import org.openrdf.model.URI;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.repository.Repository;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.RepositoryException;
+import org.openrdf.rio.RDFFormat;
+import org.openrdf.rio.RDFParseException;
+
+import com.bigdata.btree.IndexMetadata;
+import com.bigdata.gom.gpo.BasicSkin;
+import com.bigdata.gom.gpo.GPO;
+import com.bigdata.gom.gpo.IGPO;
+import com.bigdata.gom.gpo.ILinkSet;
+import com.bigdata.gom.om.IObjectManager;
+import com.bigdata.gom.om.ObjectManager;
+import com.bigdata.gom.skin.GenericSkinRegistry;
+import com.bigdata.journal.BufferMode;
+import com.bigdata.journal.Journal.Options;
+import com.bigdata.rdf.sail.BigdataSail;
+import com.bigdata.rdf.sail.BigdataSailRepository;
+import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
+import com.bigdata.rdf.store.AbstractTrip...
[truncated message content] |
|
From: <tho...@us...> - 2012-05-31 15:43:24
|
Revision: 6323
http://bigdata.svn.sourceforge.net/bigdata/?rev=6323&view=rev
Author: thompsonbry
Date: 2012-05-31 15:43:17 +0000 (Thu, 31 May 2012)
Log Message:
-----------
Partial resolution of https://sourceforge.net/apps/trac/bigdata/ticket/558 (SPARQL INSERT not working in same request after INSERT DATA).
This fixes the problem when the updates are performed using the unisolated connection. There is a still a problem when they are performed in a full read/write tx. I will look into that next.
This commit includes a unit test which replicates the problem (against both the unisolated connection and a read/write tx), it includes a new AST optimizer to handle resolution of unknown terms before evaluating an updte, a test suite for that optimizer, and a patch to AST2BOpUpdate to correctly (a) flush the sail assertion and retraction buffers; and (b) perform additional terms resolution steps.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/Bigdata2ASTSPARQLParser.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLUpdateTest.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTBatchResolveTermsOptimizer.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTBatchResolveTermsOptimizer.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-05-31 07:48:48 UTC (rev 6322)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUpdate.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -115,6 +115,7 @@
import com.bigdata.rdf.sparql.ast.UpdateRoot;
import com.bigdata.rdf.sparql.ast.UpdateType;
import com.bigdata.rdf.sparql.ast.VarNode;
+import com.bigdata.rdf.sparql.ast.optimizers.ASTBatchResolveTermsOptimizer;
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.store.BD;
@@ -216,8 +217,10 @@
final ASTContainer astContainer = context.astContainer;
- // * Note: Change this to the optimized AST if we start doing AST
- // optimizations for UPDATE. */
+ /*
+ * Note: Change this to the optimized AST if we start doing AST
+ * optimizations for UPDATE.
+ */
final UpdateRoot updateRoot = astContainer.getOriginalUpdateAST();
// Set as annotation on the ASTContainer.
@@ -227,10 +230,45 @@
* Evaluate each update operation in the optimized UPDATE AST in turn.
*/
PipelineOp left = null;
- for (Update op : updateRoot) {
+ int updateIndex = 0;
+ for (Update op : updateRoot) {
- left = convertUpdateSwitch(left, op, context);
+ if (updateIndex > 0) {
+ /*
+ * There is more than one update operation in this request.
+ */
+
+ /*
+ * Note: We need to flush the assertion / retraction buffers if
+ * the Sail is local since some of the code paths supporting
+ * UPDATEs do not go through the BigdataSail and would otherwise
+ * not have their updates flushed until the commit (which does
+ * go through the BigdataSail).
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/558
+ */
+ context.conn.flush();
+
+ // log.error("\nafter op=" + op + "\n" + context.db.dumpStore());
+
+ /*
+ * We need to re-resolve any RDF Values appearing in this UPDATE
+ * operation which have a 0L term identifier in case they have
+ * become defined through the previous update(s).
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/558
+ */
+ op = (Update) new ASTBatchResolveTermsOptimizer().optimize(context,
+ op/* queryNode */, null/* bindingSets */);
+
+ }
+
+ // convert/run the update operation.
+ left = convertUpdateSwitch(left, op, context);
+
+ updateIndex++;
+
}
/*
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTBatchResolveTermsOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTBatchResolveTermsOptimizer.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/optimizers/ASTBatchResolveTermsOptimizer.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -0,0 +1,182 @@
+package com.bigdata.rdf.sparql.ast.optimizers;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.BOpUtility;
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IConstant;
+import com.bigdata.bop.IVariable;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.sail.sparql.Bigdata2ASTSPARQLParser;
+import com.bigdata.rdf.sparql.ast.ConstantNode;
+import com.bigdata.rdf.sparql.ast.IQueryNode;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+
+/**
+ * Optimizer attempts to resolve any {@link BigdataValue}s in the AST which are
+ * associated with a mock IV. This resolution step is normally performed by the
+ * parser. However, when there is a sequence of UPDATE operations in the same
+ * request, previous UPDATE operations in that sequence MIGHT have caused terms
+ * to become declared in the lexicon so we must attempt to re-resolve any which
+ * are mock IVs.
+ *
+ * TODO We could have this always run and not do resolution in
+ * {@link Bigdata2ASTSPARQLParser}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: BatchRDFValueResolver.java 6160 2012-03-18 19:57:37Z
+ * thompsonbry $
+ */
+public class ASTBatchResolveTermsOptimizer implements IASTOptimizer {
+
+ private static final transient Logger log = Logger
+ .getLogger(ASTBatchResolveTermsOptimizer.class);
+
+ @Override
+ public IQueryNode optimize(final AST2BOpContext context,
+ final IQueryNode queryNode, final IBindingSet[] bindingSets) {
+
+ /*
+ * Look for unknown terms and attempt to resolve them now.
+ */
+
+ /*
+ * The list of the constants with mock IVs and null unless we find
+ * something.
+ *
+ * Note: This is a list and not a set because we need to swap in the
+ * resolve IValueExpression for each ConstantNode in the query (there
+ * can be distinct ConstantNodes for the same unknown value and we would
+ * miss the duplicates unless this is a simple list).
+ */
+ List<ConstantNode> unknown = null;
+
+ // Look for unknown terms.
+ {
+
+ final Iterator<ConstantNode> itr = BOpUtility.visitAll(
+ (BOp) queryNode, ConstantNode.class);
+
+ while (itr.hasNext()) {
+
+ final ConstantNode cnode = itr.next();
+
+ final IConstant<?> c = cnode.getValueExpression();
+
+ final IV<?, ?> iv = (IV<?, ?>) c.get();
+
+ if (iv.isNullIV()) {
+
+ if (unknown == null) {
+
+ // Lazy allocation.
+ unknown = new LinkedList<ConstantNode>();
+
+ }
+
+ unknown.add(cnode);
+
+ }
+
+ }
+
+ }
+
+ if (unknown != null) {
+
+ /*
+ * Batch resolve any terms which are currently unknown.
+ */
+
+ final BigdataValue[] values = new BigdataValue[unknown.size()];
+ {
+
+ /*
+ * First, build up the set of unknown terms.
+ */
+ int i = 0;
+
+ for (ConstantNode cnode : unknown) {
+
+ final IConstant<?> c = cnode.getValueExpression();
+
+ final IV<?, ?> iv = (IV<?, ?>) c.get();
+
+ final BigdataValue v = iv.getValue();
+
+ v.clearInternalValue();
+
+ values[i++] = v;
+
+ }
+
+ /*
+ * Batch resolution.
+ */
+
+ if (log.isDebugEnabled())
+ log.debug("UNKNOWNS: " + Arrays.toString(values));
+
+ context.getAbstractTripleStore().getLexiconRelation()
+ .addTerms(values, values.length, true/* readOnly */);
+
+ }
+
+ /*
+ * Replace the value expression with one which uses the resolved IV.
+ */
+
+ {
+ int i = 0;
+ for (ConstantNode cnode : unknown) {
+
+ final BigdataValue v = values[i++];
+
+ if (v.isRealIV()) {
+
+ if (log.isInfoEnabled())
+ log.info("RESOLVED: " + v + " => " + v.getIV());
+
+ /*
+ * Note: If the constant is an effective constant
+ * because it was given in the binding sets then we also
+ * need to capture the variable name associated with
+ * that constant.
+ */
+
+ final IV<?, ?> iv = v.getIV();
+
+ final Constant<?> oldc = (Constant<?>) cnode
+ .getValueExpression();
+
+ final IVariable<?> var = (IVariable<?>) oldc
+ .getProperty(Constant.Annotations.VAR);
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ final Constant<?> newc = (var == null ? new Constant(iv)
+ : new Constant(var, iv));
+
+ // cnode.setValueExpression(newc);
+ cnode.setArg(0, newc);
+
+ }
+
+ }
+
+ }
+
+ }
+
+ return queryNode;
+
+ }
+
+}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTBatchResolveTermsOptimizer.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTBatchResolveTermsOptimizer.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestASTBatchResolveTermsOptimizer.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -0,0 +1,315 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 29, 2011
+ */
+
+package com.bigdata.rdf.sparql.ast.optimizers;
+
+import org.openrdf.query.algebra.StatementPattern.Scope;
+
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IValueExpression;
+import com.bigdata.bop.Var;
+import com.bigdata.bop.bindingSet.ListBindingSet;
+import com.bigdata.rdf.internal.VTE;
+import com.bigdata.rdf.internal.impl.TermId;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.sparql.ast.ASTContainer;
+import com.bigdata.rdf.sparql.ast.AbstractASTEvaluationTestCase;
+import com.bigdata.rdf.sparql.ast.ConstantNode;
+import com.bigdata.rdf.sparql.ast.IQueryNode;
+import com.bigdata.rdf.sparql.ast.JoinGroupNode;
+import com.bigdata.rdf.sparql.ast.ProjectionNode;
+import com.bigdata.rdf.sparql.ast.QueryRoot;
+import com.bigdata.rdf.sparql.ast.QueryType;
+import com.bigdata.rdf.sparql.ast.StatementPatternNode;
+import com.bigdata.rdf.sparql.ast.VarNode;
+import com.bigdata.rdf.sparql.ast.eval.AST2BOpContext;
+
+/**
+ * Test suite for {@link ASTBatchResolveTermsOptimizer}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: TestASTServiceNodeOptimizer.java 6080 2012-03-07 18:38:55Z thompsonbry $
+ */
+public class TestASTBatchResolveTermsOptimizer extends AbstractASTEvaluationTestCase {
+
+ /**
+ *
+ */
+ public TestASTBatchResolveTermsOptimizer() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestASTBatchResolveTermsOptimizer(String name) {
+ super(name);
+ }
+
+ /**
+ * Given
+ *
+ * <pre>
+ * SELECT VarNode(s) VarNode(p) VarNode(v)
+ * JoinGroupNode {
+ * JoinGroupNode {
+ * JoinGroupNode [context=ConstantNode(TermId(0L)[http://example/out])] {
+ * StatementPatternNode(VarNode(s), VarNode(p), VarNode(v), ConstantNode(TermId(0L)[http://example/out])) [scope=NAMED_CONTEXTS]
+ * }
+ * }
+ * }
+ * </pre>
+ *
+ * where the unknown term is <code>http://example/out</code> and IS in fact
+ * in the lexicon, the {@link IValueExpression} for the {@link ConstantNode}
+ * associated with that mock IV is rewritten to the resolved IV.
+ */
+ public void test_batchResolveTerms_01() {
+
+ /*
+ * Note: DO NOT share structures in this test!!!!
+ *
+ * Note: This test depends on having multiple BigdataURIs for the
+ * unknown term. In one case the IV is known and in the other case it is
+ * not known.
+ */
+
+ final BigdataValueFactory f = store.getValueFactory();
+
+ // A version where a mock IV is associated with the term.
+ final BigdataURI unknown1 = f.createURI("http://example/out");
+ unknown1.setIV(TermId.mockIV(VTE.URI));
+ assertFalse(unknown1.isRealIV());
+ unknown1.getIV().setValue(unknown1);
+
+ // A version where a real IV is associated with the term.
+ final BigdataURI known1 = f.createURI("http://example/out");
+ store.addTerms(new BigdataValue[]{known1});
+ assertTrue(known1.isRealIV());
+
+ final IBindingSet[] bsets = new IBindingSet[] { //
+ new ListBindingSet()
+ };
+
+ /**
+ * The source AST.
+ */
+ final QueryRoot given = new QueryRoot(QueryType.SELECT);
+ {
+
+ final ProjectionNode projection = new ProjectionNode();
+ given.setProjection(projection);
+
+ projection.addProjectionVar(new VarNode("s"));
+ projection.addProjectionVar(new VarNode("p"));
+ projection.addProjectionVar(new VarNode("v"));
+
+ final JoinGroupNode whereClause = new JoinGroupNode();
+ given.setWhereClause(whereClause);
+
+ {
+
+ final JoinGroupNode graphPattern = new JoinGroupNode();
+ graphPattern.setContext(new ConstantNode(new Constant(unknown1
+ .getIV())));
+ whereClause.addChild(graphPattern);
+
+ final JoinGroupNode innerGroup = new JoinGroupNode();
+ graphPattern.addChild(innerGroup);
+
+ innerGroup.addChild(new StatementPatternNode(new VarNode("s"),
+ new VarNode("p"), new VarNode("v"), new ConstantNode(
+ new Constant(unknown1.getIV()))/* c */,
+ Scope.NAMED_CONTEXTS));
+
+ }
+
+ }
+
+ /**
+ * The expected AST after the rewrite.
+ */
+ final QueryRoot expected = new QueryRoot(QueryType.SELECT);
+ {
+
+ final ProjectionNode projection = new ProjectionNode();
+ expected.setProjection(projection);
+
+ projection.addProjectionVar(new VarNode("s"));
+ projection.addProjectionVar(new VarNode("p"));
+ projection.addProjectionVar(new VarNode("v"));
+
+ final JoinGroupNode whereClause = new JoinGroupNode();
+ expected.setWhereClause(whereClause);
+
+ {
+
+ final JoinGroupNode graphPattern = new JoinGroupNode();
+ graphPattern.setContext(new ConstantNode(new Constant(known1
+ .getIV())));
+ whereClause.addChild(graphPattern);
+
+ final JoinGroupNode innerGroup = new JoinGroupNode();
+ graphPattern.addChild(innerGroup);
+
+ innerGroup.addChild(new StatementPatternNode(new VarNode("s"),
+ new VarNode("p"), new VarNode("v"), new ConstantNode(
+ new Constant(known1.getIV()))/* c */,
+ Scope.NAMED_CONTEXTS));
+
+ }
+
+ }
+
+ final IASTOptimizer rewriter = new ASTBatchResolveTermsOptimizer();
+
+ final AST2BOpContext context = new AST2BOpContext(new ASTContainer(
+ given), store);
+
+ final IQueryNode actual = rewriter.optimize(context,
+ given/* queryNode */, bsets);
+
+ assertSameAST(expected, actual);
+
+ }
+
+ /**
+ * A variant of the test above where the Constant/2 constructor was used and
+ * we need to propagate the variable associated with that constant.
+ */
+ public void test_batchResolveTerms_02() {
+
+ /*
+ * Note: DO NOT share structures in this test!!!!
+ *
+ * Note: This test depends on having multiple BigdataURIs for the
+ * unknown term. In one case the IV is known and in the other case it is
+ * not known.
+ */
+
+ final BigdataValueFactory f = store.getValueFactory();
+
+ // A version where a mock IV is associated with the term.
+ final BigdataURI unknown1 = f.createURI("http://example/out");
+ unknown1.setIV(TermId.mockIV(VTE.URI));
+ assertFalse(unknown1.isRealIV());
+ unknown1.getIV().setValue(unknown1);
+
+ // A version where a real IV is associated with the term.
+ final BigdataURI known1 = f.createURI("http://example/out");
+ store.addTerms(new BigdataValue[]{known1});
+ assertTrue(known1.isRealIV());
+
+ final IBindingSet[] bsets = new IBindingSet[] { //
+ new ListBindingSet()
+ };
+
+ /**
+ * The source AST.
+ */
+ final QueryRoot given = new QueryRoot(QueryType.SELECT);
+ {
+
+ final ProjectionNode projection = new ProjectionNode();
+ given.setProjection(projection);
+
+ projection.addProjectionVar(new VarNode("s"));
+ projection.addProjectionVar(new VarNode("p"));
+ projection.addProjectionVar(new VarNode("v"));
+
+ final JoinGroupNode whereClause = new JoinGroupNode();
+ given.setWhereClause(whereClause);
+
+ {
+
+ final JoinGroupNode graphPattern = new JoinGroupNode();
+ graphPattern.setContext(new ConstantNode(new Constant(Var
+ .var("x"), unknown1.getIV())));
+ whereClause.addChild(graphPattern);
+
+ final JoinGroupNode innerGroup = new JoinGroupNode();
+ graphPattern.addChild(innerGroup);
+
+ innerGroup.addChild(new StatementPatternNode(new VarNode("s"),
+ new VarNode("p"), new VarNode("v"),
+ new ConstantNode(new Constant(Var.var("x"), unknown1
+ .getIV()))/* c */, Scope.NAMED_CONTEXTS));
+
+ }
+
+ }
+
+ /**
+ * The expected AST after the rewrite.
+ */
+ final QueryRoot expected = new QueryRoot(QueryType.SELECT);
+ {
+
+ final ProjectionNode projection = new ProjectionNode();
+ expected.setProjection(projection);
+
+ projection.addProjectionVar(new VarNode("s"));
+ projection.addProjectionVar(new VarNode("p"));
+ projection.addProjectionVar(new VarNode("v"));
+
+ final JoinGroupNode whereClause = new JoinGroupNode();
+ expected.setWhereClause(whereClause);
+
+ {
+
+ final JoinGroupNode graphPattern = new JoinGroupNode();
+ graphPattern.setContext(new ConstantNode(new Constant(Var
+ .var("x"), known1.getIV())));
+ whereClause.addChild(graphPattern);
+
+ final JoinGroupNode innerGroup = new JoinGroupNode();
+ graphPattern.addChild(innerGroup);
+
+ innerGroup.addChild(new StatementPatternNode(new VarNode("s"),
+ new VarNode("p"), new VarNode("v"),
+ new ConstantNode(new Constant(Var.var("x"), known1
+ .getIV()))/* c */, Scope.NAMED_CONTEXTS));
+
+ }
+
+ }
+
+ final IASTOptimizer rewriter = new ASTBatchResolveTermsOptimizer();
+
+ final AST2BOpContext context = new AST2BOpContext(new ASTContainer(
+ given), store);
+
+ final IQueryNode actual = rewriter.optimize(context,
+ given/* queryNode */, bsets);
+
+ assertSameAST(expected, actual);
+
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestAll.java 2012-05-31 07:48:48 UTC (rev 6322)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/rdf/sparql/ast/optimizers/TestAll.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -136,6 +136,9 @@
// Unit tests for optimizer which attaches join filters to SPs.
suite.addTestSuite(TestASTRangeOptimizer.class);
+ // Test suite for resolving mock IVs.
+ suite.addTestSuite(TestASTBatchResolveTermsOptimizer.class);
+
return suite;
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/Bigdata2ASTSPARQLParser.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/Bigdata2ASTSPARQLParser.java 2012-05-31 07:48:48 UTC (rev 6322)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/java/com/bigdata/rdf/sail/sparql/Bigdata2ASTSPARQLParser.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -223,17 +223,19 @@
BlankNodeVarProcessor.process(uc);
/*
- * Batch resolve ASTRDFValue to BigdataValues with their
- * associated IVs.
- *
- * TODO IV resolution might need to proceed separately for each
- * UPDATE operation in a sequence since some operations can
- * cause new IVs to be declared in the lexicon. Resolution
- * before those IVs have been declared would produce a different
- * result than resolution afterward (it will be a null IV before
- * the Value is added to the lexicon and a TermId or BlobIV
- * afterward).
- */
+ * Batch resolve ASTRDFValue to BigdataValues with their
+ * associated IVs.
+ *
+ * Note: IV resolution must proceed separately (or be
+ * re-attempted) for each UPDATE operation in a sequence since
+ * some operations can cause new IVs to be declared in the
+ * lexicon. Resolution before those IVs have been declared would
+ * produce a different result than resolution afterward (it will
+ * be a null IV before the Value is added to the lexicon and a
+ * TermId or BlobIV afterward).
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/558
+ */
new BatchRDFValueResolver(context, true/* readOnly */)
.process(uc);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest.java 2012-05-31 07:48:48 UTC (rev 6322)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSPARQLUpdateTest.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -32,6 +32,12 @@
import java.util.Properties;
import org.apache.log4j.Logger;
+import org.openrdf.model.Resource;
+import org.openrdf.model.URI;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.UpdateExecutionException;
import org.openrdf.query.parser.sparql.SPARQLUpdateTest;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryException;
@@ -40,6 +46,7 @@
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.IIndexManager;
+import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSail.Options;
import com.bigdata.rdf.sail.BigdataSailRepository;
@@ -207,5 +214,86 @@
}
logger.debug("dataset loaded.");
}
-
+
+ /**
+ * Unit test for isolation semantics for a sequences of updates.
+ *
+ * @throws UpdateExecutionException
+ * @throws MalformedQueryException
+ * @throws RepositoryException
+ * @throws QueryEvaluationException
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/558
+ */
+ public void test_ticket538() throws UpdateExecutionException,
+ RepositoryException, MalformedQueryException, QueryEvaluationException {
+
+ // the [in] and [out] graphs.
+ final URI gin = f.createURI("http://example/in");
+ final URI gout = f.createURI("http://example/out");
+
+// ((BigdataSailRepository) con.getRepository()).getDatabase().addTerms(
+// new BigdataValue[] { (BigdataValue) gin, (BigdataValue) gout });
+
+// // Make sure the target graphs are empty.
+// {
+// final String s = "# Update 1\n"
+// + "DROP SILENT GRAPH <http://example/in>;\n"
+// + "DROP SILENT GRAPH <http://example/out> ;\n";
+//
+// con.prepareUpdate(QueryLanguage.SPARQL, s).execute();
+// }
+
+ con.prepareUpdate(QueryLanguage.SPARQL, "DROP SILENT ALL").execute();
+ // Make sure the graphs are empty.
+ assertFalse(con.hasStatement(null,null,null, true, (Resource)gin));
+ assertFalse(con.hasStatement(null,null,null, true, (Resource)gout));
+
+ /*
+ * The mutation.
+ *
+ * A statement is inserted into the [in] graph in one operation. Then
+ * all statements in the [in] graph are copied into the [out] graph.
+ */
+ final String s = "# Update 2\n"//
+ + "PREFIX foaf: <http://xmlns.com/foaf/0.1/>\n"//
+ + "INSERT DATA {\n"//
+ + " GRAPH <http://example/in> {\n"//
+ + " <http://example/president25> foaf:givenName \"William\" .\n"//
+ + " }\n"//
+ + "};\n"//
+ + "INSERT {\n"//
+ + " GRAPH <http://example/out> {\n"//
+ + " ?s ?p ?v .\n"//
+ + " }\n"//
+ + " }\n"//
+ + "WHERE {\n"//
+ + " GRAPH <http://example/in> {\n"//
+ + " ?s ?p ?v .\n"//
+ + " }\n"//
+ + " }\n"//
+ + ";";
+
+// // The query - how many statements are in the [out] graph.
+// final String q = "# Query\n"//
+// + "SELECT (COUNT(*) as ?cnt) {\n"//
+// + " GRAPH <http://example/out> {\n"//
+// + " ?s ?p ?v .\n"//
+// + " }\n"//
+// + "} LIMIT 10";//
+
+ // run update once.
+ con.prepareUpdate(QueryLanguage.SPARQL, s).execute();
+
+ // Both graphs should now have some data.
+ assertTrue(con.hasStatement(null,null,null, true, (Resource)gin));
+
+ // Note: Succeeds if you do this a 2nd time.
+ if (false)
+ con.prepareUpdate(QueryLanguage.SPARQL, s).execute();
+
+ assertTrue(con.hasStatement(null,null,null, true, (Resource)gout));
+
+ }
+
}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLUpdateTest.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLUpdateTest.java 2012-05-31 07:48:48 UTC (rev 6322)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLUpdateTest.java 2012-05-31 15:43:17 UTC (rev 6323)
@@ -49,7 +49,7 @@
protected RepositoryConnection con;
- private ValueFactory f;
+ protected ValueFactory f;
private URI bob;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-06-12 14:59:04
|
Revision: 6338
http://bigdata.svn.sourceforge.net/bigdata/?rev=6338&view=rev
Author: thompsonbry
Date: 2012-06-12 14:58:53 +0000 (Tue, 12 Jun 2012)
Log Message:
-----------
Per email thread with MikeP, modified the CUTOFF_LIMIT interpretation to pass the effective cutoffLimit into the IAccessPath#solutions() method. This works since the default is MAX_LONG, which has the same semantics as ZERO (no cutoff is imposed).
Modified AccessPath#solutions() to (a) remove the old implementation and (b) modify the new implementation to NOT pass the [limit] as the capacity into AccessPath#iterator(). The latter method has logic to make decent choices about the capacity of the iterator which was being overridden when the CUTOFF_LIMIT was specified.
Removed the old solutions() method on IBindingSetAccessPath. Everything uses the new method which accepts the CUTOFF_LIMIT.
Removed FIXMEs in TestJoiNGraphOnBSBMData since MikeP had remove the range variable for range bops from the API.
Reconciled the SIDs changes with the range bop changes in AST2BOpUtility.
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinOp.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -390,17 +390,14 @@
final UnsyncLocalOutputBuffer<IBindingSet> unsyncBuffer = new UnsyncLocalOutputBuffer<IBindingSet>(
op.getChunkCapacity(), sink);
- final long cutoffLimit = pred.getProperty(
- IPredicate.Annotations.CUTOFF_LIMIT,
- IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
-
- // Obtain the iterator for the current join dimension.
- final ICloseableIterator<IBindingSet> itr;
- if (cutoffLimit == Long.MAX_VALUE)
- itr = ((IBindingSetAccessPath<?>) accessPath).solutions(stats);
- else
- itr = ((IBindingSetAccessPath<?>) accessPath).solutions(cutoffLimit, stats);
-
+ final long cutoffLimit = pred.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
+ // Obtain the iterator for the current join dimension.
+ final ICloseableIterator<IBindingSet> itr = ((IBindingSetAccessPath<?>) accessPath)
+ .solutions(cutoffLimit, stats);
+
state.hashJoin(
itr,// left
unsyncBuffer// out
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/JVMHashJoinOp.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -326,17 +326,14 @@
final UnsyncLocalOutputBuffer<IBindingSet> unsyncBuffer = new UnsyncLocalOutputBuffer<IBindingSet>(
op.getChunkCapacity(), sink);
- final long cutoffLimit = pred.getProperty(
- IPredicate.Annotations.CUTOFF_LIMIT,
- IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
-
+ final long cutoffLimit = pred.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
// Obtain the iterator for the current join dimension.
- final ICloseableIterator<IBindingSet> itr;
- if (cutoffLimit == Long.MAX_VALUE)
- itr = ((IBindingSetAccessPath<?>) accessPath).solutions(stats);
- else
- itr = ((IBindingSetAccessPath<?>) accessPath).solutions(cutoffLimit, stats);
-
+ final ICloseableIterator<IBindingSet> itr = ((IBindingSetAccessPath<?>) accessPath)
+ .solutions(cutoffLimit, stats);
+
state.hashJoin(
itr,// left
unsyncBuffer // where to write the solutions which join.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -1527,16 +1527,13 @@
*/
protected void handleJoin() {
- final long cutoffLimit = predicate.getProperty(
- IPredicate.Annotations.CUTOFF_LIMIT,
- IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
-
+ final long cutoffLimit = predicate.getProperty(
+ IPredicate.Annotations.CUTOFF_LIMIT,
+ IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
+
// Obtain the iterator for the current join dimension.
- final IChunkedOrderedIterator<?> itr;
- if (cutoffLimit == Long.MAX_VALUE)
- itr = accessPath.iterator();
- else
- itr = accessPath.iterator(0, limit, (int) limit);
+ final IChunkedOrderedIterator<?> itr = accessPath.iterator(0,
+ cutoffLimit, 0/* capacity */);
try {
@@ -1646,16 +1643,13 @@
*/
protected void handleJoin2() {
- final long cutoffLimit = predicate.getProperty(
+ final long cutoffLimit = predicate.getProperty(
IPredicate.Annotations.CUTOFF_LIMIT,
IPredicate.Annotations.DEFAULT_CUTOFF_LIMIT);
// Obtain the iterator for the current join dimension.
- final ICloseableIterator<IBindingSet> itr;
- if (cutoffLimit == Long.MAX_VALUE)
- itr = ((IBindingSetAccessPath<?>) accessPath).solutions(stats);
- else
- itr = ((IBindingSetAccessPath<?>) accessPath).solutions(cutoffLimit, stats);
+ final ICloseableIterator<IBindingSet> itr = ((IBindingSetAccessPath<?>) accessPath)
+ .solutions(cutoffLimit, stats);
try {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -717,20 +717,20 @@
}
- /**
- * {@inheritDoc}
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/209 (Access path
- * should visit solutions for high level query).
- */
- public ICloseableIterator<IBindingSet> solutions(final BaseJoinStats stats) {
-
-// final IVariable<?>[] vars = BOpUtility
-// .getDistinctArgumentVariables(predicate);
-
- return BOpContext.solutions(iterator(), predicate, /*vars,*/ stats);
-
- }
+// /**
+// * {@inheritDoc}
+// *
+// * @see https://sourceforge.net/apps/trac/bigdata/ticket/209 (Access path
+// * should visit solutions for high level query).
+// */
+// public ICloseableIterator<IBindingSet> solutions(final BaseJoinStats stats) {
+//
+//// final IVariable<?>[] vars = BOpUtility
+//// .getDistinctArgumentVariables(predicate);
+//
+// return BOpContext.solutions(iterator(), predicate, /*vars,*/ stats);
+//
+// }
/**
* {@inheritDoc}
@@ -744,7 +744,8 @@
// final IVariable<?>[] vars = BOpUtility
// .getDistinctArgumentVariables(predicate);
- return BOpContext.solutions(iterator(0, limit, (int) limit), predicate, /*vars,*/ stats);
+ return BOpContext.solutions(iterator(0, limit, 0/* (int) limit */),
+ predicate, /* vars, */stats);
}
@@ -1195,7 +1196,6 @@
* buffer.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
static private class ChunkConsumerTask<R> implements Callable<Void> {
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/accesspath/IBindingSetAccessPath.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -45,17 +45,6 @@
* @version $Id$
*/
public interface IBindingSetAccessPath<R> extends IAbstractAccessPath<R> {
-
- /**
- * Return an iterator which will visit the solutions drawn from the access
- * path.
- *
- * TODO Should this be visiting IBindingSet[]s?
- *
- * @see https://sourceforge.net/apps/trac/bigdata/ticket/209 (Access path
- * should visit solutions for high level query).
- */
- ICloseableIterator<IBindingSet> solutions(BaseJoinStats stats);
/**
* Return an iterator which will visit the solutions drawn from the access
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/sparql/ast/eval/AST2BOpUtility.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -3803,12 +3803,26 @@
anns.add(new NV(IPredicate.Annotations.OPTIONAL, Boolean.TRUE));
}
+ /*
+ * Statements about statements.
+ *
+ * <a href="https://sourceforge.net/apps/trac/bigdata/ticket/526">
+ * Reification Done Right</a>
+ */
+ final VarNode sidVar = sp.sid();
+ if (QueryHints.DEFAULT_REIFICATION_DONE_RIGHT && sidVar != null) {
+
+ anns.add(new NV(SPOPredicate.Annotations.SID, sidVar
+ .getValueExpression()));
+
+ }
+
final RangeNode range = sp.getRange();
if (range != null) {
// Add the RangeBOp
anns.add(new NV(IPredicate.Annotations.RANGE, range.getRangeBOp()));
}
-
+
final String cutoffLimit = sp.getQueryHint(QueryHints.CUTOFF_LIMIT);
if (cutoffLimit != null) {
// Add the cutoff limit
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2012-06-12 14:34:22 UTC (rev 6337)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2012-06-12 14:58:53 UTC (rev 6338)
@@ -409,7 +409,6 @@
new NV(BOp.Annotations.BOP_ID, nextId++),//
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RANGE, new RangeBOp(//
-// origProperty1,// FIXME verify correct var w/ MikeP
new MathBOp(origProperty1, new Constant(
new XSDIntegerIV(BigInteger.valueOf(120))),
MathOp.MINUS,globals),//
@@ -440,7 +439,6 @@
new NV(BOp.Annotations.BOP_ID, nextId++),//
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RANGE, new RangeBOp(//
-// origProperty2,// FIXME verify correct var with MikeP
new MathBOp(origProperty2, new Constant(
new XSDIntegerIV(BigInteger.valueOf(170))),
MathOp.MINUS,globals),//
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-06-12 15:38:12
|
Revision: 6339
http://bigdata.svn.sourceforge.net/bigdata/?rev=6339&view=rev
Author: thompsonbry
Date: 2012-06-12 15:38:03 +0000 (Tue, 12 Jun 2012)
Log Message:
-----------
I have added the com.bigdata.gom.TestAll suite into CI. Changes to build.xml and com.bigdata.TestAll.
https://sourceforge.net/apps/trac/bigdata/ticket/560 (Graph API/GOM)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/TestAll.java
branches/BIGDATA_RELEASE_1_2_0/build.xml
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/TestAll.java 2012-06-12 14:58:53 UTC (rev 6338)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/TestAll.java 2012-06-12 15:38:03 UTC (rev 6339)
@@ -130,6 +130,12 @@
// The REST API test suite.
suite.addTest(com.bigdata.rdf.sail.webapp.TestAll.suite());
+ /*
+ * The Generic Object Model and Graph API (includes remote tests against
+ * the NanoSparqlServer layer).
+ */
+ suite.addTest(com.bigdata.gom.TestAll.suite());
+
return suite;
}
Modified: branches/BIGDATA_RELEASE_1_2_0/build.xml
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/build.xml 2012-06-12 14:58:53 UTC (rev 6338)
+++ branches/BIGDATA_RELEASE_1_2_0/build.xml 2012-06-12 15:38:03 UTC (rev 6339)
@@ -2077,6 +2077,8 @@
<test name="com.bigdata.rdf.sail.TestAll" todir="${test.results.dir}" unless="testName" />
<test name="com.bigdata.rdf.sail.webapp.TestAll" todir="${test.results.dir}" unless="testName" />
+ <test name="com.bigdata.gom.TestAll" todir="${test.results.dir}" unless="testName" />
+
<!-- All tests.
<test name="com.bigdata.TestAll" todir="${test.results.dir}" unless="testName" />
-->
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-06-12 16:18:31
|
Revision: 6340
http://bigdata.svn.sourceforge.net/bigdata/?rev=6340&view=rev
Author: thompsonbry
Date: 2012-06-12 16:18:20 +0000 (Tue, 12 Jun 2012)
Log Message:
-----------
Modified the top-level build.xml file to appropriate stage the GOM files for CI.
Added a bigdata-gom/src/samples directory (nothing there yet).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/.classpath
branches/BIGDATA_RELEASE_1_2_0/build.xml
Modified: branches/BIGDATA_RELEASE_1_2_0/.classpath
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/.classpath 2012-06-12 15:38:03 UTC (rev 6339)
+++ branches/BIGDATA_RELEASE_1_2_0/.classpath 2012-06-12 16:18:20 UTC (rev 6340)
@@ -1,87 +1,88 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" path="bigdata-rdf/src/java"/>
- <classpathentry kind="src" path="bigdata-rdf/src/samples"/>
- <classpathentry kind="src" path="dsi-utils/src/java"/>
- <classpathentry kind="src" path="bigdata/src/resources/logging"/>
- <classpathentry kind="src" path="bigdata-sails/src/samples"/>
- <classpathentry kind="src" path="bigdata-jini/src/test"/>
- <classpathentry kind="src" path="bigdata-sails/src/java"/>
- <classpathentry kind="src" path="bigdata/src/java"/>
- <classpathentry kind="src" path="bigdata-rdf/src/test"/>
- <classpathentry kind="src" path="bigdata/src/test"/>
- <classpathentry kind="src" path="bigdata-sails/src/test"/>
- <classpathentry kind="src" path="bigdata-jini/src/java"/>
- <classpathentry kind="src" path="contrib/src/problems"/>
- <classpathentry kind="src" path="bigdata/src/samples"/>
- <classpathentry kind="src" path="dsi-utils/src/test"/>
- <classpathentry kind="src" path="ctc-striterators/src/java"/>
- <classpathentry kind="src" path="ctc-striterators/src/test"/>
- <classpathentry kind="src" path="junit-ext/src/test"/>
- <classpathentry kind="src" path="junit-ext/src/java"/>
- <classpathentry kind="src" path="lgpl-utils/src/java"/>
- <classpathentry kind="src" path="lgpl-utils/src/test"/>
- <classpathentry kind="src" path="bigdata-war/src"/>
- <classpathentry kind="src" path="bigdata-ganglia/src/java"/>
- <classpathentry kind="src" path="bigdata-ganglia/src/test"/>
- <classpathentry kind="src" path="bigdata-rdf/src/resources/service-providers"/>
- <classpathentry kind="src" path="bigdata-gom/src/java"/>
- <classpathentry kind="src" path="bigdata-gom/src/test"/>
- <classpathentry kind="src" path="bigdata-gom/src/resources"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/dsi-utils-1.0.6-020610.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/lgpl-utils-1.0.6-020610.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/apache/zookeeper-3.3.3.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-rdf/lib/nxparser-6-22-2010.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-continuation-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-http-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-io-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-server-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-util-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/servlet-api-2.5.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-servlet-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-security-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-webapp-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-xml-7.2.2.v20101205.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/unimi/colt-1.2.0.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/junit-3.8.1.jar" sourcepath="/root/.m2/repository/junit/junit/3.8.1/junit-3.8.1-sources.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/apache/log4j-1.2.15.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/browser.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/classserver.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/fiddler.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jini-core.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jini-ext.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jsk-lib.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jsk-platform.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jsk-resources.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/mahalo.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/mercury.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/norm.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/outrigger.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/reggie.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/start.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/sun-util.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/tools.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/unimi/fastutil-5.1.5.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/lucene/lucene-analyzers-3.0.0.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/lucene/lucene-core-3.0.0.jar"/>
- <classpathentry kind="lib" path="bigdata-rdf/lib/sesame-rio-testsuite-2.6.3.jar" sourcepath="/org.openrdf.sesame-2.6.1"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/high-scale-lib-v1.1.2.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/icu/icu4j-4_8.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/icu/icu4j-charset-4_8.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/junit-ext-1.1-b3-dev.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-rdf/lib/slf4j-api-1.6.1.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata-rdf/lib/slf4j-log4j12-1.6.1.jar"/>
- <classpathentry kind="lib" path="bigdata-rdf/lib/openrdf-sesame-2.6.3-onejar.jar" sourcepath="/org.openrdf.sesame-2.6.1"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/sesame-sparql-testsuite-2.6.3.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/sesame-store-testsuite-2.6.3.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-codec-1.4.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-logging-1.1.1.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpclient-4.1.3.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpclient-cache-4.1.3.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpcore-4.1.4.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpmime-4.1.3.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-fileupload-1.2.2.jar"/>
- <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-io-2.1.jar"/>
- <classpathentry kind="output" path="bin"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="bigdata-rdf/src/java"/>
+ <classpathentry kind="src" path="bigdata-rdf/src/samples"/>
+ <classpathentry kind="src" path="dsi-utils/src/java"/>
+ <classpathentry kind="src" path="bigdata/src/resources/logging"/>
+ <classpathentry kind="src" path="bigdata-sails/src/samples"/>
+ <classpathentry kind="src" path="bigdata-jini/src/test"/>
+ <classpathentry kind="src" path="bigdata-sails/src/java"/>
+ <classpathentry kind="src" path="bigdata/src/java"/>
+ <classpathentry kind="src" path="bigdata-rdf/src/test"/>
+ <classpathentry kind="src" path="bigdata/src/test"/>
+ <classpathentry kind="src" path="bigdata-sails/src/test"/>
+ <classpathentry kind="src" path="bigdata-jini/src/java"/>
+ <classpathentry kind="src" path="contrib/src/problems"/>
+ <classpathentry kind="src" path="bigdata/src/samples"/>
+ <classpathentry kind="src" path="dsi-utils/src/test"/>
+ <classpathentry kind="src" path="ctc-striterators/src/java"/>
+ <classpathentry kind="src" path="ctc-striterators/src/test"/>
+ <classpathentry kind="src" path="junit-ext/src/test"/>
+ <classpathentry kind="src" path="junit-ext/src/java"/>
+ <classpathentry kind="src" path="lgpl-utils/src/java"/>
+ <classpathentry kind="src" path="lgpl-utils/src/test"/>
+ <classpathentry kind="src" path="bigdata-war/src"/>
+ <classpathentry kind="src" path="bigdata-ganglia/src/java"/>
+ <classpathentry kind="src" path="bigdata-ganglia/src/test"/>
+ <classpathentry kind="src" path="bigdata-rdf/src/resources/service-providers"/>
+ <classpathentry kind="src" path="bigdata-gom/src/java"/>
+ <classpathentry kind="src" path="bigdata-gom/src/test"/>
+ <classpathentry kind="src" path="bigdata-gom/src/resources"/>
+ <classpathentry kind="src" path="bigdata-gom/src/samples"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/dsi-utils-1.0.6-020610.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/lgpl-utils-1.0.6-020610.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/apache/zookeeper-3.3.3.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-rdf/lib/nxparser-6-22-2010.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-continuation-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-http-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-io-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-server-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-util-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/servlet-api-2.5.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-servlet-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-security-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-webapp-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/jetty/jetty-xml-7.2.2.v20101205.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/unimi/colt-1.2.0.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/junit-3.8.1.jar" sourcepath="/root/.m2/repository/junit/junit/3.8.1/junit-3.8.1-sources.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/apache/log4j-1.2.15.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/browser.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/classserver.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/fiddler.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jini-core.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jini-ext.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jsk-lib.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jsk-platform.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/jsk-resources.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/mahalo.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/mercury.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/norm.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/outrigger.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/reggie.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/start.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/sun-util.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-jini/lib/jini/lib/tools.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/unimi/fastutil-5.1.5.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/lucene/lucene-analyzers-3.0.0.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/lucene/lucene-core-3.0.0.jar"/>
+ <classpathentry kind="lib" path="bigdata-rdf/lib/sesame-rio-testsuite-2.6.3.jar" sourcepath="/org.openrdf.sesame-2.6.3"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/high-scale-lib-v1.1.2.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/icu/icu4j-4_8.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/icu/icu4j-charset-4_8.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata/lib/junit-ext-1.1-b3-dev.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-rdf/lib/slf4j-api-1.6.1.jar"/>
+ <classpathentry exported="true" kind="lib" path="bigdata-rdf/lib/slf4j-log4j12-1.6.1.jar"/>
+ <classpathentry kind="lib" path="bigdata-rdf/lib/openrdf-sesame-2.6.3-onejar.jar" sourcepath="/Users/bryan/Documents/workspace/org.openrdf.sesame-2.6.3"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/sesame-sparql-testsuite-2.6.3.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/sesame-store-testsuite-2.6.3.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-codec-1.4.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-logging-1.1.1.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpclient-4.1.3.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpclient-cache-4.1.3.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpcore-4.1.4.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/httpmime-4.1.3.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-fileupload-1.2.2.jar"/>
+ <classpathentry kind="lib" path="bigdata-sails/lib/httpcomponents/commons-io-2.1.jar"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
Modified: branches/BIGDATA_RELEASE_1_2_0/build.xml
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/build.xml 2012-06-12 15:38:03 UTC (rev 6339)
+++ branches/BIGDATA_RELEASE_1_2_0/build.xml 2012-06-12 16:18:20 UTC (rev 6340)
@@ -65,6 +65,9 @@
<fileset dir="${bigdata.dir}/bigdata-sails/lib">
<include name="**/*.jar" />
</fileset>
+ <fileset dir="${bigdata.dir}/bigdata-gom/lib">
+ <include name="**/*.jar" />
+ </fileset>
<!--
<fileset dir="${bigdata.dir}/ctc-striterator/lib">
<include name="**/*.jar" />
@@ -224,8 +227,9 @@
<!-- note: must also specify -bootclasspath and -extdirs when cross-compiling -->
<src path="${bigdata.dir}/bigdata/src/java" />
<src path="${bigdata.dir}/bigdata-jini/src/java" />
- <src path="${bigdata.dir}/bigdata-rdf/src/java" />
+ <src path="${bigdata.dir}/bigdata-rdf/src/java" />
<src path="${bigdata.dir}/bigdata-sails/src/java" />
+ <src path="${bigdata.dir}/bigdata-gom/src/java" />
<src path="${bigdata.dir}/ctc-striterators/src/java" />
<!-- Do not include the unit tests @todo conditionally include?
<src path="${bigdata.dir}/bigdata/src/test"/>
@@ -254,6 +258,10 @@
<exclude name="**/*.java" />
<exclude name="**/package.html" />
</fileset>
+ <fileset dir="${bigdata.dir}/bigdata-gom/src/java">
+ <exclude name="**/*.java" />
+ <exclude name="**/package.html" />
+ </fileset>
<!-- Note: This simple copy works so long as there is just one service
provider file per interface. It will not combine (append) multiple
files for the same interface. -->
@@ -286,6 +294,8 @@
<fileset dir="${bigdata.dir}/bigdata-rdf/src/samples" />
<fileset dir="${bigdata.dir}/bigdata-sails/src/java" />
<fileset dir="${bigdata.dir}/bigdata-sails/src/samples" />
+ <fileset dir="${bigdata.dir}/bigdata-gom/src/java" />
+ <fileset dir="${bigdata.dir}/bigdata-gom/src/samples" />
<fileset dir="${bigdata.dir}/ctc-striterators/src/java" />
</jar>
</target>
@@ -304,7 +314,7 @@
<fileset dir="${bigdata.dir}/bigdata">
<include name="LEGAL/apache-license-2_0.txt"/>
</fileset>
- <fileset dir="${bigdata.dir}/bigdata-rdf">
+ <fileset dir="${bigdata.dir}/bigdata-rdf">
<include name="LEGAL/sesame2.x-license.txt"/>
</fileset>
<manifest>
@@ -330,8 +340,9 @@
</manifest>
<fileset dir="bigdata/src/java" />
<fileset dir="bigdata-jini/src/java" />
- <fileset dir="bigdata-rdf/src/java" />
+ <fileset dir="bigdata-rdf/src/java" />
<fileset dir="bigdata-sails/src/java" />
+ <fileset dir="bigdata-gom/src/java" />
</jar>
<bnd output="${build.dir}/bundles/com.bigata-${osgi.version}.jar" classpath="${build.dir}/classes" eclipse="false" failok="false" exceptions="true" files="${basedir}/osgi/bigdata.bnd" />
@@ -368,6 +379,8 @@
<packageset dir="${bigdata.dir}/bigdata-rdf/src/samples" />
<packageset dir="${bigdata.dir}/bigdata-sails/src/java" />
<packageset dir="${bigdata.dir}/bigdata-sails/src/samples" />
+ <packageset dir="${bigdata.dir}/bigdata-gom/src/java" />
+ <packageset dir="${bigdata.dir}/bigdata-gom/src/samples" />
<packageset dir="${bigdata.dir}/ctc-striterators/src/java" />
<doctitle>
<![CDATA[<h1>bigdata® v${build.ver}</h1>]]></doctitle>
@@ -405,6 +418,9 @@
<fileset dir="${bigdata.dir}/bigdata-sails/lib">
<include name="**/*.jar" />
</fileset>
+ <fileset dir="${bigdata.dir}/bigdata-gom/lib">
+ <include name="**/*.jar" />
+ </fileset>
</copy>
<!-- Do NOT flatten the jini jars. We need the to preserve the -->
<!-- lib, lib-dl, and lib-ext distinctions. -->
@@ -470,6 +486,7 @@
<fileset dir="${bigdata.dir}/bigdata" includes="LEGAL/*"/>
<fileset dir="${bigdata.dir}/bigdata-rdf" includes="LEGAL/*"/>
<fileset dir="${bigdata.dir}/bigdata-sails" includes="LEGAL/*"/>
+ <fileset dir="${bigdata.dir}/bigdata-gom" includes="LEGAL/*"/>
<fileset dir="${bigdata.dir}/bigdata-jini" includes="LEGAL/*"/>
<!-- Basic RWStore configuration. -->
<file file="bigdata-war/src/resources/RWStore.properties"/>
@@ -578,8 +595,9 @@
<include name="README-JINI" />
<include name="bigdata/LEGAL/*" />
<include name="bigdata-jini/LEGAL/*" />
- <include name="bigdata-rdf/LEGAL/*" />
+ <include name="bigdata-rdf/LEGAL/*" />
<include name="bigdata-sails/LEGAL/*" />
+ <include name="bigdata-gom/LEGAL/*" />
</fileset>
</copy>
<!-- install JARs. -->
@@ -841,6 +859,7 @@
<property name="bigdata-jini.lib" location="${bigdata.dir}/bigdata-jini/lib/jini/lib" />
<property name="bigdata-rdf.lib" location="${bigdata.dir}/bigdata-rdf/lib" />
<property name="bigdata-sails.lib" location="${bigdata.dir}/bigdata-sails/lib" />
+ <property name="bigdata-gom.lib" location="${bigdata.dir}/bigdata-gom/lib" />
<property name="bigdata-jetty.lib" location="${bigdata.dir}/bigdata/lib/jetty" />
<property name="bigdata-http.lib" location="${bigdata.dir}/bigdata-sails/lib/httpcomponents" />
<property name="bigdata-zookeeper.lib" location="${bigdata.dir}/bigdata-jini/lib/apache" />
@@ -878,6 +897,9 @@
<copy file="${bigdata-rdf.lib}/slf4j-log4j12-${slf4j.version}.jar"
tofile="${dist.lib}/slf4j-log4j.jar" />
+ <!-- GOM library -->
+ <!-- Note: Nothing yet for GOM -->
+
<!-- jetty library -->
<copy file="${bigdata-jetty.lib}/jetty-continuation-${jetty.version}.jar"
tofile="${dist.lib}/jetty-continuation.jar" />
@@ -1153,6 +1175,9 @@
<copy toDir="${build.dir}/bigdata-sails/src">
<fileset dir="${bigdata.dir}/bigdata-sails/src" />
</copy>
+ <copy toDir="${build.dir}/bigdata-gom/src">
+ <fileset dir="${bigdata.dir}/bigdata-gom/src" />
+ </copy>
<copy toDir="${build.dir}/bigdata-war/src">
<fileset dir="${bigdata.dir}/bigdata-war/src" />
</copy>
@@ -1179,6 +1204,10 @@
<copy toDir="${build.dir}/bigdata-rdf/lib">
<fileset dir="${bigdata.dir}/bigdata-rdf/lib" />
</copy>
+ <mkdir dir="${build.dir}/bigdata-gom/lib" />
+ <copy toDir="${build.dir}/bigdata-gom/lib">
+ <fileset dir="${bigdata.dir}/bigdata-gom/lib" />
+ </copy>
<mkdir dir="${build.dir}/bigdata-sails/lib" />
<copy toDir="${build.dir}/bigdata-sails/lib">
<fileset dir="${bigdata.dir}/bigdata-sails/lib" />
@@ -1236,6 +1265,7 @@
<include name="bigdata-jini/src/**" />
<include name="bigdata-rdf/src/**" />
<include name="bigdata-sails/src/**" />
+ <include name="bigdata-gom/src/**" />
<include name="bigdata-war/src/**" />
<include name="ctc-striterators/src/**" />
<include name="lgpl-utils/src/**" />
@@ -1244,6 +1274,7 @@
<include name="bigdata-jini/lib/**" />
<include name="bigdata-rdf/lib/**" />
<include name="bigdata-sails/lib/**" />
+ <include name="bigdata-gom/lib/**" />
<include name="src/**" />
<exclude name="classes/**" />
<exclude name="${version}.jar" />
@@ -1301,6 +1332,7 @@
<include name="bigdata-jini/LEGAL/*" />
<include name="bigdata-rdf/LEGAL/*" />
<include name="bigdata-sails/LEGAL/*" />
+ <include name="bigdata-gom/LEGAL/*" />
</fileset>
</copy>
@@ -1553,6 +1585,7 @@
<src path="${bigdata.dir}/bigdata-jini/src/test" />
<src path="${bigdata.dir}/bigdata-rdf/src/test" />
<src path="${bigdata.dir}/bigdata-sails/src/test" />
+ <src path="${bigdata.dir}/bigdata-gom/src/test" />
<src path="${bigdata.dir}/ctc-striterators/src/test" />
<!--
<src path="${bigdata.dir}/bigdata-gom/src/test" />
@@ -1602,6 +1635,9 @@
<fileset dir="${bigdata.dir}/bigdata-sails/src/test">
<exclude name="**/*.java" />
</fileset>
+ <fileset dir="${bigdata.dir}/bigdata-gom/src/test">
+ <exclude name="**/*.java" />
+ </fileset>
<fileset dir="${bigdata.dir}/bigdata-jini/src/test">
<exclude name="**/*.java" />
</fileset>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2012-06-14 18:08:22
|
Revision: 6352
http://bigdata.svn.sourceforge.net/bigdata/?rev=6352&view=rev
Author: thompsonbry
Date: 2012-06-14 18:08:09 +0000 (Thu, 14 Jun 2012)
Log Message:
-----------
- Added support for the HTree on the AbstractJournal and
TemporaryStore. This sets up a pattern which can be used to provide
support for other persistence capable data structures. There are
some API conflicts which I was not able to resolve. The main
conflict is
public IIndex IBTreeManager:registerIndex(String name, IndexMetadata indexMetadata);
That method returns an IIndex, but HTree does not implement IIndex.
The method can also return an object which is not an ICommitter,
such as a FusedView.
The IndexMetadata class was subclassed to create a version for the
HTree, HTreeIndexMetadata. You must now use the HTree variant in
order to create an HTree. This change is backwards compatible with
existing BTree records (there are no durable HTree records since it
was only used with the MemoryManager historically). The
IndexMetadata now includes the IndexTypeEnum and passes that value
through to the Checkpoint class. A single unit test was written to
verify that an HTree can be registered against a Journal, recovered
by its name, and dropped. More tests should be added. No tests
were written for the TemporaryStore, but it also supports the HTree
now.
Name2Addr is now completely written to the ICheckpointProtocol
interface. It is no longer BTree specific. AbstractJournal was
changed to be generic (at the ICheckpointProtocol layer) as much as
possible, but the registerIndex(name, metadata): IIndex method
prevents a fully generic solution.
Much of the logic for creating a persistence capable data structure
from an IndexMetadata object, and for loading one from a checkpoint
record, is now isolated in static methods on the Checkpoint class.
I have run through the majority of the test suites and everything is
green.
@see https://sourceforge.net/apps/trac/bigdata/ticket/531 (SPARQL UPDATE for SOLUTION SETS).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinUtility.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/BTree.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/Checkpoint.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/htree/AbstractHTree.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/htree/HTree.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/AbstractTask.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/IBTreeManager.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/Journal.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/Name2Addr.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/journal/TemporaryStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/relation/AbstractRelation.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/search/FullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/AbstractHTreeTestCase.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/HTreeVersusHashMapDemo.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/ShowHTreeResourceUsage.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestHTreeRecycle.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestHTreeWithMemStore.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestHTree_init.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestIncrementalWrite.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestRawRecords.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestRemovals.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestReopen.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/htree/TestTransientHTree.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/test/com/bigdata/journal/TestNamedIndices.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/bop/rdf/filter/NativeDistinctFilter.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataSubjectCentricFullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/BigdataValueCentricFullTextIndex.java
branches/BIGDATA_RELEASE_1_2_0/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/HTreeIndexMetadata.java
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/IndexTypeEnum.java
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2012-06-14 14:14:07 UTC (rev 6351)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2012-06-14 18:08:09 UTC (rev 6352)
@@ -26,6 +26,7 @@
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
+import com.bigdata.btree.IndexTypeEnum;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.btree.proc.ISimpleIndexProcedure;
@@ -457,7 +458,7 @@
final String name = getNamespace()+"."+FILE_METADATA_INDEX_BASENAME;
final IndexMetadata md = new IndexMetadata(indexManager, tmp,
- name, UUID.randomUUID());
+ name, UUID.randomUUID(), IndexTypeEnum.BTree);
// Ensure that splits do not break logical rows.
md.setSplitHandler(LogicalRowSplitHandler.INSTANCE);
@@ -481,7 +482,7 @@
final String name = getNamespace()+"."+FILE_DATA_INDEX_BASENAME;
final IndexMetadata md = new IndexMetadata(indexManager, tmp,
- name, UUID.randomUUID());
+ name, UUID.randomUUID(), IndexTypeEnum.BTree);
/*
* @todo unit tests for correct copying of blobs during overflow.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinUtility.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinUtility.java 2012-06-14 14:14:07 UTC (rev 6351)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/bop/join/HTreeHashJoinUtility.java 2012-06-14 18:08:09 UTC (rev 6352)
@@ -50,6 +50,7 @@
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.Checkpoint;
import com.bigdata.btree.DefaultTupleSerializer;
+import com.bigdata.btree.HTreeIndexMetadata;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.ITupleSerializer;
@@ -427,9 +428,10 @@
* Setup the {@link IndexMetadata} for {@link #rightSolutions} or
* {@link #joinSet}.
*/
- static private IndexMetadata getIndexMetadata(final PipelineOp op) {
+ static private HTreeIndexMetadata getIndexMetadata(final PipelineOp op) {
- final IndexMetadata metadata = new IndexMetadata(UUID.randomUUID());
+ final HTreeIndexMetadata metadata = new HTreeIndexMetadata(
+ UUID.randomUUID());
final int addressBits = op.getProperty(HTreeAnnotations.ADDRESS_BITS,
HTreeAnnotations.DEFAULT_ADDRESS_BITS);
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/BTree.java 2012-06-14 14:14:07 UTC (rev 6351)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/BTree.java 2012-06-14 18:08:09 UTC (rev 6352)
@@ -1568,6 +1568,10 @@
* You can use {@link IndexMetadata#clone()} to obtain a new
* copy of the metadata object with the metadata address set
* to <code>0L</code>.
+ * @exception IllegalStateException
+ * if the {@link IndexTypeEnum} in the supplied
+ * {@link IndexMetadata} object is not
+ * {@link IndexTypeEnum#BTree}.
*/
public static BTree create(final IRawStore store, final IndexMetadata metadata) {
@@ -1577,6 +1581,13 @@
}
+ if (metadata.getIndexType() != IndexTypeEnum.BTree) {
+
+ throw new IllegalStateException("Wrong index type: "
+ + metadata.getIndexType());
+
+ }
+
/*
* Write metadata record on store. The address of that record is set as
* a side-effect on the metadata object.
@@ -1751,8 +1762,8 @@
+ store.toString(addrCheckpoint), t);
}
-// if (checkpoint.getIndexType() != IndexTypeEnum.BTree)
-// throw new RuntimeException("Not a BTree checkpoint: " + checkpoint);
+ if (checkpoint.getIndexType() != IndexTypeEnum.BTree)
+ throw new RuntimeException("Not a BTree checkpoint: " + checkpoint);
/*
* Read metadata record from store.
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/Checkpoint.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/Checkpoint.java 2012-06-14 14:14:07 UTC (rev 6351)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/Checkpoint.java 2012-06-14 18:08:09 UTC (rev 6352)
@@ -1,3 +1,26 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
package com.bigdata.btree;
import java.io.Externalizable;
@@ -53,43 +76,6 @@
private IndexTypeEnum indexType;
/**
- * Type safe enumeration of index types.
- */
- public static enum IndexTypeEnum {
-
- /** BTree. */
- BTree((short)0),
-
- /** Extendable hash tree. */
- HTree((short)1);
-
- private IndexTypeEnum(final short code) {
-
- this.code = code;
-
- }
-
- private final short code;
-
- public short getCode() {
-
- return code;
-
- }
-
- static public IndexTypeEnum valueOf(final short code) {
- switch (code) {
- case 0:
- return BTree;
- case 1:
- return HTree;
- default:
- throw new IllegalArgumentException("code=" + code);
- }
- }
- }
-
- /**
* The address used to read this {@link Checkpoint} record from the
* store.
* <p>
@@ -300,7 +286,7 @@
0L, // nentries
0L, // counter
0L, // recordVersion
- IndexTypeEnum.BTree // indexType
+ metadata.getIndexType() // indexType
);
@@ -331,7 +317,7 @@
0L, // nentries
oldCheckpoint.counter,//
0L, // recordVersion
- IndexTypeEnum.BTree//
+ metadata.getIndexType()//
);
}
@@ -395,7 +381,7 @@
*/
btree.counter.get(),//
btree.getRecordVersion(),//
- IndexTypeEnum.BTree//
+ IndexTypeEnum.BTree // IndexTypeEnum
);
}
@@ -450,7 +436,7 @@
htree.getEntryCount(),//
htree.getCounter().get(),//
htree.getRecordVersion(),//
- IndexTypeEnum.HTree//
+ IndexTypeEnum.HTree // IndexTypeEnum
);
}
@@ -736,4 +722,88 @@
}
+ /**
+ * Utility method reads the {@link Checkpoint} record and then loads and
+ * returns a view of the associated read-only persistence capable data
+ * structure.
+ * <p>
+ * <strong>Note: The caller is responsible for marking the returned object
+ * as read-only or not depending on the context. This method should be used
+ * from trusted code such as {@link Name2Addr} and {@link AbstractJournal}
+ * which can make this decision.</strong>
+ *
+ * @param store
+ * The backing store.
+ * @param checkpointAddr
+ * The address of the checkpoint record.
+ * @param readOnly
+ * <code>true</code> if the object will be read-only.
+ *
+ * @return The persistence capable data structure loaded from that
+ * checkpoint.
+ */
+ public static ICheckpointProtocol loadFromCheckpoint(final IRawStore store,
+ final long checkpointAddr, final boolean readOnly) {
+
+ /*
+ * Read checkpoint record from store.
+ */
+ final Checkpoint checkpoint;
+ try {
+ checkpoint = Checkpoint.load(store, checkpointAddr);
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not load Checkpoint: store="
+ + store + ", addrCheckpoint="
+ + store.toString(checkpointAddr), t);
+ }
+
+ // re-load from the store.
+ final ICheckpointProtocol ndx;
+ switch (checkpoint.getIndexType()) {
+ case BTree:
+ ndx = BTree.load(store, checkpointAddr, readOnly);
+ break;
+ case HTree:
+ ndx = HTree.load(store, checkpointAddr, readOnly);
+ break;
+ default:
+ throw new AssertionError("Unknown: " + checkpoint.getIndexType());
+ }
+
+ // // set the lastCommitTime on the index.
+ // btree.setLastCommitTime(lastCommitTime);
+
+ return ndx;
+
+ }
+
+ /**
+ * Create a persistence capable data structure.
+ *
+ * @param store
+ * The backing store.
+ * @param metadata
+ * The metadata that describes the data structure to be created.
+ *
+ * @return The persistence capable data structure.
+ */
+ public static ICheckpointProtocol create(final IRawStore store,
+ final IndexMetadata metadata) {
+
+ final ICheckpointProtocol ndx;
+ switch (metadata.getIndexType()) {
+ case BTree:
+ ndx = BTree.create(store, metadata);
+ break;
+ case HTree:
+ ndx = HTree.create(store, (HTreeIndexMetadata) metadata);
+ break;
+ default:
+ throw new AssertionError("Unknown: " + metadata.getIndexType());
+ }
+
+ return ndx;
+
+ }
+
}
Added: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/HTreeIndexMetadata.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/HTreeIndexMetadata.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/HTreeIndexMetadata.java 2012-06-14 18:08:09 UTC (rev 6352)
@@ -0,0 +1,312 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+package com.bigdata.btree;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Properties;
+import java.util.UUID;
+
+import com.bigdata.btree.raba.codec.IRabaCoder;
+import com.bigdata.htree.HTree;
+import com.bigdata.io.LongPacker;
+import com.bigdata.journal.IIndexManager;
+import com.bigdata.service.IBigdataFederation;
+
+/**
+ * HTree specific implementation.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+public class HTreeIndexMetadata extends IndexMetadata {
+
+ private static final long serialVersionUID = -1L;
+
+ /**
+ * {@link HTree} specific options.
+ */
+ public interface Options extends IndexMetadata.Options {
+
+ /**
+ * The name of a class derived from {@link HTree} that will be used to
+ * re-load the index.
+ */
+ String HTREE_CLASS_NAME = HTree.class.getName()+".className";
+
+ /**
+ * The name of an optional property whose value specifies the number of
+ * address bits for an {@link HTree} (default
+ * {@value #DEFAULT_HTREE_ADDRESS_BITS}).
+ * <p>
+ * The #of children for a directory is <code>2^addressBits</code>. For
+ * example, a value of <code>10</code> means a <code>10</code> bit
+ * address space in the directory. Such a directory would provide direct
+ * addressing for <code>1024</code> child references. Given an overhead
+ * of <code>8</code> bytes per child address, that would result in an
+ * expected page size of 8k before compression.
+ *
+ * @see #DEFAULT_HTREE_ADDRESS_BITS
+ */
+ String HTREE_ADDRESS_BITS = HTree.class.getPackage().getName()
+ + ".addressBits";
+
+ String DEFAULT_HTREE_ADDRESS_BITS = "10";
+
+ /**
+ * The name of an optional property whose value specifies the fixed by
+ * length of the keys in the {@link HTree} -or- ZERO (0) if the key
+ * length is unconstrained, in which case variable length keys may be
+ * used (default {@value #DEFAULT_HTREE_KEY_LEN}). This may be used in
+ * combination with an appropriate {@link IRabaCoder} to optimize to
+ * search and encoding of int32 or int64 keys.
+ *
+ * @see #DEFAULT_HTREE_KEY_LEN
+ */
+ String HTREE_KEY_LEN = HTree.class.getPackage().getName()
+ + ".keyLen";
+
+ String DEFAULT_HTREE_KEY_LEN = "0";
+
+ }
+
+ /**
+ * @see Options#HTREE_CLASS_NAME
+ */
+ private String htreeClassName;
+
+ /**
+ * @see Options#HTREE_ADDRESS_BITS
+ */
+ private int addressBits;
+
+ /**
+ * @see Options#HTREE_KEY_LEN
+ */
+ private int keyLen;
+
+ /**
+ * The name of a class derived from {@link HTree} that will be used to
+ * re-load the index.
+ *
+ * @see Options#HTREE_CLASS_NAME
+ */
+ public final String getHTreeClassName() {
+
+ return htreeClassName;
+
+ }
+
+ public void setHTreeClassName(final String className) {
+
+ if (className == null)
+ throw new IllegalArgumentException();
+
+ this.htreeClassName = className;
+
+ }
+
+ public int getAddressBits() {
+
+ return addressBits;
+
+ }
+
+ public void setAddressBits(final int addressBits) {
+
+ this.addressBits = addressBits;
+
+ }
+
+ public int getKeyLen() {
+
+ return keyLen;
+
+ }
+
+ public void setKeyLen(final int keyLen) {
+
+ this.keyLen = keyLen;
+
+ }
+
+ /**
+ * <strong>De-serialization constructor only</strong> - DO NOT use this ctor
+ * for creating a new instance! It will result in a thrown exception,
+ * typically from {@link #firstCheckpoint()}.
+ */
+ public HTreeIndexMetadata() {
+
+ super();
+
+ }
+
+ /**
+ * Constructor used to configure a new <em>unnamed</em> {@link HTree}. The
+ * index UUID is set to the given value and all other fields are defaulted
+ * as explained at {@link #HTreeIndexMetadata(Properties, String, UUID)}.
+ * Those defaults may be overridden using the various setter methods, but
+ * some values can not be safely overridden after the index is in use.
+ *
+ * @param indexUUID
+ * The indexUUID.
+ *
+ * @throws IllegalArgumentException
+ * if the indexUUID is <code>null</code>.
+ */
+ public HTreeIndexMetadata(final UUID indexUUID) {
+
+ this(null/* name */, indexUUID);
+
+ }
+
+ /**
+ * Constructor used to configure a new <em>named</em> {@link BTree}. The
+ * index UUID is set to the given value and all other fields are defaulted
+ * as explained at {@link #IndexMetadata(Properties, String, UUID)}. Those
+ * defaults may be overridden using the various setter methods, but some
+ * values can not be safely overridden after the index is in use.
+ *
+ * @param name
+ * The index name. When this is a scale-out index, the same
+ * <i>name</i> is specified for each index resource. However they
+ * will be registered on the journal under different names
+ * depending on the index partition to which they belong.
+ *
+ * @param indexUUID
+ * The indexUUID. The same index UUID MUST be used for all
+ * component indices in a scale-out index.
+ *
+ * @throws IllegalArgumentException
+ * if the indexUUID is <code>null</code>.
+ */
+ public HTreeIndexMetadata(final String name, final UUID indexUUID) {
+
+ this(null/* name */, System.getProperties(), name, indexUUID);
+
+ }
+
+ /**
+ * Constructor used to configure a new <em>named</em> B+Tree. The index UUID
+ * is set to the given value and all other fields are defaulted as explained
+ * at {@link #getProperty(Properties, String, String, String)}. Those
+ * defaults may be overridden using the various setter methods.
+ *
+ * @param indexManager
+ * Optional. When given and when the {@link IIndexManager} is a
+ * scale-out {@link IBigdataFederation}, this object will be used
+ * to interpret the {@link Options#INITIAL_DATA_SERVICE}
+ * property.
+ * @param properties
+ * Properties object used to overridden the default values for
+ * this {@link IndexMetadata} instance.
+ * @param namespace
+ * The index name. When this is a scale-out index, the same
+ * <i>name</i> is specified for each index resource. However they
+ * will be registered on the journal under different names
+ * depending on the index partition to which they belong.
+ * @param indexUUID
+ * component indices in a scale-out index.
+ * The indexUUID. The same index UUID MUST be used for all
+ * @param indexType
+ * Type-safe enumeration specifying the type of the persistence
+ * class data structure (historically, this was always a B+Tree).
+ *
+ * @throws IllegalArgumentException
+ * if <i>properties</i> is <code>null</code>.
+ * @throws IllegalArgumentException
+ * if <i>indexUUID</i> is <code>null</code>.
+ */
+ public HTreeIndexMetadata(final IIndexManager indexManager,
+ final Properties properties, final String namespace,
+ final UUID indexUUID) {
+
+ super(indexManager, properties, namespace, indexUUID,
+ IndexTypeEnum.HTree);
+
+ /*
+ * Intern'd to reduce duplication on the heap.
+ */
+ this.htreeClassName = getProperty(indexManager, properties, namespace,
+ Options.HTREE_CLASS_NAME, HTree.class.getName()).intern();
+
+ this.addressBits = Integer.parseInt(getProperty(indexManager,
+ properties, namespace, Options.HTREE_ADDRESS_BITS,
+ Options.DEFAULT_HTREE_ADDRESS_BITS));
+
+ this.keyLen = Integer
+ .parseInt(getProperty(indexManager, properties, namespace,
+ Options.HTREE_KEY_LEN, Options.DEFAULT_HTREE_KEY_LEN));
+
+ }
+
+ @Override
+ protected void toString(final StringBuilder sb) {
+
+ super.toString(sb);
+
+ // htree
+ sb.append(", htreeClassName=" + htreeClassName);
+ sb.append(", addressBits=" + addressBits);
+ sb.append(", keyLen=" + keyLen);
+
+ }
+
+ @Override
+ public void readExternal(final ObjectInput in) throws IOException,
+ ClassNotFoundException {
+
+ super.readExternal(in);
+
+ keyLen = LongPacker.unpackInt(in);
+
+ addressBits = LongPacker.unpackInt(in);
+
+ htreeClassName = in.readUTF();
+
+ }
+
+ @Override
+ public void writeExternal(final ObjectOutput out) throws IOException {
+
+ final int version = CURRENT_VERSION;
+
+ super.writeExternal(out);
+
+ LongPacker.packLong(out, keyLen);
+
+ LongPacker.packLong(out, addressBits);
+
+ out.writeUTF(htreeClassName);
+
+ }
+
+ @Override
+ public HTreeIndexMetadata clone() {
+
+ return (HTreeIndexMetadata) super.clone();
+
+ }
+
+}
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2012-06-14 14:14:07 UTC (rev 6351)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2012-06-14 18:08:09 UTC (rev 6352)
@@ -38,7 +38,6 @@
import org.apache.log4j.Logger;
import com.bigdata.LRUNexus;
-import com.bigdata.btree.Checkpoint.IndexTypeEnum;
import com.bigdata.btree.data.ILeafData;
import com.bigdata.btree.data.INodeData;
import com.bigdata.btree.isolation.IConflictResolver;
@@ -78,7 +77,7 @@
* <p>
* The persistent and mostly immutable metadata for a {@link AbstractBTree}.
* This class allows you to configured several very important aspects of the
- * B+Tree behavior. Read on.
+ * B+Tree (and other persistence capable data structures) behavior. Read on.
* </p>
* <p>
* An instance of this class is required in order to create a {@link BTree} or
@@ -222,7 +221,7 @@
private static final long serialVersionUID = 4370669592664382720L;
- protected static final transient Logger log = Logger
+ private static final transient Logger log = Logger
.getLogger(IndexMetadata.class);
/**
@@ -1003,50 +1002,6 @@
String DEFAULT_SCATTER_SPLIT_INDEX_PARTITION_COUNT = "0";
- /*
- * htree package options.
- */
-
- /**
- * The name of a class derived from {@link HTree} that will be used to
- * re-load the index.
- */
- String HTREE_CLASS_NAME = HTree.class.getName()+".className";
-
- /**
- * The name of an optional property whose value specifies the number of
- * address bits for an {@link HTree} (default
- * {@value #DEFAULT_HTREE_ADDRESS_BITS}).
- * <p>
- * The #of children for a directory is <code>2^addressBits</code>. For
- * example, a value of <code>10</code> means a <code>10</code> bit
- * address space in the directory. Such a directory would provide direct
- * addressing for <code>1024</code> child references. Given an overhead
- * of <code>8</code> bytes per child address, that would result in an
- * expected page size of 8k before compression.
- *
- * @see #DEFAULT_HTREE_ADDRESS_BITS
- */
- String HTREE_ADDRESS_BITS = HTree.class.getPackage().getName()
- + ".addressBits";
-
- String DEFAULT_HTREE_ADDRESS_BITS = "10";
-
- /**
- * The name of an optional property whose value specifies the fixed by
- * length of the keys in the {@link HTree} -or- ZERO (0) if the key
- * length is unconstrained, in which case variable length keys may be
- * used (default {@value #DEFAULT_HTREE_KEY_LEN}). This may be used in
- * combination with an appropriate {@link IRabaCoder} to optimize to
- * search and encoding of int32 or int64 keys.
- *
- * @see #DEFAULT_HTREE_KEY_LEN
- */
- String HTREE_KEY_LEN = HTree.class.getPackage().getName()
- + ".keyLen";
-
- String DEFAULT_HTREE_KEY_LEN = "0";
-
}
/**
@@ -1103,6 +1058,12 @@
private UUID indexUUID;
private String name;
+ /**
+ * The type of the index.
+ *
+ * @see #VERSION4
+ */
+ private IndexTypeEnum indexType;
private int branchingFactor;
private int writeRetentionQueueCapacity;
private int writeRetentionQueueScan;
@@ -1111,13 +1072,11 @@
private LocalPartitionMetadata pmd;
private String btreeClassName;
private String checkpointClassName;
-// private IAddressSerializer addrSer;
private IRabaCoder nodeKeysCoder;
private ITupleSerializer<?, ?> tupleSer;
private IRecordCompressorFactory<?> btreeRecordCompressorFactory;
private IRecordCompressorFactory<?> indexSegmentRecordCompressorFactory;
private IConflictResolver conflictResolver;
-// private boolean childLocks;
private boolean deleteMarkers;
private boolean versionTimestamps;
private boolean versionTimestampFilters;
@@ -1125,9 +1084,7 @@
private short maxRecLen;
private BloomFilterFactory bloomFilterFactory;
private IOverflowHandler overflowHandler;
-// private ISplitHandler splitHandler;
private ISimpleSplitHandler splitHandler2;
-// private Object historyPolicy;
private AsynchronousIndexWriteConfiguration asynchronousIndexWriteConfiguration;
private ScatterSplitConfiguration scatterSplitConfiguration;
@@ -1145,36 +1102,7 @@
*/
private boolean indexSegmentBufferNodes;
-// /**
-// * @see Options#INDEX_SEGMENT_LEAF_CACHE_CAPACITY
-// */
-// private int indexSegmentLeafCacheCapacity;
-//
-// /**
-// * @see Options#INDEX_SEGMENT_LEAF_CACHE_TIMEOUT
-// */
-// private long indexSegmentLeafCacheTimeout;
-
- /*
- * HTree
- */
-
/**
- * @see Options#HTREE_CLASS_NAME
- */
- private String htreeClassName;
-
- /**
- * @see Options#HTREE_ADDRESS_BITS
- */
- private int addressBits;
-
- /**
- * @see Options#HTREE_KEY_LEN
- */
- private int keyLen;
-
- /**
* The unique identifier for the (scale-out) index whose data is stored in
* this B+Tree data structure.
* <p>
@@ -1187,6 +1115,13 @@
public final UUID getIndexUUID() {return indexUUID;}
/**
+ * The type of the associated persistence capable data structure.
+ */
+ public final IndexTypeEnum getIndexType() {
+ return indexType;
+ }
+
+ /**
* The name associated with the index -or- <code>null</code> iff the index
* is not named (internal indices are generally not named while application
* indices are always named).
@@ -1310,54 +1245,6 @@
}
-// /**
-// * Return the capacity of the LRU cache of leaves for an
-// * {@link IndexSegment}.
-// *
-// * @see Options#INDEX_SEGMENT_LEAF_CACHE_CAPACITY
-// */
-// public final int getIndexSegmentLeafCacheCapacity() {
-//
-// return indexSegmentLeafCacheCapacity;
-//
-// }
-//
-// public final void setIndexSegmentLeafCacheCapacity(final int newValue) {
-//
-// if (newValue <= 0) {
-//
-// throw new IllegalArgumentException();
-//
-// }
-//
-// this.indexSegmentLeafCacheCapacity = newValue;
-//
-// }
-//
-// /**
-// * Return the timeout in nanoseconds of the LRU cache of leaves of
-// * an {@link IndexSegment}.
-// *
-// * @see Options#INDEX_SEGMENT_LEAF_CACHE_TIMEOUT
-// */
-// public final long getIndexSegmentLeafCacheTimeout() {
-//
-// return indexSegmentLeafCacheTimeout;
-//
-// }
-//
-// public final void setIndexSegmentLeafCacheTimeout(final long nanos) {
-//
-// if (nanos <= 0) {
-//
-// throw new IllegalArgumentException();
-//
-// }
-//
-// this.indexSegmentLeafCacheTimeout = nanos;
-//
-// }
-
/**
* @see Options#WRITE_RETENTION_QUEUE_CAPACITY
*/
@@ -1463,11 +1350,6 @@
}
-// /**
-// * Object used to (de-)serialize the addresses of the children of a node.
-// */
-// public final IAddressSerializer getAddressSerializer() {return addrSer;}
-
/**
* Object used to code (compress) the keys in a node.
* <p>
@@ -1510,17 +1392,6 @@
*/
public final IConflictResolver getConflictResolver() {return conflictResolver;}
-// /**
-// * @see Options#CHILD_LOCKS
-// */
-// public final boolean getChildLocks() {return childLocks;}
-//
-// public final void setChildLocks(final boolean newValue) {
-//
-// this.childLocks = newValue;
-//
-// }
-
/**
* When <code>true</code> the index will write a delete marker when an
* attempt is made to delete the entry under a key. Delete markers will be
@@ -1714,15 +1585,6 @@
}
-// public void setAddressSerializer(final IAddressSerializer addrSer) {
-//
-// if (addrSer == null)
-// throw new IllegalArgumentException();
-//
-// this.addrSer = addrSer;
-//
-// }
-
public void setNodeKeySerializer(final IRabaCoder nodeKeysCoder) {
if (nodeKeysCoder == null)
@@ -1914,57 +1776,7 @@
}
- /*
- * HTree
- */
-
-
/**
- * The name of a class derived from {@link HTree} that will be used to
- * re-load the index.
- *
- * @see Options#HTREE_CLASS_NAME
- */
- public final String getHTreeClassName() {
-
- return htreeClassName;
-
- }
-
- public void setHTreeClassName(final String className) {
-
- if (className == null)
- throw new IllegalArgumentException();
-
- this.htreeClassName = className;
-
- }
-
- public int getAddressBits() {
-
- return addressBits;
-
- }
-
- public void setAddressBits(final int addressBits) {
-
- this.addressBits = addressBits;
-
- }
-
- public int getKeyLen() {
-
- return keyLen;
-
- }
-
- public void setKeyLen(final int keyLen) {
-
- this.keyLen = keyLen;
-
- }
-
- /**
* Create an instance of a class known to implement the specified interface
* from a class name.
*
@@ -2037,73 +1849,82 @@
*/
public IndexMetadata(final UUID indexUUID) {
- this(null, indexUUID);
-
+ this(null/* name */, indexUUID);
+
}
- /**
- * Constructor used to configure a new <em>named</em> B+Tree. The index
- * UUID is set to the given value and all other fields are defaulted as
- * explained at {@link #IndexMetadata(Properties, String, UUID)}. Those
- * defaults may be overridden using the various setter methods, but some
- * values can not be safely overridden after the index is in use.
- *
- * @param name
- * The index name. When this is a scale-out index, the same
- * <i>name</i> is specified for each index resource. However
- * they will be registered on the journal under different names
- * depending on the index partition to which they belong.
- *
- * @param indexUUID
- * The indexUUID. The same index UUID MUST be used for all
- * component indices in a scale-out index.
- *
- * @throws IllegalArgumentException
- * if the indexUUID is <code>null</code>.
- */
- public IndexMetadata(final String name, final UUID indexUUID) {
+ /**
+ * Constructor used to configure a new <em>named</em> {@link BTree}. The
+ * index UUID is set to the given value and all other fields are defaulted
+ * as explained at {@link #IndexMetadata(Properties, String, UUID)}. Those
+ * defaults may be overridden using the various setter methods, but some
+ * values can not be safely overridden after the index is in use.
+ *
+ * @param name
+ * The index name. When this is a scale-out index, the same
+ * <i>name</i> is specified for each index resource. However they
+ * will be registered on the journal under different names
+ * depending on the index partition to which they belong.
+ *
+ * @param indexUUID
+ * The indexUUID. The same index UUID MUST be used for all
+ * component indices in a scale-out index.
+ *
+ * @throws IllegalArgumentException
+ * if the indexUUID is <code>null</code>.
+ */
+ public IndexMetadata(final String name, final UUID indexUUID) {
- this(null, System.getProperties(), name, indexUUID);
-
- }
+ this(null/* name */, System.getProperties(), name, indexUUID,
+ IndexTypeEnum.BTree);
+ }
+
/**
- * Constructor used to configure a new <em>named</em> B+Tree. The index
- * UUID is set to the given value and all other fields are defaulted as
- * explained at {@link #getProperty(Properties, String, String, String)}.
- * Those defaults may be overridden using the various setter methods.
- *
- * @param indexManager
- * Optional. When given and when the {@link IIndexManager} is a
- * scale-out {@link IBigdataFederation}, this object will be
- * used to interpret the {@link Options#INITIAL_DATA_SERVICE}
- * property.
- * @param properties
- * Properties object used to overridden the default values for
- * this {@link IndexMetadata} instance.
- * @param namespace
- * The index name. When this is a scale-out index, the same
- * <i>name</i> is specified for each index resource. However
- * they will be registered on the journal under different names
- * depending on the index partition to which they belong.
- * @param indexUUID
- * The indexUUID. The same index UUID MUST be used for all
- * component indices in a scale-out index.
- *
- * @throws IllegalArgumentException
- * if <i>properties</i> is <code>null</code>.
- * @throws IllegalArgumentException
- * if <i>indexUUID</i> is <code>null</code>.
- */
- public IndexMetadata(final IIndexManager indexManager,
- final Properties properties, final String namespace,
- final UUID indexUUID) {
+ * Constructor used to configure a new <em>named</em> B+Tree. The index UUID
+ * is set to the given value and all other fields are defaulted as explained
+ * at {@link #getProperty(Properties, String, String, String)}. Those
+ * defaults may be overridden using the various setter methods.
+ *
+ * @param indexManager
+ * Optional. When given and when the {@link IIndexManager} is a
+ * scale-out {@link IBigdataFederation}, this object will be used
+ * to interpret the {@link Options#INITIAL_DATA_SERVICE}
+ * property.
+ * @param properties
+ * Properties object used to overridden the default values for
+ * this {@link IndexMetadata} instance.
+ * @param namespace
+ * The index name. When this is a scale-out index, the same
+ * <i>name</i> is specified for each index resource. However they
+ * will be registered on the journal under different names
+ * depending on the index partition to which they belong.
+ * @param indexUUID
+ * The indexUUID. The same index UUID MUST be used for all
+ * component indices in a scale-out index.
+ * @param indexType
+ * Type-safe enumeration specifying the type of the persistence
+ * class data structure (historically, this was always a B+Tree).
+ *
+ * @throws IllegalArgumentException
+ * if <i>properties</i> is <code>null</code>.
+ * @throws IllegalArgumentException
+ * if <i>indexUUID</i> is <code>null</code>.
+ */
+ public IndexMetadata(final IIndexManager indexManager,
+ final Properties properties, final String namespace,
+ final UUID indexUUID, final IndexTypeEnum indexType) {
if (indexUUID == null)
throw new IllegalArgumentException();
-
+
+ if (indexType == null)
+ throw new IllegalArgumentException();
+
this.name = namespace;
+ this.indexType = indexType;
+
this.indexUUID = indexUUID;
{
@@ -2484,24 +2305,6 @@
}
- /*
- * HTree
- */
-
- /* Intern'd to reduce duplication on the heap. Will be com.bigdata.btree.BTree or
- * com.bigdata.btree.IndexSegment and occasionally a class derived from BTree.
- */
- this.htreeClassName = getProperty(indexManager, properties, namespace,
- Options.HTREE_CLASS_NAME, HTree.class.getName()).intern();
-
- this.addressBits = Integer.parseInt(getProperty(indexManager,
- properties, namespace, Options.HTREE_ADDRESS_BITS,
- Options.DEFAULT_HTREE_ADDRESS_BITS));
-
- this.keyLen = Integer.parseInt(getProperty(indexManager,
- properties, namespace, Options.HTREE_KEY_LEN,
- Options.DEFAULT_HTREE_KEY_LEN));
-
if (log.isInfoEnabled())
log.info(toString());
@@ -2580,6 +2383,7 @@
// persistent
sb.append(", name=" + (name == null ? "N/A" : name));
+ sb.append(", indexType=" + indexType);
sb.append(", indexUUID=" + indexUUID);
if (initialDataServiceUUID != null) {
sb.append(", initialDataServiceUUID=" + initialDataServiceUUID);
@@ -2619,19 +2423,28 @@
: indexSegmentRecordCompressorFactory));
sb.append(", asynchronousIndexWriteConfiguration=" + asynchronousIndexWriteConfiguration);
sb.append(", scatterSplitConfiguration=" + scatterSplitConfiguration);
- // htree
- sb.append(", htreeClassName=" + htreeClassName);
- sb.append(", addressBits=" + addressBits);
- sb.append(", keyLen=" + keyLen);
+ toString(sb); // extension hook
return sb.toString();
}
+ /**
+ * Extension hook for {@link #toString()}.
+ *
+ * @param sb
+ * Where to write additional metadata.
+ */
+ protected void toString(final StringBuilder sb) {
+
+ // NOP
+
+ }
+
/**
* The initial version.
*/
- private static transient final int VERSION0 = 0x0;
+ protected static transient final int VERSION0 = 0x0;
/**
* This version adds support for {@link ILeafData#getRawRecord(int)} and
@@ -2639,101 +2452,40 @@
* earlier versions and {@link IndexMetadata#getMaxRecLen()} will report
* {@link Options#DEFAULT_MAX_REC_LEN}.
*/
- private static transient final int VERSION1 = 0x1;
+ protected static transient final int VERSION1 = 0x1;
/**
* This version adds support for {@link HTree}. This includes
* {@link #addressBits} and {@link #htreeClassName}.
*/
- private static transient final int VERSION2 = 0x2;
+ protected static transient final int VERSION2 = 0x2;
/**
* This version adds support for a fixed length key option for the
* {@link HTree} using {@link #keyLen}.
*/
- private static transient final int VERSION3 = 0x3;
+ protected static transient final int VERSION3 = 0x3;
-// /**
-// * This version introduced the {@link #asynchronousIndexWriteConfiguration}.
-// * Reads of an earlier version create a instance of that field based on a
-// * default configuration.
-// */
-// private static transient final int VERSION1 = 0x1;
-//
-// /**
-// * This version introduced the {@link #scatterSplitConfiguration}. Reads of
-// * an earlier version create a instance of that field based on a default
-// * configuration.
-// */
-// private static transient final int VERSION2 = 0x2;
-//
-// /**
-// * This version introduced {@link #indexSegmentLeafCacheTimeout}. Reads of
-// * an earlier version use the
-// * {@link Options#DEFAULT_INDEX_SEGMENT_LEAF_CACHE_TIMEOUT} for this field.
-// */
-// private static transient final int VERSION3 = 0x3;
-//
-// /**
-// * This version introduced {@link #btreeRecordCompressorFactory} and
-// * {@link #indexSegmentRecordCompressorFactory}. Both of these fields are
-// * optional, which implies no compression provider. Reads of prior versions
-// * set these fields to <code>null</code>.
-// *
-// * @see Options#BTREE_RECORD_COMPRESSOR_FACTORY
-// * @see Options#INDEX_SEGMENT_RECORD_COMPRESSOR_FACTORY
-// */
-// private static transient final int VERSION4 = 0x04;
-//
-// /**
-// * This version introduced {@link #childLocks}. Reads of prior versions set
-// * this field to <code>true</code>.
-// *
-// * @see Options#CHILD_LOCKS
-// */
-// private static transient final int VERSION5 = 0x05;
-//
-// /**
-// * This version introduced {@link #versionTimestampFilters}. Reads of prior
-// * versions set this field to <code>false</code>.
-// */
-// private static transient final int VERSION6 = 0x06;
-//
-// /**
-// * This version gets rid of the read-retention queue capacity and nscan
-// * properties and the index segment leaf cache capacity and timeout
-// * properties.
-// */
-// private static transient final int VERSION7 = 0x07;
-//
-// /**
-// * This version gets rid of the IAddressSerializer interface used by the
-// * older {@link NodeSerializer} class to (de-)serialize the child addresses
-// * for a {@link Node}.
-// */
-// private static transient final int VERSION8 = 0x08;
-//
-// /**
-// * The childLocks feature was dropped in this version.
-// */
-// private static transient final int VERSION9 = 0x09;
-//
-// /**
-// * The split handler was changed from an implementation based on the #of
-// * tuples to one based on the size on disk of an index segment after a
-// * compacting merge. The old split handlers are replaced by a
-// * <code>null</code> reference when they are de-serialized.
-// *
-// * @see ISplitHandler
-// * @see ISimpleSplitHandler
-// */
-// private static transient final int VERSION10 = 0x10;
+ /**
+ * This version moves the {@link HTree} specific metadata into a derived
+ * class. Prior to this version, the {@link HTree} was not used in a durable
+ * context. Thus, there is no need to recover HTree specific index metadata
+ * records before {@link #VERSION4}. This version also introduces the
+ * {@link #indexType} field. This field defaults to
+ * {@link IndexTypeEnum#BTree} for all prior versions.
+ */
+ protected static transient final int VERSION4 = 0x4;
/**
* The version that will be serialized by this class.
*/
- private static transient final int CURRENT_VERSION = VERSION3;
-// private static transient final int CURRENT_VERSION = VERSION10;
+ protected static transient final int CURRENT_VERSION = VERSION4;
+
+ /**
+ * The actual version as set by {@link #readExternal(ObjectInput)} and
+ * {@link #writeExternal(ObjectOutput)}.
+ */
+ protected transient int version;
/**
* @todo review generated record for compactness.
@@ -2741,14 +2493,14 @@
public void readExternal(final ObjectInput in) throws IOException,
ClassNotFoundException {
- final int version = (int) LongPacker.unpackLong(in);
+ final int version = this.version = (int) LongPacker.unpackLong(in);
switch (version) {
case VERSION0:
case VERSION1:
case VERSION2:
case VERSION3:
-// case VERSION4:
+ case VERSION4:
// case VERSION5:
// case VERSION6:
// case VERSION7:
@@ -2767,15 +2519,25 @@
name = in.readUTF();
}
+
+ if (version >= VERSION4) {
+
+ indexType = IndexTypeEnum.valueOf(in.readShort());
+
+ } else {
+
+ indexType = IndexTypeEnum.BTree;
+
+ }
indexUUID = new UUID(in.readLong()/* MSB */, in.readLong()/* LSB */);
- branchingFactor = (int)LongPacker.unpackLong(in);
+ branchingFactor = (int) LongPacker.unpackLong(in);
- writeRetentionQueueCapacity = (int)LongPacker.unpackLong(in);
-
- writeRetentionQueueScan = (int)LongPacker.unpackLong(in);
-
+ writeRetentionQueueCapacity = (int) LongPacker.unpackLong(in);
+
+ writeRetentionQueueScan = (int) LongPacker.unpackLong(in);
+
// if (version < VERSION7) {
//
// /* btreeReadRetentionQueueCapacity = (int) */LongPacker
@@ -2785,47 +2547,21 @@
//
// }
- pmd = (LocalPartitionMetadata)in.readObject();
-
+ pmd = (LocalPartitionMetadata) in.readObject();
+
btreeClassName = in.readUTF();
-
+
checkpointClassName = in.readUTF();
-// if (version < VERSION8) {
-//
-// // Read and discard the IAddressSerializer object.
-// in.readObject();
-//
-// }
-
nodeKeysCoder = (IRabaCoder) in.readObject();
- tupleSer = (ITupleSerializer) in.readObject();
-
-// if (version < VERSION4) {
-//
-// btreeRecordCompressorFactory = null;
-//
-// } else {
+ tupleSer = (ITupleSerializer<?,?>) in.readObject();
- btreeRecordCompressorFactory = (IRecordCompressorFactory) in
- .readObject();
+ btreeRecordCompressorFactory = (IRecordCompressorFactory<?>) in
+ .readObject();
-// }
-
- conflictResolver = (IConflictResolver)in.readObject();
+ conflictResolver = (IConflictResolver) in.readObject();
-// if (version < VERSION5 || version >= VERSION9) {
-//
-//// childLocks = true;
-//
-// } else {
-//
-//// childLocks =
-// in.readBoolean();
-//
-// }
-
deleteMarkers = in.readBoolean();
if (version >= VERSION1) {
@@ -2838,191 +2574,59 @@
versionTimestamps = in.readBoolean();
-// if (version < VERSION6) {
-//
-// versionTimestampFilters = false;
-//
-// } else {
+ versionTimestampFilters = in.readBoolean();
- versionTimestampFilters = in.readBoolean();
-//
-// }
-
bloomFilterFactory = (BloomFilterFactory) in.readObject();
- overflowHandler = (IOverflowHandler)in.readObject();
+ overflowHandler = (IOverflowHandler) in.readObject();
-// if (version < VERSION10) {
-//
-// /*
-// * The old style of split handler is discarded. The default behavior
-// * for the new style of split handler covers all known uses of the
-// * old style split handler. While some indices (the sparse row store
-// * for example) will have to register a new split handler for
-// * safety, those indices were not safe for splits historically.
-// */
-//
-// // read and discard the old split handler.
-// in.readObject();
-//
-// splitHandler2 = null;
-//
-// } else {
+ splitHandler2 = (ISimpleSplitHandler) in.readObject();
- splitHandler2 = (ISimpleSplitHandler) in.readObject();
-
-// }
-
/*
* IndexSegment.
*/
indexSegmentBranchingFactor = (int) LongPacker.unpackLong(in);
-// if (version < VERSION7) {
-//
-// /* indexSegmentLeafCacheCapacity = (int) */LongPacker
-// .unpackLong(in);
-//
-// if (version < VERSION3) {
-//
-// /*
-// * indexSegmentLeafCacheTimeout = Long
-// * .parseLong(Options.DEFAULT_INDEX_SEGMENT_LEAF_CACHE_TIMEOUT);
-// */
-//
-// } else {
-//
-// /* indexSegmentLeafCacheTimeout = (long) */LongPacker
-// .unpackLong(in);
-//
-// }
-//
-// }
-
indexSegmentBufferNodes = in.readBoolean();
-// if (version < VERSION4) {
-//
-// indexSegmentRecordCompressorFactory = null;
-//
-// } else {
+ indexSegmentRecordCompressorFactory = (IRecordCompressorFactory<?>) in
+ .readObject();
- indexSegmentRecordCompressorFactory = (IRecordCompressorFactory) in
- .readObject();
+ asynchronousIndexWriteConfiguration = (AsynchronousIndexWriteConfiguration) in
+ .readObject();
-// }
-
-// if (version < VERSION1) {
-//
-// /*
-// * Use the default configuration since not present in the serialized
-// * form before VERSION1.
-// */
-//
-// final int masterQueueCapacity = Integer
-// .parseInt(Options.DEFAULT_MASTER_QUEUE_CAPACITY);
-//
-// final int masterChunkSize = Integer
-// .parseInt(Options.DEFAULT_MASTER_CHUNK_SIZE);
-//
-// final long masterChunkTimeoutNanos = Long
-// .parseLong(Options.DEFAULT_MASTER_CHUNK_TIMEOUT_NANOS);
-//
-// final long sinkIdleTimeoutNanos = Long
-// .parseLong(Options.DEFAULT_SINK_IDLE_TIMEOUT_NANOS);
-//
-// final long sinkPollTimeoutNanos = Long
-// .parseLong(Options.DEFAULT_SINK_POLL_TIMEOUT_NANOS);
-//
-// final int sinkQueueCapacity = Integer
-// .parseInt(Options.DEFAULT_SINK_QUEUE_CAPACITY);
-//
-// final int sinkChunkSize = Integer
-// .parseInt(Options.DEFAULT_SINK_CHUNK_SIZE);
-//
-// final long sinkChunkTimeoutNanos = Long
-// .parseLong(Options.DEFAULT_SINK_CHUNK_TIMEOUT_NANOS);
-//
-// asynchronousIndexWriteConfiguration = new AsynchronousIndexWriteConfiguration(
-// masterQueueCapacity,//
-// masterChunkSize,//
-// masterChunkTimeoutNanos,//
-// sinkIdleTimeoutNanos,//
-// sinkPollTimeoutNanos,//
-// sinkQueueCapacity,//
-// sinkChunkSize,//
-// sinkChunkTimeoutNanos//
-// );
-//
-// } else {
-
- asynchronousIndexWriteConfiguration = (AsynchronousIndexWriteConfiguration) in
- .readObject();
-
-// }
+ scatterSplitConfiguration = (ScatterSplitConfiguration) in.readObject();
-// if (version < VERSION2) {
-//
-// /*
-// * Use the default configuration since not present in the serialized
-// * form before VERSION2.
-// */
-//
-// final boolean scatterSplitEnabled = Boolean
-// .parseBoolean(Options.DEFAULT_SCATTER_SPLIT_ENABLED);
-//
-// final double scatterSplitPercentOfSplitThreshold = Double
-// .parseDouble(Options.DEFAULT_SCATTER_SPLIT_PERCENT_OF_SPLIT_THRESHOLD);
-//
-// final int scatterSplitDataServicesCount = Integer
-// .parseInt(Options.DEFAULT_SCATTER_SPLIT_DATA_SERVICE_COUNT);
-//
-// final int scatterSplitIndexPartitionsCount = Integer
-// .parseInt(Options.DEFAULT_SCATTER_SPLIT_INDEX_PARTITION_COUNT);
-//
-// this.scatterSplitConfiguration = new ScatterSplitConfiguration(
-// scatterSplitEnabled, scatterSplitPercentOfSplitThreshold,
-// scatterSplitDataServicesCount,
-// scatterSplitIndexPartitionsCount);
-//
-// } else {
+ if (version >= VERSION2 && version < VERSION4) {
- scatterSplitConfiguration = (ScatterSplitConfiguration) in.readObject();
-
-// }
-
- if (version >= VERSION2) {
+ /*
+ * These data were moved into the HTreeIndexMetadata subclass
+ * in VERSION4. The HTree was only used against the memory
+ * manager before VERSION4. Therefore, we never have durable
+ * data for an HTree before VERSION4.
+ */
+
+ if (version >= VERSION3) {
- if (version >= VERSION3) {
+ // keyLen
+ LongPacker.unpackInt(in);
- keyLen = LongPacker.unpackInt(in);
+ }
- } else {
-
- keyLen = 0;
-
- }
-
- addressBits = LongPacker.unpackInt(in);
-
- htreeClassName = in.readUTF();
+ // addressBits
+ LongPacker.unpackInt(in);
- } else {
+ // htreeClassName
+ in.readUTF();
- addressBits = 10;
-
- htreeClassName = HTree.class.getName();
-
- keyLen = 0;
-
- }
-
+ }
+
}
public void writeExternal(final ObjectOutput out) throws IOException {
-
- final int version = CURRENT_VERSION;
+
+ final int version = CURRENT_VERSION;
LongPacker.packLong(out, version);
@@ -3036,6 +2640,12 @@
}
+ if (version >= VERSION4) {
+
+ out.writeShort(indexType.getCode());
+
+ }
+
out.writeLong(indexUUID.getMostSignificantBits());
out.writeLong(indexUUID.getLeastSignificantBits());
@@ -3051,50 +2661,32 @@
// LongPacker.packLong(out, btreeReadRetentionQueueScan);
out.writeObject(pmd);
-
+
out.writeUTF(btreeClassName);
out.writeUTF(checkpointClassName);
-
- // Note: This field was dropped as of VERSION8.
-// out.writeObject(addrSer);
-
+
out.writeObject(nodeKeysCoder);
-
+
out.writeObject(tupleSer);
-
-// if (version >= VERSION4) {
-
- out.writeObject(btreeRecordCompressorFactory);
-
-// }
+ out.writeObject(btreeRecordCompressorFactory);
+
out.writeObject(conflictResolver);
-// if (version >= VERSION5 && version < VERSION9 ) {
-//
-//// out.writeBoolean(childLocks);
-// out.writeBoolean(false/* childLocks */);
-//
-// }
-
out.writeBoolean(deleteMarkers);
-
- if (version >= VERSION1) {
- out.writeBoolean(rawRecords);
- out.writeShort(maxRecLen);
- }
-
+
+ if (version >= VERSION1) {
+ out.writeBoolean(rawRecords);
+ out.writeShort(maxRecLen);
+ }
+
out.writeBoolean(versionTimestamps);
-
-// if (version >= VERSION6) {
- out.writeBoolean(versionTime...
[truncated message content] |
|
From: <tho...@us...> - 2012-06-14 14:14:17
|
Revision: 6351
http://bigdata.svn.sourceforge.net/bigdata/?rev=6351&view=rev
Author: thompsonbry
Date: 2012-06-14 14:14:07 +0000 (Thu, 14 Jun 2012)
Log Message:
-----------
updated dev logging properties and build properties (location of javacc).
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_2_0/bigdata/src/resources/logging/log4j-dev.properties
branches/BIGDATA_RELEASE_1_2_0/build.properties
Modified: branches/BIGDATA_RELEASE_1_2_0/bigdata/src/resources/logging/log4j-dev.properties
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/bigdata/src/resources/logging/log4j-dev.properties 2012-06-14 14:11:36 UTC (rev 6350)
+++ branches/BIGDATA_RELEASE_1_2_0/bigdata/src/resources/logging/log4j-dev.properties 2012-06-14 14:14:07 UTC (rev 6351)
@@ -39,7 +39,11 @@
#log4j.logger.com.bigdata.rdf.sparql.ast=INFO
#log4j.logger.com.bigdata.rdf.sparql.ast.cache=ALL
#log4j.logger.com.bigdata.rdf.sail.sparql=ALL
+#log4j.logger.com.bigdata.rdf.sparql.ast.optimizers.ASTBatchResolveTerms=ALL
+#log4j.logger.com.bigdata.rdf.rio.ntriples.BigdataNTriplesParserTestCase=ALL
+#log4j.logger.com.bigdata.rdf.rio.StatementBuffer=ALL
+
#log4j.logger.com.bigdata.rdf.sail.webapp=ALL
# To see the masked type errors.
Modified: branches/BIGDATA_RELEASE_1_2_0/build.properties
===================================================================
--- branches/BIGDATA_RELEASE_1_2_0/build.properties 2012-06-14 14:11:36 UTC (rev 6350)
+++ branches/BIGDATA_RELEASE_1_2_0/build.properties 2012-06-14 14:14:07 UTC (rev 6351)
@@ -29,7 +29,7 @@
javac.encoding=Cp1252
# javacc is required to compile the SPARQL grammar.
-javacc.home=C:/usr/local/javacc-5.0
+javacc.home=/usr/java/javacc-5.0
# Versions to use of various dependencies.
icu.version=4_8
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X