Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4181] branches/QUADS_QUERY_BRANCH/bigdata-sails/src
|
From: <mrp...@us...> - 2011-02-07 20:30:50
|
Revision: 4181
http://bigdata.svn.sourceforge.net/bigdata/?rev=4181&view=rev
Author: mrpersonick
Date: 2011-02-07 20:30:43 +0000 (Mon, 07 Feb 2011)
Log Message:
-----------
fixing a problem with nested unions
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-02-07 16:46:53 UTC (rev 4180)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-02-07 20:30:43 UTC (rev 4181)
@@ -349,6 +349,10 @@
new NV(SliceOp.Annotations.EVALUATION_CONTEXT,
BOpEvaluationContext.CONTROLLER),//
})),queryHints);
+
+ if (rule.getTailCount() == 0) {
+ return startOp;
+ }
/*
* First put the tails in the correct order based on the logic in
@@ -569,12 +573,10 @@
for (int i = 0; i < order.length; i++) {
- final int joinId = idFactory.incrementAndGet();
-
// assign a bop id to the predicate
Predicate<?> pred = (Predicate<?>) rule.getTail(order[i]).setBOpId(
idFactory.incrementAndGet());
-
+
/*
* Decorate the predicate with the assigned index (this is purely
* informative).
@@ -585,7 +587,7 @@
Annotations.ORIGINAL_INDEX, keyOrder[order[i]]);
}
- // decorate the predicate with the cardinality estimate.
+ // decorate the predicate with the cardinality estimate.
if (cardinality != null) {
pred = (Predicate<?>) pred.setProperty(
Annotations.ESTIMATED_CARDINALITY,
@@ -628,107 +630,9 @@
}
}
- // annotations for this join.
- final List<NV> anns = new LinkedList<NV>();
-
- anns.add(new NV(BOp.Annotations.BOP_ID, joinId));
+ left = join(queryEngine, left, pred, constraints, context,
+ idFactory, queryHints);
-// anns.add(new NV(PipelineJoin.Annotations.SELECT,
-// selectVars[order[i]]));
-
- // No. The join just looks at the Predicate's optional annotation.
-// if (pred.isOptional())
-// anns.add(new NV(PipelineJoin.Annotations.OPTIONAL, pred
-// .isOptional()));
-
- /*
- * Pull off annotations before we clear them from the predicate.
- */
- final Scope scope = (Scope) pred.getProperty(Annotations.SCOPE);
-
- // true iff this is a quads access path.
- final boolean quads = pred.getProperty(Annotations.QUADS,
- Annotations.DEFAULT_QUADS);
-
- // pull of the Sesame dataset before we strip the annotations.
- final Dataset dataset = (Dataset) pred
- .getProperty(Annotations.DATASET);
-
- // strip off annotations that we do not want to propagate.
- pred = pred.clearAnnotations(new String[] { Annotations.SCOPE,
- Annotations.QUADS, Annotations.DATASET });
-
- if (!constraints.isEmpty()) {
-// // decorate the predicate with any constraints.
-// pred = (Predicate<?>) pred.setProperty(
-// IPredicate.Annotations.CONSTRAINTS, constraints
-// .toArray(new IConstraint[constraints.size()]));
- // add constraints to the join for that predicate.
- anns.add(new NV(PipelineJoin.Annotations.CONSTRAINTS,
- constraints
- .toArray(new IConstraint[constraints.size()])));
-
- }
-
- if (quads) {
-
- /*
- * Quads mode.
- */
-
- if (enableDecisionTree) {
- /*
- * Strip off the named graph or default graph expander (in
- * the long term it will simply not be generated.)
- */
- pred = pred
- .clearAnnotations(new String[] { IPredicate.Annotations.ACCESS_PATH_EXPANDER });
-
- switch (scope) {
- case NAMED_CONTEXTS:
- left = namedGraphJoin(queryEngine, context, idFactory,
- left, anns, pred, dataset, queryHints);
- break;
- case DEFAULT_CONTEXTS:
- left = defaultGraphJoin(queryEngine, context, idFactory,
- left, anns, pred, dataset, queryHints);
- break;
- default:
- throw new AssertionError();
- }
-
- } else {
-
- /*
- * This is basically the old way of handling quads query
- * using expanders which were attached by
- * BigdataEvaluationStrategyImpl.
- */
-
- final boolean scaleOut = queryEngine.isScaleOut();
- if (scaleOut)
- throw new UnsupportedOperationException();
-
- anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.ANY));
-
- anns.add(new NV(PipelineJoin.Annotations.PREDICATE,pred));
-
- left = applyQueryHints(new PipelineJoin(new BOp[] { left },
- anns.toArray(new NV[anns.size()])), queryHints);
-
- }
-
- } else {
-
- /*
- * Triples or provenance mode.
- */
-
- left = triplesModeJoin(queryEngine, left, anns, pred, queryHints);
-
- }
-
}
// if (rule.getConstraintCount() > 0) {
@@ -758,7 +662,130 @@
return left;
}
+
+ public static PipelineOp join(final QueryEngine queryEngine,
+ PipelineOp left, Predicate pred, final AtomicInteger idFactory,
+ final Properties queryHints) {
+
+ return join(queryEngine, left, pred, null,
+ new BOpContextBase(queryEngine), idFactory, queryHints);
+
+ }
+
+ public static PipelineOp join(final QueryEngine queryEngine,
+ PipelineOp left, Predicate pred,
+ final Collection<IConstraint> constraints,
+ final BOpContextBase context, final AtomicInteger idFactory,
+ final Properties queryHints) {
+
+ final int joinId = idFactory.incrementAndGet();
+
+ // annotations for this join.
+ final List<NV> anns = new LinkedList<NV>();
+
+ anns.add(new NV(BOp.Annotations.BOP_ID, joinId));
+// anns.add(new NV(PipelineJoin.Annotations.SELECT,
+// selectVars[order[i]]));
+
+ // No. The join just looks at the Predicate's optional annotation.
+// if (pred.isOptional())
+// anns.add(new NV(PipelineJoin.Annotations.OPTIONAL, pred
+// .isOptional()));
+
+ if (constraints != null && !constraints.isEmpty()) {
+// // decorate the predicate with any constraints.
+// pred = (Predicate<?>) pred.setProperty(
+// IPredicate.Annotations.CONSTRAINTS, constraints
+// .toArray(new IConstraint[constraints.size()]));
+
+ // add constraints to the join for that predicate.
+ anns.add(new NV(
+ PipelineJoin.Annotations.CONSTRAINTS,
+ constraints.toArray(new IConstraint[constraints.size()])));
+
+ }
+
+ /*
+ * Pull off annotations before we clear them from the predicate.
+ */
+ final Scope scope = (Scope) pred.getProperty(Annotations.SCOPE);
+
+ // true iff this is a quads access path.
+ final boolean quads = pred.getProperty(Annotations.QUADS,
+ Annotations.DEFAULT_QUADS);
+
+ // pull of the Sesame dataset before we strip the annotations.
+ final Dataset dataset = (Dataset) pred
+ .getProperty(Annotations.DATASET);
+
+ // strip off annotations that we do not want to propagate.
+ pred = pred.clearAnnotations(new String[] { Annotations.SCOPE,
+ Annotations.QUADS, Annotations.DATASET });
+
+ if (quads) {
+
+ /*
+ * Quads mode.
+ */
+
+ if (enableDecisionTree) {
+ /*
+ * Strip off the named graph or default graph expander (in
+ * the long term it will simply not be generated.)
+ */
+ pred = pred
+ .clearAnnotations(new String[] { IPredicate.Annotations.ACCESS_PATH_EXPANDER });
+
+ switch (scope) {
+ case NAMED_CONTEXTS:
+ left = namedGraphJoin(queryEngine, context, idFactory,
+ left, anns, pred, dataset, queryHints);
+ break;
+ case DEFAULT_CONTEXTS:
+ left = defaultGraphJoin(queryEngine, context, idFactory,
+ left, anns, pred, dataset, queryHints);
+ break;
+ default:
+ throw new AssertionError();
+ }
+
+ } else {
+
+ /*
+ * This is basically the old way of handling quads query
+ * using expanders which were attached by
+ * BigdataEvaluationStrategyImpl.
+ */
+
+ final boolean scaleOut = queryEngine.isScaleOut();
+ if (scaleOut)
+ throw new UnsupportedOperationException();
+
+ anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.ANY));
+
+ anns.add(new NV(PipelineJoin.Annotations.PREDICATE,pred));
+
+ left = applyQueryHints(new PipelineJoin(new BOp[] { left },
+ anns.toArray(new NV[anns.size()])), queryHints);
+
+ }
+
+ } else {
+
+ /*
+ * Triples or provenance mode.
+ */
+
+ left = triplesModeJoin(queryEngine, left, anns, pred, queryHints);
+
+ }
+
+ return left;
+
+ }
+
/**
* Generate a {@link PipelineJoin} for a triples mode access path.
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java 2011-02-07 16:46:53 UTC (rev 4180)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java 2011-02-07 20:30:43 UTC (rev 4181)
@@ -40,6 +40,7 @@
import org.openrdf.query.algebra.StatementPattern;
import com.bigdata.bop.BOp;
+import com.bigdata.bop.BOpContextBase;
import com.bigdata.bop.BOpEvaluationContext;
import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IConstraint;
@@ -132,7 +133,8 @@
private static boolean isNonOptionalJoinGroup(final SOpGroup sopGroup) {
- return !(isUnion(sopGroup) || isOptional(sopGroup));
+ return sopGroup.size() > 0 &&
+ !(isUnion(sopGroup) || isOptional(sopGroup));
}
@@ -184,14 +186,37 @@
* Start with left=<this join group> and add a SubqueryOp for each
* sub group.
*/
+// final SOpGroups children = join.getChildren();
+// if (children != null) {
+// for (SOpGroup child : children) {
+// if (isSingleOptional(child)) {
+// // handled by the rule() conversion above
+// continue;
+// }
+// final PipelineOp subquery = convert(
+// child, idFactory, db, queryEngine, queryHints);
+// final boolean optional = isOptional(child);
+// final int subqueryId = idFactory.incrementAndGet();
+// left = new SubqueryOp(new BOp[]{left},
+// new NV(Predicate.Annotations.BOP_ID, subqueryId),//
+// new NV(SubqueryOp.Annotations.SUBQUERY, subquery),//
+// new NV(SubqueryOp.Annotations.OPTIONAL,optional)//
+// );
+// if (log.isInfoEnabled()) {
+// log.info("adding a subquery: " + subqueryId + "\n" + left);
+// }
+// }
+// }
final SOpGroups children = join.getChildren();
if (children != null) {
- for (SOpGroup child : join.getChildren()) {
- if (isSingleOptional(child)) {
- // handled by the rule() conversion above
+ /*
+ * First do the non-optional subqueries (UNIONs)
+ */
+ for (SOpGroup child : children) {
+ if (!isUnion(child))
continue;
- }
+
final PipelineOp subquery = convert(
child, idFactory, db, queryEngine, queryHints);
final boolean optional = isOptional(child);
@@ -205,6 +230,39 @@
log.info("adding a subquery: " + subqueryId + "\n" + left);
}
}
+
+ /*
+ * Next do the optional subqueries and optional tails
+ */
+ for (SOpGroup child : children) {
+ if (isUnion(child))
+ continue;
+
+ if (isSingleOptional(child)) {
+ final SOp sop = child.getSingletonSOp();
+ final BOp bop = sop.getBOp();
+ Predicate pred = (Predicate) bop.setProperty(
+ IPredicate.Annotations.OPTIONAL, Boolean.TRUE);
+ pred = pred.setBOpId(idFactory.incrementAndGet());
+ left = Rule2BOpUtility.join(
+ queryEngine, left, pred,
+ idFactory,
+ queryHints);
+ } else {
+ final PipelineOp subquery = convert(
+ child, idFactory, db, queryEngine, queryHints);
+ final boolean optional = isOptional(child);
+ final int subqueryId = idFactory.incrementAndGet();
+ left = new SubqueryOp(new BOp[]{left},
+ new NV(Predicate.Annotations.BOP_ID, subqueryId),//
+ new NV(SubqueryOp.Annotations.SUBQUERY, subquery),//
+ new NV(SubqueryOp.Annotations.OPTIONAL,optional)//
+ );
+ if (log.isInfoEnabled()) {
+ log.info("adding a subquery: " + subqueryId + "\n" + left);
+ }
+ }
+ }
}
for (IConstraint c : postConditionals) {
@@ -313,23 +371,23 @@
}
}
- /*
- * The way that the Sesame operator tree is parsed, optional tails
- * become single-operator (predicate) join groups without any children
- * of their own.
- */
- final SOpGroups children = group.getChildren();
- if (children != null) {
- for (SOpGroup child : group.getChildren()) {
- if (isSingleOptional(child)) {
- final SOp sop = child.getSingletonSOp();
- final BOp bop = sop.getBOp();
- final IPredicate pred = (IPredicate) bop.setProperty(
- IPredicate.Annotations.OPTIONAL, Boolean.TRUE);
- preds.add(pred);
- }
- }
- }
+// /*
+// * The way that the Sesame operator tree is parsed, optional tails
+// * become single-operator (predicate) join groups without any children
+// * of their own.
+// */
+// final SOpGroups children = group.getChildren();
+// if (children != null) {
+// for (SOpGroup child : children) {
+// if (isSingleOptional(child)) {
+// final SOp sop = child.getSingletonSOp();
+// final BOp bop = sop.getBOp();
+// final IPredicate pred = (IPredicate) bop.setProperty(
+// IPredicate.Annotations.OPTIONAL, Boolean.TRUE);
+// preds.add(pred);
+// }
+// }
+// }
/*
* Gather up all the variables used by predicates in this group
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java 2011-02-07 16:46:53 UTC (rev 4180)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java 2011-02-07 20:30:43 UTC (rev 4181)
@@ -43,10 +43,14 @@
}
group.add(sop);
}
+ if (!groups.containsKey(0)) {
+ // need a dummy root group
+ groups.put(0, new LinkedList<SOp>());
+ }
for (Integer g : groups.keySet()) {
final List<SOp> group = groups.get(g);
- final int pg = group.get(0).getParentGroup();
+ final int pg = group.isEmpty() ? -1 : group.get(0).getParentGroup();
allGroups.put(g, new SOpGroup(g, pg, group));
}
@@ -195,7 +199,10 @@
}
sb.append("SOp -> parent:").append(nl);
for (Map.Entry<Integer, SOpGroup> e : this.parents.entrySet()) {
- sb.append(e.getKey() + " -> " + e.getValue().getGroup()).append(nl);
+ sb.append(e.getKey());
+ sb.append(" -> ");
+ sb.append(e.getValue() == null ? "null" : e.getValue().getGroup());
+ sb.append(nl);
}
sb.append("SOp -> children:").append(nl);
for (Map.Entry<Integer, SOpGroups> e : this.children.entrySet()) {
@@ -206,7 +213,9 @@
}
sb2.setLength(sb2.length()-2);
sb.append(e.getKey() + " -> {" + sb2.toString() + "}");
+ sb.append(nl);
}
+ sb.setLength(sb.length()-1);
return sb.toString();
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java 2011-02-07 16:46:53 UTC (rev 4180)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java 2011-02-07 20:30:43 UTC (rev 4181)
@@ -121,6 +121,8 @@
collectSOps(sops, (LeftJoin) left, rslj, g, pg);
} else if (left instanceof SingletonSet){
// do nothing
+ } else if (left instanceof Union){
+ collectSOps(sops, (Union) left, rslj, groupId.incrementAndGet(), g);
} else {
throw new UnsupportedOperatorException(left);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java 2011-02-07 16:46:53 UTC (rev 4180)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java 2011-02-07 20:30:43 UTC (rev 4181)
@@ -28,14 +28,12 @@
import java.util.Collection;
import java.util.LinkedList;
-import java.util.Map;
import java.util.Properties;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.log4j.Logger;
-import org.openrdf.model.Literal;
import org.openrdf.model.URI;
import org.openrdf.model.ValueFactory;
+import org.openrdf.model.impl.LiteralImpl;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.model.vocabulary.RDFS;
import org.openrdf.query.BindingSet;
@@ -48,17 +46,9 @@
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.sail.SailTupleQuery;
-import com.bigdata.bop.BOpUtility;
-import com.bigdata.bop.PipelineOp;
-import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.rdf.axioms.NoAxioms;
-import com.bigdata.rdf.sail.sop.SOp;
-import com.bigdata.rdf.sail.sop.SOp2BOpUtility;
import com.bigdata.rdf.sail.sop.SOpTree;
-import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
-import com.bigdata.rdf.sail.sop.SOpTree.SOpGroups;
import com.bigdata.rdf.sail.sop.SOpTreeBuilder;
-import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.store.BD;
import com.bigdata.rdf.vocab.NoVocabulary;
@@ -229,4 +219,118 @@
}
+ public void testNestedUnionWithOptionals() throws Exception {
+
+// final Sail sail = new MemoryStore();
+// sail.initialize();
+// final Repository repo = new SailRepository(sail);
+
+ final BigdataSail sail = getSail();
+ try {
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+
+ final RepositoryConnection cxn = repo.getConnection();
+
+ try {
+
+ cxn.setAutoCommit(false);
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ /*
+ * Create some terms.
+ */
+ final URI john = vf.createURI(BD.NAMESPACE + "john");
+ final URI mary = vf.createURI(BD.NAMESPACE + "mary");
+ final URI leon = vf.createURI(BD.NAMESPACE + "leon");
+ final URI paul = vf.createURI(BD.NAMESPACE + "paul");
+ final URI brad = vf.createURI(BD.NAMESPACE + "brad");
+ final URI fred = vf.createURI(BD.NAMESPACE + "fred");
+ final URI knows = vf.createURI(BD.NAMESPACE + "knows");
+
+ /*
+ * Create some statements.
+ */
+ cxn.add(mary, knows, fred);
+ cxn.add(john, knows, leon);
+ cxn.add(john, RDFS.LABEL, vf.createLiteral("John"));
+ cxn.add(mary, RDF.TYPE, RDFS.RESOURCE);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any
+ * operations that go around the sail.
+ */
+ cxn.commit();
+
+ {
+
+ String query =
+ "prefix bd: <"+BD.NAMESPACE+"> " +
+ "prefix rdf: <"+RDF.NAMESPACE+"> " +
+ "prefix rdfs: <"+RDFS.NAMESPACE+"> " +
+ "select * " +
+ "where { " +
+ " { " +
+ " ?a bd:knows bd:fred . " +
+ " } UNION { " +
+ " ?a bd:knows bd:leon . " +
+ " } " +
+ " OPTIONAL { ?a rdf:type ?type } " +
+ " OPTIONAL { ?a rdfs:label ?label } " +
+ "}";
+
+ final SailTupleQuery tupleQuery = (SailTupleQuery)
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(false /* includeInferred */);
+
+ if (log.isInfoEnabled()) {
+
+ final BigdataSailTupleQuery bdTupleQuery =
+ (BigdataSailTupleQuery) tupleQuery;
+ final QueryRoot root = (QueryRoot) bdTupleQuery.getTupleExpr();
+ final Projection p = (Projection) root.getArg();
+ final TupleExpr tupleExpr = p.getArg();
+
+ log.info(tupleExpr);
+
+ final SOpTreeBuilder stb = new SOpTreeBuilder();
+ final SOpTree tree = stb.collectSOps(tupleExpr);
+
+ log.info(tree);
+ log.info(query);
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ log.info("results:");
+ while (result.hasNext()) {
+ log.info(result.next());
+ }
+
+ }
+
+ final Collection<BindingSet> answer = new LinkedList<BindingSet>();
+ answer.add(createBindingSet(
+ new BindingImpl("a", john),
+ new BindingImpl("label", vf.createLiteral("John"))
+ ));
+ answer.add(createBindingSet(
+ new BindingImpl("a", mary),
+ new BindingImpl("type", RDFS.RESOURCE)
+ ));
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ }
+ } finally {
+ sail.__tearDownUnitTest();//shutDown();
+ }
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
