Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4159] branches/QUADS_QUERY_BRANCH/bigdata
|
From: <tho...@us...> - 2011-01-21 16:46:48
|
Revision: 4159
http://bigdata.svn.sourceforge.net/bigdata/?rev=4159&view=rev
Author: thompsonbry
Date: 2011-01-21 16:46:41 +0000 (Fri, 21 Jan 2011)
Log Message:
-----------
Modified UNION/STEPS and AbstractSubqueryOp to use an annotation for the subqueries rather than the arguments of the UNION/STEPS. This makes it possible to use these operators at positions other than the start of the pipeline.
Bug fix to PipelineJoin where it was applying constraints for optional joins to the original solution even when the constraints was a null reference.
Fix to the semantics of an optional join test in TestQueryEngine to reflect the fact that the constraint is applied on all paths through a join.
Turning on the new eval strategy by default in the SAIL for CI builds.
13 sail failures in this commit. some appear to be new while some have disappeared.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Steps.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Union.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestUnion.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -38,7 +38,6 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.BOpContext;
-import com.bigdata.bop.BOpEvaluationContext;
import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.PipelineOp;
@@ -74,7 +73,7 @@
*
* <pre>
* SLICE[1](
- * UNION[2]([a{sinkRef=1},b{sinkRef=1},c{sinkRef=1}],{})
+ * UNION[2]([...],{subqueries=[a{sinkRef=1},b{sinkRef=1},c{sinkRef=1}]})
* )
* </pre>
*
@@ -96,6 +95,12 @@
public interface Annotations extends PipelineOp.Annotations {
/**
+ * The ordered {@link BOp}[] of subqueries to be evaluated for each
+ * binding set presented (required).
+ */
+ String SUBQUERIES = SubqueryOp.class.getName() + ".subqueries";
+
+ /**
* The maximum parallelism with which the subqueries will be evaluated
* (default is unlimited).
*/
@@ -132,13 +137,19 @@
super(args, annotations);
- if (!getEvaluationContext().equals(BOpEvaluationContext.CONTROLLER))
- throw new IllegalArgumentException(Annotations.EVALUATION_CONTEXT
- + "=" + getEvaluationContext());
+// if (!getEvaluationContext().equals(BOpEvaluationContext.CONTROLLER))
+// throw new IllegalArgumentException(Annotations.EVALUATION_CONTEXT
+// + "=" + getEvaluationContext());
- if (!getProperty(Annotations.CONTROLLER, Annotations.DEFAULT_CONTROLLER))
- throw new IllegalArgumentException(Annotations.CONTROLLER);
-
+// if (!getProperty(Annotations.CONTROLLER, Annotations.DEFAULT_CONTROLLER))
+// throw new IllegalArgumentException(Annotations.CONTROLLER);
+
+ // verify required annotation.
+ final BOp[] subqueries = (BOp[]) getRequiredProperty(Annotations.SUBQUERIES);
+
+ if (subqueries.length == 0)
+ throw new IllegalArgumentException(Annotations.SUBQUERIES);
+
// // The id of this operator (if any).
// final Integer thisId = (Integer)getProperty(Annotations.BOP_ID);
//
@@ -170,6 +181,7 @@
private static class ControllerTask implements Callable<Void> {
private final AbstractSubqueryOp controllerOp;
+ private final BOp[] subqueries;
private final BOpContext<IBindingSet> context;
private final int nparallel;
private final Executor executor;
@@ -187,6 +199,9 @@
this.context = context;
+ this.subqueries = (BOp[]) controllerOp
+ .getRequiredProperty(Annotations.SUBQUERIES);
+
this.nparallel = controllerOp.getProperty(Annotations.MAX_PARALLEL,
Annotations.DEFAULT_MAX_PARALLEL);
@@ -244,8 +259,8 @@
try {
- final CountDownLatch latch = new CountDownLatch(controllerOp
- .arity());
+ final CountDownLatch latch = new CountDownLatch(
+ subqueries.length);
/*
* Create FutureTasks for each subquery. The futures are not
@@ -253,7 +268,7 @@
* deferring the evaluation until call() we gain the ability to
* cancel all subqueries if any subquery fails.
*/
- for (BOp op : controllerOp.args()) {
+ for (BOp op : subqueries) {
/*
* Task runs subquery and cancels all subqueries in [tasks]
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Steps.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Steps.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Steps.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -37,7 +37,7 @@
* STEPS(ops)
*
* <pre>
- * STEPS([a,b,c],{})
+ * STEPS([],{subqueries=[a,b,c]})
* </pre>
*
* Will run the subqueries <i>a</i>, <i>b</i>, and <i>c</i> in sequence. Each
@@ -62,12 +62,11 @@
public Steps(Steps op) {
super(op);
}
-
+
/**
* Shallow copy constructor.
*
* @param args
- * Two or more operators whose union is desired.
* @param annotations
*/
public Steps(final BOp[] args,
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Union.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Union.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/Union.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -34,10 +34,10 @@
import com.bigdata.bop.PipelineOp;
/**
- * UNION(ops)[maxParallel(default all)]
+ * UNION()[maxParallel(default all); subqueries=ops]
*
* <pre>
- * UNION([a,b,c],{})
+ * UNION([],{subqueries=[a,b,c]})
* </pre>
*
* Will run the subqueries <i>a</i>, <i>b</i>, and <i>c</i> in parallel for each
@@ -68,15 +68,14 @@
* Shallow copy constructor.
*
* @param args
- * Two or more operators whose union is desired.
* @param annotations
*/
public Union(final BOp[] args, final Map<String, Object> annotations) {
super(args, annotations);
- if (args.length < 2)
- throw new IllegalArgumentException();
+// if (args.length < 2)
+// throw new IllegalArgumentException();
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestUnion.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestUnion.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestUnion.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -45,6 +45,7 @@
import com.bigdata.bop.bindingSet.ArrayBindingSet;
import com.bigdata.bop.bindingSet.EmptyBindingSet;
import com.bigdata.bop.bindingSet.HashBindingSet;
+import com.bigdata.bop.bindingSet.ListBindingSet;
import com.bigdata.bop.bset.StartOp;
import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.QueryEngine;
@@ -119,7 +120,7 @@
// data to insert (in key order for convenience).
final E[] a = {//
- new E("John", "Mary"),// [0]
+ new E("John", "Mary"),// [0]
new E("Leon", "Paul"),// [1]
new E("Mary", "Paul"),// [2]
new E("Paul", "Leon"),// [3]
@@ -176,12 +177,14 @@
BOpEvaluationContext.CONTROLLER),//
}));
- final BOp unionOp = new Union(new BOp[] { startOp1, startOp2 }, NV
+ final BOp unionOp = new Union(new BOp[0], NV
.asMap(new NV[] {//
new NV(Union.Annotations.BOP_ID, unionId),//
- new NV(Union.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.CONTROLLER),//
- new NV(Union.Annotations.CONTROLLER, true),//
+ new NV(Union.Annotations.SUBQUERIES, new BOp[] {
+ startOp1, startOp2 }),//
+// new NV(Union.Annotations.EVALUATION_CONTEXT,
+// BOpEvaluationContext.CONTROLLER),//
+// new NV(Union.Annotations.CONTROLLER, true),//
}));
final BOp query = unionOp;
@@ -203,6 +206,60 @@
}
+ public void test_union_consumesSource() throws Exception {
+
+ final int startId1 = 1;
+ final int startId2 = 2;
+ final int unionId = 3;
+
+ final BOp startOp1 = new StartOp(new BOp[] {}, NV.asMap(new NV[] {//
+ new NV(StartOp.Annotations.BOP_ID, startId1),//
+ new NV(StartOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER),//
+ }));
+
+ final BOp startOp2 = new StartOp(new BOp[] {}, NV.asMap(new NV[] {//
+ new NV(StartOp.Annotations.BOP_ID, startId2),//
+ new NV(StartOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER),//
+ }));
+
+ final BOp unionOp = new Union(new BOp[]{}, NV
+ .asMap(new NV[] {//
+ new NV(Union.Annotations.BOP_ID, unionId),//
+ new NV(Union.Annotations.SUBQUERIES, new BOp[] {
+ startOp1, startOp2 }) //
+// new NV(Union.Annotations.EVALUATION_CONTEXT,
+// BOpEvaluationContext.CONTROLLER),//
+// new NV(Union.Annotations.CONTROLLER, true),//
+ }));
+
+ final BOp query = unionOp;
+
+ /*
+ * Create an initial non-empty binding set.
+ */
+ final IBindingSet bset = new ListBindingSet();
+ bset.set(Var.var("x"), new Constant<String>("John"));
+ bset.set(Var.var("y"), new Constant<String>("Mary"));
+
+ // the expected solutions.
+ final IBindingSet[] expected = new IBindingSet[] {//
+ bset, // one copy from the left side of the union.
+ bset, // one copy from the right side of the union.
+ };
+
+ final IRunningQuery runningQuery = queryEngine.eval(query, bset);
+
+ // verify solutions.
+ TestQueryEngine.assertSameSolutionsAnyOrder(expected,
+ new Dechunkerator<IBindingSet>(runningQuery.iterator()));
+
+ // Wait until the query is done.
+ runningQuery.get();
+
+ }
+
/**
* Verifies that the UNION of two operators is computed. The operators do
* not route around the UNION, so their solutions are copied to the UNION
@@ -249,12 +306,14 @@
new NV(StartOp.Annotations.BINDING_SETS,bindingSets2)
}));
- final BOp unionOp = new Union(new BOp[] { startOp1, startOp2 }, NV
+ final BOp unionOp = new Union(new BOp[] {}, NV
.asMap(new NV[] {//
new NV(Union.Annotations.BOP_ID, unionId),//
- new NV(Union.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.CONTROLLER),//
- new NV(Union.Annotations.CONTROLLER, true),//
+ new NV(Union.Annotations.SUBQUERIES, new BOp[] {
+ startOp1, startOp2 }) //
+// new NV(Union.Annotations.EVALUATION_CONTEXT,
+// BOpEvaluationContext.CONTROLLER),//
+// new NV(Union.Annotations.CONTROLLER, true),//
}));
final BOp sliceOp = new SliceOp(new BOp[]{unionOp},NV.asMap(
@@ -336,12 +395,14 @@
new NV(StartOp.Annotations.BINDING_SETS,bindingSets2)
}));
- final BOp unionOp = new Union(new BOp[] { startOp1, startOp2 }, NV
+ final BOp unionOp = new Union(new BOp[] {}, NV
.asMap(new NV[] {//
new NV(Union.Annotations.BOP_ID, unionId),//
- new NV(Union.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.CONTROLLER),//
- new NV(Union.Annotations.CONTROLLER, true),//
+ new NV(Union.Annotations.SUBQUERIES, new BOp[] {
+ startOp1, startOp2 }) //
+// new NV(Union.Annotations.EVALUATION_CONTEXT,
+// BOpEvaluationContext.CONTROLLER),//
+// new NV(Union.Annotations.CONTROLLER, true),//
}));
final BOp sliceOp = new SliceOp(new BOp[]{unionOp},NV.asMap(
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -1643,17 +1643,24 @@
new Constant<String>("Leon"),
new Constant<String>("Paul") }//
),
- // plus anything we read from the first access path which did not join.
- new ArrayBindingSet(//
- new IVariable[] { Var.var("x"), Var.var("y") },//
- new IConstant[] { new Constant<String>("John"),
- new Constant<String>("Mary") }//
- ),
- new ArrayBindingSet(//
- new IVariable[] { Var.var("x"), Var.var("y") },//
- new IConstant[] { new Constant<String>("Mary"),
- new Constant<String>("Paul") }//
- )
+ /*
+ * No. The CONSTRAINT on the 2nd join [x == y] filters all
+ * solutions. For solutions where the optional join fails, [y] is
+ * not bound. Since [y] is part of the constraint on that join we DO
+ * NOT observe those solutions which only join on the first access
+ * path.
+ */
+// // plus anything we read from the first access path which did not join.
+// new ArrayBindingSet(//
+// new IVariable[] { Var.var("x"), Var.var("y") },//
+// new IConstant[] { new Constant<String>("John"),
+// new Constant<String>("Mary") }//
+// ),
+// new ArrayBindingSet(//
+// new IVariable[] { Var.var("x"), Var.var("y") },//
+// new IConstant[] { new Constant<String>("Mary"),
+// new Constant<String>("Paul") }//
+// )
};
assertSameSolutionsAnyOrder(expected,
@@ -1714,7 +1721,7 @@
// verify query solution stats details.
// assertEquals(1L, stats.chunksIn.get());
assertEquals(4L, stats.unitsIn.get());
- assertEquals(4L, stats.unitsOut.get());
+ assertEquals(2L, stats.unitsOut.get());
// assertEquals(1L, stats.chunksOut.get());
}
@@ -1727,8 +1734,8 @@
// verify query solution stats details.
// assertEquals(2L, stats.chunksIn.get());
- assertEquals(4L, stats.unitsIn.get());
- assertEquals(4L, stats.unitsOut.get());
+ assertEquals(2L, stats.unitsIn.get());
+ assertEquals(2L, stats.unitsOut.get());
// assertEquals(1L, stats.chunksOut.get());
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -382,7 +382,7 @@
public static final String NEW_EVAL_STRATEGY = BigdataSail.class.getPackage()
.getName()+ ".newEvalStrategy";
- public static final String DEFAULT_NEW_EVAL_STRATEGY = "false";
+ public static final String DEFAULT_NEW_EVAL_STRATEGY = "true";
/**
* Option as to whether or not to allow Sesame evaluation of queries
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -65,6 +65,7 @@
import com.bigdata.bop.bindingSet.HashBindingSet;
import com.bigdata.bop.bset.ConditionalRoutingOp;
import com.bigdata.bop.bset.StartOp;
+import com.bigdata.bop.controller.AbstractSubqueryOp;
import com.bigdata.bop.controller.Steps;
import com.bigdata.bop.controller.Union;
import com.bigdata.bop.cost.ScanCostReport;
@@ -1304,13 +1305,11 @@
// The bopId for the UNION or STEP.
final int thisId = idFactory.incrementAndGet();
- final int arity = program.stepCount();
-
- final IStep[] steps = program.toArray();
+ final IStep[] steps = program.toArray();
- final BOp[] args = new BOp[arity];
+ final BOp[] subqueries = new BOp[steps.length];
- for (int i = 0; i < arity; i++) {
+ for (int i = 0; i < steps.length; i++) {
// convert the child IStep
final BOpBase tmp = convert(steps[i], idFactory, db, queryEngine,
@@ -1324,28 +1323,31 @@
*/
// tmp = tmp.setProperty(PipelineOp.Annotations.SINK_REF, thisId);
- args[i] = tmp;
+ subqueries[i] = tmp;
}
final LinkedList<NV> anns = new LinkedList<NV>();
- anns.add(new NV(Union.Annotations.BOP_ID, thisId));
+ anns.add(new NV(BOp.Annotations.BOP_ID, thisId));
+
+ // the subqueries.
+ anns.add(new NV(AbstractSubqueryOp.Annotations.SUBQUERIES, subqueries));
+
+// anns.add(new NV(Union.Annotations.EVALUATION_CONTEXT,
+// BOpEvaluationContext.CONTROLLER));
+//
+// anns.add(new NV(Union.Annotations.CONTROLLER, true));
- anns.add(new NV(Union.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.CONTROLLER));
-
- anns.add(new NV(Union.Annotations.CONTROLLER, true));
-
if (!isParallel)
anns.add(new NV(Union.Annotations.MAX_PARALLEL, 1));
final PipelineOp thisOp;
if (isParallel) {
- thisOp = new Union(args, NV
+ thisOp = new Union(new BOp[]{}, NV
.asMap(anns.toArray(new NV[anns.size()])));
} else {
- thisOp = new Steps(args, NV
+ thisOp = new Steps(new BOp[]{}, NV
.asMap(anns.toArray(new NV[anns.size()])));
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java 2011-01-21 16:41:12 UTC (rev 4158)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java 2011-01-21 16:46:41 UTC (rev 4159)
@@ -273,12 +273,13 @@
}
final LinkedList<NV> anns = new LinkedList<NV>();
- anns.add(new NV(Union.Annotations.BOP_ID, thisId));
- anns.add(new NV(Union.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.CONTROLLER));
- anns.add(new NV(Union.Annotations.CONTROLLER, true));
+ anns.add(new NV(BOp.Annotations.BOP_ID, thisId));
+ anns.add(new NV(Union.Annotations.SUBQUERIES,args));
+// anns.add(new NV(Union.Annotations.EVALUATION_CONTEXT,
+// BOpEvaluationContext.CONTROLLER));
+// anns.add(new NV(Union.Annotations.CONTROLLER, true));
- final Union thisOp = new Union(args, NV
+ final Union thisOp = new Union(new BOp[]{}, NV
.asMap(anns.toArray(new NV[anns.size()])));
return thisOp;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
