[Bigdata-commit] SF.net SVN: bigdata:[4270] branches/QUADS_QUERY_BRANCH/bigdata

[<tho...@us...>]

Revision: 4270
          http://bigdata.svn.sourceforge.net/bigdata/?rev=4270&view=rev
Author:   thompsonbry
Date:     2011-03-03 21:36:04 +0000 (Thu, 03 Mar 2011)

Log Message:
-----------
Javadoc edits and notes on future direction for the RTO.

Modified Paths:
--------------
    branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java
    branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JGraph.java
    branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/AbstractJoinGraphTestCase.java
    branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java

Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java	2011-03-03 21:32:03 UTC (rev 4269)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java	2011-03-03 21:36:04 UTC (rev 4270)
@@ -1041,6 +1041,8 @@
 
         PipelineOp lastOp = null;
 
+        final Set<IVariable<?>> knownBound = new LinkedHashSet<IVariable<?>>();
+        
         for (int i = 0; i < preds.length; i++) {
 
             // The next vertex in the selected join order.
@@ -1065,11 +1067,49 @@
 						assignedConstraints[i]));
 			}
 
+			// collect variables used as arguments by this predicate.
+			final Set<IVariable<?>> pvars = new LinkedHashSet<IVariable<?>>();
+			{
+				final Iterator<IVariable<?>> vitr = BOpUtility
+						.getArgumentVariables(p);
+				while (vitr.hasNext()) {
+					pvars.add(vitr.next());
+				}
+			}
+
+			// figure out if there are ANY shared variables.
+			boolean shared = false;
+			{
+				for(IVariable<?> v : pvars) {
+					if(knownBound.contains(v)) {
+						shared = true;
+						break;
+					}
+				}
+			}
+
+			/*
+			 * FIXME Explore the merit of this optimization with MikeP,
+			 * including consideration of the PIPELINE_QUEUE_CAPACITY and
+			 * whether or not to request an analytic join (hash join).
+			 */
+			if (false && !shared) {
+				System.err.println("Full cross product join: " + p);
+				/*
+				 * Force at-once evaluation to ensure that we evaluate the AP
+				 * for [p] exactly once.
+				 */
+				anns.add(new NV(PipelineOp.Annotations.PIPELINED, false));
+			}
+
 			final PipelineJoin<?> joinOp = new PipelineJoin(//
 					lastOp == null ? new BOp[0] : new BOp[] { lastOp }, //
 					anns.toArray(new NV[anns.size()])//
 			);
 
+			// Add predicate argument variables to [knownBound].
+			knownBound.addAll(pvars);
+			
             lastOp = joinOp;
 
         }

Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JGraph.java	2011-03-03 21:32:03 UTC (rev 4269)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JGraph.java	2011-03-03 21:36:04 UTC (rev 4270)
@@ -423,6 +423,14 @@
 			 * #of paths with cardinality estimate underflow to jump up and down
 			 * due to the sample which is making its way through each path in
 			 * each round.
+			 * 
+			 * TODO The RTO needs an escape hatch here. FOr example, if the sum
+			 * of the expected IOs for some path(s) strongly dominates all other
+			 * paths sharing the same vertices, then we should prune those paths
+			 * even if there is a cardinality estimate underflow in those paths.
+			 * This will allow us to focus our efforts on those paths having
+			 * less IO cost while we seek cardinality estimates which do not
+			 * underflow.
 			 */
 			int nunderflow;
 

Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/AbstractJoinGraphTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/AbstractJoinGraphTestCase.java	2011-03-03 21:32:03 UTC (rev 4269)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/AbstractJoinGraphTestCase.java	2011-03-03 21:36:04 UTC (rev 4270)
@@ -493,6 +493,16 @@
                 + ", elapsed=" + q.getElapsed() + ", nout=" + nout
                 + ", nchunks=" + nchunks + ", stats=" + stats);
 
+//        if(false) {
+//
+//        	final StringBuilder sb = new StringBuilder();
+//			
+//        	QueryLog.log(true/* tableHeader */, q, sb);
+//			
+//        	System.err.println(sb);
+//        	
+//		}
+
         return stats;
 
     }

Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java	2011-03-03 21:32:03 UTC (rev 4269)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java	2011-03-03 21:36:04 UTC (rev 4270)
@@ -534,7 +534,8 @@
         if(false){
 			// Run some fixed order.
 //			final IPredicate<?>[] path = { p5, p6, p0, p2, p1, p4, p3 }; 
-			final IPredicate<?>[] path = { p5, p3, p1, p2, p4, p6, p0 };
+//			final IPredicate<?>[] path = { p5, p3, p1, p2, p4, p6, p0 };
+			final IPredicate<?>[] path = { p3, p5, p1, p2, p6, p4, p0 };
 			runQuery("FIXED ORDER", queryEngine, distinct, selected, path,
 					constraints);
 		}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.