bigdata-commit

[Bigdata-commit] SF.net SVN: bigdata:[7704] branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/ com/bigdata/bop

[<tho...@us...>]

Revision: 7704
          http://bigdata.svn.sourceforge.net/bigdata/?rev=7704&view=rev
Author:   thompsonbry
Date:     2013-12-30 19:52:04 +0000 (Mon, 30 Dec 2013)
Log Message:
-----------
ChunkedRunningQuery - added more information for an assert that is being tripped by the RTO for plans involving an OPTIONAL join group.

QueryLog - added conditional logic for a cause with a null message.

PartitionedJoinGroup - removed some dead code and organized imports.

JoinGraph - added some logic to attach the generated query plan to the IRunningQuery's attributes. This is forward looking to an integration where the RTO runs in parallel with the head of the query and then floods all solutions into a subquery that runs the RTO determined join ordering.  We need to get the integration going first with the different join modes (triples versus quads), conditional materialization for filters, etc.

@see #64 (RTO).

Modified Paths:
--------------
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JoinGraph.java

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java	2013-12-30 19:07:46 UTC (rev 7703)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java	2013-12-30 19:52:04 UTC (rev 7704)
@@ -1149,7 +1149,7 @@
                 stats = op.newStats();
 //        		log.warn("bopId=" + bopId + ", stats=" + stats);
             }
-            assert stats != null;
+            assert stats != null : "No stats: op=" + op;
 
 //            // The groupId (if any) for this operator.
 //            final Integer fromGroupId = (Integer) op

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java	2013-12-30 19:07:46 UTC (rev 7703)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java	2013-12-30 19:52:04 UTC (rev 7704)
@@ -964,8 +964,12 @@
             w.write(TDx);
         }
         w.write(TD);
-        if (cause != null)
-            w.write(cause.getLocalizedMessage());
+        if (cause != null) {
+            String msg = cause.getLocalizedMessage();
+            if (msg == null)
+                msg = cause.toString();
+            w.write(cdata(msg));
+        }
         w.write(TDx);
         
         final Map<Integer, BOp> bopIndex = q.getBOpIndex();

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java	2013-12-30 19:07:46 UTC (rev 7703)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/PartitionedJoinGroup.java	2013-12-30 19:52:04 UTC (rev 7704)
@@ -21,7 +21,6 @@
 import com.bigdata.bop.IVariable;
 import com.bigdata.bop.NV;
 import com.bigdata.bop.PipelineOp;
-import com.bigdata.bop.bset.EndOp;
 import com.bigdata.bop.join.PipelineJoin;
 import com.bigdata.bop.joinGraph.rto.JoinGraph;
 import com.bigdata.bop.solutions.JVMDistinctBindingSetsOp;
@@ -1134,24 +1133,6 @@
 			);
 		}
 
-        /*
-         * FIXME Why does wrapping with this slice appear to be
-         * necessary? (It is causing runtime errors when not wrapped).
-         * Is this a bopId collision which is not being detected?
-         * 
-         * @see https://sourceforge.net/apps/trac/bigdata/ticket/227
-         * 
-         * [This should perhaps be moved into the caller.]
-         */
-//        lastOp = new EndOp(new BOp[] { lastOp }, NV
-//                .asMap(new NV[] {
-//                        new NV(JoinGraph.Annotations.BOP_ID, idFactory.nextId()), //
-//                        new NV(JoinGraph.Annotations.EVALUATION_CONTEXT,
-//                                BOpEvaluationContext.CONTROLLER)//
-////                        new NV(PipelineOp.Annotations.SHARED_STATE,true),//
-//                        }) //
-//        );
-
         return lastOp;
 
     }

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JoinGraph.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JoinGraph.java	2013-12-30 19:07:46 UTC (rev 7703)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/joinGraph/rto/JoinGraph.java	2013-12-30 19:52:04 UTC (rev 7704)
@@ -52,7 +52,6 @@
 import com.bigdata.bop.engine.IRunningQuery;
 import com.bigdata.bop.engine.QueryEngine;
 import com.bigdata.bop.joinGraph.PartitionedJoinGroup;
-import com.bigdata.relation.accesspath.BufferClosedException;
 import com.bigdata.util.concurrent.Haltable;
 
 import cutthecrap.utils.striterators.ICloseableIterator;
@@ -81,7 +80,10 @@
 
 	private static final long serialVersionUID = 1L;
 
-	/**
+    private static final transient Logger log = Logger
+            .getLogger(JoinGraph.class);
+
+    /**
 	 * Known annotations.
 	 */
 	public interface Annotations extends PipelineOp.Annotations {
@@ -157,6 +159,13 @@
          */
         String SAMPLES = JoinGraph.class.getName() + ".samples";
 
+        /**
+         * The physical query plan generated from the RTO determined best join
+         * ordering (output). This is used to specify the query plan to be
+         * executed by a downstream operator.
+         */
+        String QUERY_PLAN = JoinGraph.class.getName() + ".queryPlan";
+
 	}
 	
 	/**
@@ -252,6 +261,26 @@
     }
 
     /**
+     * Return the query plan to be executed based on the RTO determined join
+     * ordering.
+     * 
+     * @see Attributes#QUERY_PLAN
+     */
+    public PipelineOp getQueryPlan(final IRunningQuery q) {
+
+        return (PipelineOp) q.getAttributes().get(
+                getId() + "-" + Attributes.QUERY_PLAN);
+
+    }
+
+    private void setQueryPlan(final IRunningQuery q,
+            final PipelineOp queryPlan) {
+        
+        q.getAttributes().put(getId() + "-" + Attributes.QUERY_PLAN, queryPlan);
+        
+    }
+
+    /**
      * Deep copy constructor.
      * 
      * @param op
@@ -327,14 +356,6 @@
 
 	    private final BOpContext<IBindingSet> context;
 
-	//  private final JGraph g;
-
-//	    final private int limit;
-//	    
-//	    final private int nedges;
-//	    
-//	    final private SampleType sampleType;
-
 	    JoinGraphTask(final BOpContext<IBindingSet> context) {
 
 	        if (context == null)
@@ -342,30 +363,11 @@
 
 	        this.context = context;
 
-//	        // The initial cutoff sampling limit.
-//	        limit = getLimit();
-//
-//	        // The initial number of edges (1 step paths) to explore.
-//	        nedges = getNEdges();
-//
-//	        sampleType = getSampleType();
-	        
-//	      if (limit <= 0)
-//	          throw new IllegalArgumentException();
-	//
-//	      if (nedges <= 0)
-//	          throw new IllegalArgumentException();
-
-//	        g = new JGraph(getVertices(), getConstraints());
-
 	    }
 
         /**
          * {@inheritDoc}
          * 
-         * 
-         * TODO where to handle DISTINCT, ORDER BY, GROUP BY for join graph?
-         * 
          * FIXME When run as sub-query, we need to fix point the upstream
          * solutions and then flood them into the join graph. Samples of the
          * known bound variables can be pulled from those initial solutions.
@@ -414,13 +416,22 @@
 	        // Factory avoids reuse of bopIds assigned to the predicates.
 	        final BOpIdFactory idFactory = new BOpIdFactory();
 
-			// Generate the query from the join path.
-	        // FIXME Update this using StaticAnalysis logic.
+            /*
+             * Generate the query from the join path.
+             * 
+             * FIXME Update this using StaticAnalysis logic. Also, both this and
+             * the JGraph need to handle triples versus named graph versus
+             * default graph APs. Further, JGraph should handle filters that
+             * require conditional materialization.
+             */
 			final PipelineOp queryOp = PartitionedJoinGroup.getQuery(idFactory,
 					false/* distinct */, getSelected(), p.getPredicates(),
 					getConstraints());
 
-	        // Run the query, blocking until it is done.
+            // Set attribute for the join path samples.
+            setQueryPlan(context.getRunningQuery(), queryOp);
+
+            // Run the query, blocking until it is done.
 	        JoinGraph.runSubquery(context, queryOp);
 
 	        final long elapsed_queryExecution = System.nanoTime() - mark;
@@ -436,7 +447,6 @@
 	    }
 
 	} // class JoinGraphTask
-    private static final transient Logger log = Logger.getLogger(JGraph.class);
 
     /**
      * Execute the selected join path.
@@ -461,10 +471,6 @@
      * @todo If there are source binding sets then they need to be applied above
      *       (when we are sampling) and below (when we evaluate the selected
      *       join path).
-     * 
-     *       FIXME runQuery() is not working correctly. The query is being
-     *       halted by a {@link BufferClosedException} which appears before it
-     *       has materialized the necessary results.
      */
     static private void runSubquery(
             final BOpContext<IBindingSet> parentContext,

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Bigdata-commit] SF.net SVN: bigdata:[7709] branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/ com/bigdata/bop

[<tho...@us...>]

Revision: 7709
          http://bigdata.svn.sourceforge.net/bigdata/?rev=7709&view=rev
Author:   thompsonbry
Date:     2013-12-31 19:43:09 +0000 (Tue, 31 Dec 2013)
Log Message:
-----------
final and @Override attributes

Modified Paths:
--------------
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/BOpStats.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/BaseJoinStats.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoinStats.java

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/BOpStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/BOpStats.java	2013-12-31 17:11:39 UTC (rev 7708)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/BOpStats.java	2013-12-31 19:43:09 UTC (rev 7709)
@@ -155,6 +155,7 @@
         mutationCount.add(o.mutationCount.get());
     }
     
+    @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder();
         sb.append(super.toString());

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/BaseJoinStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/BaseJoinStats.java	2013-12-31 17:11:39 UTC (rev 7708)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/BaseJoinStats.java	2013-12-31 19:43:09 UTC (rev 7709)
@@ -84,6 +84,7 @@
      */
     public final CAT accessPathUnitsIn = new CAT();
 
+    @Override
     public void add(final BOpStats o) {
 
         super.add(o);

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoinStats.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoinStats.java	2013-12-31 17:11:39 UTC (rev 7708)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/join/PipelineJoinStats.java	2013-12-31 19:43:09 UTC (rev 7709)
@@ -115,6 +115,7 @@
         
     }
 
+    @Override
 	public void add(final BOpStats o) {
 
 		super.add(o);

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.