[Bigdata-commit] SF.net SVN: bigdata:[4276] branches/QUADS_QUERY_BRANCH/bigdata

[<tho...@us...>]

Revision: 4276
          http://bigdata.svn.sourceforge.net/bigdata/?rev=4276&view=rev
Author:   thompsonbry
Date:     2011-03-07 15:04:34 +0000 (Mon, 07 Mar 2011)

Log Message:
-----------
Working on query performance statistics reporting.

Modified Paths:
--------------
    branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
    branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
    branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java
    branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
    branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailBooleanQuery.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailQuery.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
    branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java

Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -49,9 +49,11 @@
 import com.bigdata.bop.BOpUtility;
 import com.bigdata.bop.IBindingSet;
 import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.engine.QueryEngine.Counters;
 import com.bigdata.bop.solutions.SliceOp;
 import com.bigdata.journal.IIndexManager;
 import com.bigdata.journal.ITx;
+import com.bigdata.rdf.sail.QueryHints;
 import com.bigdata.relation.accesspath.IAsynchronousIterator;
 import com.bigdata.relation.accesspath.IBlockingBuffer;
 import com.bigdata.service.IBigdataFederation;
@@ -1046,9 +1048,28 @@
                 // log summary statistics for the query.
                 if (isController())
                     QueryLog.log(this);
+				final String tag = getQuery().getProperty(QueryHints.TAG,
+						QueryHints.DEFAULT_TAG);
+				final Counters c = tag == null ? null : queryEngine
+						.getCounters(tag);
+				// track #of done queries.
+				queryEngine.counters.doneCount.increment();
+				if (c != null)
+					c.doneCount.increment();
+				// track elapsed run time of done queries.
+				final long elapsed = getElapsed();
+				queryEngine.counters.elapsedMillis.add(elapsed);
+				if (c != null)
+					c.elapsedMillis.add(elapsed);
+				if (future.getCause() != null) {
+					// track #of queries with abnormal termination.
+					queryEngine.counters.errorCount.increment();
+					if (c != null)
+						c.errorCount.increment();
+				}
+				// remove from the collection of running queries.
+				queryEngine.halt(this);
             }
-            // remove from the collection of running queries.
-            queryEngine.halt(this);
             // true iff we cancelled something.
             return cancelled;
         } finally {

Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -30,6 +30,7 @@
 import java.lang.reflect.Constructor;
 import java.rmi.RemoteException;
 import java.util.Comparator;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -57,7 +58,11 @@
 import com.bigdata.btree.BTree;
 import com.bigdata.btree.IndexSegment;
 import com.bigdata.btree.view.FusedView;
+import com.bigdata.counters.CAT;
+import com.bigdata.counters.CounterSet;
+import com.bigdata.counters.Instrument;
 import com.bigdata.journal.IIndexManager;
+import com.bigdata.rdf.sail.QueryHints;
 import com.bigdata.rdf.sail.bench.NanoSparqlServer;
 import com.bigdata.relation.accesspath.IAsynchronousIterator;
 import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
@@ -212,6 +217,16 @@
      */
     public interface Annotations extends PipelineOp.Annotations {
 
+		/**
+		 * Annotation may be used to impose a specific {@link UUID} for a query.
+		 * This may be used by an external process such that it can then use
+		 * {@link QueryEngine#getRunningQuery(UUID)} to gain access to the
+		 * running query instance. It is an error if there is a query already
+		 * running with the same {@link UUID}.
+		 */
+		String QUERY_ID = (QueryEngine.class.getName() + ".runningQueryClass")
+				.intern();
+    	
         /**
          * The name of the {@link IRunningQuery} implementation class which will
          * be used to evaluate a query marked by this annotation (optional). The
@@ -235,8 +250,177 @@
         String DEFAULT_RUNNING_QUERY_CLASS = ChunkedRunningQuery.class.getName();
 
     }
+
+	/**
+	 * Statistics collected for queries.
+	 * 
+	 * @author thompsonbry
+	 */
+    protected static class Counters {
+
+		/**
+		 * The #of queries which have been executed (set on completion).
+		 */
+		final CAT startCount = new CAT();
+		
+		/**
+		 * The #of queries which have been executed (set on completion).
+		 */
+		final CAT doneCount = new CAT();
+
+		/**
+		 * The #of instances of the query which terminated abnormally.
+		 */
+		final CAT errorCount = new CAT();
+
+		/**
+		 * The total elapsed time (millis) for evaluation queries. This is the
+		 * wall clock time per query. The aggregated wall clock time per query
+		 * will sum to greater than the elapsed wall clock time in any interval
+		 * where there is more than one query running concurrently.
+		 */
+		final CAT elapsedMillis = new CAT();
+
+		public CounterSet getCounters() {
+
+			final CounterSet root = new CounterSet();
+
+			// #of queries started on this server.
+			root.addCounter("startCount", new Instrument<Long>() {
+				public void sample() {
+					setValue(startCount.get());
+				}
+			});
+
+			// #of queries retired on this server.
+			root.addCounter("doneCount", new Instrument<Long>() {
+				public void sample() {
+					setValue(doneCount.get());
+				}
+			});
+
+			// #of queries with abnormal termination on this server.
+			root.addCounter("errorCount", new Instrument<Long>() {
+				public void sample() {
+					setValue(errorCount.get());
+				}
+			});
+
+			// #of queries retired per second on this server.
+			root.addCounter("queriesPerSecond", new Instrument<Double>() {
+				public void sample() {
+					final long ms = elapsedMillis.get();
+					final long n = doneCount.get();
+					// compute throughput, normalized to q/s := (q*1000)/ms.
+					final double d = ms == 0 ? 0d : ((1000d * n) / ms);
+					setValue(d);
+				}
+			});
+
+			return root;
+
+		}
+		
+    }
+
+    /**
+     * Return a {@link CounterSet} which reports various statistics for the
+     * {@link QueryEngine}.
+     */
+    public CounterSet getCounters() {
+
+		final CounterSet root = new CounterSet();
+
+		// global counters.
+        root.attach(counters.getCounters());
+		
+        // counters per tagged query group.
+		{
+
+			final CounterSet groups = root.makePath("groups");
+
+			final Iterator<Map.Entry<String, Counters>> itr = groupCounters
+					.entrySet().iterator();
+
+			while (itr.hasNext()) {
+
+				final Map.Entry<String, Counters> e = itr.next();
+
+				final String tag = e.getKey();
+
+				final Counters counters = e.getValue();
+
+				groups.makePath(tag).attach(counters.getCounters());
+
+			}
+
+		}
+
+		return root;
+
+    }
     
     /**
+     * Counters at the global level.
+     */
+	final protected Counters counters = newCounters();
+
+	/**
+	 * Statistics for queries which are "tagged" so we can recognize their
+	 * instances as members of some group.
+	 */
+	final protected ConcurrentHashMap<String/* groupId */, Counters> groupCounters = new ConcurrentHashMap<String, Counters>();
+
+	/**
+	 * Factory for {@link Counters} instances associated with a query group. A
+	 * query is marked as a member of a group using {@link QueryHints#TAG}. This
+	 * is typically used to mark queries which are instances of the same query
+	 * template.
+	 * 
+	 * @param tag
+	 *            The tag identifying a query group.
+	 * 
+	 * @return The {@link Counters} for that query group.
+	 * 
+	 * @throws IllegalArgumentException
+	 *             if the argument is <code>null</code>.
+	 */
+	protected Counters getCounters(final String tag) {
+
+		if(tag == null)
+			throw new IllegalArgumentException();
+		
+		Counters c = groupCounters.get(tag);
+
+		if (c == null) {
+
+			c = new Counters();
+
+			final Counters tmp = groupCounters.putIfAbsent(tag, c);
+
+			if (tmp != null) {
+
+				// someone else won the data race.
+				c = tmp;
+
+			}
+
+		}
+
+		return c;
+
+	}
+
+	/**
+	 * Extension hook for new {@link Counters} instances.
+	 */
+	protected Counters newCounters() {
+		
+		return new Counters();
+		
+	}
+	
+    /**
      * Access to the indices.
      * <p>
      * Note: You MUST NOT use unisolated indices without obtaining the necessary
@@ -708,7 +892,7 @@
         
     }
     
-    public void bufferReady(IChunkMessage<IBindingSet> msg) {
+    public void bufferReady(final IChunkMessage<IBindingSet> msg) {
 
         throw new UnsupportedOperationException();
 
@@ -719,7 +903,7 @@
      * <p>
      * The default implementation is a NOP.
      */
-    public void cancelQuery(UUID queryId, Throwable cause) {
+    public void cancelQuery(final UUID queryId, final Throwable cause) {
         // NOP
     }
 
@@ -811,14 +995,20 @@
      */
     public AbstractRunningQuery eval(final BOp op, final IBindingSet bset)
             throws Exception {
+    	
+    	// Use a random UUID unless the UUID was specified on the query.
+    	final UUID queryId = op.getProperty(QueryEngine.Annotations.QUERY_ID,
+				UUID.randomUUID());
 
-        return eval(op, newBindingSetIterator(bset));
+        return eval(queryId, op, newBindingSetIterator(bset));
 
     }
 
     /**
      * Evaluate a query. This node will serve as the controller for the query.
      * 
+     * @param queryId
+     *            The unique identifier for the query.
      * @param query
      *            The query to evaluate.
      * @param bsets
@@ -830,15 +1020,13 @@
      *             if the {@link QueryEngine} has been {@link #shutdown()}.
      * @throws Exception
      */
-    public AbstractRunningQuery eval(final BOp op,
+    public AbstractRunningQuery eval(final UUID queryId, final BOp op,
             final IAsynchronousIterator<IBindingSet[]> bsets) throws Exception {
 
         final BOp startOp = BOpUtility.getPipelineStart(op);
 
         final int startId = startOp.getId();
 
-        final UUID queryId = UUID.randomUUID();
-
         return eval(queryId, (PipelineOp) op,
                 new LocalChunkMessage<IBindingSet>(this/* queryEngine */,
                         queryId, startId, -1 /* partitionId */, bsets));
@@ -916,9 +1104,32 @@
         // verify query engine is running.
         assertRunning();
 
-        // add to running query table.
-        putIfAbsent(queryId, runningQuery);
+		// add to running query table.
+		if (putIfAbsent(queryId, runningQuery) != runningQuery) {
 
+			/*
+			 * UUIDs should not collide when assigned randomly. However, the
+			 * UUID may be imposed by an exterior process, such as a SPARQL end
+			 * point, so it can access metadata about the running query even
+			 * when it is not a direct client of the QueryEngine. This provides
+			 * a safety check against UUID collisions which might be non-random.
+			 */
+			throw new RuntimeException("Query exists with that UUID: uuid="
+					+ runningQuery.getQueryId());
+
+		}
+
+		final String tag = query.getProperty(QueryHints.TAG,
+				QueryHints.DEFAULT_TAG);
+
+		final Counters c = tag == null ? null : getCounters(tag);
+
+		// track #of started queries.
+		counters.startCount.increment();
+
+		if (c != null)
+			c.startCount.increment();
+
         // notify query start
         runningQuery.startQuery(msg);
         
@@ -1156,7 +1367,8 @@
      * 
      * @see Annotations#RUNNING_QUERY_CLASS
      */
-    protected AbstractRunningQuery newRunningQuery(
+    @SuppressWarnings("unchecked")
+	protected AbstractRunningQuery newRunningQuery(
             /*final QueryEngine queryEngine,*/ final UUID queryId,
             final boolean controller, final IQueryClient clientProxy,
             final PipelineOp query) {

Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -40,6 +40,8 @@
 import com.bigdata.bop.IPredicate;
 import com.bigdata.bop.join.PipelineJoin;
 import com.bigdata.bop.join.PipelineJoin.PipelineJoinStats;
+import com.bigdata.rawstore.Bytes;
+import com.bigdata.rdf.sail.QueryHints;
 import com.bigdata.rdf.sail.Rule2BOpUtility;
 import com.bigdata.striterator.IKeyOrder;
 
@@ -52,7 +54,9 @@
  */
 public class QueryLog {
 
-    protected static final transient Logger log = Logger
+    private static final String NA = "N/A";
+    
+	protected static final transient Logger log = Logger
             .getLogger(QueryLog.class);
 
     static {
@@ -63,14 +67,18 @@
     	if(log.isInfoEnabled())
     		log.info(QueryLog.getTableHeader());
     }
-    
+
     /**
+     * A single buffer is reused to keep down the heap churn.
+     */
+	final private static StringBuilder sb = new StringBuilder(
+			Bytes.kilobyte32 * 4);
+
+    /**
      * Log rule execution statistics.
      * 
      * @param q
      *            The running query.
-     *            
-     *            @todo need start and end time for the query.
      */
     static public void log(final IRunningQuery q) {
 
@@ -78,13 +86,25 @@
 			
 			try {
 
-				final StringBuilder sb = new StringBuilder(1024);
+				/*
+				 * Note: We could use a striped lock here over a small pool of
+				 * StringBuilder's to decrease contention for the single buffer
+				 * while still avoiding heap churn for buffer allocation. Do
+				 * this if the monitor for this StringBuilder shows up as a hot
+				 * spot when query logging is enabled.
+				 */
+				synchronized(sb) {
 				
-				logDetailRows(q,sb);
+					// clear the buffer.
+					sb.setLength(0);
 
-				logSummaryRow(q,sb);
+					logDetailRows(q, sb);
 
-				log.info(sb);
+					logSummaryRow(q, sb);
+
+					log.info(sb);
+
+				}
 				
 			} catch (RuntimeException t) {
 
@@ -160,6 +180,7 @@
          * Common columns for the overall query and for each pipeline operator.
          */
         sb.append("queryId");
+        sb.append("\ttag");
         sb.append("\tbeginTime");
         sb.append("\tdoneTime");
         sb.append("\tdeadline");
@@ -233,6 +254,9 @@
         
         sb.append(q.getQueryId());
         sb.append('\t');
+		sb.append(q.getQuery().getProperty(QueryHints.TAG,
+				QueryHints.DEFAULT_TAG));
+		sb.append('\t');
         sb.append(dateFormat.format(new Date(q.getStartTime())));
         sb.append('\t');
         sb.append(dateFormat.format(new Date(q.getDoneTime())));
@@ -242,7 +266,7 @@
         sb.append('\t');
         sb.append(elapsed);
         sb.append('\t');
-        sb.append(serviceId == null ? "N/A" : serviceId.toString());
+        sb.append(serviceId == null ? NA : serviceId.toString());
         sb.append('\t');
         if (cause != null) 
             sb.append(cause.getLocalizedMessage());
@@ -299,12 +323,13 @@
 		 */
 		{
 
+			@SuppressWarnings("unchecked")
 			final IPredicate pred = (IPredicate<?>) bop
 					.getProperty(PipelineJoin.Annotations.PREDICATE);
 			
 			if (pred != null) {
 			
-				final IKeyOrder keyOrder = (IKeyOrder<?>) pred
+				final IKeyOrder<?> keyOrder = (IKeyOrder<?>) pred
 						.getProperty(Rule2BOpUtility.Annotations.ORIGINAL_INDEX);
 				
 				final Long rangeCount = (Long) pred
@@ -364,7 +389,7 @@
 		sb.append('\t');
 		sb.append(stats.unitsOut.get());
 		sb.append('\t');
-		sb.append(unitsIn == 0 ? "N/A" : unitsOut / (double) unitsIn);
+		sb.append(unitsIn == 0 ? NA : unitsOut / (double) unitsIn);
 		sb.append('\t');
 		sb.append(stats.accessPathDups.get());
 		sb.append('\t');

Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -409,7 +409,6 @@
                 /*
                  * Unshared for any other view of the triple store.
                  */
-            	System.err.println("Unshared: "+termCacheCapacity);//FIXME remove stderr.
                 termCache = new ConcurrentWeakValueCacheWithBatchedUpdates<IV, BigdataValue>(//
                         termCacheCapacity, // queueCapacity
                         .75f, // loadFactor (.75 is the default)
@@ -2262,10 +2261,8 @@
         @Override
         protected ConcurrentWeakValueCacheWithBatchedUpdates<IV, BigdataValue> newInstance(
                 NT key, Integer termCacheCapacity) {
-        	final int queueCapacity = 50000;// FIXME termCacheCapacity.intValue();
-        	System.err.println("Shared  : "+termCacheCapacity);//FIXME remove stderr.
             return new ConcurrentWeakValueCacheWithBatchedUpdates<IV, BigdataValue>(//
-                    queueCapacity,// backing hard reference LRU queue capacity.
+                    termCacheCapacity.intValue(),// backing hard reference LRU queue capacity.
                     .75f, // loadFactor (.75 is the default)
                     16 // concurrency level (16 is the default)
             );

Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -593,7 +593,7 @@
         String TERM_CACHE_CAPACITY = (AbstractTripleStore.class.getName()
                 + ".termCache.capacity").intern();
         
-        String DEFAULT_TERM_CACHE_CAPACITY = "500";//"50000";
+        String DEFAULT_TERM_CACHE_CAPACITY = "10000";//"50000";
         
         /**
          * The name of the class that will establish the pre-defined

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -28,7 +28,6 @@
 import org.openrdf.query.algebra.And;
 import org.openrdf.query.algebra.Bound;
 import org.openrdf.query.algebra.Compare;
-import org.openrdf.query.algebra.Compare.CompareOp;
 import org.openrdf.query.algebra.Filter;
 import org.openrdf.query.algebra.Group;
 import org.openrdf.query.algebra.IsBNode;
@@ -50,13 +49,14 @@
 import org.openrdf.query.algebra.Regex;
 import org.openrdf.query.algebra.SameTerm;
 import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.StatementPattern.Scope;
 import org.openrdf.query.algebra.TupleExpr;
 import org.openrdf.query.algebra.UnaryTupleOperator;
 import org.openrdf.query.algebra.Union;
 import org.openrdf.query.algebra.ValueConstant;
 import org.openrdf.query.algebra.ValueExpr;
 import org.openrdf.query.algebra.Var;
+import org.openrdf.query.algebra.Compare.CompareOp;
+import org.openrdf.query.algebra.StatementPattern.Scope;
 import org.openrdf.query.algebra.evaluation.impl.EvaluationStrategyImpl;
 import org.openrdf.query.algebra.evaluation.iterator.FilterIterator;
 import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
@@ -68,12 +68,12 @@
 import com.bigdata.bop.IConstant;
 import com.bigdata.bop.IConstraint;
 import com.bigdata.bop.IPredicate;
-import com.bigdata.bop.IPredicate.Annotations;
 import com.bigdata.bop.IValueExpression;
 import com.bigdata.bop.IVariable;
 import com.bigdata.bop.IVariableOrConstant;
 import com.bigdata.bop.NV;
 import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.IPredicate.Annotations;
 import com.bigdata.bop.ap.Predicate;
 import com.bigdata.bop.constraint.INBinarySearch;
 import com.bigdata.bop.engine.IRunningQuery;
@@ -84,33 +84,29 @@
 import com.bigdata.rdf.internal.DummyIV;
 import com.bigdata.rdf.internal.IV;
 import com.bigdata.rdf.internal.XSDBooleanIV;
-import com.bigdata.rdf.internal.XSDDecimalIV;
-import com.bigdata.rdf.internal.XSDDoubleIV;
-import com.bigdata.rdf.internal.XSDIntIV;
-import com.bigdata.rdf.internal.XSDIntegerIV;
 import com.bigdata.rdf.internal.constraints.AndBOp;
 import com.bigdata.rdf.internal.constraints.CompareBOp;
-import com.bigdata.rdf.internal.constraints.SPARQLConstraint;
 import com.bigdata.rdf.internal.constraints.EBVBOp;
 import com.bigdata.rdf.internal.constraints.IsBNodeBOp;
 import com.bigdata.rdf.internal.constraints.IsBoundBOp;
 import com.bigdata.rdf.internal.constraints.IsLiteralBOp;
 import com.bigdata.rdf.internal.constraints.IsURIBOp;
 import com.bigdata.rdf.internal.constraints.MathBOp;
-import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
 import com.bigdata.rdf.internal.constraints.NotBOp;
 import com.bigdata.rdf.internal.constraints.OrBOp;
 import com.bigdata.rdf.internal.constraints.RangeBOp;
+import com.bigdata.rdf.internal.constraints.SPARQLConstraint;
 import com.bigdata.rdf.internal.constraints.SameTermBOp;
+import com.bigdata.rdf.internal.constraints.MathBOp.MathOp;
 import com.bigdata.rdf.lexicon.LexiconRelation;
 import com.bigdata.rdf.model.BigdataValue;
 import com.bigdata.rdf.sail.BigdataSail.Options;
 import com.bigdata.rdf.sail.sop.SOp;
 import com.bigdata.rdf.sail.sop.SOp2BOpUtility;
 import com.bigdata.rdf.sail.sop.SOpTree;
-import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
 import com.bigdata.rdf.sail.sop.SOpTreeBuilder;
 import com.bigdata.rdf.sail.sop.UnsupportedOperatorException;
+import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
 import com.bigdata.rdf.spo.DefaultGraphSolutionExpander;
 import com.bigdata.rdf.spo.ExplicitSPOFilter;
 import com.bigdata.rdf.spo.ISPO;
@@ -592,7 +588,6 @@
 			 */
 			throw new UnsupportedOperatorException(ex);
 		} catch (Throwable ex) {
-//			log.error("Remove log stmt:"+ex,ex);// FIXME remove this - I am just looking for the root cause of something in the SAIL.
 			throw new QueryEvaluationException(ex);
 		}
 	}
@@ -814,7 +809,7 @@
     	 */
     	attachNamedGraphsFilterToSearches(sopTree);
     	
-		if (false) {
+		if (true) {
 			/*
 			 * Look for numerical filters that can be rotated inside predicates
 			 */
@@ -891,6 +886,9 @@
 
     }
     
+    /*
+     * FIXME What is [bs]?  It is not being used within this context.
+     */
     CloseableIteration<BindingSet, QueryEvaluationException> 
 		doEvaluateNatively(final PipelineOp query, final BindingSet bs,
 			final QueryEngine queryEngine, final IVariable[] required
@@ -925,7 +923,6 @@
 				// ensure query is halted.
 				runningQuery.cancel(true/* mayInterruptIfRunning */);
 			}
-//			log.error("Remove log stmt"+t,t);// FIXME remove this - I am just looking for the root cause of something in the SAIL.
 			throw new QueryEvaluationException(t);
 		}
     	

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailBooleanQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailBooleanQuery.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailBooleanQuery.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -1,7 +1,9 @@
 package com.bigdata.rdf.sail;
 
 import info.aduna.iteration.CloseableIteration;
+
 import java.util.Properties;
+
 import org.openrdf.query.BindingSet;
 import org.openrdf.query.Dataset;
 import org.openrdf.query.QueryEvaluationException;
@@ -9,8 +11,8 @@
 import org.openrdf.query.parser.ParsedBooleanQuery;
 import org.openrdf.repository.sail.SailBooleanQuery;
 import org.openrdf.repository.sail.SailRepositoryConnection;
-import org.openrdf.sail.SailConnection;
 import org.openrdf.sail.SailException;
+
 import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
 
 public class BigdataSailBooleanQuery extends SailBooleanQuery 
@@ -22,10 +24,19 @@
      */
     private final Properties queryHints;
     
+    public Properties getQueryHints() {
+    	
+    	return queryHints;
+    	
+    }
+
     public BigdataSailBooleanQuery(ParsedBooleanQuery tupleQuery,
             SailRepositoryConnection con, Properties queryHints) {
-        super(tupleQuery, con);
+
+    	super(tupleQuery, con);
+    	
         this.queryHints = queryHints;
+        
     }
 
     /**

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -3,7 +3,7 @@
 import info.aduna.iteration.CloseableIteration;
 import info.aduna.iteration.ConvertingIteration;
 import info.aduna.iteration.FilterIteration;
-import java.util.Arrays;
+
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -43,6 +43,7 @@
 import org.openrdf.repository.sail.SailGraphQuery;
 import org.openrdf.repository.sail.SailRepositoryConnection;
 import org.openrdf.sail.SailException;
+
 import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
 
 public class BigdataSailGraphQuery extends SailGraphQuery 
@@ -64,6 +65,12 @@
      */
     private boolean useNativeConstruct = false;
     
+    public Properties getQueryHints() {
+    	
+    	return queryHints;
+    	
+    }
+
     public BigdataSailGraphQuery(final ParsedGraphQuery tupleQuery,
             final SailRepositoryConnection con, final Properties queryHints, 
             final boolean describe) {

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailQuery.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailQuery.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -23,10 +23,14 @@
 
 package com.bigdata.rdf.sail;
 
+import java.util.Properties;
+
 import org.openrdf.query.QueryEvaluationException;
 import org.openrdf.query.algebra.TupleExpr;
 
 /**
+ * Extension API for bigdata queries.
+ * 
  * @author <a href="mailto:mrp...@us...">Mike Personick</a>
  */
 public interface BigdataSailQuery {
@@ -37,4 +41,11 @@
      */
     TupleExpr getTupleExpr() throws QueryEvaluationException;
     
+	/**
+	 * Return query hints associated with this query. Query hints are embedded
+	 * in query strings as namespaces. See {@link QueryHints#NAMESPACE} for more
+	 * information.
+	 */
+    Properties getQueryHints(); 
+    
 }

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -1,10 +1,9 @@
 package com.bigdata.rdf.sail;
 
-import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
 import java.util.StringTokenizer;
-import org.apache.log4j.Logger;
+
 import org.openrdf.query.MalformedQueryException;
 import org.openrdf.query.QueryLanguage;
 import org.openrdf.query.parser.ParsedBooleanQuery;
@@ -16,13 +15,11 @@
 import org.openrdf.query.parser.sparql.ast.ParseException;
 import org.openrdf.query.parser.sparql.ast.SyntaxTreeBuilder;
 import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailBooleanQuery;
-import org.openrdf.repository.sail.SailGraphQuery;
 import org.openrdf.repository.sail.SailQuery;
 import org.openrdf.repository.sail.SailRepositoryConnection;
-import org.openrdf.repository.sail.SailTupleQuery;
 import org.openrdf.sail.SailConnection;
 import org.openrdf.sail.SailException;
+
 import com.bigdata.rdf.changesets.IChangeLog;
 import com.bigdata.rdf.changesets.IChangeRecord;
 import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
@@ -33,10 +30,18 @@
 import com.bigdata.rdf.sail.sparql.StringEscapesProcessor;
 import com.bigdata.rdf.store.AbstractTripleStore;
 
+/**
+ * Class with support for a variety of bigdata specific extensions,
+ * <p>
+ * {@inheritDoc}
+ * 
+ * @author mrpersonick
+ */
 public class BigdataSailRepositoryConnection extends SailRepositoryConnection {
    
-    protected Logger log = Logger.getLogger(BigdataSailRepositoryConnection.class);
-   
+//	private static transient final Logger log = Logger
+//			.getLogger(BigdataSailRepositoryConnection.class);
+
     public BigdataSailRepositoryConnection(BigdataSailRepository repository,
             SailConnection sailConnection) {
     
@@ -46,100 +51,128 @@
     
     @Override
     public BigdataSailRepository getRepository() {
-        return (BigdataSailRepository)super.getRepository();
+
+    	return (BigdataSailRepository)super.getRepository();
+    	
     }
     
     @Override
     public BigdataSailConnection getSailConnection() {
+
     	return (BigdataSailConnection)super.getSailConnection();
+    	
     }
-    
-    /**
-     * {@inheritDoc}
-     * <p>
-     * Overridden to capture query hints from SPARQL queries. Query hints are
-     * embedded in query strings as namespaces.  
-     * See {@link QueryHints#NAMESPACE} for more information.
-     */
+
+	/**
+	 * {@inheritDoc}
+	 * <p>
+	 * Overridden to capture query hints from SPARQL queries. Query hints are
+	 * embedded in query strings as namespaces. See {@link QueryHints#NAMESPACE}
+	 * for more information.
+	 */
     @Override
-    public SailGraphQuery prepareGraphQuery(final QueryLanguage ql,
+    public BigdataSailGraphQuery prepareGraphQuery(final QueryLanguage ql,
             final String qs, final String baseURI) 
             throws MalformedQueryException {
-        final ParsedGraphQuery parsedQuery = QueryParserUtil.parseGraphQuery(
-                ql, qs, baseURI);
-        final Properties queryHints = parseQueryHints(ql, qs, baseURI);
-        final boolean describe = 
-            ql == QueryLanguage.SPARQL &&
-            NanoSparqlClient.QueryType.fromQuery(qs) == QueryType.DESCRIBE;
-        return new BigdataSailGraphQuery(parsedQuery, this, queryHints, describe);
+
+		final ParsedGraphQuery parsedQuery = QueryParserUtil.parseGraphQuery(
+				ql, qs, baseURI);
+
+		final Properties queryHints = parseQueryHints(ql, qs, baseURI);
+
+		final boolean describe = ql == QueryLanguage.SPARQL
+				&& NanoSparqlClient.QueryType.fromQuery(qs) == QueryType.DESCRIBE;
+
+		return new BigdataSailGraphQuery(parsedQuery, this, queryHints,
+				describe);
+
     }
 
-    /**
-     * {@inheritDoc}
-     * <p>
-     * Overridden to capture query hints from SPARQL queries. Query hints are
-     * embedded in query strings as namespaces.  
-     * See {@link QueryHints#NAMESPACE} for more information.
-     */
+	/**
+	 * {@inheritDoc}
+	 * <p>
+	 * Overridden to capture query hints from SPARQL queries. Query hints are
+	 * embedded in query strings as namespaces. See {@link QueryHints#NAMESPACE}
+	 * for more information.
+	 */
     @Override
-    public SailTupleQuery prepareTupleQuery(final QueryLanguage ql,
-            final String queryString, final String baseURI) 
-            throws MalformedQueryException {
-        final ParsedTupleQuery parsedQuery = QueryParserUtil.parseTupleQuery(
-                ql, queryString, baseURI);
-        final Properties queryHints = parseQueryHints(ql, queryString, baseURI);
-        return new BigdataSailTupleQuery(parsedQuery, this, queryHints);
+	public BigdataSailTupleQuery prepareTupleQuery(final QueryLanguage ql,
+			final String queryString, final String baseURI)
+			throws MalformedQueryException {
+
+		final ParsedTupleQuery parsedQuery = QueryParserUtil.parseTupleQuery(
+				ql, queryString, baseURI);
+
+		final Properties queryHints = parseQueryHints(ql, queryString, baseURI);
+
+		return new BigdataSailTupleQuery(parsedQuery, this, queryHints);
+
     }
 
-    /**
-     * {@inheritDoc}
-     * <p>
-     * Overridden to capture query hints from SPARQL queries. Query hints are
-     * embedded in query strings as namespaces. See
-     * {@link QueryHints#NAMESPACE} for more information.
-     */
+	/**
+	 * {@inheritDoc}
+	 * <p>
+	 * Overridden to capture query hints from SPARQL queries. Query hints are
+	 * embedded in query strings as namespaces. See {@link QueryHints#NAMESPACE}
+	 * for more information.
+	 */
     @Override
-    public SailBooleanQuery prepareBooleanQuery(final QueryLanguage ql,
-            final String queryString, final String baseURI) 
-            throws MalformedQueryException {
-        final ParsedBooleanQuery parsedQuery = QueryParserUtil
-                .parseBooleanQuery(ql, queryString, baseURI);
-        final Properties queryHints = parseQueryHints(ql, queryString, baseURI);
-        return new BigdataSailBooleanQuery(parsedQuery, this, queryHints);
-    }
+	public BigdataSailBooleanQuery prepareBooleanQuery(final QueryLanguage ql,
+			final String queryString, final String baseURI)
+			throws MalformedQueryException {
 
-    /**
-     * {@inheritDoc}
-     * <p>
-     * Overridden to capture query hints from SPARQL queries. Query hints are
-     * embedded in query strings as namespaces.  
-     * See {@link QueryHints#NAMESPACE} for more information.
-     */
-    @Override
-    public SailQuery prepareQuery(final QueryLanguage ql, final String qs,
-            final String baseURI) 
-            throws MalformedQueryException {
-        final ParsedQuery parsedQuery = QueryParserUtil.parseQuery(ql, 
-                qs, baseURI);
-        final Properties queryHints = parseQueryHints(ql, qs, baseURI);
-        if (parsedQuery instanceof ParsedTupleQuery) {
-            return new BigdataSailTupleQuery(
-                    (ParsedTupleQuery) parsedQuery, this, queryHints);
-        } else if (parsedQuery instanceof ParsedGraphQuery) {
-            final boolean describe = 
-                ql == QueryLanguage.SPARQL &&
-                NanoSparqlClient.QueryType.fromQuery(qs) == QueryType.DESCRIBE;
-            return new BigdataSailGraphQuery(
-                    (ParsedGraphQuery) parsedQuery, this, queryHints, describe);
-        } else if (parsedQuery instanceof ParsedBooleanQuery) {
-            return new BigdataSailBooleanQuery(
-                    (ParsedBooleanQuery) parsedQuery, this, queryHints);
-        } else {
-            throw new RuntimeException("Unexpected query type: "
-                    + parsedQuery.getClass());
-        }
+		final ParsedBooleanQuery parsedQuery = QueryParserUtil
+				.parseBooleanQuery(ql, queryString, baseURI);
+
+		final Properties queryHints = parseQueryHints(ql, queryString, baseURI);
+
+		return new BigdataSailBooleanQuery(parsedQuery, this, queryHints);
+
     }
 
+	/**
+	 * {@inheritDoc}
+	 * <p>
+	 * Overridden to capture query hints from SPARQL queries. Query hints are
+	 * embedded in query strings as namespaces. See {@link QueryHints#NAMESPACE}
+	 * for more information.
+	 */
+	@Override
+	public SailQuery prepareQuery(final QueryLanguage ql, final String qs,
+			final String baseURI) throws MalformedQueryException {
+
+		final ParsedQuery parsedQuery = QueryParserUtil.parseQuery(ql, qs,
+				baseURI);
+
+		final Properties queryHints = parseQueryHints(ql, qs, baseURI);
+
+		if (parsedQuery instanceof ParsedTupleQuery) {
+
+			return new BigdataSailTupleQuery((ParsedTupleQuery) parsedQuery,
+					this, queryHints);
+
+		} else if (parsedQuery instanceof ParsedGraphQuery) {
+
+			final boolean describe = ql == QueryLanguage.SPARQL
+					&& NanoSparqlClient.QueryType.fromQuery(qs) == QueryType.DESCRIBE;
+
+			return new BigdataSailGraphQuery((ParsedGraphQuery) parsedQuery,
+					this, queryHints, describe);
+
+		} else if (parsedQuery instanceof ParsedBooleanQuery) {
+
+			return new BigdataSailBooleanQuery(
+					(ParsedBooleanQuery) parsedQuery, this, queryHints);
+
+		} else {
+
+			throw new RuntimeException("Unexpected query type: "
+					+ parsedQuery.getClass());
+
+		}
+
+	}
+
     /**
      * {@inheritDoc}
      * <p>
@@ -288,19 +321,20 @@
      * query string via special namespaces.      
      * See {@link QueryHints#NAMESPACE} for more information.
      */
-    private Properties parseQueryHints(QueryLanguage ql, String queryString, 
-            String baseURI) 
-                throws MalformedQueryException {
+	private Properties parseQueryHints(final QueryLanguage ql,
+			final String queryString, final String baseURI)
+			throws MalformedQueryException {
         try {
-            Properties queryHints = new Properties();
+            final Properties queryHints = new Properties();
             // currently only supporting SPARQL
             if (ql == QueryLanguage.SPARQL) {
                 // the next four lines were taken directly from
                 // org.openrdf.query.parser.sparql.SPARQLParser.parseQuery(String queryStr, String baseURI)
-                ASTQueryContainer qc = SyntaxTreeBuilder.parseQuery(queryString);
+                final ASTQueryContainer qc = SyntaxTreeBuilder.parseQuery(queryString);
                 StringEscapesProcessor.process(qc);
                 BaseDeclProcessor.process(qc, baseURI);
-                Map<String, String> prefixes = PrefixDeclProcessor.process(qc);
+				final Map<String, String> prefixes = PrefixDeclProcessor
+						.process(qc);
                 // iterate the namespaces
                 for (Map.Entry<String, String> prefix : prefixes.entrySet()) {
                     // if we see one that matches the magic namespace, try
@@ -314,15 +348,15 @@
                         }
                         hints = hints.substring(i+1);
                         // properties are separated by &
-                        StringTokenizer st = new StringTokenizer(hints, "&");
+                        final StringTokenizer st = new StringTokenizer(hints, "&");
                         while (st.hasMoreTokens()) {
                             String hint = st.nextToken();
                             i = hint.indexOf('=');
                             if (i < 0 || i == hint.length()-1) {
                                 throw new MalformedQueryException("bad query hint: " + hint);
                             }
-                            String key = hint.substring(0, i);
-                            String val = hint.substring(i+1);
+                            final String key = hint.substring(0, i);
+                            final String val = hint.substring(i+1);
                             queryHints.put(key, val);
                         }
                     }

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -1,8 +1,10 @@
 package com.bigdata.rdf.sail;
 
 import info.aduna.iteration.CloseableIteration;
+
 import java.util.ArrayList;
 import java.util.Properties;
+
 import org.openrdf.query.BindingSet;
 import org.openrdf.query.QueryEvaluationException;
 import org.openrdf.query.TupleQueryResult;
@@ -11,8 +13,8 @@
 import org.openrdf.query.parser.ParsedTupleQuery;
 import org.openrdf.repository.sail.SailRepositoryConnection;
 import org.openrdf.repository.sail.SailTupleQuery;
-import org.openrdf.sail.SailConnection;
 import org.openrdf.sail.SailException;
+
 import com.bigdata.rdf.sail.BigdataSail.BigdataSailConnection;
 
 public class BigdataSailTupleQuery extends SailTupleQuery 
@@ -23,18 +25,29 @@
      * See {@link QueryHints#NAMESPACE} for more information.
      */
     private final Properties queryHints;
+
+    public Properties getQueryHints() {
+    	
+    	return queryHints;
+    	
+    }
     
-    public BigdataSailTupleQuery(ParsedTupleQuery tupleQuery,
-            SailRepositoryConnection con, Properties queryHints) {
-        super(tupleQuery, con);
+	public BigdataSailTupleQuery(final ParsedTupleQuery tupleQuery,
+			final SailRepositoryConnection con, final Properties queryHints) {
+
+    	super(tupleQuery, con);
+    	
         this.queryHints = queryHints;
+        
     }
 
-    /**
-     * Overridden to use query hints from SPARQL queries. Query hints are
-     * embedded in query strings as namespaces.  
-     * See {@link QueryHints#NAMESPACE} for more information.
-     */
+	/**
+	 *{@inheritDoc}
+	 * <p>
+	 * Overridden to use query hints from SPARQL queries. Query hints are
+	 * embedded in query strings as namespaces. See {@link QueryHints#NAMESPACE}
+	 * for more information.
+	 */
     @Override
     public TupleQueryResult evaluate() throws QueryEvaluationException {
         
@@ -58,23 +71,31 @@
 		} catch (SailException e) {
 
 			throw new QueryEvaluationException(e);
+
+		}
+
+	}
+
+	public TupleExpr getTupleExpr() throws QueryEvaluationException {
+
+		TupleExpr tupleExpr = getParsedQuery().getTupleExpr();
+
+		try {
+
+			final BigdataSailConnection sailCon = (BigdataSailConnection) getConnection()
+					.getSailConnection();
+
+			tupleExpr = sailCon.optimize(tupleExpr, getActiveDataset(),
+					getBindings(), getIncludeInferred(), queryHints);
+
+			return tupleExpr;
+
+		} catch (SailException e) {
+
+			throw new QueryEvaluationException(e.getMessage(), e);
 			
-        }
+		}
 
-    }
+	}
 
-    public TupleExpr getTupleExpr() throws QueryEvaluationException {
-        TupleExpr tupleExpr = getParsedQuery().getTupleExpr();
-        try {
-            BigdataSailConnection sailCon =
-                (BigdataSailConnection) getConnection().getSailConnection();
-            tupleExpr = sailCon.optimize(tupleExpr, getActiveDataset(), 
-                    getBindings(), getIncludeInferred(), queryHints);
-            return tupleExpr;
-        }
-        catch (SailException e) {
-            throw new QueryEvaluationException(e.getMessage(), e);
-        }
-    }
-
 }

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -78,15 +78,22 @@
 
 	/**
 	 * A label which may be used to tag the instances of some SPARQL query
-	 * template in manner which makes sense to the application. The tag is used
-	 * to aggregate performance statistics for tagged queries.
+	 * template in manner which makes sense to the application (default
+	 * {@value #DEFAULT_TAG}). The tag is used to aggregate performance
+	 * statistics for tagged queries.
 	 * 
 	 * <pre>
 	 * PREFIX BIGDATA_QUERY_HINTS: &lt;http://www.bigdata.com/queryHints#com.bigdata.rdf.sail.QueryHints.tag=Query12&gt;
 	 * </pre>
 	 * 
-	 * @see http://sourceforge.net/apps/trac/bigdata/ticket/207
+	 * @see http://sourceforge.net/apps/trac/bigdata/ticket/207 (Report on Top-N queries)
+	 * @see http://sourceforge.net/apps/trac/bigdata/ticket/256 (Amortize RTO cost)
 	 */
     String TAG = QueryHints.class.getName() + ".tag";
 
+    /**
+     * @see #TAG
+     */
+    String DEFAULT_TAG = "N/A";
+
 }

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -107,7 +107,7 @@
 
     protected static final Logger log = Logger.getLogger(Rule2BOpUtility.class);
 
-    private static final transient IConstraint[][] NO_ASSIGNED_CONSTRAINTS = new IConstraint[0][];
+//    private static final transient IConstraint[][] NO_ASSIGNED_CONSTRAINTS = new IConstraint[0][];
     
     /**
      * Flag to conditionally enable the new named and default graph support.
@@ -221,17 +221,6 @@
 
     }
 
-//    /**
-//     * A list of annotations to be cleared from {@link Predicate} when they are
-//     * copied into a query plan.
-//     */
-//    private static final String[] ANNS_TO_CLEAR_FROM_PREDICATE = new String[] {
-//            Annotations.QUADS,//
-//            Annotations.DATASET,//
-//            Annotations.SCOPE,//
-//            IPredicate.Annotations.OPTIONAL //
-//    };
-    
     /**
      * Convert an {@link IStep} into an operator tree. This should handle
      * {@link IRule}s and {@link IProgram}s as they are currently implemented
@@ -296,7 +285,7 @@
      *       really all that accessible.
      */
     private static PipelineOp applyQueryHints(PipelineOp op,
-            Properties queryHints) {
+            final Properties queryHints) {
 
         final Enumeration<?> pnames = queryHints.propertyNames();
 

Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java	2011-03-05 20:38:35 UTC (rev 4275)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java	2011-03-07 15:04:34 UTC (rev 4276)
@@ -64,13 +64,11 @@
 import org.apache.log4j.Logger;
 import org.openrdf.query.MalformedQueryException;
 import org.openrdf.query.QueryLanguage;
-import org.openrdf.query.TupleQuery;
 import org.openrdf.query.parser.ParsedQuery;
 import org.openrdf.query.parser.QueryParser;
 import org.openrdf.query.parser.sparql.SPARQLParserFactory;
 import org.openrdf.query.resultio.sparqlxml.SPARQLResultsXMLWriter;
 import org.openrdf.repository.RepositoryException;
-import org.openrdf.repository.sail.SailRepositoryConnection;
 import org.openrdf.rio.rdfxml.RDFXMLWriter;
 import org.openrdf.sail.SailException;
 
@@ -83,7 +81,9 @@
 import com.bigdata.bop.fed.QueryEngineFactory;
 import com.bigdata.bop.join.PipelineJoin;
 import com.bigdata.btree.IndexMetadata;
+import com.bigdata.counters.httpd.CounterSetHTTPD;
 import com.bigdata.journal.AbstractJournal;
+import com.bigdata.journal.IAtomicStore;
 import com.bigdata.journal.IBufferStrategy;
 import com.bigdata.journal.IIndexManager;
 import com.bigdata.journal.IJournal;
@@ -97,6 +97,7 @@
 import com.bigdata.rdf.sail.BigdataSailGraphQuery;
 import com.bigdata.rdf.sail.BigdataSailRepository;
 import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
+import com.bigdata.rdf.sail.BigdataSailTupleQuery;
 import com.bigdata.rdf.sail.bench.NanoSparqlClient.QueryType;
 import com.bigdata.rdf.store.AbstractTripleStore;
 import com.bigdata.relation.AbstractResource;
@@ -118,16 +119,26 @@
  * @todo Allow configuration options for the sparql endpoint either as URI
  *       parameters, in the property file, as request headers, or as query hints
  *       using the PREFIX mechanism.
-
- * @todo Allow timestamp for server reads as protocol parameter (URL query
- *       parameter or header).
  * 
  * @todo Add an "?explain" URL query parameter and show the execution plan and
  *       costs (or make this a navigable option from the set of running queries
  *       to drill into their running costs and offer an opportunity to kill them
  *       as well).
  * 
- * @todo Add command to kill a running query.
+ * @todo Add command to kill a running query, e.g., from the view of the long
+ *       running queries.
+ * 
+ * @todo Report other performance counters using {@link CounterSetHTTPD}
+ * 
+ * @todo Simple update protocol.
+ * 
+ * @todo Remote command to bulk load data from a remote or local resource (it's
+ *       pretty much up to people handling deployment to secure access to
+ *       queries, update requests, and bulk load requests).
+ * 
+ * @todo Remote command to advance the read-behind point. This will let people
+ *       bulk load a bunch of stuff before advancing queries to read from the
+ *       new consistent commit point.
  */
 public class NanoSparqlServer extends AbstractHTTPD {
 
@@ -151,17 +162,7 @@
 	 * The character set used for the response (not negotiated).
 	 */
     static private final String charset = "UTF-8";
-    
-//    /**
-//     * The target Sail.
-//     */
-//    private final BigdataSail sail;
-//
-//    /**
-//	 * The target repository.
-//	 */
-//	private final BigdataSailRepository repo;
-
+   
     /**
      * The configuration object.
      */
@@ -185,32 +186,64 @@
     /**
      * Metadata about running queries.
      */
-    private static class RunningQuery {
-    	/** The unique identifier for this query. */
-    	final long queryId;
-    	/** The query. */
-    	final String query;
-    	/** The timestamp when the query was accepted (ns). */
-    	final long begin;
-        public RunningQuery(long queryId, String query, long begin) {
-    		this.queryId = queryId;
-    		this.query = query;
-    		this.begin = begin;
-    	}
-    }
+	private static class RunningQuery {
 
+		/**
+		 * The unique identifier for this query for the {@link NanoSparqlServer}.
+		 */
+		final long queryId;
+
+		/**
+		 * The unique identifier for this query for the {@link QueryEngine}.
+		 * 
+		 * @see QueryEngine#getRunningQuery(UUID)
+		 */
+		final UUID queryId2;
+
+		/** The query. */
+		final String query;
+		
+		/** The timestamp when the query was accepted (ns). */
+		final long begin;
+
+		public RunningQuery(final long queryId, final UUID queryId2,
+				final String query, final long begin) {
+
+			this.queryId = queryId;
+
+			this.queryId2 = queryId2;
+			
+			this.query = query;
+
+			this.begin = begin;
+
+		}
+
+	}
+
     /**
      * The currently executing queries (does not include queries where a client
      * has established a connection but the query is not running because the
      * {@link #queryService} is blocking).
      */
     private final ConcurrentHashMap<Long/* queryId */, RunningQuery> queries = new ConcurrentHashMap<Long, RunningQuery>();
-
+    
     /**
      * Factory for the query identifiers.
      */
-    private final AtomicLong queryIdFactory = new AtomicLong(); 
-    
+    private final AtomicLong queryIdFactory = new AtomicLong();
+
+	/**
+	 * 
+	 * @param config
+	 *            The configuration for the server.
+	 * @param indexManager
+	 *            The database instance that the server will operate against.
+	 * 
+	 * @throws IOException
+	 * @throws SailException
+	 * @throws RepositoryException
+	 */
 	public NanoSparqlServer(final Config config,
 			final IIndexManager indexManager) throws IOException,
 			SailException, RepositoryException {
@@ -277,7 +310,7 @@
 					.getValue();
 
 			try {
-				final Class cls = Class.forName(className);
+				final Class<?> cls = Class.forName(className);
 				if (AbstractTripleStore.class.isAssignableFrom(cls)) {
 					// this is a triple store (vs something else).
 					namespaces.add(namespace);
@@ -695,24 +728,37 @@
         // return the namespace.
         return uri.substring(beginIndex + 1, endIndex);
 
-	}
+    }
 
-    /**
-     * Return the timestamp which will be used to execute the query.
-     * 
-     * @todo the configured timestamp should only be used for the default
-     *       namespace (or it should be configured for each graph explicitly, or
-     *       we should bundle the (namespace,timestamp) together as a single
-     *       object).
-     * 
-     * @todo use path for the timestamp or acquire read lock when the server
-     *       starts against a specific namespace?
-     */
-    private long getTimestamp(final String uri,
-            final LinkedHashMap<String, Vector<String>> params) {
+	/**
+	 * Return the timestamp which will be used to execute the query. The uri
+	 * query parameter <code>timestamp</code> may be used to communicate the
+	 * desired commit time against which the query will be issued. If that uri
+	 * query parameter is not given then the default configured commit time will
+	 * be used. Applications may create protocols for sharing interesting commit
+	 * times as reported by {@link IAtomicStore#commit()} or by a distributed
+	 * data loader (for scale-out).
+	 * 
+	 * @todo the configured timestamp should only be used for the default
+	 *       namespace (or it should be configured for each graph explicitly, or
+	 *       we should bundle the (namespace,timestamp) together as a single
+	 *       object).
+	 */
+	private long getTimestamp(final String uri,
+			final LinkedHashMap<String, Vector<String>> params) {
 
-        return config.timestamp;
-        
+		final Vector<String> tmp = params.get("timestamp");
+
+		if (tmp == null || tmp.size() == 0 || tmp.get(0) == null) {
+
+			return config.timestamp;
+
+		}
+
+		final String val = tmp.get(0);
+
+		return Long.valueOf(val);
+
     }
 
 	/**
@@ -970,17 +1016,21 @@
 
         final String namespace = getNamespace(uri);
 
-        final long timestamp = getTimestamp(uri, params);
-	    
-		final String queryStr = params.get("query").get(0);
+		final long timestamp = getTimestamp(uri, params);
 
-		if (queryStr == null) {
+		final String queryStr;
+		{
 
-			return new Response(HTTP_BADREQUEST, MIME_TEXT_PLAIN,
-					"Specify query using ?query=....");
+			final Vector<String> tmp = params.get("query");
 
+			if (tmp == null || tmp.isEmpty() || tmp.get(0) == null)
+				return new Response(HTTP_BADREQUEST, MIME_TEXT_PLAIN,
+						"Specify query using ?query=....");
+
+			queryStr = tmp.get(0);
+
 		}
-
+		
 		if (log.isDebugEnabled())
 			log.debug("query: " + queryStr);
 		
@@ -1201,7 +1251,24 @@
         /** A pipe used to incrementally deliver the results to the client. */
         private final PipedOutputStream os;
 
+		/**
+		 * Sesame has an option for a base URI during query evaluation. This
+		 * provides a symbolic placeholder for that URI in case we ever provide
+		 * a hook to set it.
+		 */
+        protected final String baseURI = null;
+        
         /**
+         * The queryId used by the {@link NanoSparqlServer}.
+         */
+        protected final Long queryId;
+        
+        /**
+         * The queryId used by the {@link QueryEngine}.
+         */
+        protected final UUID queryId2;
+        
+        /**
          * 
          * @param namespace
          *            The namespace against which the query will be run.
@@ -1222,6 +1289,8 @@
             this.timestamp = timestamp;
             this.queryStr = queryStr;
             this.os = os;
+            this.queryId = Long.valueOf(queryIdFactory.incrementAndGet());
+            this.queryId2 = UUID.randomUUID();
 
         }
 
@@ -1235,16 +1304,15 @@
          * 
          * @throws Exception
          */
-        abstract protected void doQuery(SailRepositoryConnection cxn,
+        abstract protected void doQuery(BigdataSailRepositoryConnection cxn,
                 OutputStream os) throws Exception;
 
         final public Void call() throws Exception {
-            final Long queryId = Long.valueOf(queryIdFactory.incrementAndGet());
-            final SailRepositoryConnection cxn = getQueryConnection(namespace,
-                    timestamp);
-            final long begin = System.nanoTime();
+			final BigdataSailRepositoryConnection cxn = getQueryConnection(
+					namespace, timestamp);
+			final long begin = System.nanoTime();
             try {
-                queries.put(queryId, new RunningQuery(queryId.longValue(),
+                queries.put(queryId, new RunningQuery(queryId.longValue(),queryId2,
                         queryStr, begin));
                 try {
                 	doQuery(cxn, os);
@@ -1276,51 +1344,28 @@
 
     }
 
-    /**
-     * Executes a tuple query.
-     */
+	/**
+	 * Executes a tuple query.
+	 */
 	private class TupleQueryTask extends AbstractQueryTask {
 
-        public TupleQueryTask(final String namespace, final long timestamp,
-                final String queryStr, final PipedOutputStream os) {
+		public TupleQueryTask(final String namespace, final long timestamp,
+				final String queryStr, final PipedOutputStream os) {
 
-            super(namespace, timestamp, queryStr, os);
+			super(namespace, timestamp, queryStr, os);
 
 		}
 
-        protected void doQuery(final SailRepositoryConnection cxn,
-                final OutputStream os) throws Exception {
+		protected void doQuery(final BigdataSailRepositoryConnection cxn,
+				final OutputStream os) throws Exception {
 
-            final TupleQuery query = cxn.prepareTupleQuery(
-                    QueryLanguage.SPARQL, queryStr);
-            
-            query.evaluate(new SPARQLResultsXMLWriter(new XMLWriter(os)));
-            
-        }
-        
-//		public Void call() throws Exception {
-//			final Long queryId = Long.valueOf(queryIdFactory.incrementAndGet());
-//			final SailRepositoryConnection cxn = getQueryConnection();
-//			try {
-//				final long begin = System.nanoTime();
-//				queries.put(queryId, new RunningQuery(queryId.longValue(), queryStr, begin));
-//				final TupleQuery query = cxn.prepareTupleQuery(
-//						QueryLanguage.SPARQL, queryStr);
-//				query.evaluate(new SPARQLResultsXMLWriter(new XMLWriter(os)));
-//				os.close();
-//				return null;
-//			} catch (Throwable t) {
-//				// launder and rethrow the exception.
-//				throw launderThrowable(t,os);
-//			} finally {
-//				try {
-//					cxn.close();
-//				} finally {
-//					queries.remove(queryId);
-//				}
-//			}
-//		}
+			final BigdataSailTupleQuery query = cxn.prepareTupleQuery(
+					QueryLanguage.SPARQL, queryStr, baseURI);
+			
+			query.evaluate(new SPARQLResultsXMLWriter(new XMLWriter(os)));
 
+		}
+
 	}
 
 	/**
@@ -1328,44 +1373,22 @@
 	 */
 	private class GraphQueryTask exte...
 
[truncated message content]