Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4717] branches/QUADS_QUERY_BRANCH/bigdata
|
From: <tho...@us...> - 2011-06-16 21:44:00
|
Revision: 4717
http://bigdata.svn.sourceforge.net/bigdata/?rev=4717&view=rev
Author: thompsonbry
Date: 2011-06-16 21:43:52 +0000 (Thu, 16 Jun 2011)
Log Message:
-----------
Added an "?explain"
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/HTMLUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlClient.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/NullOutputStream.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -1258,6 +1258,11 @@
*
* @return The {@link AbstractRunningQuery} -or- <code>null</code> if there
* is no query associated with that query identifier.
+ *
+ * @throws InterruptedException
+ * if the query halted normally.
+ * @throws RuntimeException
+ * if the query halted with an error.
*/
public /*protected*/ AbstractRunningQuery getRunningQuery(final UUID queryId) {
@@ -1452,5 +1457,89 @@
return runningQueries.keySet().toArray(new UUID[0]);
}
-
+
+// synchronized public void addListener(final IQueryEngineListener listener) {
+//
+// if (m_listeners == null) {
+//
+// m_listeners = new Vector<IQueryEngineListener>();
+//
+// m_listeners.add(listener);
+//
+// } else {
+//
+// if (m_listeners.contains(listener)) {
+//
+// throw new IllegalStateException("Already registered: listener="
+// + listener);
+//
+// }
+//
+// m_listeners.add(listener);
+//
+// }
+//
+// }
+//
+// synchronized public void removeListener(IQueryEngineListener listener) {
+//
+// if( m_listeners == null ) {
+//
+// throw new IllegalStateException
+// ( "Not registered: listener="+listener
+// );
+//
+// }
+//
+// if( ! m_listeners.remove( listener ) ) {
+//
+// throw new IllegalStateException
+// ( "Not registered: listener="+listener
+// );
+//
+// }
+//
+// if(m_listeners.isEmpty()) {
+//
+// /*
+// * Note: We can test whether or not listeners need to be notified
+// * simply by testing m_listeners != null.
+// */
+//
+// m_listeners = null;
+//
+// }
+//
+// }
+//
+// // TODO Must deliver events in another thread!
+// // TODO Must drop and drop any errors.
+// // TODO Optimize with CopyOnWriteArray
+// // Note: Security hole if we allow notification for queries w/o queryId.
+// protected void fireQueryEndedEvent(final IRunningQuery query) {
+//
+// if (m_listeners == null)
+// return;
+//
+// final IQueryEngineListener[] listeners = (IQueryEngineListener[]) m_listeners
+// .toArray(new IQueryEngineListener[] {});
+//
+// for (int i = 0; i < listeners.length; i++) {
+//
+// final IQueryEngineListener l = listeners[i];
+//
+// l.queryEnded(query);
+//
+// }
+//
+// }
+//
+// private Vector<IQueryEngineListener> m_listeners;
+//
+// public interface IQueryEngineListener {
+//
+// void queryEnded(final IRunningQuery q);
+//
+// }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryLog.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -58,6 +58,8 @@
public class QueryLog {
private static final String NA = "N/A";
+ private static final String TD = "<td>";
+ private static final String TDx = "</td\n>";
protected static final transient Logger log = Logger
.getLogger(QueryLog.class);
@@ -429,9 +431,59 @@
}
- public static void getTableHeaderXHTML(final IRunningQuery q, final Writer w)
- throws IOException {
+ /**
+ * Format the data as an (X)HTML table. The table will include a header
+ * which declares the columns, a detail row for each operator (optional),
+ * and a summary row for the query as a whole.
+ *
+ * @param queryStr
+ * The original text of the query (e.g., a SPARQL query)
+ * (optional).
+ * @param q
+ * The {@link IRunningQuery}.
+ * @param w
+ * Where to write the table.
+ * @param summaryOnly
+ * When <code>true</code> only the summary row will be written.
+ * @param maxBopLength
+ * The maximum length to display from {@link BOp#toString()} and
+ * ZERO (0) to display everything. Data longer than this value
+ * will be accessible from a flyover, but not directly visible
+ * in the page.
+ * @throws IOException
+ */
+ public static void getTableXHTML(final String queryStr,
+ final IRunningQuery q, final Writer w, final boolean summaryOnly,
+ final int maxBopLength)
+ throws IOException {
+ // the table start tag.
+ w.write("<table border=\"1\" summary=\"" + attrib("Query Statistics")
+ + "\"\n>");
+
+ getTableHeaderXHTML(q, w);
+
+ if(summaryOnly) {
+
+ getSummaryRowXHTML(queryStr, q, w, maxBopLength);
+
+ } else {
+
+ // Summary first.
+ getSummaryRowXHTML(queryStr, q, w, maxBopLength);
+
+ // Then the detail rows.
+ getTableRowsXHTML(queryStr, q, w, maxBopLength);
+
+ }
+
+ w.write("</table\n>");
+
+ }
+
+ public static void getTableHeaderXHTML(final IRunningQuery q, final Writer w)
+ throws IOException {
+
// header row.
w.write("<tr\n>");
/*
@@ -445,6 +497,7 @@
w.write("<th>elapsed</th>");
w.write("<th>serviceId</th>");
w.write("<th>cause</th>");
+ w.write("<th>query</th>");
w.write("<th>bop</th>");
/*
* Columns for each pipeline operator.
@@ -487,18 +540,26 @@
}
- /**
- * Write the table rows.
- *
- * @param q
- * The query.
- * @param w
- * Where to write the rows.
- *
- * @throws IOException
- */
- public static void getTableRowsXHTML(final IRunningQuery q, final Writer w)
- throws IOException {
+ /**
+ * Write the table rows.
+ *
+ * @param queryStr
+ * The query text (optional).
+ * @param q
+ * The {@link IRunningQuery}.
+ * @param w
+ * Where to write the rows.
+ * @param maxBopLength
+ * The maximum length to display from {@link BOp#toString()} and
+ * ZERO (0) to display everything. Data longer than this value
+ * will be accessible from a flyover, but not directly visible in
+ * the page.
+ *
+ * @throws IOException
+ */
+ public static void getTableRowsXHTML(final String queryStr,
+ final IRunningQuery q, final Writer w, final int maxBopLength)
+ throws IOException {
final Integer[] order = BOpUtility.getEvaluationOrder(q.getQuery());
@@ -506,34 +567,41 @@
for (Integer bopId : order) {
- getTableRowXHTML(q, w, orderIndex, bopId, false/* summary */);
-
+ getTableRowXHTML(queryStr, q, w, orderIndex, bopId,
+ false/* summary */, maxBopLength);
+
orderIndex++;
}
}
- private static final String TD = "<td>";
- private static final String TDx = "</td\n>";
-
- /**
- * Return a tabular representation of the query {@link RunState}.
- *
- * @param q
- * The {@link IRunningQuery}.
- * @param evalOrder
- * The evaluation order for the operator.
- * @param bopId
- * The identifier for the operator.
- * @param summary
- * <code>true</code> iff the summary for the query should be
- * written.
- * @return The row of the table.
- */
- static private void getTableRowXHTML(final IRunningQuery q, final Writer w,
- final int evalOrder, final Integer bopId, final boolean summary)
- throws IOException {
+ /**
+ * Return a tabular representation of the query {@link RunState}.
+ *
+ * @param queryStr
+ * The query text (optional).
+ * @param q
+ * The {@link IRunningQuery}.
+ * @param evalOrder
+ * The evaluation order for the operator.
+ * @param bopId
+ * The identifier for the operator.
+ * @param summary
+ * <code>true</code> iff the summary for the query should be
+ * written.
+ * @param maxBopLength
+ * The maximum length to display from {@link BOp#toString()} and
+ * ZERO (0) to display everything. Data longer than this value
+ * will be accessible from a flyover, but not directly visible
+ * in the page.
+ *
+ * @return The row of the table.
+ */
+ static private void getTableRowXHTML(final String queryStr,
+ final IRunningQuery q, final Writer w, final int evalOrder,
+ final Integer bopId, final boolean summary, final int maxBopLength)
+ throws IOException {
final DateFormat dateFormat = DateFormat.getDateTimeInstance(
DateFormat.FULL, DateFormat.FULL);
@@ -564,24 +632,29 @@
if (cause != null)
w.write(cause.getLocalizedMessage());
w.write(TDx);
-
+
final Map<Integer, BOp> bopIndex = q.getBOpIndex();
final Map<Integer, BOpStats> statsMap = q.getStats();
final BOp bop = bopIndex.get(bopId);
// the operator.
if (summary) {
+ w.write(TD);
+ w.write(queryStr == null ? cdata(NA) : prettyPrintSparql(queryStr));
+ w.write(TDx);
/*
* The entire query (recursively).
*/
- final String queryStr = BOpUtility.toString(q.getQuery());
+ final String bopStr = BOpUtility.toString(q.getQuery());
w.write(TD);
w.write("<a href=\"#\" title=\"");
- w.write(attrib(queryStr));// the entire query as a tooltip.
+ w.write(attrib(bopStr));// the entire query as a tooltip.
w.write("\"\n>");
- // A slice of the query inline on the page.
- w.write(cdata(queryStr.substring(0/* begin */, Math.min(64, queryStr
- .length()))));
+ // A slice of the query inline on the page or everything if
+ // maxBopLength<=0.
+ w.write(cdata(bopStr.substring(0/* begin */,
+ maxBopLength <= 0 ? bopStr.length() : Math.min(
+ maxBopLength, bopStr.length()))));
w.write("...");
w.write("</a>");
w.write(TDx);
@@ -589,14 +662,17 @@
w.write("total"); // summary line.
w.write(TDx);
} else {
+ w.write(TD);
+ w.write("...");// elide the original query string on a detail row.
+ w.write(TDx);
// Otherwise show just this bop.
- final String queryStr = bopIndex.get(bopId).toString();
+ final String bopStr = bopIndex.get(bopId).toString();
w.write(TD);
w.write("<a href=\"#\" title=\"");
- w.write(attrib(queryStr));// the entire query as a tooltip.
+ w.write(attrib(bopStr));// the entire query as a tooltip.
w.write("\"\n>");
// A slice of the query inline on the page.
- w.write(cdata(queryStr.substring(0/* begin */, Math.min(64, queryStr
+ w.write(cdata(bopStr.substring(0/* begin */, Math.min(64, bopStr
.length()))));
w.write("...");
w.write("</a>");
@@ -773,66 +849,25 @@
/**
* Write a summary row for the query. The table element, header, and footer
* must be written separately.
- * @param q
- * @param w
- * @param sb
+ * @param queryStr The original query text (optional).
+ * @param q The {@link IRunningQuery}.
+ * @param w Where to write the data.
+ * @param maxBopLength
+ * The maximum length to display from {@link BOp#toString()} and
+ * ZERO (0) to display everything. Data longer than this value
+ * will be accessible from a flyover, but not directly visible
+ * in the page.
* @throws IOException
*/
- static public void getSummaryRowXHTML(final IRunningQuery q,
- final Writer w, final StringBuilder sb) throws IOException {
+ static public void getSummaryRowXHTML(final String queryStr,
+ final IRunningQuery q, final Writer w, final int maxBopLength)
+ throws IOException {
- getTableRowXHTML(q, w, -1/* orderIndex */, q.getQuery().getId(), true/* summary */);
+ getTableRowXHTML(queryStr, q, w, -1/* orderIndex */, q.getQuery()
+ .getId(), true/* summary */, maxBopLength);
}
- /**
- * Format the data as an (X)HTML table. The table will include a header
- * which declares the columns, a detail row for each operator (optional),
- * and a summary row for the query as a whole.
- *
- * @param q
- * The query.
- * @param w
- * Where to write the table.
- * @param summaryOnly
- * When <code>true</code> only the summary row will be written.
- * @throws IOException
- */
- public static void getTableXHTML(final IRunningQuery q, final Writer w,
- final boolean summaryOnly) throws IOException {
-
- // the table start tag.
- {
- /*
- * Summary for the table.
- */
- final String summary = "Query details";
-
- /*
- * Format the entire table now that we have all the data on hand.
- */
-
- w.write("<table border=\"1\" summary=\"" + attrib(summary)
- + "\"\n>");
-
- }
-
- getTableHeaderXHTML(q, w);
-
- if(summaryOnly) {
-
- getSummaryRowXHTML(q, w, sb);
-
- } else {
-
- getTableRowsXHTML(q, w);
-
- }
-
- w.write("</table\n>");
-
- }
-
private static String cdata(String s) {
return XHTMLRenderer.cdata(s);
@@ -845,4 +880,31 @@
}
+ private static String prettyPrintSparql(String s) {
+
+// return cdata(s);
+//
+// }
+
+ s = s.replace("\n", " ");
+
+ s = s.replace("PREFIX", "\nPREFIX");
+ s = s.replace("select", "\nselect");
+ s = s.replace("where", "\nwhere");
+ s = s.replace("{","{\n");
+ s = s.replace("}","\n}");
+ s = s.replace(" ."," .\n"); // TODO Must not match within quotes (literals) or <> (URIs).
+// s = s.replace("||","||\n");
+// s = s.replace("&&","&&\n");
+
+ s = cdata(s);
+
+ s = s.replace("\n", "<br>");
+
+// return "<pre>"+s+"</pre>";
+
+ return s;
+
+ }
+
}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/NullOutputStream.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/NullOutputStream.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/NullOutputStream.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -0,0 +1,68 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on May 26, 2011
+ */
+
+package com.bigdata.io;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An {@link OutputStream} which discards anything written on it.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: NullOutputStream.java 4582 2011-05-31 19:12:53Z thompsonbry $
+ */
+public class NullOutputStream extends OutputStream {
+
+ private boolean open = true;
+
+ public NullOutputStream() {
+ }
+
+ @Override
+ final public void write(int b) throws IOException {
+ if (!open)
+ throw new IOException();
+ }
+
+ @Override
+ final public void write(byte[] b) throws IOException {
+ if (!open)
+ throw new IOException();
+ }
+
+ @Override
+ final public void write(byte[] b, int len, int off) throws IOException {
+ if (!open)
+ throw new IOException();
+ }
+
+ final public void close() {
+ open = false;
+ }
+
+}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/HTMLUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/HTMLUtility.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/HTMLUtility.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -78,7 +78,7 @@
* </p>
*/
- public static String escapeForXHTML(String s) {
+ public static String escapeForXHTML(final String s) {
if( s == null ) {
@@ -86,16 +86,16 @@
}
- int len = s.length();
+ final int len = s.length();
if (len == 0)
return s;
- StringBuffer sb = new StringBuffer(len + 20);
+ final StringBuffer sb = new StringBuffer(len + 20);
for (int i = 0; i < len; i++) {
- char ch = s.charAt(i);
+ final char ch = s.charAt(i);
switch (ch) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlClient.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlClient.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlClient.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -246,7 +246,8 @@
// Fully formed and encoded URL @todo use */* for ASK.
final String urlString = opts.serviceURL
+ "?query="
- + URLEncoder.encode(opts.queryStr, "UTF-8")
+ + URLEncoder.encode(opts.queryStr, "UTF-8")//
+ + (opts.explain?"&explain=":"")//
+ (opts.defaultGraphUri == null ? ""
: ("&default-graph-uri=" + URLEncoder.encode(
opts.defaultGraphUri, "UTF-8")));
@@ -324,7 +325,7 @@
log.debug("Status Line: " + conn.getResponseMessage());
}
- if (opts.showResults) {
+ if(opts.explain || opts.showResults) {
// Write the response body onto stdout.
showResults(conn);
@@ -756,6 +757,8 @@
public String baseURI;
/** The default graph URI (optional). */
public String defaultGraphUri = null;
+ /** When true, request an explanation for the query. */
+ public boolean explain = false;
/** The connection timeout (ms). */
public int timeout = DEFAULT_TIMEOUT;
/**
@@ -1291,6 +1294,10 @@
opts.showQuery = true;
+ } else if (arg.equals("-explain")) {
+
+ opts.explain = true;
+
} else if (arg.equals("-showParseTree")) {
opts.showParseTree = true;
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -43,9 +43,11 @@
import com.bigdata.bop.BufferAnnotations;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IndexMetadata;
+import com.bigdata.io.NullOutputStream;
import com.bigdata.journal.IBufferStrategy;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITx;
@@ -55,9 +57,11 @@
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSailBooleanQuery;
import com.bigdata.rdf.sail.BigdataSailGraphQuery;
+import com.bigdata.rdf.sail.BigdataSailQuery;
import com.bigdata.rdf.sail.BigdataSailRepository;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
import com.bigdata.rdf.sail.BigdataSailTupleQuery;
+import com.bigdata.rdf.sail.QueryHints;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.relation.AbstractResource;
import com.bigdata.relation.RelationSchema;
@@ -78,8 +82,14 @@
static private final transient Logger log = Logger
.getLogger(BigdataRDFContext.class);
+ /**
+ * URL Query parameter used to request the explanation of a query rather
+ * than its results.
+ */
+ protected static final String EXPLAIN = "explain";
+
private final SparqlEndpointConfig m_config;
- private final QueryParser m_engine;
+ private final QueryParser m_queryParser;
/**
* A thread pool for running accepted queries against the
@@ -128,7 +138,7 @@
m_config = config;
// used to parse queries.
- m_engine = new SPARQLParserFactory().getParser();
+ m_queryParser = new SPARQLParserFactory().getParser();
if (config.queryThreadPoolSize == 0) {
@@ -282,17 +292,33 @@
*/
protected final String baseURI;
+// /**
+// * Set to the timestamp as reported by {@link System#nanoTime()} when
+// * the query begins to execute.
+// */
+// final AtomicLong beginTime = new AtomicLong();
+
/**
* The queryId as assigned by the SPARQL end point (rather than the
* {@link QueryEngine}).
*/
protected final Long queryId;
-
- /**
- * The queryId used by the {@link QueryEngine}.
- */
- protected final UUID queryId2;
+ /**
+ * The queryId used by the {@link QueryEngine}. If the application has
+ * not specified this using {@link QueryHints#QUERYID} then this is
+ * assigned and set on the query using {@link QueryHints#QUERYID}. This
+ * decision can not be made until we parse the query so the behavior is
+ * handled by the subclasses.
+ */
+ volatile protected UUID queryId2;
+
+ /**
+ * When true, provide an "explanation" for the query (query plan, query
+ * evaluation statistics) rather than the results of the query.
+ */
+ final boolean explain;
+
/**
*
* @param namespace
@@ -354,9 +380,10 @@
this.charset = charset;
this.fileExt = fileExt;
this.req = req;
+ this.explain = req.getParameter(EXPLAIN) != null;
this.os = os;
this.queryId = Long.valueOf(m_queryIdFactory.incrementAndGet());
- this.queryId2 = UUID.randomUUID();
+// this.queryId2 = UUID.randomUUID();
/*
* Setup the baseURI for this request. It will be set to the
@@ -408,6 +435,39 @@
}
+ /**
+ * Sets {@link #queryId2} to the {@link UUID} which will be associated
+ * with the {@link IRunningQuery}. If {@link QueryHints#QUERYID} has
+ * already been used by the application to specify the {@link UUID} then
+ * that {@link UUID} is noted. Otherwise, a random {@link UUID} is
+ * generated and assigned to the query by binding it on the query hints.
+ * <p>
+ * Note: This is also responsible for noticing the time at which the
+ * query begins to execute and storing the {@link RunningQuery} in the
+ * {@link #m_queries} map.
+ *
+ * @param query
+ * The query.
+ */
+ protected void setQueryId(final BigdataSailQuery query) {
+ assert queryId2 == null; // precondition.
+ // Note the begin time for the query.
+ final long begin = System.nanoTime();
+ // Figure out the effective UUID under which the query will run.
+ final String queryIdStr = query.getQueryHints().getProperty(
+ QueryHints.QUERYID);
+ if (queryIdStr == null) {
+ queryId2 = UUID.randomUUID();
+ query.getQueryHints().setProperty(QueryHints.QUERYID,
+ queryId2.toString());
+ } else {
+ queryId2 = UUID.fromString(queryIdStr);
+ }
+ // Stuff it in the map of running queries.
+ m_queries.put(queryId, new RunningQuery(queryId.longValue(),
+ queryId2, queryStr, begin));
+ }
+
/**
* Execute the query.
*
@@ -422,28 +482,40 @@
OutputStream os) throws Exception;
final public Void call() throws Exception {
- final long begin = System.nanoTime();
BigdataSailRepositoryConnection cxn = null;
try {
cxn = getQueryConnection(namespace, timestamp);
- m_queries.put(queryId, new RunningQuery(queryId.longValue(),
- queryId2, queryStr, begin));
if(log.isTraceEnabled())
log.trace("Query running...");
// try {
- doQuery(cxn, os);
+ if(explain) {
+ /*
+ * The data goes to a bit bucket and we send an
+ * "explanation" of the query evaluation back to the caller.
+ *
+ * Note: The trick is how to get hold of the IRunningQuery
+ * object. It is created deep within the Sail when we
+ * finally submit a query plan to the query engine. We have
+ * the queryId (on queryId2), so we can look up the
+ * IRunningQuery in [m_queries] while it is running, but
+ * once it is terminated the IRunningQuery will have been
+ * cleared from the internal map maintained by the
+ * QueryEngine, at which point we can not longer find it.
+ */
+ doQuery(cxn, new NullOutputStream());
+ } else {
+ doQuery(cxn, os);
+ os.flush();
+ os.close();
+ }
+ if(log.isTraceEnabled())
+ log.trace("Query done.");
// } catch(Throwable t) {
// /*
// * Log the query and the exception together.
// */
// log.error(t.getLocalizedMessage() + ":\n" + queryStr, t);
// }
- if(log.isTraceEnabled())
- log.trace("Query done - flushing results.");
- os.flush();
- os.close();
- if(log.isTraceEnabled())
- log.trace("Query done - output stream closed.");
return null;
// } catch (Throwable t) {
// // launder and rethrow the exception.
@@ -493,6 +565,9 @@
final BigdataSailBooleanQuery query = cxn.prepareBooleanQuery(
QueryLanguage.SPARQL, queryStr, baseURI);
+ // Figure out the UUID under which the query will execute.
+ setQueryId(query);
+
// Override query if data set protocol parameters were used.
overrideDataset(query);
@@ -534,6 +609,9 @@
final BigdataSailTupleQuery query = cxn.prepareTupleQuery(
QueryLanguage.SPARQL, queryStr, baseURI);
+ // Figure out the UUID under which the query will execute.
+ setQueryId(query);
+
// Override query if data set protocol parameters were used.
overrideDataset(query);
@@ -574,6 +652,9 @@
final BigdataSailGraphQuery query = cxn.prepareGraphQuery(
QueryLanguage.SPARQL, queryStr, baseURI);
+ // Figure out the UUID under which the query will execute.
+ setQueryId(query);
+
// Override query if data set protocol parameters were used.
overrideDataset(query);
@@ -647,13 +728,19 @@
* Therefore, we are in the position of having to parse the query here
* and then again when it is executed.]
*/
- final ParsedQuery q = m_engine.parseQuery(queryStr, null/*baseURI*/);
+ final ParsedQuery q = m_queryParser.parseQuery(queryStr, null/*baseURI*/);
if(log.isDebugEnabled())
log.debug(q.toString());
final QueryType queryType = QueryType.fromQuery(queryStr);
+ /*
+ * When true, provide an "explanation" for the query (query plan, query
+ * evaluation statistics) rather than the results of the query.
+ */
+ final boolean explain = req.getParameter(EXPLAIN) != null;
+
/*
* CONNEG for the MIME type.
*
@@ -667,7 +754,8 @@
* has some stuff related to generating Accept headers in their
* RDFFormat which could bear some more looking into in this regard.)
*/
- final String acceptStr = req.getHeader("Accept");
+ final String acceptStr = explain ? "text/html" : req
+ .getHeader("Accept");
switch (queryType) {
case ASK: {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -1,15 +1,27 @@
package com.bigdata.rdf.sail.webapp;
import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.UUID;
import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
+import com.bigdata.bop.engine.IRunningQuery;
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.bop.engine.QueryLog;
+import com.bigdata.bop.fed.QueryEngineFactory;
+import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.TimestampUtility;
+import com.bigdata.rawstore.Bytes;
import com.bigdata.rdf.sail.webapp.BigdataRDFContext.AbstractQueryTask;
+import com.bigdata.util.InnerCause;
/**
* SPARQL query handler for GET or POST verbs.
@@ -80,6 +92,7 @@
final long timestamp = getTimestamp(req);
+ // The SPARQL query.
final String queryStr = req.getParameter("query");
if(queryStr == null) {
@@ -101,10 +114,15 @@
*/
try {
- final AbstractQueryTask queryTask = getBigdataRDFContext()
- .getQueryTask(namespace, timestamp, queryStr, req,
- resp.getOutputStream());
+ final OutputStream os = resp.getOutputStream();
+
+ final BigdataRDFContext context = getBigdataRDFContext();
+
+ final boolean explain = req.getParameter(BigdataRDFContext.EXPLAIN) != null;
+ final AbstractQueryTask queryTask = context.getQueryTask(namespace,
+ timestamp, queryStr, req, os);
+
final FutureTask<Void> ft = new FutureTask<Void>(queryTask);
if (log.isTraceEnabled())
@@ -170,12 +188,17 @@
}
- // Begin executing the query (asynchronous)
- getBigdataRDFContext().queryService.execute(ft);
-
- // wait for the Future.
- ft.get();
+ // Begin executing the query (asynchronous)
+ getBigdataRDFContext().queryService.execute(ft);
+ if (explain) {
+ // Send an explanation instead of the query results.
+ explainQuery(queryStr, queryTask, ft, os);
+ } else {
+ // Wait for the Future.
+ ft.get();
+ }
+
} catch (Throwable e) {
try {
throw BigdataRDFServlet.launderThrowable(e, resp, queryStr);
@@ -186,5 +209,124 @@
}
+ /**
+ * Sends an explanation for the query rather than the query results. The
+ * query is still run, but the query statistics are reported instead of the
+ * query results.
+ *
+ * @param queryStr
+ * @param queryTask
+ * @param ft
+ * @param os
+ * @throws Exception
+ */
+ private void explainQuery(final String queryStr,
+ final AbstractQueryTask queryTask, final FutureTask<Void> ft,
+ final OutputStream os) throws Exception {
+
+ /*
+ * Spin until either we have the IRunningQuery or the Future of the
+ * query is done (in which case we won't get it).
+ */
+ if(log.isDebugEnabled())
+ log.debug("Will build explanation");
+ UUID queryId2 = null;
+ IRunningQuery q = null;
+ while (!ft.isDone() && queryId2 == null) {
+ try {
+ // Wait a bit for queryId2 to be assigned.
+ ft.get(1/* timeout */, TimeUnit.MILLISECONDS);
+ } catch(TimeoutException ex) {
+ // Ignore.
+ }
+ if (queryTask.queryId2 != null) {
+ queryId2 = queryTask.queryId2;
+ break;
+ }
+ }
+ if (queryId2 != null) {
+ if(log.isDebugEnabled())
+ log.debug("Resolving IRunningQuery: queryId2=" + queryId2);
+ final IIndexManager indexManager = getBigdataRDFContext()
+ .getIndexManager();
+ final QueryEngine queryEngine = QueryEngineFactory
+ .getQueryController(indexManager);
+ while (!ft.isDone() && q == null) {
+ try {
+ // Wait a bit for the IRunningQuery to *start*.
+ ft.get(1/* timeout */, TimeUnit.MILLISECONDS);
+ } catch(TimeoutException ex) {
+ // Ignore.
+ }
+ // Resolve the IRunningQuery.
+ try {
+ q = queryEngine.getRunningQuery(queryId2);
+ } catch (RuntimeException ex) {
+ if (InnerCause.isInnerCause(ex, InterruptedException.class)) {
+ // Ignore. Query terminated normally, but we don't have
+ // it.
+ } else {
+ // Ignore. Query has error, but we will get err from
+ // Future.
+ }
+ }
+ }
+ if (q != null)
+ if(log.isDebugEnabled())
+ log.debug("Resolved IRunningQuery: query=" + q);
+ }
+
+ // wait for the Future (will toss any exceptions).
+ ft.get();
+
+ /*
+ * Build the explanation.
+ */
+ final HTMLBuilder doc = new HTMLBuilder();
+ {
+
+ XMLBuilder.Node current = doc.root("html");
+ {
+ current = current.node("head");
+ current.node("meta").attr("http-equiv", "Content-Type")
+ .attr("content", "text/html;charset=utf-8").close();
+ current.node("title").text("bigdata®").close();
+ current = current.close();// close the head.
+ }
+ current = current.node("body");
+
+ if (q != null) {
+ // Format query statistics as a table.
+ final StringWriter w = new StringWriter(
+ 8 * Bytes.kilobyte32);
+ QueryLog.getTableXHTML(queryStr, q, w,
+ true/* showQueryDetails */, 64/* maxBopLength */);
+
+ // Add into the HTML document.
+ current.text(w.toString());
+ } else {
+ current.node("p",
+ "Query ran too quickly to collect statistics.");
+ }
+ doc.closeAll(current);
+ }
+
+ /*
+ * Send the response.
+ *
+ * TODO It would be better to stream this rather than buffer it in
+ * RAM. That also opens up the opportunity for real-time updates for
+ * long-running (analytic) queries, incremental information from the
+ * runtime query optimizer, etc.
+ */
+ if(log.isDebugEnabled())
+ log.debug("Sending explanation.");
+ os.write(doc.toString().getBytes("UTF-8"));
+ os.flush();
+ os.close();
+ if(log.isDebugEnabled())
+ log.debug("Sent explanation.");
+
+ }
+
}
-
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -4,6 +4,7 @@
import java.io.StringWriter;
import java.util.Comparator;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -12,7 +13,6 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.engine.QueryLog;
@@ -39,47 +39,56 @@
// static private final transient Logger log = Logger
// .getLogger(StatusServlet.class);
- /**
- * <p>
- * A status page. Options include:
- * <dl>
- * <dt>showQueries</dt>
- * <dd>List SPARQL queries accepted by the SPARQL end point. The queries are
- * listed in order of decreasing elapsed time.</dd>
- * <dt>showRunningQueries</dt>
- * <dd>List SPARQL queries accepted by the SPARQL end point which are
- * currently executing on the {@link QueryEngine}. The queries are listed in
- * order of decreasing elapsed time.</dd>
- * <dt>showKBInfo</dt>
- * <dd>Show some information about the {@link AbstractTripleStore} instance
- * being served by this SPARQL end point.</dd>
- * <dt>showNamespaces</dt>
- * <dd>List the namespaces for the registered {@link AbstractTripleStore}s.</dd>
- * </dl>
- * </p>
- *
- * @todo This status page combines information about the addressed KB and
- * the backing store. Those items should be split out onto different
- * status requests. One should be at a URI for the database. The other
- * should be at the URI of the SPARQL end point.
- */
+ /**
+ * <p>
+ * A status page. Options include:
+ * <dl>
+ * <dt>showQueries</dt>
+ * <dd>List SPARQL queries accepted by the SPARQL end point which are
+ * currently executing on the {@link QueryEngine}. The queries are listed in
+ * order of decreasing elapsed time. You can also specify
+ * <code>showQueries=details</code> to get a detailed breakdown of the query
+ * execution.</dd>
+ * <dt>showKBInfo</dt>
+ * <dd>Show some information about the {@link AbstractTripleStore} instance
+ * being served by this SPARQL end point.</dd>
+ * <dt>showNamespaces</dt>
+ * <dd>List the namespaces for the registered {@link AbstractTripleStore}s.</dd>
+ * </dl>
+ * </p>
+ *
+ * @todo This status page combines information about the addressed KB and
+ * the backing store. Those items should be split out onto different
+ * status requests. One should be at a URI for the database. The other
+ * should be at the URI of the SPARQL end point.
+ */
@Override
protected void doGet(final HttpServletRequest req,
final HttpServletResponse resp) throws IOException {
- // SPARQL queries accepted by the SPARQL end point.
+ // IRunningQuery objects currently running on the query controller.
final boolean showQueries = req.getParameter("showQueries") != null;
- // IRunningQuery objects currently running on the query controller.
- final boolean showRunningQueries = req
- .getParameter("showRunningQueries") != null;
+ boolean showQueryDetails = false;
+ if (showQueries) {
+ for (String tmp : req.getParameterValues("showQueries")) {
+ if (tmp.equals("details"))
+ showQueryDetails = true;
+ }
+ }
- final boolean showRunningQueryStats = req
- .getParameter("showRunningQueryStats") != null;
+ /*
+ * The maximum inline length of BOp#toString() visible on the page. The
+ * entire thing is accessible via the title attribute (a flyover). Use
+ * ZERO (0) to see everything.
+ */
+ int maxBopLength = 64;
+ if (req.getParameter("maxBopLength") != null) {
+ maxBopLength = Integer.valueOf(req.getParameter("maxBopLength"));
+ if (maxBopLength < 0)
+ maxBopLength = 0;
+ }
- final boolean showRunningQueryDetailStats = req
- .getParameter("showRunningQueryDetailStats") != null;
-
// Information about the KB (stats, properties).
final boolean showKBInfo = req.getParameter("showKBInfo") != null;
@@ -136,157 +145,136 @@
.getCounters().toString()));
}
+
+ if (!showQueries) {
+ // Nothing more to do.
+ return;
+ }
- if (showQueries) {
+ // Marker timestamp used to report the age of queries.
+ final long now = System.nanoTime();
- /*
- * Show the queries which are currently executing (accepted by the
- * NanoSparqlServer).
- */
+ /*
+ * Map providing a cross walk from the QueryEngine's
+ * IRunningQuery.getQueryId() to NanoSparqlServer's
+ * RunningQuery.queryId.
+ */
+ final Map<UUID,RunningQuery> crosswalkMap = new LinkedHashMap<UUID, RunningQuery>();
- final long now = System.nanoTime();
+ /*
+ * Map providing the accepted RunningQuery objects in descending order
+ * by their elapsed run time.
+ */
+ final TreeMap<Long, RunningQuery> acceptedQueryAge = newQueryMap();
- final TreeMap<Long, RunningQuery> ages = newQueryMap();
+ {
- {
+ final Iterator<RunningQuery> itr = getBigdataRDFContext()
+ .getQueries().values().iterator();
- final Iterator<RunningQuery> itr = getBigdataRDFContext()
- .getQueries().values().iterator();
+ while (itr.hasNext()) {
- while (itr.hasNext()) {
+ final RunningQuery query = itr.next();
- final RunningQuery query = itr.next();
+ crosswalkMap.put(query.queryId2, query);
- final long age = now - query.begin;
+ final long age = now - query.begin;
- ages.put(age, query);
+ acceptedQueryAge.put(age, query);
- }
+ }
- }
+ }
- {
+ /*
+ * Show the queries which are currently executing (actually running on
+ * the QueryEngine).
+ */
- final Iterator<RunningQuery> itr = ages.values().iterator();
+ final QueryEngine queryEngine = (QueryEngine) QueryEngineFactory
+ .getQueryController(getIndexManager());
- while (itr.hasNext()) {
+ final UUID[] queryIds = queryEngine.getRunningQueries();
- final RunningQuery query = (RunningQuery) itr.next();
+ // final long now = System.nanoTime();
- final long age = now - query.begin;
+ /*
+ * Map providing the QueryEngine's IRunningQuery objects in order by
+ * descending elapsed evaluation time.
+ */
+ final TreeMap<Long, IRunningQuery> runningQueryAge = newQueryMap();
- current = current.node(
- "p",
- "age="
- + java.util.concurrent.TimeUnit.NANOSECONDS
- .toMillis(age) + "ms, queryId="
- + query.queryId + "\n").node("p",
- HTMLUtility.escapeForXHTML(query.query) + "\n");
+ for (UUID queryId : queryIds) {
- }
+ final IRunningQuery query;
+ try {
- }
+ query = queryEngine.getRunningQuery(queryId);
- }
+ if (query == null) {
- if (showRunningQueries || showRunningQueryStats
- || showRunningQueryDetailStats) {
+ // Already terminated.
+ continue;
- /*
- * Show the queries which are currently executing (actually running
- * on the QueryEngine).
- */
+ }
- final QueryEngine queryEngine = (QueryEngine) QueryEngineFactory
- .getQueryController(getIndexManager());
+ } catch (RuntimeException e) {
- final UUID[] queryIds = queryEngine.getRunningQueries();
+ if (InnerCause.isInnerCause(e, InterruptedException.class)) {
- // final long now = System.nanoTime();
+ // Already terminated.
+ continue;
- final TreeMap<Long, IRunningQuery> ages = newQueryMap();
-
- for (UUID queryId : queryIds) {
-
- final IRunningQuery query;
- try {
- query = queryEngine.getRunningQuery(queryId);
-
- if (query == null) {
-
- // Already terminated.
- continue;
-
- }
-
- } catch (RuntimeException e) {
-
- if (InnerCause.isInnerCause(e, InterruptedException.class)) {
-
- // Already terminated.
- continue;
-
- }
-
- throw new RuntimeException(e);
-
}
- ages.put(query.getElapsed(), query);
+ throw new RuntimeException(e);
}
- {
+ runningQueryAge.put(query.getElapsed(), query);
- final Iterator<IRunningQuery> itr = ages.values().iterator();
+ }
- final StringWriter w = showRunningQueryStats
- || showRunningQueryDetailStats ? new StringWriter(
- Bytes.kilobyte32 * 8) : null;
-
- while (itr.hasNext()) {
+ {
- final IRunningQuery query = itr.next();
+ final Iterator<IRunningQuery> itr = runningQueryAge.values()
+ .iterator();
- if (query.isDone() && query.getCause() != null) {
- // Already terminated (normal completion).
- continue;
- }
+ final StringWriter w = new StringWriter(Bytes.kilobyte32 * 8);
- if (showRunningQueries) {
- current = current.node("p",
- "age=" + query.getElapsed() + "ms").node("p",
- "queryId=" + query.getQueryId()).node("p",
- HTMLUtility.escapeForXHTML(query.toString()))
- .node(
- "p",
- HTMLUtility.escapeForXHTML(BOpUtility
- .toString(query.getQuery())));
- }
+ while (itr.hasNext()) {
- if (showRunningQueryStats || showRunningQueryDetailStats) {
+ final IRunningQuery query = itr.next();
- // Format as a table.
- QueryLog.getTableXHTML(query, w,
- !showRunningQueryDetailStats);
+ if (query.isDone() && query.getCause() != null) {
+ // Already terminated (normal completion).
+ continue;
+ }
- // Extract as String
- final String s = w.getBuffer().toString();
+ // Lookup the NanoSparqlServer's RunningQuery object.
+ final RunningQuery acceptedQuery = crosswalkMap.get(query
+ .getQueryId());
- // Add into the HTML document.
- current.text(s);
+ final String queryStr = acceptedQuery == null ? "N/A"
+ : acceptedQuery.query;
- // Clear the buffer.
- w.getBuffer().setLength(0);
+ // Format as a table.
+ QueryLog.getTableXHTML(queryStr, query, w, !showQueryDetails,
+ maxBopLength);
- }
-
- } // next IRunningQuery.
+ // Extract as String
+ final String s = w.getBuffer().toString();
- }
+ // Add into the HTML document.
+ current.text(s);
- }
+ // Clear the buffer.
+ w.getBuffer().setLength(0);
+ } // next IRunningQuery.
+
+ }
+
doc.closeAll(current);
buildResponse(resp, HTTP_OK, MIME_TEXT_HTML, doc.toString());
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java 2011-06-16 16:13:05 UTC (rev 4716)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java 2011-06-16 21:43:52 UTC (rev 4717)
@@ -169,9 +169,10 @@
return tmp.close();
}
-
+
/**
* Close the open element.
+ *
* @return The parent element.
* @throws IOException
*/
@@ -185,7 +186,7 @@
* @param simpleEnd
* When <code>true</code> an open tag without a body will be
* closed by a single > symbol rather than the XML style
- * &47;>.
+ * />.
*
* @return The parent element.
* @throws IOException
@@ -200,7 +201,7 @@
m_writer.write("/>");
}
} else {
- m_writer.write("</" + m_tag + ">");
+ m_writer.write("</" + m_tag + "\n>");
}
m_open = false;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
