From: <tho...@us...> - 2011-06-17 17:09:01
|
Revision: 4726 http://bigdata.svn.sourceforge.net/bigdata/?rev=4726&view=rev Author: thompsonbry Date: 2011-06-17 17:08:54 +0000 (Fri, 17 Jun 2011) Log Message: ----------- Added the original SPARQL and parsed SPARQL operator tree in their own sections of the page. Modified to stream the explanation and status page results rather than buffering in memory. Removed the XMLBuilder and HTMLBuilder constructor variants which did not require the caller to provide the target on which to write the data. The XMLBuilder#toString() assumed that the Writer was a StringWriter. That assumption was not compatible with the use of the class when building a servlet response. See https://sourceforge.net/apps/trac/bigdata/ticket/331 Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HTMLBuilder.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestXMLBuilder.java Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFContext.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -264,7 +264,8 @@ protected final QueryType queryType; /** - * The negotiated MIME type to be used for the query response. + * The negotiated MIME type to be used for the query response (this + * does not include the charset encoding). */ protected final String mimeType; @@ -314,6 +315,15 @@ volatile protected UUID queryId2; /** + * The parsed query. It will be one of the {@link BigdataSailQuery} + * implementations. They all extend {@link AbstractQuery}. + * <p> + * Note: This field is made visible by the volatile write on + * {@link #queryId2}. + */ + protected AbstractQuery sailQuery; + + /** * When true, provide an "explanation" for the query (query plan, query * evaluation statistics) rather than the results of the query. */ @@ -435,24 +445,53 @@ } - /** - * Sets {@link #queryId2} to the {@link UUID} which will be associated - * with the {@link IRunningQuery}. If {@link QueryHints#QUERYID} has - * already been used by the application to specify the {@link UUID} then - * that {@link UUID} is noted. Otherwise, a random {@link UUID} is - * generated and assigned to the query by binding it on the query hints. + /** + * * <p> * Note: This is also responsible for noticing the time at which the * query begins to execute and storing the {@link RunningQuery} in the * {@link #m_queries} map. * + * @param query + */ + protected void setupQuery(final AbstractQuery query) { + + // Note the begin time for the query. + final long begin = System.nanoTime(); + + // Figure out the UUID under which the query will execute. + final UUID queryId2 = setQueryId((BigdataSailQuery)query); + + // Override query if data set protocol parameters were used. + overrideDataset(query); + + // Set the query object. + this.sailQuery = query; + + // Set the IRunningQuery's UUID (volatile write!) + this.queryId2 = queryId2; + + // Stuff it in the map of running queries. + m_queries.put(queryId, new RunningQuery(queryId.longValue(), + queryId2, queryStr, begin)); + + } + + /** + * Determines the {@link UUID} which will be associated with the + * {@link IRunningQuery}. If {@link QueryHints#QUERYID} has already been + * used by the application to specify the {@link UUID} then that + * {@link UUID} is noted. Otherwise, a random {@link UUID} is generated + * and assigned to the query by binding it on the query hints. + * * @param query * The query. + * + * @return The {@link UUID} which will be associated with the + * {@link IRunningQuery}. */ - protected void setQueryId(final BigdataSailQuery query) { + protected UUID setQueryId(final BigdataSailQuery query) { assert queryId2 == null; // precondition. - // Note the begin time for the query. - final long begin = System.nanoTime(); // Figure out the effective UUID under which the query will run. final String queryIdStr = query.getQueryHints().getProperty( QueryHints.QUERYID); @@ -463,9 +502,7 @@ } else { queryId2 = UUID.fromString(queryIdStr); } - // Stuff it in the map of running queries. - m_queries.put(queryId, new RunningQuery(queryId.longValue(), - queryId2, queryStr, begin)); + return queryId2; } /** @@ -564,13 +601,9 @@ final BigdataSailBooleanQuery query = cxn.prepareBooleanQuery( QueryLanguage.SPARQL, queryStr, baseURI); - - // Figure out the UUID under which the query will execute. - setQueryId(query); + + setupQuery(query); - // Override query if data set protocol parameters were used. - overrideDataset(query); - // Note: getQueryTask() verifies that format will be non-null. final BooleanQueryResultFormat format = BooleanQueryResultWriterRegistry .getInstance().getFileFormatForMIMEType(mimeType); @@ -609,12 +642,8 @@ final BigdataSailTupleQuery query = cxn.prepareTupleQuery( QueryLanguage.SPARQL, queryStr, baseURI); - // Figure out the UUID under which the query will execute. - setQueryId(query); + setupQuery(query); - // Override query if data set protocol parameters were used. - overrideDataset(query); - // Note: getQueryTask() verifies that format will be non-null. final TupleQueryResultFormat format = TupleQueryResultWriterRegistry .getInstance().getFileFormatForMIMEType(mimeType); @@ -652,11 +681,7 @@ final BigdataSailGraphQuery query = cxn.prepareGraphQuery( QueryLanguage.SPARQL, queryStr, baseURI); - // Figure out the UUID under which the query will execute. - setQueryId(query); - - // Override query if data set protocol parameters were used. - overrideDataset(query); + setupQuery(query); /* * FIXME An error thrown here (such as if format is null and we do Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/BigdataRDFServlet.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -32,6 +32,7 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.PrintWriter; +import java.io.StringWriter; import javax.servlet.Servlet; import javax.servlet.http.HttpServletRequest; @@ -267,12 +268,14 @@ protected void reportModifiedCount(final HttpServletResponse resp, final long nmodified, final long elapsed) throws IOException { - final XMLBuilder t = new XMLBuilder(); + final StringWriter w = new StringWriter(); + + final XMLBuilder t = new XMLBuilder(w); t.root("data").attr("modified", nmodified) .attr("milliseconds", elapsed).close(); - buildResponse(resp, HTTP_OK, MIME_APPLICATION_XML, t.toString()); + buildResponse(resp, HTTP_OK, MIME_APPLICATION_XML, w.toString()); } Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HTMLBuilder.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HTMLBuilder.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/HTMLBuilder.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -1,6 +1,7 @@ package com.bigdata.rdf.sail.webapp; import java.io.IOException; +import java.io.Writer; /** * Variant of {@link XMLBuilder} for HTML output. @@ -9,12 +10,23 @@ */ public class HTMLBuilder extends XMLBuilder { - public HTMLBuilder() throws IOException { - super(false); - } + public HTMLBuilder(final Writer w) throws IOException { - public Node body() throws IOException { - return root("html").node("body"); + super(false/* isXML */, null/* encoding */, w); + + } + + public HTMLBuilder(final String encoding, final Writer w) + throws IOException { + + super(false/* isXML */, encoding, w); + + } + + public Node body() throws IOException { + + return root("html").node("body"); + } } \ No newline at end of file Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/QueryServlet.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -2,7 +2,8 @@ import java.io.IOException; import java.io.OutputStream; -import java.io.StringWriter; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.util.UUID; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; @@ -19,8 +20,8 @@ import com.bigdata.bop.fed.QueryEngineFactory; import com.bigdata.journal.IIndexManager; import com.bigdata.journal.TimestampUtility; -import com.bigdata.rawstore.Bytes; import com.bigdata.rdf.sail.webapp.BigdataRDFContext.AbstractQueryTask; +import com.bigdata.util.HTMLUtility; import com.bigdata.util.InnerCause; /** @@ -192,8 +193,16 @@ getBigdataRDFContext().queryService.execute(ft); if (explain) { - // Send an explanation instead of the query results. - explainQuery(queryStr, queryTask, ft, os); + final Writer w = new OutputStreamWriter(os, queryTask.charset); + try { + // Send an explanation instead of the query results. + explainQuery(queryStr, queryTask, ft, w); + } finally { + w.flush(); + w.close(); + os.flush(); + os.close(); + } } else { // Wait for the Future. ft.get(); @@ -222,11 +231,11 @@ */ private void explainQuery(final String queryStr, final AbstractQueryTask queryTask, final FutureTask<Void> ft, - final OutputStream os) throws Exception { - + final Writer w) throws Exception { + /* - * Spin until either we have the IRunningQuery or the Future of the - * query is done (in which case we won't get it). + * Spin until either we have the UUID of the IRunningQuery or the Future + * of the query is done. */ if(log.isDebugEnabled()) log.debug("Will build explanation"); @@ -240,49 +249,44 @@ // Ignore. } if (queryTask.queryId2 != null) { + // Got it. queryId2 = queryTask.queryId2; break; } } - if (queryId2 != null) { - if(log.isDebugEnabled()) - log.debug("Resolving IRunningQuery: queryId2=" + queryId2); - final IIndexManager indexManager = getBigdataRDFContext() - .getIndexManager(); - final QueryEngine queryEngine = QueryEngineFactory - .getQueryController(indexManager); - while (!ft.isDone() && q == null) { - try { - // Wait a bit for the IRunningQuery to *start*. - ft.get(1/* timeout */, TimeUnit.MILLISECONDS); - } catch(TimeoutException ex) { - // Ignore. + + if(ft.isDone()) { + /* + * If the query is done, the check for an error before we build up + * the explanation document. + */ + ft.get(); + + /* + * No error and the Future is done. The UUID of the IRunningQuery + * MUST have been assigned. If we do not have it yet, then check + * once more. If it is not set then that is an error. + */ + if (queryTask.queryId2 != null) { + // Check once more. + queryId2 = queryTask.queryId2; + if (queryId2 == null) { + /* + * This should have been assigned unless the query failed + * during the setup. + */ + throw new AssertionError(); } - // Resolve the IRunningQuery. - try { - q = queryEngine.getRunningQuery(queryId2); - } catch (RuntimeException ex) { - if (InnerCause.isInnerCause(ex, InterruptedException.class)) { - // Ignore. Query terminated normally, but we don't have - // it. - } else { - // Ignore. Query has error, but we will get err from - // Future. - } - } } - if (q != null) - if(log.isDebugEnabled()) - log.debug("Resolved IRunningQuery: query=" + q); } - - // wait for the Future (will toss any exceptions). - ft.get(); + assert queryId2 != null; /* * Build the explanation. + * + * Note: The query may still be executing while we do this. */ - final HTMLBuilder doc = new HTMLBuilder(); + final HTMLBuilder doc = new HTMLBuilder(queryTask.charset.name(), w); { XMLBuilder.Node current = doc.root("html"); @@ -295,37 +299,93 @@ } current = current.node("body"); - if (q != null) { - // Format query statistics as a table. - final StringWriter w = new StringWriter( - 8 * Bytes.kilobyte32); - QueryLog.getTableXHTML(queryStr, q, w, - true/* showQueryDetails */, 64/* maxBopLength */); + current.node("h2", "SPARQL").node("p", + HTMLUtility.escapeForXHTML(queryTask.queryStr)); - // Add into the HTML document. - current.text(w.toString()); - } else { - current.node("p", - "Query ran too quickly to collect statistics."); + current.node("h2", "Parsed Query").node("pre", + HTMLUtility.escapeForXHTML(queryTask.sailQuery.toString())); + + /* + * Spin until we get the IRunningQuery reference or the query is + * done, in which case we won't get it. + */ + if (queryId2 != null) { + if(log.isDebugEnabled()) + log.debug("Resolving IRunningQuery: queryId2=" + queryId2); + final IIndexManager indexManager = getBigdataRDFContext() + .getIndexManager(); + final QueryEngine queryEngine = QueryEngineFactory + .getQueryController(indexManager); + while (!ft.isDone() && q == null) { + try { + // Wait a bit for the IRunningQuery to *start*. + ft.get(1/* timeout */, TimeUnit.MILLISECONDS); + } catch(TimeoutException ex) { + // Ignore. + } + // Resolve the IRunningQuery. + try { + q = queryEngine.getRunningQuery(queryId2); + } catch (RuntimeException ex) { + if (InnerCause.isInnerCause(ex, InterruptedException.class)) { + // Ignore. Query terminated normally, but we don't have + // it. + } else { + // Ignore. Query has error, but we will get err from + // Future. + } + } + } + if (q != null) + if(log.isDebugEnabled()) + log.debug("Resolved IRunningQuery: query=" + q); } + + // wait for the Future (will toss any exceptions). + ft.get(); + + { + current.node("h2", + "Query Evaluation Statistics").node("p"); + if (q != null) { + /* + * Format query statistics as a table. + * + * Note: This is writing on the Writer so it goes directly + * into the HTML document we are building for the client. + */ + QueryLog.getTableXHTML(queryStr, q, w, + true/* showQueryDetails */, 64/* maxBopLength */); + +// // Add into the HTML document. +// statsNode.text(w.toString()); + } else { + /* + * This can happen if we fail to get the IRunningQuery + * reference before the query terminates. E.g., if the + * query runs too quickly there is a data race and the + * reference may not be available anymore. + */ + current + .text("Not available."); + } + } doc.closeAll(current); } - /* - * Send the response. - * - * TODO It would be better to stream this rather than buffer it in - * RAM. That also opens up the opportunity for real-time updates for - * long-running (analytic) queries, incremental information from the - * runtime query optimizer, etc. - */ - if(log.isDebugEnabled()) - log.debug("Sending explanation."); - os.write(doc.toString().getBytes("UTF-8")); - os.flush(); - os.close(); - if(log.isDebugEnabled()) - log.debug("Sent explanation."); +// /* +// * Send the response. +// * +// * TODO It would be better to stream this rather than buffer it in +// * RAM. That also opens up the opportunity for real-time updates for +// * long-running (analytic) queries, incremental information from the +// * runtime query optimizer, etc. +// */ +// if(log.isDebugEnabled()) +// log.debug("Sending explanation."); +// os.write(doc.toString().getBytes("UTF-8")); +// if(log.isDebugEnabled()) +// log.debug("Sent explanation."); } Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/StatusServlet.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -1,7 +1,8 @@ package com.bigdata.rdf.sail.webapp; import java.io.IOException; -import java.io.StringWriter; +import java.io.OutputStreamWriter; +import java.io.Writer; import java.util.Comparator; import java.util.Iterator; import java.util.LinkedHashMap; @@ -17,7 +18,6 @@ import com.bigdata.bop.engine.QueryEngine; import com.bigdata.bop.engine.QueryLog; import com.bigdata.bop.fed.QueryEngineFactory; -import com.bigdata.rawstore.Bytes; import com.bigdata.rdf.sail.webapp.BigdataRDFContext.RunningQuery; import com.bigdata.rdf.store.AbstractTripleStore; import com.bigdata.util.HTMLUtility; @@ -95,7 +95,11 @@ // bigdata namespaces known to the index manager. final boolean showNamespaces = req.getParameter("showNamespaces") != null; - final HTMLBuilder doc = new HTMLBuilder(); + resp.setContentType(MIME_TEXT_HTML); + final Writer w = new OutputStreamWriter(resp.getOutputStream(), "UTF-8"); + try { + + final HTMLBuilder doc = new HTMLBuilder("UTF-8", w); XMLBuilder.Node current = doc.root("html"); { @@ -240,7 +244,7 @@ final Iterator<IRunningQuery> itr = runningQueryAge.values() .iterator(); - final StringWriter w = new StringWriter(Bytes.kilobyte32 * 8); +// final StringWriter w = new StringWriter(Bytes.kilobyte32 * 8); while (itr.hasNext()) { @@ -258,26 +262,33 @@ final String queryStr = acceptedQuery == null ? "N/A" : acceptedQuery.query; - // Format as a table. + // Format as a table, writing onto the response. QueryLog.getTableXHTML(queryStr, query, w, !showQueryDetails, maxBopLength); - // Extract as String - final String s = w.getBuffer().toString(); +// // Extract as String +// final String s = w.getBuffer().toString(); +// +// // Add into the HTML document. +// current.text(s); +// +// // Clear the buffer. +// w.getBuffer().setLength(0); - // Add into the HTML document. - current.text(s); - - // Clear the buffer. - w.getBuffer().setLength(0); - } // next IRunningQuery. } doc.closeAll(current); + + } finally { + + w.flush(); + w.close(); + + } - buildResponse(resp, HTTP_OK, MIME_TEXT_HTML, doc.toString()); +// buildResponse(resp, HTTP_OK, MIME_TEXT_HTML, doc.toString()); } Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/webapp/XMLBuilder.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -1,9 +1,6 @@ package com.bigdata.rdf.sail.webapp; import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.StringWriter; import java.io.Writer; /** @@ -23,7 +20,7 @@ * </div> * </xml> * - * XMLBuilder.Node closed = new XMLBuilder(false) + * XMLBuilder.Node closed = new XMLBuilder(false,writer) * .root("xml") * .node("div") * .attr("attr", "attr1") @@ -49,42 +46,59 @@ // private boolean m_pp = false; - public XMLBuilder() throws IOException { - this(true, (OutputStream) null); +// public XMLBuilder() throws IOException { +// +// this(true, (OutputStream) null); +// +// } +// +// public XMLBuilder(boolean xml) throws IOException { +// +// this(xml, (OutputStream) null); +// +// } +// +// public XMLBuilder(final boolean xml, final OutputStream outstr) +// throws IOException { +// +// this(xml, null/* encoding */, outstr); +// +// } +// +// public XMLBuilder(final boolean xml, final String encoding) +// throws IOException { +// +// this(xml, encoding, (OutputStream) null); +// +// } + + public XMLBuilder(final Writer w) throws IOException { + + this(true/* xml */, null/* encoding */, w/* writer */); + } - public XMLBuilder(boolean xml) throws IOException { - this(xml, (OutputStream) null); - } - - public XMLBuilder(boolean xml, OutputStream outstr) throws IOException { - this(xml,null/*encoding*/,outstr); - } - - public XMLBuilder(boolean xml, String encoding) throws IOException { + public XMLBuilder(final boolean xml, final String encoding, + final Writer w) throws IOException { - this(xml, encoding, (OutputStream) null); - - } - - public XMLBuilder(boolean xml, String encoding, OutputStream outstr) throws IOException { - + if(w == null) + throw new IllegalArgumentException(); + this.xml = xml; - if (outstr == null) { - m_writer = new StringWriter(); - } else { - m_writer = new OutputStreamWriter(outstr); - } + this.m_writer = w; if (xml) { - if(encoding!=null) { + if (encoding != null) { m_writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>"); } else { m_writer.write("<?xml version=\"1.0\"?>"); } } else { - // TODO Note the optional encoding for use in a meta tag. + /* + * Note: The optional encoding should also be included in a meta tag + * for an HTML document. + */ m_writer.write("<!DOCTYPE HTML PUBLIC"); m_writer.write(" \"-//W3C//DTD HTML 4.01 Transitional//EN\""); m_writer.write(" \"http://www.w3.org/TR/html4/loose.dtd\">"); @@ -99,9 +113,10 @@ // private void initWriter(OutputStream outstr) { // } - public String toString() { - return m_writer.toString(); - } + // Note: This method assumed that m_writer was a StringWriter!!! +// public String toString() { +// return m_writer.toString(); +// } public Node root(String name) throws IOException { return new Node(name, null); Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestXMLBuilder.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestXMLBuilder.java 2011-06-17 17:02:38 UTC (rev 4725) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/webapp/TestXMLBuilder.java 2011-06-17 17:08:54 UTC (rev 4726) @@ -28,6 +28,7 @@ package com.bigdata.rdf.sail.webapp; import java.io.IOException; +import java.io.StringWriter; import junit.framework.TestCase2; @@ -61,7 +62,9 @@ */ public void testXMLBuilder() throws IOException { - final XMLBuilder xml = new XMLBuilder(); + final StringWriter w = new StringWriter(); + + final XMLBuilder xml = new XMLBuilder(w); XMLBuilder.Node close = xml.root("data") .attr("id", "TheRoot") @@ -79,7 +82,7 @@ assertTrue(close == null); if(log.isInfoEnabled()) - log.info(xml.toString()); + log.info(w.toString()); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |