Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[8549] branches/BIGDATA_RELEASE_1_3_0/bigdata/src
|
From: <tho...@us...> - 2014-07-15 13:01:43
|
Revision: 8549
http://sourceforge.net/p/bigdata/code/8549
Author: thompsonbry
Date: 2014-07-15 13:01:35 +0000 (Tue, 15 Jul 2014)
Log Message:
-----------
Bug fix to the query deadline support. The remaining time until the next check of the deadline queue was computed incorrectly. This resulted in a progressive increase in the time until the next deadline check.
The logic in QueryEngine.QueryTask.run() to check the deadline queue has been fixed.
The code has been converted to use nanoseconds rather than milliseconds for checking deadlines.
The deadline check code now runs every 100ms, as intended. We should recheck hot query performance for large concurrent query workloads to ensure that this does not translate into a query penalty (it should not as the code is polling on the query chunks available for processing and will grab a check before 100ms has expired if there is one available).
See #242 (Deadlines do not play well with GROUP_BY, ORDER_BY, etc.)
See #772 (Query timeout only checked at operator start/stop)
See #865 (OutOfMemoryError instead of Timeout for SPARQL Property Paths)
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java
branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -99,7 +99,6 @@
* first result when compared with pipelined evaluation.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*/
abstract public class AbstractRunningQuery implements IRunningQuery {
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -114,8 +114,8 @@
Map<Integer/* bopId */, BOpStats> getStats();
/**
- * Return the query deadline (the time at which it will terminate regardless
- * of its run state).
+ * Return the query deadline in milliseconds (the time at which it will
+ * terminate regardless of its run state).
*
* @return The query deadline (milliseconds since the epoch) and
* {@link Long#MAX_VALUE} if no explicit deadline was specified.
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryDeadline.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -19,9 +19,9 @@
class QueryDeadline implements Comparable<QueryDeadline> {
/**
- * The deadline for this query.
+ * The deadline for this query (in nanoseconds).
*/
- final long deadline;
+ final long deadlineNanos;
/**
* A reference to the query.
@@ -33,14 +33,14 @@
/**
*
- * @param deadline
- * The deadline.
+ * @param deadlineNanos
+ * The deadline for this query (in nanoseconds).
* @param query
* The query.
*/
- public QueryDeadline(final long deadline, final AbstractRunningQuery query) {
+ public QueryDeadline(final long deadlineNanos, final AbstractRunningQuery query) {
- this.deadline = deadline;
+ this.deadlineNanos = deadlineNanos;
this.queryRef = new WeakReference<AbstractRunningQuery>(query);
@@ -61,8 +61,8 @@
*/
@Override
public int compareTo(final QueryDeadline o) {
- final long d0 = this.deadline;
- final long d1 = o.deadline;
+ final long d0 = this.deadlineNanos;
+ final long d1 = o.deadlineNanos;
if (d0 < d1)
return -1;
if (d0 > d1)
@@ -74,13 +74,13 @@
* Check the deadline on the query. If the query is not terminated and the
* deadline has expired, then the query is terminated as a side-effect.
*
- * @param now
+ * @param nowNanosIsIgnored
* A current timestamp.
*
* @return <code>null</code> if the query is terminated and
* <code>this</code> if the query is not terminated.
*/
- QueryDeadline checkDeadline(final long now) {
+ QueryDeadline checkDeadline(final long nowNanosIsIgnored) {
final AbstractRunningQuery q = queryRef.get();
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -195,7 +195,6 @@
* query manager task for the terminated join.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @todo Expander patterns will continue to exist until we handle the standalone
* backchainers in a different manner for scale-out so add support for
@@ -669,15 +668,17 @@
throw new IllegalArgumentException();
}
- deadlineQueue.add(new QueryDeadline(deadline, query));
+ final long deadlineNanos = TimeUnit.MILLISECONDS.toNanos(deadline);
+ deadlineQueue.add(new QueryDeadline(deadlineNanos, query));
+
}
/**
* Scan the priority queue of queries with a specified deadline, halting any
* queries whose deadline has expired.
*/
- static private void checkDeadlines(final long now,
+ static private void checkDeadlines(final long nowNanos,
final PriorityBlockingQueue<QueryDeadline> deadlineQueue) {
/*
@@ -690,7 +691,7 @@
* Check the head of the deadline queue for any queries whose
* deadline has expired.
*/
- checkHeadOfDeadlineQueue(now, deadlineQueue);
+ checkHeadOfDeadlineQueue(nowNanos, deadlineQueue);
if (deadlineQueue.size() > DEADLINE_QUEUE_SCAN_SIZE) {
@@ -698,7 +699,7 @@
* Scan the deadline queue, removing entries for expired
* queries.
*/
- scanDeadlineQueue(now, deadlineQueue);
+ scanDeadlineQueue(nowNanos, deadlineQueue);
}
@@ -710,7 +711,7 @@
* Check the head of the deadline queue for any queries whose deadline has
* expired.
*/
- static private void checkHeadOfDeadlineQueue(final long now,
+ static private void checkHeadOfDeadlineQueue(final long nowNanos,
final PriorityBlockingQueue<QueryDeadline> deadlineQueue) {
QueryDeadline x;
@@ -719,7 +720,7 @@
while ((x = deadlineQueue.poll()) != null) {
// test for query done or deadline expired.
- if (x.checkDeadline(now) == null) {
+ if (x.checkDeadline(nowNanos) == null) {
/*
* This query is known to be done. It was removed from the
@@ -731,7 +732,7 @@
}
- if (x.deadline > now) {
+ if (x.deadlineNanos > nowNanos) {
/*
* This query has not yet reached its deadline. That means that
@@ -757,7 +758,7 @@
* has not be reached. Therefore, periodically, we need to scan the queue
* and clear out entries for terminated queries.
*/
- static private void scanDeadlineQueue(final long now,
+ static private void scanDeadlineQueue(final long nowNanos,
final PriorityBlockingQueue<QueryDeadline> deadlineQueue) {
final List<QueryDeadline> c = new ArrayList<QueryDeadline>(
@@ -770,7 +771,7 @@
for (QueryDeadline x : c) {
- if (x.checkDeadline(now) != null) {
+ if (x.checkDeadline(nowNanos) != null) {
// return this query to the deadline queue.
deadlineQueue.add(x);
@@ -939,27 +940,31 @@
if(log.isInfoEnabled())
log.info("Running: " + this);
try {
- long mark = System.currentTimeMillis();
- long remaining = DEADLINE_CHECK_MILLIS;
+ final long deadline = TimeUnit.MILLISECONDS
+ .toNanos(DEADLINE_CHECK_MILLIS);
+ long mark = System.nanoTime();
+ long remaining = deadline;
while (true) {
try {
+ //log.warn("Polling deadline queue: remaining="+remaining+", deadlinkCheckMillis="+DEADLINE_CHECK_MILLIS);
final AbstractRunningQuery q = priorityQueue.poll(
- remaining, TimeUnit.MILLISECONDS);
- final long now = System.currentTimeMillis();
- if ((remaining = now - mark) < 0) {
+ remaining, TimeUnit.NANOSECONDS);
+ final long now = System.nanoTime();
+ if ((remaining = deadline - (now - mark)) < 0) {
+ //log.error("Checking deadline queue");
/*
* Check for queries whose deadline is expired.
- *
+ *
* Note: We only do this every DEADLINE_CHECK_MILLIS
* and then reset [mark] and [remaining].
- *
+ *
* Note: In queue.pool(), we only wait only up to
* the [remaining] time before the next check in
* queue.poll().
*/
checkDeadlines(now, deadlineQueue);
mark = now;
- remaining = DEADLINE_CHECK_MILLIS;
+ remaining = deadline;
}
// Consume chunk already on queue for this query.
if (q != null && !q.isDone())
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryDeadlineOrder.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -25,6 +25,7 @@
import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import junit.framework.TestCase2;
@@ -155,20 +156,26 @@
final AbstractRunningQuery runningQuery1 = queryEngine.eval(UUID.randomUUID(),
query1, new ListBindingSet());
- runningQuery1.setDeadline(now + 10000);
+ final long deadline1Millis = now + 10000/* millis */;
+ runningQuery1.setDeadline(deadline1Millis);
+
Thread.sleep(2);
final AbstractRunningQuery runningQuery2 = queryEngine.eval(UUID.randomUUID(),
query2, new ListBindingSet());
- runningQuery2.setDeadline(now + 20000);
+ final long deadline2Millis = now + 20000/* millis */;
+
+ runningQuery2.setDeadline(deadline2Millis);
final QueryDeadline queryDeadline1 = new QueryDeadline(
- runningQuery1.getDeadline(), runningQuery1);
+ TimeUnit.MILLISECONDS.toNanos(runningQuery1.getDeadline()),
+ runningQuery1);
final QueryDeadline queryDeadline2 = new QueryDeadline(
- runningQuery2.getDeadline(), runningQuery2);
+ TimeUnit.MILLISECONDS.toNanos(runningQuery2.getDeadline()),
+ runningQuery2);
// The earlier deadline is LT the later deadline.
assertTrue(queryDeadline1.compareTo(queryDeadline2) < 0);
@@ -180,6 +187,15 @@
assertEquals(0, queryDeadline1.compareTo(queryDeadline1));
assertEquals(0, queryDeadline2.compareTo(queryDeadline2));
+ /*
+ * Verify that the query deadline (millis) was converted to nanos for
+ * QueryDeadline object.
+ */
+ assertEquals(TimeUnit.MILLISECONDS.toNanos(deadline1Millis),
+ queryDeadline1.deadlineNanos);
+ assertEquals(TimeUnit.MILLISECONDS.toNanos(deadline2Millis),
+ queryDeadline2.deadlineNanos);
+
}
}
Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2014-07-14 22:21:43 UTC (rev 8548)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2014-07-15 13:01:35 UTC (rev 8549)
@@ -87,7 +87,6 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
*
* @see TestFederatedQueryEngine
*/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
