Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2011-01-18 21:57:53
|
Revision: 4125
http://bigdata.svn.sourceforge.net/bigdata/?rev=4125&view=rev
Author: thompsonbry
Date: 2011-01-18 21:57:47 +0000 (Tue, 18 Jan 2011)
Log Message:
-----------
Modified the build file to try and stop zookeeper before starting it. This addresses an issue with the zkServer script. That script writes a pid file. If the zookeeper service does, then that file will be lying around until it is removed. A "zkServer stop" will remove the pid file, so we do that before trying a "zkServer start".
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/build.xml
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 21:50:42 UTC (rev 4124)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 21:57:47 UTC (rev 4125)
@@ -1619,6 +1619,10 @@
<copy file="${build.properties.from.file}"
todir="${build.properties.test.to.path}" />
+ <!-- Issue stop on zookeeper first since zkServer leaves a pid file -->
+ <!-- around if the JVM is killed. That pid file needs to be cleaned -->
+ <!-- up before we can start a new instance. -->
+ <antcall target="stopZookeeper" />
<antcall target="startZookeeper" />
<antcall target="startHttpd" />
<antcall target="startLookup" />
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-18 21:50:48
|
Revision: 4124
http://bigdata.svn.sourceforge.net/bigdata/?rev=4124&view=rev
Author: thompsonbry
Date: 2011-01-18 21:50:42 +0000 (Tue, 18 Jan 2011)
Log Message:
-----------
Put the parameterized clientPort and tickTime into the environment when running junit.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/DestroyTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
branches/QUADS_QUERY_BRANCH/build.xml
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2011-01-18 20:24:39 UTC (rev 4123)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2011-01-18 21:50:42 UTC (rev 4124)
@@ -86,13 +86,13 @@
// ACL used for the unit tests.
protected final List<ACL> acl = Ids.OPEN_ACL_UNSAFE;
- Configuration config;
+ protected Configuration config;
final protected MockListener listener = new MockListener();
- JiniFederation fed;
+ protected JiniFederation<?> fed;
- String zrootname = null;
+ protected String zrootname = null;
public void setUp() throws Exception {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/DestroyTransactionService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/DestroyTransactionService.java 2011-01-18 20:24:39 UTC (rev 4123)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/DestroyTransactionService.java 2011-01-18 21:50:42 UTC (rev 4124)
@@ -56,11 +56,11 @@
public static void main(String[] args) throws InterruptedException,
RemoteException {
- JiniFederation fed = JiniClient.newInstance(args).connect();
+ final JiniFederation<?> fed = JiniClient.newInstance(args).connect();
try {
- IService service = fed.getTransactionService();
+ final IService service = fed.getTransactionService();
if (service == null) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2011-01-18 20:24:39 UTC (rev 4123)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2011-01-18 21:50:42 UTC (rev 4124)
@@ -27,21 +27,17 @@
package com.bigdata.zookeeper;
-import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.BindException;
import java.net.InetAddress;
import java.net.ServerSocket;
-import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import junit.framework.TestCase2;
-import net.jini.config.Configuration;
-import net.jini.config.ConfigurationProvider;
import org.apache.log4j.Level;
import org.apache.zookeeper.CreateMode;
@@ -53,14 +49,9 @@
import org.apache.zookeeper.KeeperException.SessionExpiredException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.server.quorum.QuorumPeerMain;
import com.bigdata.jini.start.MockListener;
-import com.bigdata.jini.start.config.ZookeeperServerConfiguration;
import com.bigdata.jini.start.process.ProcessHelper;
-import com.bigdata.jini.start.process.ZookeeperProcessHelper;
-import com.bigdata.jini.util.ConfigMath;
-import com.bigdata.resources.ResourceFileFilter;
import com.bigdata.util.config.NicUtil;
/**
@@ -150,7 +141,7 @@
// private File dataDir = null;
// the chosen client port.
- int clientPort = -1;
+ protected int clientPort = -1;
public void setUp() throws Exception {
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 20:24:39 UTC (rev 4123)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 21:50:42 UTC (rev 4124)
@@ -1846,6 +1846,9 @@
<sysproperty key="app.home" value="${app.home}" />
<sysproperty key="default.nic" value="${default.nic}" />
+ <sysproperty key="test.zookeeper.clientPort" value="${test.zookeeper.clientPort}" />
+ <sysproperty key="test.zookeeper.tickTime" value="${test.zookeeper.tickTime}" />
+
<sysproperty key="classserver.jar" value="${dist.lib}/classserver.jar" />
<sysproperty key="colt.jar" value="${dist.lib}/colt.jar" />
<sysproperty key="cweb-commons.jar" value="${dist.lib}/cweb-commons.jar" />
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-18 20:24:46
|
Revision: 4123
http://bigdata.svn.sourceforge.net/bigdata/?rev=4123&view=rev
Author: thompsonbry
Date: 2011-01-18 20:24:39 +0000 (Tue, 18 Jan 2011)
Log Message:
-----------
CI using the default zookeeper client port.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/build.properties
Modified: branches/QUADS_QUERY_BRANCH/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.properties 2011-01-18 17:36:58 UTC (rev 4122)
+++ branches/QUADS_QUERY_BRANCH/build.properties 2011-01-18 20:24:39 UTC (rev 4123)
@@ -381,4 +381,4 @@
# file for the zookeeper instance used to run CI.
test.zookeeper.installDir=/Users/bryan/zookeeper-3.2.1
test.zookeeper.tickTime=2000
-test.zookeeper.clientPort=2888
+test.zookeeper.clientPort=2181
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-18 17:37:07
|
Revision: 4122
http://bigdata.svn.sourceforge.net/bigdata/?rev=4122&view=rev
Author: thompsonbry
Date: 2011-01-18 17:36:58 +0000 (Tue, 18 Jan 2011)
Log Message:
-----------
Working on removing zookeeper start/stop from the individual unit tests. It will now be started/stopped once per CI run by the top-level ant build.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/testfed.config
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/service/jini/util/JiniServicesHelper.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
branches/QUADS_QUERY_BRANCH/build.properties
branches/QUADS_QUERY_BRANCH/build.xml
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2011-01-18 15:30:00 UTC (rev 4121)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2011-01-18 17:36:58 UTC (rev 4122)
@@ -28,6 +28,7 @@
package com.bigdata.jini.start;
import java.io.File;
+import java.net.InetAddress;
import java.util.List;
import java.util.UUID;
@@ -35,18 +36,17 @@
import net.jini.config.Configuration;
import net.jini.config.ConfigurationProvider;
-import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.ACL;
import com.bigdata.jini.start.config.ZookeeperClientConfig;
import com.bigdata.jini.start.process.ProcessHelper;
-import com.bigdata.jini.start.process.ZookeeperProcessHelper;
import com.bigdata.resources.ResourceFileFilter;
import com.bigdata.service.jini.JiniClient;
import com.bigdata.service.jini.JiniFederation;
+import com.bigdata.util.config.NicUtil;
+import com.bigdata.zookeeper.ZooHelper;
/**
* Abstract base class for unit tests requiring a running zookeeper and a
@@ -122,9 +122,24 @@
config = ConfigurationProvider.getInstance(args);
- // if necessary, start zookeeper (a server instance).
- ZookeeperProcessHelper.startZookeeper(config, listener);
+// // if necessary, start zookeeper (a server instance).
+// ZookeeperProcessHelper.startZookeeper(config, listener);
+ final int clientPort = Integer.valueOf(System
+ .getProperty("test.zookeeper.clientPort","2181"));
+
+ // Verify zookeeper is running on the local host at the client port.
+ {
+ final InetAddress localIpAddr = NicUtil.getInetAddress(null, 0,
+ null, true);
+ try {
+ ZooHelper.ruok(localIpAddr, clientPort);
+ } catch (Throwable t) {
+ fail("Zookeeper not running:: " + localIpAddr + ":"
+ + clientPort, t);
+ }
+ }
+
/*
* FIXME We need to start a jini lookup service for groups = {fedname}
* for this test to succeed.
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/testfed.config
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/testfed.config 2011-01-18 15:30:00 UTC (rev 4121)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/testfed.config 2011-01-18 17:36:58 UTC (rev 4122)
@@ -39,11 +39,10 @@
/**
* Zookeeper server configuration.
- */
+ *
org.apache.zookeeper.server.quorum.QuorumPeerMain {
- /* Directory for zookeeper's persistent state.
- */
+ // Directory for zookeeper's persistent state.
dataDir = new File(bigdata.fedname+"/zookeeper");
// required.
@@ -60,19 +59,14 @@
//classpath = new String[] { "${zookeeper.jar}", "${log4j.jar}" };
- /* Optional command line arguments for the JVM used to execute
- * zookeeper.
- *
- * Note: swapping for zookeeper is especially bad since the
- * operations are serialized, so if anything hits then disk then
- * all operations in the queue will have that latency as well.
- */
+ // Optional command line arguments for the JVM used to execute
//args=new String[]{"-Xmx2G"};
// zookeeper server logging configuration (value is a URI!)
log4j = bigdata.log4j;
}
+*/
/*
* Service configuration defaults. These can also be specified on a
@@ -127,15 +121,14 @@
*/
org.apache.zookeeper.ZooKeeper {
- /* Root znode for the federation instance. */
+ // Root znode for the federation instance.
zroot = "/test/"+bigdata.fedname;
- /* A comma separated list of host:port pairs, where the port is
- * the CLIENT port for the zookeeper server instance.
- */
+ // A comma separated list of host:port pairs, where the port is
+ // the CLIENT port for the zookeeper server instance.
servers="localhost:2181";
- /* Session timeout (optional). */
+ // Session timeout (optional).
//sessionTimeout=xxxx;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/service/jini/util/JiniServicesHelper.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/service/jini/util/JiniServicesHelper.java 2011-01-18 15:30:00 UTC (rev 4121)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/service/jini/util/JiniServicesHelper.java 2011-01-18 17:36:58 UTC (rev 4122)
@@ -10,6 +10,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.net.BindException;
+import java.net.InetAddress;
import java.net.ServerSocket;
import java.util.Arrays;
import java.util.Queue;
@@ -32,7 +33,6 @@
import com.bigdata.jini.start.config.ZookeeperClientConfig;
import com.bigdata.jini.start.config.ZookeeperServerConfiguration;
import com.bigdata.jini.start.process.ProcessHelper;
-import com.bigdata.jini.start.process.ZookeeperProcessHelper;
import com.bigdata.jini.util.ConfigMath;
import com.bigdata.jini.util.JiniUtil;
import com.bigdata.resources.ResourceFileFilter;
@@ -47,6 +47,7 @@
import com.bigdata.service.jini.MetadataServer;
import com.bigdata.service.jini.TransactionServer;
import com.bigdata.util.concurrent.DaemonThreadFactory;
+import com.bigdata.util.config.NicUtil;
import com.bigdata.zookeeper.ZooHelper;
/**
@@ -267,10 +268,10 @@
*/
private int clientPort;
- /**
- * The directory in which zookeeper is running.
- */
- private File zooDataDir;
+// /**
+// * The directory in which zookeeper is running.
+// */
+// private File zooDataDir;
/**
* Starts all services and connects the {@link JiniClient} to the
@@ -374,43 +375,46 @@
final String[] options;
{
- // the zookeeper service directory.
- zooDataDir = new File(fedServiceDir, "zookeeper");
-
- if(zooDataDir.exists()) {
-
- // clear out old zookeeper state first.
- recursiveDelete(zooDataDir);
-
- }
-
- // create.
- zooDataDir.mkdirs();
+// // the zookeeper service directory.
+// zooDataDir = new File(fedServiceDir, "zookeeper");
+//
+// if(zooDataDir.exists()) {
+//
+// // clear out old zookeeper state first.
+// recursiveDelete(zooDataDir);
+//
+// }
+//
+// // create.
+// zooDataDir.mkdirs();
- try {
+// try {
- // find ports that are not in use.
- clientPort = getPort(2181/* suggestedPort */);
- final int peerPort = getPort(2888/* suggestedPort */);
- final int leaderPort = getPort(3888/* suggestedPort */);
- final String servers = "1=localhost:" + peerPort + ":"
- + leaderPort;
+// // find ports that are not in use.
+// clientPort = getPort(2181/* suggestedPort */);
+// final int peerPort = getPort(2888/* suggestedPort */);
+// final int leaderPort = getPort(3888/* suggestedPort */);
+// final String servers = "1=localhost:" + peerPort + ":"
+// + leaderPort;
+ clientPort = Integer.valueOf(System
+ .getProperty("test.zookeeper.clientPort","2181"));
+
options = new String[] {
// overrides the clientPort to be unique.
QuorumPeerMain.class.getName()
+ "."
+ ZookeeperServerConfiguration.Options.CLIENT_PORT
+ "=" + clientPort,
- // overrides servers declaration.
- QuorumPeerMain.class.getName() + "."
- + ZookeeperServerConfiguration.Options.SERVERS
- + "=\"" + servers + "\"",
- // overrides the dataDir
- QuorumPeerMain.class.getName() + "."
- + ZookeeperServerConfiguration.Options.DATA_DIR
- + "=new java.io.File("
- + ConfigMath.q(zooDataDir.toString()) + ")"//
+// // overrides servers declaration.
+// QuorumPeerMain.class.getName() + "."
+// + ZookeeperServerConfiguration.Options.SERVERS
+// + "=\"" + servers + "\"",
+// // overrides the dataDir
+// QuorumPeerMain.class.getName() + "."
+// + ZookeeperServerConfiguration.Options.DATA_DIR
+// + "=new java.io.File("
+// + ConfigMath.q(zooDataDir.toString()) + ")"//
};
System.err.println("options=" + Arrays.toString(options));
@@ -418,27 +422,39 @@
final Configuration config = ConfigurationProvider
.getInstance(concat(args, options));
- // start zookeeper (a server instance).
- final int nstarted = ZookeeperProcessHelper.startZookeeper(
- config, serviceListener);
+// // start zookeeper (a server instance).
+// final int nstarted = ZookeeperProcessHelper.startZookeeper(
+// config, serviceListener);
+//
+// if (nstarted != 1) {
+//
+// throw new RuntimeException(
+// "Expected to start one zookeeper instance, not "
+// + nstarted);
+//
+// }
- if (nstarted != 1) {
-
- throw new RuntimeException(
- "Expected to start one zookeeper instance, not "
- + nstarted);
-
+ // Verify zookeeper is running on the local host at the client port.
+ {
+ final InetAddress localIpAddr = NicUtil.getInetAddress(null, 0,
+ null, true);
+ try {
+ ZooHelper.ruok(localIpAddr, clientPort);
+ } catch (Throwable t) {
+ throw new RuntimeException("Zookeeper not running:: "
+ + localIpAddr + ":" + clientPort, t);
+ }
}
- } catch (Throwable t) {
+// } catch (Throwable t) {
+//
+// // don't leave around the dataDir if the setup fails.
+// recursiveDelete(zooDataDir);
+//
+// throw new RuntimeException(t);
+//
+// }
- // don't leave around the dataDir if the setup fails.
- recursiveDelete(zooDataDir);
-
- throw new RuntimeException(t);
-
- }
-
}
/*
@@ -679,30 +695,30 @@
}
- try {
-
- ZooHelper.kill(clientPort);
-
- } catch (Throwable t) {
- log.error("Could not kill zookeeper: clientPort=" + clientPort
- + " : " + t, t);
- }
-
- if (zooDataDir != null && zooDataDir.exists()) {
-
- /*
- * Wait a bit and then try and delete the zookeeper directory.
- */
-
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
-
- recursiveDelete(zooDataDir);
-
- }
+// try {
+//
+// ZooHelper.kill(clientPort);
+//
+// } catch (Throwable t) {
+// log.error("Could not kill zookeeper: clientPort=" + clientPort
+// + " : " + t, t);
+// }
+//
+// if (zooDataDir != null && zooDataDir.exists()) {
+//
+// /*
+// * Wait a bit and then try and delete the zookeeper directory.
+// */
+//
+// try {
+// Thread.sleep(250);
+// } catch (InterruptedException e) {
+// throw new RuntimeException(e);
+// }
+//
+// recursiveDelete(zooDataDir);
+//
+// }
// // Stop the lookup service.
// new Thread(new Runnable() {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2011-01-18 15:30:00 UTC (rev 4121)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2011-01-18 17:36:58 UTC (rev 4122)
@@ -32,6 +32,7 @@
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.BindException;
+import java.net.InetAddress;
import java.net.ServerSocket;
import java.util.Arrays;
import java.util.List;
@@ -60,6 +61,7 @@
import com.bigdata.jini.start.process.ZookeeperProcessHelper;
import com.bigdata.jini.util.ConfigMath;
import com.bigdata.resources.ResourceFileFilter;
+import com.bigdata.util.config.NicUtil;
/**
* Abstract base class for zookeeper integration tests.
@@ -145,7 +147,7 @@
protected final MockListener listener = new MockListener();
- private File dataDir = null;
+// private File dataDir = null;
// the chosen client port.
int clientPort = -1;
@@ -157,53 +159,71 @@
if (log.isInfoEnabled())
log.info(getName());
- // find ports that are not in use.
- clientPort = getPort(2181/* suggestedPort */);
- final int peerPort = getPort(2888/* suggestedPort */);
- final int leaderPort = getPort(3888/* suggestedPort */);
- final String servers = "1=localhost:" + peerPort + ":" + leaderPort;
+// // find ports that are not in use.
+// clientPort = getPort(2181/* suggestedPort */);
+// final int peerPort = getPort(2888/* suggestedPort */);
+// final int leaderPort = getPort(3888/* suggestedPort */);
+// final String servers = "1=localhost:" + peerPort + ":" + leaderPort;
+//
+//// // create a temporary file for zookeeper's state.
+//// dataDir = File.createTempFile("test", ".zoo");
+//// // delete the file so that it can be re-created as a directory.
+//// dataDir.delete();
+//// // recreate the file as a directory.
+//// dataDir.mkdirs();
+//
+// final String[] args = new String[] {
+// // The configuration file (overrides follow).
+// configFile,
+// // overrides the clientPort to be unique.
+// QuorumPeerMain.class.getName() + "."
+// + ZookeeperServerConfiguration.Options.CLIENT_PORT + "="
+// + clientPort,
+// // overrides servers declaration.
+// QuorumPeerMain.class.getName() + "."
+// + ZookeeperServerConfiguration.Options.SERVERS + "=\""
+// + servers + "\"",
+//// // overrides the dataDir
+//// QuorumPeerMain.class.getName() + "."
+//// + ZookeeperServerConfiguration.Options.DATA_DIR
+//// + "=new java.io.File("
+//// + ConfigMath.q(dataDir.toString()) + ")"//
+// };
+//
+// System.err.println("args=" + Arrays.toString(args));
+//
+// final Configuration config = ConfigurationProvider.getInstance(args);
+
+// final int tickTime = (Integer) config.getEntry(QuorumPeerMain.class
+// .getName(), ZookeeperServerConfiguration.Options.TICK_TIME,
+// Integer.TYPE);
- // create a temporary file for zookeeper's state.
- dataDir = File.createTempFile("test", ".zoo");
- // delete the file so that it can be re-created as a directory.
- dataDir.delete();
- // recreate the file as a directory.
- dataDir.mkdirs();
+ final int tickTime = Integer.valueOf(System
+ .getProperty("test.zookeeper.tickTime","2000"));
- final String[] args = new String[] {
- // The configuration file (overrides follow).
- configFile,
- // overrides the clientPort to be unique.
- QuorumPeerMain.class.getName() + "."
- + ZookeeperServerConfiguration.Options.CLIENT_PORT + "="
- + clientPort,
- // overrides servers declaration.
- QuorumPeerMain.class.getName() + "."
- + ZookeeperServerConfiguration.Options.SERVERS + "=\""
- + servers + "\"",
- // overrides the dataDir
- QuorumPeerMain.class.getName() + "."
- + ZookeeperServerConfiguration.Options.DATA_DIR
- + "=new java.io.File("
- + ConfigMath.q(dataDir.toString()) + ")"//
- };
-
- System.err.println("args=" + Arrays.toString(args));
+ clientPort = Integer.valueOf(System.getProperty(
+ "test.zookeeper.clientPort", "2181"));
- final Configuration config = ConfigurationProvider.getInstance(args);
-
- final int tickTime = (Integer) config.getEntry(QuorumPeerMain.class
- .getName(), ZookeeperServerConfiguration.Options.TICK_TIME,
- Integer.TYPE);
-
/*
- * Note: This is the actual session timeout that the zookeeper service
- * will impose on the client.
+ * Note: This MUST be the actual session timeout that the zookeeper
+ * service will impose on the client. Some unit tests depend on this.
*/
this.sessionTimeout = tickTime * 2;
- // if necessary, start zookeeper (a server instance).
- ZookeeperProcessHelper.startZookeeper(config, listener);
+ // Verify zookeeper is running on the local host at the client port.
+ {
+ final InetAddress localIpAddr = NicUtil.getInetAddress(null, 0,
+ null, true);
+ try {
+ ZooHelper.ruok(localIpAddr, clientPort);
+ } catch (Throwable t) {
+ fail("Zookeeper not running:: " + localIpAddr + ":"
+ + clientPort, t);
+ }
+ }
+
+// // if necessary, start zookeeper (a server instance).
+// ZookeeperProcessHelper.startZookeeper(config, listener);
zookeeperAccessor = new ZooKeeperAccessor("localhost:" + clientPort, sessionTimeout);
@@ -227,8 +247,8 @@
} catch (Throwable t) {
- // don't leave around the dataDir if the setup fails.
- recursiveDelete(dataDir);
+// // don't leave around the dataDir if the setup fails.
+// recursiveDelete(dataDir);
throw new Exception(t);
@@ -256,13 +276,13 @@
}
- if (dataDir != null) {
+// if (dataDir != null) {
+//
+// // clean out the zookeeper data dir.
+// recursiveDelete(dataDir);
+//
+// }
- // clean out the zookeeper data dir.
- recursiveDelete(dataDir);
-
- }
-
} catch (Throwable t) {
log.error(t, t);
@@ -527,48 +547,48 @@
}
- /**
- * Recursively removes any files and subdirectories and then removes the
- * file (or directory) itself.
- * <p>
- * Note: Files that are not recognized will be logged by the
- * {@link ResourceFileFilter}.
- *
- * @param f
- * A file or directory.
- */
- private void recursiveDelete(final File f) {
+// /**
+// * Recursively removes any files and subdirectories and then removes the
+// * file (or directory) itself.
+// * <p>
+// * Note: Files that are not recognized will be logged by the
+// * {@link ResourceFileFilter}.
+// *
+// * @param f
+// * A file or directory.
+// */
+// private void recursiveDelete(final File f) {
+//
+// if (f.isDirectory()) {
+//
+// final File[] children = f.listFiles();
+//
+// if (children == null) {
+//
+// // The directory does not exist.
+// return;
+//
+// }
+//
+// for (int i = 0; i < children.length; i++) {
+//
+// recursiveDelete(children[i]);
+//
+// }
+//
+// }
+//
+// if(log.isInfoEnabled())
+// log.info("Removing: " + f);
+//
+// if (f.exists() && !f.delete()) {
+//
+// log.warn("Could not remove: " + f);
+//
+// }
+//
+// }
- if (f.isDirectory()) {
-
- final File[] children = f.listFiles();
-
- if (children == null) {
-
- // The directory does not exist.
- return;
-
- }
-
- for (int i = 0; i < children.length; i++) {
-
- recursiveDelete(children[i]);
-
- }
-
- }
-
- if(log.isInfoEnabled())
- log.info("Removing: " + f);
-
- if (f.exists() && !f.delete()) {
-
- log.warn("Could not remove: " + f);
-
- }
-
- }
-
/**
* Recursive delete of znodes.
*
Modified: branches/QUADS_QUERY_BRANCH/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.properties 2011-01-18 15:30:00 UTC (rev 4121)
+++ branches/QUADS_QUERY_BRANCH/build.properties 2011-01-18 17:36:58 UTC (rev 4122)
@@ -377,6 +377,8 @@
# after the performance tests have run their course.
perf.run.dir=/usr/bigdata/runs
-# CI properties.
+# CI properties. These must agree with the actual installation directory and zoo.cfg
+# file for the zookeeper instance used to run CI.
test.zookeeper.installDir=/Users/bryan/zookeeper-3.2.1
-
+test.zookeeper.tickTime=2000
+test.zookeeper.clientPort=2888
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 15:30:00 UTC (rev 4121)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 17:36:58 UTC (rev 4122)
@@ -1606,7 +1606,6 @@
<!-- be running on the localhost. -->
<property name="test.zookeeper.tickTime" value="2000" />
<property name="test.zookeeper.clientPort" value="2888" />
- <property name="test.zookeeper.leaderPort" value="3888" />
<!-- The zookeeper install directory. -->
<property name="test.zookeeper.installDir" value="${zookeeper.installDir}" />
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-18 15:30:07
|
Revision: 4121
http://bigdata.svn.sourceforge.net/bigdata/?rev=4121&view=rev
Author: thompsonbry
Date: 2011-01-18 15:30:00 +0000 (Tue, 18 Jan 2011)
Log Message:
-----------
Modified the bundleJar target to run "clean" first.
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/build.xml
Modified: branches/JOURNAL_HA_BRANCH/build.xml
===================================================================
--- branches/JOURNAL_HA_BRANCH/build.xml 2011-01-16 23:16:00 UTC (rev 4120)
+++ branches/JOURNAL_HA_BRANCH/build.xml 2011-01-18 15:30:00 UTC (rev 4121)
@@ -110,7 +110,7 @@
</target>
<!-- Builds the bigdata JAR and bundles it together with all of its dependencies in the ${build.dir}/lib directory. -->
- <target name="bundleJar" depends="bundle, jar" description="Builds the bigdata JAR and bundles it together with all of its dependencies in the ${build.dir}/lib directory.">
+ <target name="bundleJar" depends="clean, bundle, jar" description="Builds the bigdata JAR and bundles it together with all of its dependencies in the ${build.dir}/lib directory.">
<copy file="${build.dir}/${version}.jar" todir="${build.dir}/lib" />
<!--<property name="myclasspath" refid="runtime.classpath" />
<echo message="${myclasspath}"/>-->
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 23:16:07
|
Revision: 4120
http://bigdata.svn.sourceforge.net/bigdata/?rev=4120&view=rev
Author: thompsonbry
Date: 2011-01-16 23:16:00 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Tagging a version for BSBM3.
Added Paths:
-----------
tags/BSBM3/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 22:59:08
|
Revision: 4119
http://bigdata.svn.sourceforge.net/bigdata/?rev=4119&view=rev
Author: thompsonbry
Date: 2011-01-16 22:59:02 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
formatting.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java 2011-01-16 22:46:27 UTC (rev 4118)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailTupleQuery.java 2011-01-16 22:59:02 UTC (rev 4119)
@@ -31,28 +31,36 @@
}
/**
- * Overriden to use query hints from SPARQL queries. Query hints are
+ * Overridden to use query hints from SPARQL queries. Query hints are
* embedded in query strings as namespaces.
* See {@link QueryHints#NAMESPACE} for more information.
*/
@Override
public TupleQueryResult evaluate() throws QueryEvaluationException {
- TupleExpr tupleExpr = getParsedQuery().getTupleExpr();
+
+ final TupleExpr tupleExpr = getParsedQuery().getTupleExpr();
try {
- CloseableIteration<? extends BindingSet, QueryEvaluationException> bindingsIter;
+
+ CloseableIteration<? extends BindingSet, QueryEvaluationException> bindingsIter;
- BigdataSailConnection sailCon =
- (BigdataSailConnection) getConnection().getSailConnection();
- bindingsIter = sailCon.evaluate(tupleExpr, getActiveDataset(), getBindings(), getIncludeInferred(), queryHints);
+ final BigdataSailConnection sailCon = (BigdataSailConnection) getConnection()
+ .getSailConnection();
- bindingsIter = enforceMaxQueryTime(bindingsIter);
+ bindingsIter = sailCon.evaluate(tupleExpr, getActiveDataset(),
+ getBindings(), getIncludeInferred(), queryHints);
- return new TupleQueryResultImpl(new ArrayList<String>(tupleExpr.getBindingNames()), bindingsIter);
+ bindingsIter = enforceMaxQueryTime(bindingsIter);
+
+ return new TupleQueryResultImpl(new ArrayList<String>(tupleExpr
+ .getBindingNames()), bindingsIter);
+
+ } catch (SailException e) {
+
+ throw new QueryEvaluationException(e);
+
}
- catch (SailException e) {
- throw new QueryEvaluationException(e.getMessage(), e);
- }
+
}
public TupleExpr getTupleExpr() throws QueryEvaluationException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 22:46:33
|
Revision: 4118
http://bigdata.svn.sourceforge.net/bigdata/?rev=4118&view=rev
Author: thompsonbry
Date: 2011-01-16 22:46:27 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
More work on [1].
[1] https://sourceforge.net/apps/trac/bigdata/ticket/230
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-16 22:45:32 UTC (rev 4117)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-16 22:46:27 UTC (rev 4118)
@@ -516,11 +516,11 @@
}
- private CloseableIteration<BindingSet, QueryEvaluationException>
+ CloseableIteration<BindingSet, QueryEvaluationException>
evaluateNatively(final TupleExpr tupleExpr, final BindingSet bs)
throws QueryEvaluationException, UnsupportedOperatorException {
try {
- return _evaluateNatively(tupleExpr, bs);
+ return doEvaluateNatively(tupleExpr, bs);
} catch (UnrecognizedValueException ex) {
if (log.isInfoEnabled()) {
log.info("unrecognized value in query: " + ex.getValue());
@@ -538,8 +538,8 @@
}
}
- private CloseableIteration<BindingSet, QueryEvaluationException>
- _evaluateNatively(final TupleExpr root, final BindingSet bs)
+ CloseableIteration<BindingSet, QueryEvaluationException>
+ doEvaluateNatively(final TupleExpr root, final BindingSet bs)
throws UnsupportedOperatorException, UnrecognizedValueException,
QueryEvaluationException {
@@ -713,7 +713,7 @@
/*
* Begin native bigdata evaluation.
*/
- CloseableIteration<BindingSet, QueryEvaluationException> result = _evaluateNatively(
+ CloseableIteration<BindingSet, QueryEvaluationException> result = doEvaluateNatively(
query, bs, queryEngine);// , sesameFilters);
/*
@@ -742,8 +742,8 @@
}
- private CloseableIteration<BindingSet, QueryEvaluationException>
- _evaluateNatively(final PipelineOp query, final BindingSet bs,
+ CloseableIteration<BindingSet, QueryEvaluationException>
+ doEvaluateNatively(final PipelineOp query, final BindingSet bs,
final QueryEngine queryEngine
// , final Collection<Filter> sesameFilters
)
@@ -794,7 +794,7 @@
*
* @throws QueryEvaluationException
*/
- private CloseableIteration<BindingSet, QueryEvaluationException> wrapQuery(
+ CloseableIteration<BindingSet, QueryEvaluationException> wrapQuery(
final IRunningQuery runningQuery
) throws QueryEvaluationException {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 22:45:39
|
Revision: 4117
http://bigdata.svn.sourceforge.net/bigdata/?rev=4117&view=rev
Author: thompsonbry
Date: 2011-01-16 22:45:32 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
More work on [1].
[1] https://sourceforge.net/apps/trac/bigdata/ticket/230
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-16 21:09:33 UTC (rev 4116)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-16 22:45:32 UTC (rev 4117)
@@ -728,6 +728,7 @@
* This is used for processing errors and also if this task is
* interrupted (because the sink has been closed).
*/
+ // ensure query halts.
halt(t);
// reset the unsync buffers.
@@ -759,7 +760,12 @@
log.error(t2.getLocalizedMessage(), t2);
}
- throw new RuntimeException(t);
+ if (getCause() != null) {
+ // abnormal termination.
+ throw new RuntimeException(t);
+ }
+ // normal termination - ignore exception.
+ return null;
// } finally {
//
@@ -1040,9 +1046,16 @@
return null;
} catch (Throwable t) {
-
- throw new RuntimeException(halt(t));
+ // ensure query halts.
+ halt(t);
+ if (getCause() != null) {
+ // abnormal termination.
+ throw new RuntimeException(t);
+ }
+ // normal terminate - ignore exception.
+ return null;
+
}
}
@@ -1674,7 +1687,13 @@
} catch (Throwable t) {
- throw new RuntimeException(halt(t));
+ // ensure query halts.
+ halt(t);
+ if (getCause() != null) {
+ // abnormal termination.
+ throw new RuntimeException(t);
+ }
+ // normal termination - ignore exception.
} finally {
@@ -1880,7 +1899,13 @@
} catch (Throwable t) {
- throw new RuntimeException(halt(t));
+ // ensure query halts.
+ halt(t);
+ if (getCause() != null) {
+ // abnormal termination.
+ throw new RuntimeException(t);
+ }
+ // normal termination - ignore exception.
} finally {
@@ -2070,7 +2095,14 @@
} catch (Throwable t) {
- throw new RuntimeException(halt(t));
+ // ensure query halts.
+ halt(t);
+ if (getCause() != null) {
+ // abnormal termination.
+ throw new RuntimeException(t);
+ }
+ // normal termination - ignore exception.
+ return null;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 21:09:39
|
Revision: 4116
http://bigdata.svn.sourceforge.net/bigdata/?rev=4116&view=rev
Author: thompsonbry
Date: 2011-01-16 21:09:33 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Updated the bsbm3 notes.
Added alternative version of Q5 with better performance (2x) and a copy of the original version of Q5 in case people lose track of which is which.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-explicit-order.txt
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-original.txt
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-explicit-order.txt
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-original.txt
Added: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-explicit-order.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-explicit-order.txt (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-explicit-order.txt 2011-01-16 21:09:33 UTC (rev 4116)
@@ -0,0 +1,22 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
+PREFIX BIGDATA_QUERY_HINTS: <http://www.bigdata.com/queryHints#com.bigdata.rdf.sail.QueryHints.optimizer=None>
+
+SELECT DISTINCT ?product ?productLabel
+WHERE {
+
+ %ProductXYZ% bsbm:productPropertyNumeric2 ?origProperty2 .
+ %ProductXYZ% bsbm:productPropertyNumeric1 ?origProperty1 .
+ %ProductXYZ% bsbm:productFeature ?prodFeature .
+ ?product bsbm:productFeature ?prodFeature .
+ ?product bsbm:productPropertyNumeric1 ?simProperty1 .
+ FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
+ ?product bsbm:productPropertyNumeric2 ?simProperty2 .
+ FILTER (?simProperty2 < (?origProperty2 + 170) && ?simProperty2 > (?origProperty2 - 170))
+ ?product rdfs:label ?productLabel .
+ FILTER (%ProductXYZ% != ?product)
+
+}
+ORDER BY ?productLabel
+LIMIT 5
\ No newline at end of file
Added: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-original.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-original.txt (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query5-original.txt 2011-01-16 21:09:33 UTC (rev 4116)
@@ -0,0 +1,19 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
+
+SELECT DISTINCT ?product ?productLabel
+WHERE {
+ ?product rdfs:label ?productLabel .
+ FILTER (%ProductXYZ% != ?product)
+ %ProductXYZ% bsbm:productFeature ?prodFeature .
+ ?product bsbm:productFeature ?prodFeature .
+ %ProductXYZ% bsbm:productPropertyNumeric1 ?origProperty1 .
+ ?product bsbm:productPropertyNumeric1 ?simProperty1 .
+ FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
+ %ProductXYZ% bsbm:productPropertyNumeric2 ?origProperty2 .
+ ?product bsbm:productPropertyNumeric2 ?simProperty2 .
+ FILTER (?simProperty2 < (?origProperty2 + 170) && ?simProperty2 > (?origProperty2 - 170))
+}
+ORDER BY ?productLabel
+LIMIT 5
\ No newline at end of file
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt 2011-01-16 20:47:09 UTC (rev 4115)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt 2011-01-16 21:09:33 UTC (rev 4116)
@@ -54,3 +54,33 @@
There are a variety of other ant tasks in that directory which may be used to
run load and run the BSBM qualification data set, etc.
+
+Performance should be extremely good for the reduced query mix, which can be
+enabled by editing:
+
+ bigdata-perf/bsbm3/bsbmtools/queries/explore/ignoreQueries
+
+For the reduced query mix, "ignoreQueries" should contain "5 6". For the full
+query mix, it should be an empty file (the reduced query mix is enabled by
+default in SVN).
+
+Notes on the queries:
+
+The static query optimizer and vectored pipelined joins do a great job on most
+of the BSBM queries. However, there are two queries which do not do so well out
+of the box:
+
+Query 5 has a bad join plan using the static query optimizer. Good performance
+for query 5 can be achieved by replacing the contents of:
+
+ bigdata-perf/bsbm3/bsbmtools/queries/explore/query5.txt
+
+ bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-explicit-order.txt
+
+The original version of query5 has also been saved as query5-original.txt
+
+Query 6 is uses a REGEX filter. Bigdata does not have index support for REGEX,
+so this winds up visiting a lot of data and then filtering using the REGEX. This
+drags the overall performance down dramatically. It is possible to integrate
+bigdata with Lucene, which does support indexed regular expressions, but that is
+not something which works out of the box.
Added: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-explicit-order.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-explicit-order.txt (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-explicit-order.txt 2011-01-16 21:09:33 UTC (rev 4116)
@@ -0,0 +1,22 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
+PREFIX BIGDATA_QUERY_HINTS: <http://www.bigdata.com/queryHints#com.bigdata.rdf.sail.QueryHints.optimizer=None>
+
+SELECT DISTINCT ?product ?productLabel
+WHERE {
+
+ %ProductXYZ% bsbm:productPropertyNumeric2 ?origProperty2 .
+ %ProductXYZ% bsbm:productPropertyNumeric1 ?origProperty1 .
+ %ProductXYZ% bsbm:productFeature ?prodFeature .
+ ?product bsbm:productFeature ?prodFeature .
+ ?product bsbm:productPropertyNumeric1 ?simProperty1 .
+ FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
+ ?product bsbm:productPropertyNumeric2 ?simProperty2 .
+ FILTER (?simProperty2 < (?origProperty2 + 170) && ?simProperty2 > (?origProperty2 - 170))
+ ?product rdfs:label ?productLabel .
+ FILTER (%ProductXYZ% != ?product)
+
+}
+ORDER BY ?productLabel
+LIMIT 5
\ No newline at end of file
Added: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-original.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-original.txt (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/query5-original.txt 2011-01-16 21:09:33 UTC (rev 4116)
@@ -0,0 +1,19 @@
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
+
+SELECT DISTINCT ?product ?productLabel
+WHERE {
+ ?product rdfs:label ?productLabel .
+ FILTER (%ProductXYZ% != ?product)
+ %ProductXYZ% bsbm:productFeature ?prodFeature .
+ ?product bsbm:productFeature ?prodFeature .
+ %ProductXYZ% bsbm:productPropertyNumeric1 ?origProperty1 .
+ ?product bsbm:productPropertyNumeric1 ?simProperty1 .
+ FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
+ %ProductXYZ% bsbm:productPropertyNumeric2 ?origProperty2 .
+ ?product bsbm:productPropertyNumeric2 ?simProperty2 .
+ FILTER (?simProperty2 < (?origProperty2 + 170) && ?simProperty2 > (?origProperty2 - 170))
+}
+ORDER BY ?productLabel
+LIMIT 5
\ No newline at end of file
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 20:47:15
|
Revision: 4115
http://bigdata.svn.sourceforge.net/bigdata/?rev=4115&view=rev
Author: thompsonbry
Date: 2011-01-16 20:47:09 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Configured for the reduced query mix by default (w/o Q5 or Q6).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/ignoreQueries.txt
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/ignoreQueries.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/ignoreQueries.txt 2011-01-16 20:45:19 UTC (rev 4114)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/queries/explore/ignoreQueries.txt 2011-01-16 20:47:09 UTC (rev 4115)
@@ -0,0 +1 @@
+5 6
\ No newline at end of file
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 20:45:26
|
Revision: 4114
http://bigdata.svn.sourceforge.net/bigdata/?rev=4114&view=rev
Author: thompsonbry
Date: 2011-01-16 20:45:19 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Fixed some spelling errors in the property files for the RW and WORM stores (sail properties mostly).
Added support for "-seed random" to the bsbm TestDriver as an convenience for scripting random runs.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/RWStore.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/WORMStore.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/src/benchmark/testdriver/TestDriver.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/RWStore.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/WORMStore.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/RWStore.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/RWStore.properties 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/RWStore.properties 2011-01-16 20:45:19 UTC (rev 4114)
@@ -24,6 +24,17 @@
com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=512
com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=470
+# Reduce the branching factor for the lexicon since BSBM uses a lot of long
+# literals. Note that you have to edit this override to specify the namespace
+# into which the BSBM data will be loaded.
+com.bigdata.namespace.BSBM_566496.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=32
+com.bigdata.namespace.BSBM_566496.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=32
+
+# 4k pages.
+com.bigdata.namespace.BSBM_566496.spo.POS.com.bigdata.btree.BTree.branchingFactor=970
+com.bigdata.namespace.BSBM_566496.spo.SPO.com.bigdata.btree.BTree.branchingFactor=512
+com.bigdata.namespace.BSBM_566496.spo.OSP.com.bigdata.btree.BTree.branchingFactor=470
+
# Override the #of write cache buffers.
com.bigdata.journal.AbstractJournal.writeCacheBufferCount=12
@@ -48,11 +59,17 @@
#com.bigdata.rdf.store.AbstractTripleStore.termCache.capacity=50000
com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms
+# xsd:DateTime needs to be inlined for BSBM
+com.bigdata.rdf.store.AbstractTripleStore.inlineDateTimes=true
+
##
## Sail options.
##
-com.bigdata.rdf.sail.BigdataSail.truthMaintenance=false
+com.bigdata.rdf.sail.truthMaintenance=false
# 10000 is default.
-com.bigdata.rdf.sail.BigdataSail.bufferCapacity=100000
+com.bigdata.rdf.sail.bufferCapacity=100000
+
+# direct sesame to bop translation.
+com.bigdata.rdf.sail.newEvalStrategy=true
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/WORMStore.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/WORMStore.properties 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/WORMStore.properties 2011-01-16 20:45:19 UTC (rev 4114)
@@ -27,11 +27,17 @@
#com.bigdata.rdf.store.AbstractTripleStore.termCache.capacity=50000
com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms
+# xsd:DateTime needs to be inlined for BSBM
+com.bigdata.rdf.store.AbstractTripleStore.inlineDateTimes=true
+
##
## Sail options.
##
-com.bigdata.rdf.sail.BigdataSail.truthMaintenance=false
+com.bigdata.rdf.sail.truthMaintenance=false
# 10000 is default.
-com.bigdata.rdf.sail.BigdataSail.bufferCapacity=100000
+com.bigdata.rdf.sail.bufferCapacity=100000
+
+# direct sesame to bop translation.
+com.bigdata.rdf.sail.newEvalStrategy=true
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/src/benchmark/testdriver/TestDriver.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/src/benchmark/testdriver/TestDriver.java 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/src/benchmark/testdriver/TestDriver.java 2011-01-16 20:45:19 UTC (rev 4114)
@@ -752,7 +752,14 @@
multithreading = true;
nrThreads = Integer.parseInt(args[i++ + 1]);
} else if (args[i].equals("-seed")) {
- seed = Long.parseLong(args[i++ + 1]);
+// Modified 1/16/2011 by BBT to recognize "random" seeds.
+// seed = Long.parseLong(args[i++ + 1]);
+ final String s = args[i++ + 1];
+ if (s.equals("random")) {
+ seed = System.nanoTime();
+ } else {
+ seed = Long.parseLong(s);
+ }
} else if (args[i].equals("-t")) {
timeout = Integer.parseInt(args[i++ + 1]);
} else if (args[i].equals("-dbdriver")) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.properties 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.properties 2011-01-16 20:45:19 UTC (rev 4114)
@@ -46,8 +46,8 @@
#bsbm.pc=10
#bsbm.pc=2785
#bsbm.pc=70812
-bsbm.pc=284826
-#bsbm.pc=566496
+#bsbm.pc=284826
+bsbm.pc=566496
# The namespace of the KB instance (multiple KBs can be in the same database).
bsbm.namespace=BSBM_${bsbm.pc}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/RWStore.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/RWStore.properties 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/RWStore.properties 2011-01-16 20:45:19 UTC (rev 4114)
@@ -66,7 +66,7 @@
# are loaded. (Actually, since we are not loading through the SAIL
# making this true does not cause incremental TM but it does disable
# closure, so "false" is what you need here).
-com.bigdata.rdf.sail.BigdataSail.truthMaintenance=false
+com.bigdata.rdf.sail.truthMaintenance=false
#
# Option to restrict ourselves to RDFS only inference. This condition
@@ -92,7 +92,7 @@
com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.RdfsAxioms
# 10000 is default.
-com.bigdata.rdf.sail.BigdataSail.bufferCapacity=100000
+com.bigdata.rdf.sail.bufferCapacity=100000
# Produce a full closure (all entailments) so that the backward chainer
# is always a NOP. Note that the configuration properties are stored in
@@ -146,7 +146,7 @@
# (x rdf:type rdfs:Resource) and owl:sameAs even through those
# entailments were not materialized during forward closure. (This
# property is interpreted by the BigdataSail).
-com.bigdata.rdf.stail.queryTimeExpander=false
+com.bigdata.rdf.sail.queryTimeExpander=false
# Note: LUBM uses blank nodes. Therefore re-loading LUBM will always
# cause new statements to be asserted and result in the closure being
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/WORMStore.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/WORMStore.properties 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/WORMStore.properties 2011-01-16 20:45:19 UTC (rev 4114)
@@ -32,7 +32,7 @@
# are loaded. (Actually, since we are not loading through the SAIL
# making this true does not cause incremental TM but it does disable
# closure, so "false" is what you need here).
-com.bigdata.rdf.sail.BigdataSail.truthMaintenance=false
+com.bigdata.rdf.sail.truthMaintenance=false
#
# Option to restrict ourselves to RDFS only inference. This condition
@@ -58,7 +58,7 @@
com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.RdfsAxioms
# 10000 is default.
-com.bigdata.rdf.sail.BigdataSail.bufferCapacity=100000
+com.bigdata.rdf.sail.bufferCapacity=100000
# Produce a full closure (all entailments) so that the backward chainer
# is always a NOP. Note that the configuration properties are stored in
@@ -112,7 +112,7 @@
# (x rdf:type rdfs:Resource) and owl:sameAs even through those
# entailments were not materialized during forward closure. (This
# property is interpreted by the BigdataSail).
-com.bigdata.rdf.stail.queryTimeExpander=false
+com.bigdata.rdf.sail.queryTimeExpander=false
# Note: LUBM uses blank nodes. Therefore re-loading LUBM will always
# cause new statements to be asserted and result in the closure being
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties 2011-01-16 20:21:58 UTC (rev 4113)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties 2011-01-16 20:45:19 UTC (rev 4114)
@@ -68,8 +68,8 @@
lubm.compressType=GZip
# Which mode to use for the Journal. (DiskRW or DiskWORM)
-#journalMode=RW
-journalMode=WORM
+journalMode=RW
+#journalMode=WORM
## The name of the directory containing the generated RDF data without the filename extension.
#lubm.outputFile=${lubm.baseDir}
@@ -128,8 +128,8 @@
#gcdebug=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:jvm_gc.log
# The record cache (empty for the default cache).
-#cache=
-cache=-Dcom.bigdata.LRUNexus.enabled=false
+cache=
+#cache=-Dcom.bigdata.LRUNexus.enabled=false
#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU2 -Dcom.bigdata.LRUNexus.threadLocalBuffers=true
#cache=-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.BCHMGlobalLRU -Dcom.bigdata.LRUNexus.accessPolicy=LIRS -Dcom.bigdata.LRUNexus.limitingCapacity=2000000
#-Dcom.bigdata.LRUNexus.class=com.bigdata.cache.StoreAndAddressLRUCache
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 20:22:04
|
Revision: 4113
http://bigdata.svn.sourceforge.net/bigdata/?rev=4113&view=rev
Author: thompsonbry
Date: 2011-01-16 20:21:58 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Added directions for running the BSBM against bigdata.
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt
Added: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/README.txt 2011-01-16 20:21:58 UTC (rev 4113)
@@ -0,0 +1,56 @@
+This directory contains a setup for running BSBM v3 against bigdata. The main
+files are:
+
+- bsbmtools - the bsbm3 source distribution.
+
+- build.properties - configuration properties for the ant script.
+
+- build.xml - an ant script which may be used to generate a BSBM data set, load
+ the data set into a bigdata database instance, start a SPARQL
+ end point for that database instance, and run the BSBM benchmark
+ against that SPARQL end point.
+
+- RWStore.properties - configuration properties for a bigdata database instance
+ suitable for BSBM and backed by the RW persistence engine
+ (single machine read/write bigdata database). This is the
+ recommended database mode for BSBM as it has significantly
+ better performance.
+
+- WORMStore.properties - configuration properties for a bigdata database instance
+ suitable for BSBM and backed by the WORM persistence
+ engine (single machine write once, read many bigdata
+ database).
+
+Other requirements include:
+
+- A 64-bit OS and a 64-bit server JVM. We have tested most extensible with Oracle
+ JDK 1.6.0_17.
+
+- Apache ant (version 1.8.0+).
+
+- Bigdata (check it out from SVN).
+
+To get started:
+
+1. Edit bigdata-perf/bsbm3/build.properties.
+
+1. In the top-level directory of the bigdata source tree, review build.properties
+ and then do:
+
+ a. "ant bundleJar".
+
+ Note: You will need to rerun this ant target any time you update the code
+ from SVN or if you make edits to the source tree.
+
+2. Change to the bigdata-perf/bsbm3 directory:
+
+ a. "ant run-generator" (generates the BSBM data set).
+
+ b. "ant run-load" (loads the generated data set into a bigdata instance).
+
+ c. "ant start-nano-server" (starts the SPARQL end point).
+
+ d. "ant run-query" (runs the benchmark).
+
+There are a variety of other ant tasks in that directory which may be used to
+run load and run the BSBM qualification data set, etc.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 19:30:59
|
Revision: 4112
http://bigdata.svn.sourceforge.net/bigdata/?rev=4112&view=rev
Author: thompsonbry
Date: 2011-01-16 19:30:52 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Moved the CONSTRAINTS annotation from the PipelineJoin operator to the IPredicate in order to support join graphs. Now that the CONSTRAINTS decorate the IPredicate, they will automatically be evaluated when that predicate is joined and we do not have to worry about raising the CONSTRAINTS onto the join operator (the join operator is still responsible for evaluating the constraints, but it picks them off of the IPredicate now).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestSubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -42,7 +42,6 @@
import com.bigdata.btree.filter.Advancer;
import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.mdi.PartitionLocator;
-import com.bigdata.rawstore.Bytes;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.ElementFilter;
@@ -102,7 +101,10 @@
* <code>true</code> iff the predicate is optional (the right operand of
* a left join).
*
- * @deprecated This flag is being moved to the join operator.
+ * @deprecated This flag is being moved to the join operator (or should
+ * it stay with the predicate so we can work with join
+ * graphs more easily, but note that join graphs never
+ * contain optional joins).
*/
String OPTIONAL = "optional";
@@ -321,6 +323,12 @@
*/
String TIMESTAMP = BOp.class.getName() + ".timestamp";
+ /**
+ * An optional {@link IConstraint}[] which places restrictions on the
+ * legal patterns in the variable bindings.
+ */
+ String CONSTRAINTS = PipelineJoin.class.getName() + ".constraints";
+
}
/**
@@ -434,6 +442,22 @@
// public IElementFilter<E> getConstraint();
/**
+ * Return the optional {@link IConstraint}[] to be applied by a join which
+ * evaluates this {@link IPredicate}.
+ * <p>
+ * Note: The {@link Annotations#CONSTRAINTS} are annotated on the
+ * {@link IPredicate} rather than the join operators so they may be used
+ * with join graphs, which are expressed solely as an unordered set of
+ * {@link IPredicate}s. Using join graphs, we are able to do nifty things
+ * such as runtime query optimization which would not be possible if the
+ * annotations were decorating the joins since we would be unable to
+ * dynamically generate the join operators with the necessary annotations.
+ *
+ * @see Annotations#CONSTRAINTS
+ */
+ public IConstraint[] constraints();
+
+ /**
* Return the optional filter to be evaluated local to the index.
*
* @see Annotations#INDEX_LOCAL_FILTER
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -35,6 +35,7 @@
import com.bigdata.bop.Constant;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstant;
+import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IElement;
import com.bigdata.bop.IPredicate;
import com.bigdata.bop.IVariable;
@@ -244,6 +245,12 @@
//
// }
+ public IConstraint[] constraints() {
+
+ return getProperty(IPredicate.Annotations.CONSTRAINTS, null/* defaultValue */);
+
+ }
+
final public IFilter getIndexLocalFilter() {
return (IFilter) getProperty(Annotations.INDEX_LOCAL_FILTER);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -73,7 +73,6 @@
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
-import com.bigdata.relation.rule.Rule;
import com.bigdata.striterator.Dechunkerator;
import com.bigdata.striterator.IChunkedIterator;
@@ -945,6 +944,13 @@
throw new IllegalStateException();
if (v2.sample == null) // vertex not sampled.
throw new IllegalStateException();
+ /*
+ * FIXME CONSTRAINT ORDERING : If a variable only appears in a
+ * CONSTRAINT for one of the two vertices then that vertex must be
+ * evaluated second. (If the vertices both have this problem then
+ * the edge can not be evaluated until some other vertex causes the
+ * variables of either one [v1] or [v2] to become bound.)
+ */
if (v1.sample.rangeCount < v2.sample.rangeCount) {
v = v1;
vp = v2;
@@ -1061,11 +1067,7 @@
new NV(PipelineJoin.Annotations.SHARED_STATE,true),
new NV(PipelineJoin.Annotations.EVALUATION_CONTEXT,BOpEvaluationContext.CONTROLLER)
);
- if (vTarget.pred.getProperty(PipelineJoin.Annotations.CONSTRAINTS) != null) {
- // Copy constraints from the predicate onto the join, which will apply them.
- anns.put(PipelineJoin.Annotations.CONSTRAINTS, vTarget.pred
- .getProperty(PipelineJoin.Annotations.CONSTRAINTS));
- }
+
final PipelineJoin joinOp = new PipelineJoin(new BOp[] {}, anns);
final PipelineOp queryOp = joinOp;
@@ -1335,7 +1337,7 @@
/*
* Scan the vertices of the caller's path. If any of those vertices
* are NOT found in this path the paths are not unordered variations
- * of one aother.
+ * of one another.
*/
for (int i = 0; i < v2.length; i++) {
@@ -1381,10 +1383,15 @@
for (Edge e : edges) {
if (tmp.isEmpty()) {
- /*
- * The first edge is handled specially in order to report
- * the minimum cardinality vertex first.
- */
+ /*
+ * The first edge is handled specially in order to report
+ * the minimum cardinality vertex first.
+ *
+ * FIXME CONSTRAINT ORDERING : A vertex can not run until
+ * all variables appearing in its CONSTRAINTS would be
+ * bound. This can cause us to use and report an ordering
+ * which does not place the minimum cardinality vertex 1st.
+ */
tmp.add(e.getMinimumCardinalityVertex());
tmp.add(e.getMaximumCardinalityVertex());
@@ -1530,26 +1537,30 @@
// The new vertex, which is not part of this path.
final Vertex targetVertex = v1Found ? e.v2 : e.v1;
- /*
- * Chain sample the edge.
- *
- * Note: ROX uses the intermediate result I(p) for the existing path
- * as the input when sampling the edge. The corresponding concept
- * for us is the sample for this Path, which will have all variable
- * bindings produced so far. In order to estimate the cardinality of
- * the new join path we have to do a one step cutoff evaluation of
- * the new Edge, given the sample available on the current Path.
- *
- * FIXME It is possible for the path sample to be empty. Unless the
- * sample also happens to be exact, this is an indication that the
- * estimated cardinality has underflowed. We track the estimated
- * cumulative cardinality, so this does not make the join path an
- * immediate winner, but it does mean that we can not probe further
- * on that join path as we lack any intermediate solutions to feed
- * into the downstream joins. [If we re-sampled the edges in the
- * join path in each round then this would help to establish a
- * better estimate in successive rounds.]
- */
+ /*
+ * Chain sample the edge.
+ *
+ * Note: ROX uses the intermediate result I(p) for the existing path
+ * as the input when sampling the edge. The corresponding concept
+ * for us is the sample for this Path, which will have all variable
+ * bindings produced so far. In order to estimate the cardinality of
+ * the new join path we have to do a one step cutoff evaluation of
+ * the new Edge, given the sample available on the current Path.
+ *
+ * FIXME It is possible for the path sample to be empty. Unless the
+ * sample also happens to be exact, this is an indication that the
+ * estimated cardinality has underflowed. We track the estimated
+ * cumulative cardinality, so this does not make the join path an
+ * immediate winner, but it does mean that we can not probe further
+ * on that join path as we lack any intermediate solutions to feed
+ * into the downstream joins. [If we re-sampled the edges in the
+ * join path in each round then this would help to establish a
+ * better estimate in successive rounds.]
+ *
+ * FIXME CONSTRAINT ORDERING : It is illegal to add a vertex to the
+ * path if any variable appearing in its CONSTRAINTS would not be
+ * bound.
+ */
final EdgeSample edgeSample = e.estimateCardinality(queryEngine,
limit, sourceVertex, targetVertex,
@@ -1741,6 +1752,14 @@
*/
private final Edge[] E;
+ /**
+ * An unordered collection of vertices which do not share any variables
+ * with the other vertices in the join graph. These vertices are run
+ * last. Once the {@link #E edges} have been exhausted, these vertices
+ * are simply appended to the join path in an arbitrary order.
+ */
+ private final Vertex[] unshared;
+
public List<Vertex> getVertices() {
return Collections.unmodifiableList(Arrays.asList(V));
}
@@ -1756,6 +1775,10 @@
for (Vertex v : V) {
sb.append("\nV[" + v.pred.getId() + "]=" + v);
}
+ sb.append("\n],unshared=[");
+ for (Vertex v : unshared) {
+ sb.append("\n" + v);
+ }
sb.append("],E=[");
for (Edge e : E) {
sb.append("\n" + e);
@@ -1780,18 +1803,39 @@
}
- /*
- * Identify the edges by looking for shared variables among the
- * predicates.
- *
- * Note: If a vertex does not share ANY variables then it is paired
- * with every other vertex. Such joins will always produce a full
- * cross product and they can be taken paired with any of the other
- * vertices.
- */
+ /*
+ * Identify the edges by looking for shared variables among the
+ * predicates.
+ *
+ * Note: Variables may appear in the arguments of the predicate,
+ * e.g., spo(?s,rdf:type,?o).
+ *
+ * Note: Variables may ALSO appear in the CONSTRAINTS (imposed on
+ * the binding sets) or FILTERS (imposed either on the local or
+ * remote access path). For example, that a variable bound by
+ * another predicate must take on a value having some mathematical
+ * relationship to a variable bound by the predicate, e.g., BSBM Q5.
+ * When a variable appears in a constraint but does not appear as an
+ * argument to the predicate, then there is an additional
+ * requirement that the variable MUST become bound before the
+ * predicate may be evaluated (again, BSBM Q5 has this form).
+ *
+ * Note: If a vertex does not share ANY variables (neither in the
+ * arguments of the predicate nor in its constraints or filters)
+ * then it can be paired with any of the other vertices. However, in
+ * such cases we always run such vertices last as they can not
+ * restrict the cardinality of the rest of the join graph. Such
+ * vertices are therefore inserted into a separate set and appended
+ * to the join path once all edges having shared variables have been
+ * exhausted.
+ */
{
- final List<Edge> tmp = new LinkedList<Edge>();
+ // The set of identified edges for vertices which share vars.
+ final List<Edge> tmp = new LinkedList<Edge>();
+
+ // The set of vertices which do not share variables.
+ final List<Vertex> unshared = new LinkedList<Vertex>();
for (int i = 0; i < v.length; i++) {
@@ -1806,12 +1850,21 @@
// consider a possible target vertex.
final IPredicate<?> p2 = v[j];
- final Set<IVariable<?>> shared = Rule.getSharedVars(p1,
- p2);
+ final Set<IVariable<?>> shared = getSharedVars(p1, p2);
if (shared != null && !shared.isEmpty()) {
- // the source and target vertices share var(s).
+ /*
+ * The source and target vertices share var(s).
+ *
+ * Note: A predicate having a variable which appears
+ * in a CONSTRAINT MUST NOT be added to the join
+ * path until that variable would be bound.
+ * Therefore, when selecting the vertices to be used
+ * to extend a join path, we must consider whether
+ * or not the join path would bind the variable(s)
+ * appearing in the CONSTRAINT.
+ */
tmp.add(new Edge(V[i], V[j], shared));
nmatched++;
@@ -1823,26 +1876,37 @@
if (nmatched == 0) {
/*
- * The source vertex does not share any variables. In
- * order to explore join paths which include that vertex
- * we therefore pair it with each of the other vertices.
+ * The source vertex does not share any variables.
*/
- for (int j = 0; j < v.length; j++) {
+
+ unshared.add(V[i]);
- if (j == i)
- continue;
-
- tmp.add(new Edge(V[i], V[j],
- Collections.EMPTY_SET));
-
- }
-
}
}
E = tmp.toArray(new Edge[0]);
+
+ this.unshared = unshared.toArray(new Vertex[0]);
+ if(!unshared.isEmpty()) {
+
+ /*
+ * FIXME This needs to be supported. We should explore and
+ * generate the join paths based on only those vertices
+ * which do share variables (and hence for which we have
+ * defined edges). Once the vertices which share variables
+ * have been exhausted, we should simply append edges for
+ * the vertices which do not share variables in an arbitrary
+ * order (they will be run last since they can not constrain
+ * the evaluation).
+ */
+
+ throw new UnsupportedOperationException(
+ "Some predicates do not share any variables with other predicates.");
+
+ }
+
}
}
@@ -1889,13 +1953,21 @@
// Setup the join graph.
Path[] paths = round0(queryEngine, limit, nedges);
- /*
- * The initial paths all have one edge, and hence two vertices. Each
- * round adds one more vertex to each path. We are done once we have
- * generated paths which include all vertices.
- *
- * This occurs at round := nvertices - 1
- */
+ /*
+ * The initial paths all have one edge, and hence two vertices. Each
+ * round adds one more vertex to each path. We are done once we have
+ * generated paths which include all vertices.
+ *
+ * This occurs at round := nvertices - 1
+ *
+ * FIXME UNSHARED VERTICES : Add [unshared] vertices after all
+ * vertices with shared variables have been incorporated into the
+ * join paths. This should happen outside of the loop since the
+ * joins with unshared variables can not constraint the solutions.
+ * Therefore choose the best join path based on the vertices with
+ * the shared variables and then simply append the [unshared]
+ * vertices.
+ */
final int nvertices = V.length;
@@ -2000,25 +2072,29 @@
}
- /**
- * Choose up to <i>nedges</i> edges to be the starting point.
- *
- * @param queryEngine
- * The query engine.
- * @param limit
- * The cutoff used when sampling the vertices and when
- * sampling the edges.
- * @param nedges
- * The maximum #of edges to choose. Those having the smallest
- * expected cardinality will be chosen.
- *
- * @return An initial set of paths starting from any most <i>nedges</i>.
- * For each of the <i>nedges</i> lowest cardinality edges, the
- * starting vertex will be the vertex with the lowest
- * cardinality for that edge.
- *
- * @throws Exception
- */
+ /**
+ * Choose up to <i>nedges</i> edges to be the starting point. For each
+ * of the <i>nedges</i> lowest cardinality edges, the starting vertex
+ * will be the vertex with the lowest cardinality for that edge.
+ * <p>
+ * Note: An edge can not serve as a starting point for exploration if it
+ * uses variables (for example, in a CONSTRAINT) which are not bound by
+ * either vertex (since the variable(s) are not bound, the constraint
+ * would always fail).
+ *
+ * @param queryEngine
+ * The query engine.
+ * @param limit
+ * The cutoff used when sampling the vertices and when
+ * sampling the edges.
+ * @param nedges
+ * The maximum #of edges to choose. Those having the smallest
+ * expected cardinality will be chosen.
+ *
+ * @return An initial set of paths starting from at most <i>nedges</i>.
+ *
+ * @throws Exception
+ */
public Path[] round0(final QueryEngine queryEngine, final int limit,
final int nedges) throws Exception {
@@ -2889,13 +2965,6 @@
//
// anns.add(new NV(PipelineJoin.Annotations.SELECT, vars.toArray(new IVariable[vars.size()])));
- if (p.getProperty(PipelineJoin.Annotations.CONSTRAINTS) != null) {
- // Copy constraints from the predicate onto the join, which will
- // apply them.
- anns.add(new NV(PipelineJoin.Annotations.CONSTRAINTS, p
- .getProperty(PipelineJoin.Annotations.CONSTRAINTS)));
- }
-
final PipelineJoin joinOp = new PipelineJoin(
lastOp == null ? new BOp[0] : new BOp[] { lastOp },
anns.toArray(new NV[anns.size()]));
@@ -3025,7 +3094,105 @@
}
- /**
+ /**
+ * Return the variables in common for two {@link IPredicate}s. All variables
+ * spanned by either {@link IPredicate} are considered.
+ * <p>
+ * Note: Variables may appear in the predicates operands, in the
+ * {@link Annotations#CONSTRAINTS} associated with the
+ * predicate, and in the {@link IPredicate.Annotations#ACCESS_PATH_FILTER}
+ * or {@link IPredicate.Annotations#INDEX_LOCAL_FILTER}.
+ * <p>
+ * Note: A variable must become bound before it may be evaluated in
+ * {@link Annotations#CONSTRAINTS}, an
+ * {@link IPredicate.Annotations#ACCESS_PATH_FILTER} or an
+ * {@link IPredicate.Annotations#INDEX_LOCAL_FILTER}. This means that the
+ * {@link IPredicate}s which can bind the variable must be ordered before
+ * those which merely test the variable.
+ *
+ *
+ * @param p1
+ * A predicate.
+ *
+ * @param p2
+ * A different predicate.
+ *
+ * @return The variables in common -or- <code>null</code> iff there are no
+ * variables in common.
+ *
+ * @throws IllegalArgumentException
+ * if the two predicates are the same reference.
+ *
+ * @todo It should be an error if a variable appear in a test is not bound
+ * by any possible join path. However, note that it may not be
+ * possible to determine this by local examination of a join graph
+ * since we do not know which variables may be presented as already
+ * bound when the join graph is evaluated (but we can only run the
+ * join graph currently against static source binding sets and for
+ * that case this is knowable).
+ *
+ * @todo When a variable is only optionally bound and it is discovered at
+ * runtime that the variable is not bound when it is considered by a
+ * CONSTRAINT, FILTER, etc., then the SPARQL semantics are that
+ * evaluation should produce a 'type' error which would cause the
+ * solution should fail (at least within its current join group). See
+ * https://sourceforge.net/apps/trac/bigdata/ticket/179.
+ *
+ * @todo Unit tests, including those which verify that variables appearing
+ * in the constraints are reported as shared with those appearing in
+ * the predicates operands.
+ */
+ static Set<IVariable<?>> getSharedVars(final IPredicate p1, final IPredicate p2) {
+
+ // The set of variables which are shared by those predicates.
+ final Set<IVariable<?>> sharedVars = new LinkedHashSet<IVariable<?>>();
+
+ /*
+ * Collect the variables appearing anyway in [p1], including the
+ * predicate's operands and its constraints, filters, etc.
+ */
+ final Set<IVariable<?>> p1vars = new LinkedHashSet<IVariable<?>>();
+ {
+
+ final Iterator<IVariable<?>> itr = BOpUtility
+ .getSpannedVariables(p1);
+
+ while (itr.hasNext()) {
+
+ p1vars.add(itr.next());
+
+ }
+
+ }
+
+ /*
+ * Consider the variables appearing anyway in [p2], including the
+ * predicate's operands and its constraints, filters, etc.
+ */
+ {
+
+ final Iterator<IVariable<?>> itr = BOpUtility
+ .getSpannedVariables(p2);
+
+ while (itr.hasNext()) {
+
+ final IVariable<?> avar = itr.next();
+
+ if(p1vars.contains(avar)) {
+
+ sharedVars.add(avar);
+
+ }
+
+ }
+
+ }
+
+ return sharedVars;
+
+ }
+
+ /**
* Exception thrown when the join graph does not have any solutions in the
* data (running the query does not produce any results).
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -27,7 +27,6 @@
package com.bigdata.bop.join;
-import java.nio.channels.ClosedByInterruptException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -78,7 +77,6 @@
import com.bigdata.service.DataService;
import com.bigdata.striterator.IChunkedOrderedIterator;
import com.bigdata.striterator.IKeyOrder;
-import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.Haltable;
import com.bigdata.util.concurrent.LatchedExecutor;
@@ -132,12 +130,6 @@
String SELECT = PipelineJoin.class.getName() + ".select";
/**
- * An optional {@link IConstraint}[] which places restrictions on the
- * legal patterns in the variable bindings.
- */
- String CONSTRAINTS = PipelineJoin.class.getName() + ".constraints";
-
- /**
* Marks the join as "optional" in the SPARQL sense. Binding sets which
* fail the join will be routed to the alternative sink as specified by
* either {@link PipelineOp.Annotations#ALT_SINK_REF} or
@@ -444,15 +436,6 @@
}
/**
- * @see Annotations#CONSTRAINTS
- */
- public IConstraint[] constraints() {
-
- return getProperty(Annotations.CONSTRAINTS, null/* defaultValue */);
-
- }
-
- /**
* @see Annotations#OPTIONAL
*/
public boolean isOptional() {
@@ -659,7 +642,7 @@
this.joinOp = joinOp;
this.predicate = joinOp.getPredicate();
- this.constraints = joinOp.constraints();
+ this.constraints = predicate.constraints();
this.maxParallel = joinOp.getMaxParallel();
if (maxParallel < 0)
throw new IllegalArgumentException(Annotations.MAX_PARALLEL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -90,6 +90,14 @@
public void test_Path_addEdge() {
fail("write test");
}
+
+ /**
+ * Test ability to identify shared variables appearing either as predicate
+ * operands or as part of CONSTRAINTS or FILTERS.
+ */
+ public void test_getSharedVariables() {
+ fail("write test");
+ }
// @Override
// public Properties getProperties() {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestSubqueryOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestSubqueryOp.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestSubqueryOp.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -596,7 +596,7 @@
// new NV(PipelineOp.Annotations.CONDITIONAL_GROUP, joinGroup1),//
new NV(PipelineJoin.Annotations.PREDICATE, pred3Op),//
// constraint d != Leon
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] { new NEConstant(d, new Constant<String>("Leon")) })
// // join is optional.
// new NV(PipelineJoin.Annotations.OPTIONAL, true),//
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -80,12 +80,10 @@
return indexManager;
}
- @Override
public void halt(Void v) {
log.warn("Mock object does not implement halt(Void)");
}
- @Override
public <T extends Throwable> T halt(T cause) {
log.warn("Mock object does not implement halt(Throwable)");
return cause;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -1128,7 +1128,7 @@
new NV(Predicate.Annotations.BOP_ID, joinId),//
new NV(PipelineJoin.Annotations.PREDICATE, predOp),//
// impose constraint on the join.
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] { new EQConstant(y,
new Constant<String>("Paul")) })//
);
@@ -1592,7 +1592,7 @@
new NV(Predicate.Annotations.BOP_ID, joinId2),//
new NV(PipelineJoin.Annotations.PREDICATE, pred2Op),//
// constraint x == z
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] { new EQ(x, z) }),
// join is optional.
new NV(PipelineJoin.Annotations.OPTIONAL, true),//
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -717,7 +717,7 @@
new NV(Predicate.Annotations.EVALUATION_CONTEXT,
BOpEvaluationContext.SHARDED),//
// impose constraint on the join.
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] { new EQConstant(y,
new Constant<String>("Paul")) }));
@@ -1233,7 +1233,7 @@
new NV(Predicate.Annotations.EVALUATION_CONTEXT,
BOpEvaluationContext.SHARDED),//
// constraint x == z
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] { new EQ(x, z) }),
// join is optional.
new NV(PipelineJoin.Annotations.OPTIONAL, true),//
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -379,7 +379,7 @@
new BOp[] { startOp },//
new NV(BOpBase.Annotations.BOP_ID, joinId),//
new NV(PipelineJoin.Annotations.PREDICATE, predOp),//
- new NV( PipelineJoin.Annotations.CONSTRAINTS,
+ new NV( Annotations.CONSTRAINTS,
new IConstraint[] { new INBinarySearch<String>(y, set) }));
// the expected solution (just one).
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -32,7 +32,6 @@
import com.bigdata.bop.engine.QueryEngine;
import com.bigdata.bop.engine.QueryLog;
import com.bigdata.bop.fed.QueryEngineFactory;
-import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.journal.Journal;
import com.bigdata.rdf.internal.XSDIntIV;
import com.bigdata.rdf.internal.constraints.CompareBOp;
@@ -341,7 +340,7 @@
* the IPredicate and apply them to the constructed join
* operator.
*/
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] {//
new NEConstant(product, new Constant(product53999
.getIV())) //
@@ -391,7 +390,7 @@
new NV(BOp.Annotations.BOP_ID, nextId++),//
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RELATION_NAME, spoRelation),
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] {//
new CompareBOp(new BOp[] {
simProperty1,
@@ -435,7 +434,7 @@
new NV(BOp.Annotations.BOP_ID, nextId++),//
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RELATION_NAME, spoRelation),
- new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new NV(Annotations.CONSTRAINTS,
new IConstraint[] {//
new CompareBOp(new BOp[] {
simProperty2,
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-16 19:26:10 UTC (rev 4111)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-16 19:30:52 UTC (rev 4112)
@@ -631,11 +631,6 @@
anns.add(new NV(PipelineJoin.Annotations.OPTIONAL, pred
.isOptional()));
- if (!constraints.isEmpty())
- anns.add(new NV(PipelineJoin.Annotations.CONSTRAINTS,
- constraints
- .toArray(new IConstraint[constraints.size()])));
-
/*
* Pull off annotations before we clear them from the predicate.
*/
@@ -653,6 +648,13 @@
pred = pred.clearAnnotations(new String[] { Annotations.SCOPE,
Annotations.QUADS, Annotations.DATASET });
+ if (!constraints.isEmpty()) {
+ // decorate the predicate with any constraints.
+ pred = (Predicate<?>) pred.setProperty(
+ IPredicate.Annotations.CONSTRAINTS, constraints
+ .toArray(new IConstraint[constraints.size()]));
+ }
+
if (quads) {
/*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 19:26:16
|
Revision: 4111
http://bigdata.svn.sourceforge.net/bigdata/?rev=4111&view=rev
Author: thompsonbry
Date: 2011-01-16 19:26:10 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Fixed unit test broken by the change to RWStore#getData() (it used to wrap things as an IllegalArgumentException, which had the wrong semantics when the underlying cause was an interrupt in the caller's thread resulting in a ClosedByInterruptException).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2011-01-16 18:59:43 UTC (rev 4110)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestRWJournal.java 2011-01-16 19:26:10 UTC (rev 4111)
@@ -41,7 +41,6 @@
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
-import com.bigdata.config.LongValidator;
import com.bigdata.journal.AbstractInterruptsTestCase;
import com.bigdata.journal.AbstractJournalTestCase;
import com.bigdata.journal.AbstractMRMWTestCase;
@@ -58,7 +57,7 @@
import com.bigdata.journal.Journal.Options;
import com.bigdata.rawstore.AbstractRawStoreTestCase;
import com.bigdata.rawstore.IRawStore;
-import com.bigdata.service.AbstractTransactionService;
+import com.bigdata.util.InnerCause;
/**
* Test suite for {@link BufferMode#DiskRW} journals.
@@ -1029,14 +1028,24 @@
assertEquals(0L, bs.getPhysicalAddress(faddr));
- try {
- rdBuf = bs.read(faddr); // should fail with illegal argument
- throw new RuntimeException("Fail");
- } catch (Exception ise) {
- assertTrue("Expected "+IllegalArgumentException.class.getName()+" reading from " + (faddr >> 32) + " instead got: "
- + ise, ise instanceof IllegalArgumentException
- );
- }
+ try {
+ // should fail with PhysicalAddressResolutionException
+ rdBuf = bs.read(faddr);
+ fail("Expecting: "
+ + PhysicalAddressResolutionException.class);
+ } catch (Throwable t) {
+ if (InnerCause.isInnerCause(t,
+ PhysicalAddressResolutionException.class)) {
+ if (log.isInfoEnabled()) {
+ log.info("Ignoring expected exception: " + t);
+ }
+ } else {
+ fail("Expected: "
+ + PhysicalAddressResolutionException.class
+ .getName() + " reading from "
+ + (faddr >> 32) + ", instead got: " + t, t);
+ }
+ }
} finally {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 18:59:49
|
Revision: 4110
http://bigdata.svn.sourceforge.net/bigdata/?rev=4110&view=rev
Author: thompsonbry
Date: 2011-01-16 18:59:43 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
More work on [1].
[1] https://sourceforge.net/apps/trac/bigdata/ticket/230
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/UnsupportedOperatorException.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-16 16:58:40 UTC (rev 4109)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-16 18:59:43 UTC (rev 4110)
@@ -35,6 +35,7 @@
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.LinkedBlockingQueue;
@@ -787,6 +788,20 @@
super.run();
+ } catch(Throwable t) {
+
+ // ensure query halts.
+ halt(t);
+
+ if (getCause() != null) {
+
+ // abnormal termination. wrap and rethrow.
+ throw new RuntimeException(t);
+
+ }
+
+ // otherwise ignore exception (normal termination).
+
} finally {
/*
@@ -859,11 +874,11 @@
final long begin = System.currentTimeMillis();
try {
t.call();
- } catch(Throwable t) {
- halt(t);
+ } catch(Throwable t2) {
+ halt(t2); // ensure query halts.
if (getCause() != null) {
- // Abnormal termination.
- throw new RuntimeException(getCause());
+ // Abnormal termination - wrap and rethrow.
+ throw new RuntimeException(t2);
}
// normal termination - swallow the exception.
} finally {
@@ -1251,8 +1266,17 @@
public Void call() throws Exception {
if (log.isDebugEnabled())
log.debug("Running chunk: " + this);
- ft.run(); // run
- ft.get(); // verify success
+ try {
+ ft.run(); // run
+ ft.get(); // verify success
+ } catch (Throwable t) {
+ halt(t); // ensure query halts.
+ if (getCause() != null) {
+ // abnormal termination - wrap and rethrow.
+ throw new Exception(t);
+ }
+ // otherwise ignore exception (normal completion).
+ }
// Done.
return null;
} // call()
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java 2011-01-16 16:58:40 UTC (rev 4109)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java 2011-01-16 18:59:43 UTC (rev 4110)
@@ -444,8 +444,8 @@
} catch(Throwable t) {
halt(t);
if (getCause() != null) {
- // Abnormal termination.
- throw getCause();
+ // Abnormal termination - wrap and rethrow.
+ throw new RuntimeException(t);
}
// normal termination - swallow the exception.
} finally {
@@ -466,10 +466,6 @@
} catch (Throwable ex1) {
- // Log an error.
- log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
- + ", bop=" + t.bop, ex1);
-
/*
* Mark the query as halted on this node regardless of whether
* we are able to communicate with the query controller.
@@ -481,7 +477,15 @@
// ensure halted.
halt(ex1);
+
+ if (getCause() != null) {
+ // Log an error.
+ log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
+ + ", bop=" + t.bop, ex1);
+
+ }
+
final HaltOpMessage msg = new HaltOpMessage(getQueryId(), t.bopId,
-1/*partitionId*/, serviceId, getCause()/*firstCauseIfError*/, t.sinkId,
0/*t.sinkMessagesOut.get()*/, t.altSinkId,
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java 2011-01-16 16:58:40 UTC (rev 4109)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java 2011-01-16 18:59:43 UTC (rev 4110)
@@ -385,6 +385,15 @@
* @see #getCause()
*/
protected boolean isNormalTerminationCause(final Throwable cause) {
+ if(isTerminationByInterrupt(cause))
+ return true;
+ if (InnerCause.isInnerCause(cause, RejectedExecutionException.class))
+ return true;
+ return false;
+ }
+
+ static public boolean isTerminationByInterrupt(final Throwable cause) {
+
if (InnerCause.isInnerCause(cause, InterruptedException.class))
return true;
if (InnerCause.isInnerCause(cause, CancellationException.class))
@@ -393,11 +402,11 @@
return true;
if (InnerCause.isInnerCause(cause, BufferClosedException.class))
return true;
- if (InnerCause.isInnerCause(cause, RejectedExecutionException.class))
- return true;
+
return false;
+
}
-
+
/**
* This logs all unexpected causes @ WARN (anything not reported as normal
* termination by {@link #isNormalTerminationCause(Throwable)}), not just
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-16 16:58:40 UTC (rev 4109)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-16 18:59:43 UTC (rev 4110)
@@ -118,6 +118,7 @@
import com.bigdata.striterator.Dechunkerator;
import com.bigdata.striterator.DistinctFilter;
import com.bigdata.striterator.IChunkedOrderedIterator;
+import com.bigdata.util.concurrent.Haltable;
/**
* Extended to rewrite Sesame {@link TupleExpr}s onto native {@link Rule}s and
@@ -393,7 +394,7 @@
} else {
// allow the query to fail
- throw ex;
+ throw new UnsupportedOperatorException(ex);
}
@@ -450,7 +451,7 @@
} else {
// allow the query to fail
- throw ex;
+ throw new UnsupportedOperatorException(ex);
}
@@ -507,7 +508,7 @@
} else {
// allow the query to fail
- throw ex;
+ throw new UnsupportedOperatorException(ex);
}
@@ -525,6 +526,12 @@
log.info("unrecognized value in query: " + ex.getValue());
}
return new EmptyIteration<BindingSet, QueryEvaluationException>();
+ } catch(UnsupportedOperatorException ex) {
+ /*
+ * Note: Do not wrap as a different exception type. The caller is
+ * looking for this.
+ */
+ throw new UnsupportedOperatorException(ex);
} catch (Throwable ex) {
// log.error("Remove log stmt:"+ex,ex);// FIXME remove this - I am just looking for the root cause of something in the SAIL.
throw new QueryEvaluationException(ex);
@@ -577,7 +584,7 @@
if (sop.isRightSideLeftJoin()) {
groupsToPrune.add(sopTree.getGroup(sop.getGroup()));
} else {
- throw ex;
+ throw new UnrecognizedValueException(ex);
}
}
}
@@ -652,8 +659,12 @@
if (sop.getGroup() == SOpTreeBuilder.ROOT_GROUP_ID) {
sopsToPrune.add(sop);
sesameFilters.add(filter);
- } else {
- throw ex;
+ } else {
+ /*
+ * Note: DO NOT wrap with a different exception type -
+ * the caller is looking for this.
+ */
+ throw new UnsupportedOperatorException(ex);
}
}
}
@@ -699,14 +710,43 @@
}
- return _evaluateNatively(query, bs, queryEngine, sesameFilters);
-
+ /*
+ * Begin native bigdata evaluation.
+ */
+ CloseableIteration<BindingSet, QueryEvaluationException> result = _evaluateNatively(
+ query, bs, queryEngine);// , sesameFilters);
+
+ /*
+ * Use the basic filter iterator for any remaining filters which will be
+ * evaluated by Sesame.
+ *
+ * Note: Some Sesame filters may pre-fetch one or more result(s). This
+ * could potentially cause the IRunningQuery to be asynchronously
+ * terminated by an interrupt. I have lifted the code to wrap the Sesame
+ * filters around the bigdata evaluation out of the code which starts
+ * the IRunningQuery evaluation in order to help clarify such
+ * circumstances as they might relate to [1].
+ *
+ * [1] https://sourceforge.net/apps/trac/bigdata/ticket/230
+ */
+ if (sesameFilters != null) {
+ for (Filter f : sesameFilters) {
+ if (log.isDebugEnabled()) {
+ log.debug("attaching sesame filter: " + f);
+ }
+ result = new FilterIterator(f, result, this);
+ }
+ }
+
+ return result;
+
}
- protected CloseableIteration<BindingSet, QueryEvaluationException>
+ private CloseableIteration<BindingSet, QueryEvaluationException>
_evaluateNatively(final PipelineOp query, final BindingSet bs,
- final QueryEngine queryEngine,
- final Collection<Filter> sesameFilters)
+ final QueryEngine queryEngine
+// , final Collection<Filter> sesameFilters
+ )
throws QueryEvaluationException {
IRunningQuery runningQuery = null;
@@ -717,10 +757,20 @@
/*
* Wrap up the native bigdata query solution iterator as Sesame
- * compatible iteration w/ any filters to be interpreted by Sesame.
+ * compatible iteration with materialized RDF Values.
*/
- return wrapQuery(runningQuery, sesameFilters);
+ return wrapQuery(runningQuery);//, sesameFilters);
+ } catch (UnsupportedOperatorException t) {
+ if (runningQuery != null) {
+ // ensure query is halted.
+ runningQuery.cancel(true/* mayInterruptIfRunning */);
+ }
+ /*
+ * Note: Do not wrap as a different exception type. The caller is
+ * looking for this.
+ */
+ throw new UnsupportedOperatorException(t);
} catch (Throwable t) {
if (runningQuery != null) {
// ensure query is halted.
@@ -734,20 +784,19 @@
/**
* Wrap the {@link IRunningQuery#iterator()}, returning a Sesame compatible
- * iteration which will visit the materialized binding sets.
+ * iteration which will visit Sesame binding sets having materialized RDF
+ * Values.
*
* @param runningQuery
* The query.
- * @param sesameFilters
- * Any filters to be applied by Sesame.
- *
+ *
* @return The iterator.
*
- * @throws QueryEvaluationException
+ * @throws QueryEvaluationException
*/
private CloseableIteration<BindingSet, QueryEvaluationException> wrapQuery(
- final IRunningQuery runningQuery,
- final Collection<Filter> sesameFilters) throws QueryEvaluationException {
+ final IRunningQuery runningQuery
+ ) throws QueryEvaluationException {
// The iterator draining the query solutions.
final IAsynchronousIterator<IBindingSet[]> it1 = runningQuery
@@ -759,7 +808,7 @@
new Dechunkerator<IBindingSet>(it1));
// Materialize IVs as RDF Values.
- CloseableIteration<BindingSet, QueryEvaluationException> result =
+ final CloseableIteration<BindingSet, QueryEvaluationException> result =
// Monitor IRunningQuery and cancel if Sesame iterator is closed.
new RunningQueryCloseableIteration<BindingSet, QueryEvaluationException>(runningQuery,
// Convert bigdata binding sets to Sesame binding sets.
@@ -768,16 +817,6 @@
new BigdataBindingSetResolverator(database, it2).start(
database.getExecutorService())));
- // use the basic filter iterator for remaining filters
- if (sesameFilters != null) {
- for (Filter f : sesameFilters) {
- if (log.isDebugEnabled()) {
- log.debug("attaching sesame filter: " + f);
- }
- result = new FilterIterator(f, result, this);
- }
- }
-
return result;
}
@@ -2286,6 +2325,18 @@
private Value value;
+ /**
+ * Wrap another instance of this exception class.
+ * @param cause
+ */
+ public UnrecognizedValueException(final UnrecognizedValueException cause) {
+
+ super(cause);
+
+ this.value = cause.value;
+
+ }
+
public UnrecognizedValueException(final Value value) {
this.value = value;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java 2011-01-16 16:58:40 UTC (rev 4109)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java 2011-01-16 18:59:43 UTC (rev 4110)
@@ -74,8 +74,8 @@
* Exception thrown by the runningQuery.
*/
if (runningQuery.getCause() != null) {
- // abnormal termination.
- throw (X) new QueryEvaluationException(runningQuery.getCause());
+ // abnormal termination - wrap and rethrow.
+ throw (X) new QueryEvaluationException(e);
}
// otherwise this is normal termination.
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/UnsupportedOperatorException.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/UnsupportedOperatorException.java 2011-01-16 16:58:40 UTC (rev 4109)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/UnsupportedOperatorException.java 2011-01-16 18:59:43 UTC (rev 4110)
@@ -2,6 +2,13 @@
import org.openrdf.query.algebra.QueryModelNode;
+/**
+ * An exception thrown when an operator can not be translated into native
+ * bigdata evaluation. This is used to detect such problems and then optionally
+ * delegate the operator to openrdf.
+ *
+ * @author mrpersonick
+ */
public class UnsupportedOperatorException extends RuntimeException {
/**
@@ -11,6 +18,16 @@
private QueryModelNode operator;
+ /**
+ * Wrap with another instance of this class.
+ *
+ * @param cause
+ */
+ public UnsupportedOperatorException(final UnsupportedOperatorException cause) {
+ super(cause);
+ this.operator = cause.operator;
+ }
+
public UnsupportedOperatorException(final QueryModelNode operator) {
this.operator = operator;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 16:58:46
|
Revision: 4109
http://bigdata.svn.sourceforge.net/bigdata/?rev=4109&view=rev
Author: thompsonbry
Date: 2011-01-16 16:58:40 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Found and closed off the source of the stack trace documented at [1]. This trace was actually just a logged error rather than being thrown out of the query to the SPARQL client. The exception occurred when startOp() attempted to start a new chunk task but the running query had been asynchronously terminated by an interrupt. The error handling logic now logs an error iff IHaltable#getCause() returns non-null.
[1] https://sourceforge.net/apps/trac/bigdata/ticket/230#comment:1
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-16 16:58:40 UTC (rev 4109)
@@ -890,10 +890,6 @@
} catch (Throwable ex1) {
- // Log an error.
- log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
- + ", bop=" + t.bop, ex1);
-
/*
* Mark the query as halted on this node regardless of whether
* we are able to communicate with the query controller.
@@ -906,6 +902,14 @@
// ensure halted.
halt(ex1);
+ if (getCause() != null) {
+
+ // Log an error (abnormal termination only).
+ log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
+ + ", bop=" + t.bop, ex1);
+
+ }
+
final HaltOpMessage msg = new HaltOpMessage(getQueryId(), t.bopId,
t.partitionId, serviceId, getCause()/*firstCauseIfError*/, t.sinkId,
t.sinkMessagesOut.get(), t.altSinkId,
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 16:38:44
|
Revision: 4108
http://bigdata.svn.sourceforge.net/bigdata/?rev=4108&view=rev
Author: thompsonbry
Date: 2011-01-16 16:38:37 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Working on https://sourceforge.net/apps/trac/bigdata/ticket/230 (occasional errors reported through to the SPARQL client from normal termination rooted in an interrupt of a query when a LIMIT is satisfied).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/CancelQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SliceOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/solutions/TestSliceOp.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsLiteral.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/IHaltable.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -305,17 +305,17 @@
public IRunningQuery call() throws Exception {
+ IRunningQuery runningSubquery = null;
IAsynchronousIterator<IBindingSet[]> subquerySolutionItr = null;
try {
final QueryEngine queryEngine = parentContext.getRunningQuery()
.getQueryEngine();
- final IRunningQuery runningQuery = queryEngine
- .eval(subQueryOp);
+ runningSubquery = queryEngine.eval(subQueryOp);
// Iterator visiting the subquery solutions.
- subquerySolutionItr = runningQuery.iterator();
+ subquerySolutionItr = runningSubquery.iterator();
// Copy solutions from the subquery to the query.
BOpUtility.copy(subquerySolutionItr, parentContext
@@ -323,20 +323,31 @@
null/* stats */);
// wait for the subquery.
- runningQuery.get();
+ runningSubquery.get();
// done.
- return runningQuery;
+ return runningSubquery;
} catch (Throwable t) {
- /*
- * If a subquery fails, then propagate the error to the
- * parent and rethrow the first cause error out of the
- * subquery.
- */
- throw new RuntimeException(ControllerTask.this.context
- .getRunningQuery().halt(t));
+ if (runningSubquery == null
+ || runningSubquery.getCause() != null) {
+ /*
+ * If things fail before we start the subquery, or if a
+ * subquery fails (due to abnormal termination), then
+ * propagate the error to the parent and rethrow the
+ * first cause error out of the subquery.
+ *
+ * Note: IHaltable#getCause() considers exceptions
+ * triggered by an interrupt to be normal termination.
+ * Such exceptions are NOT propagated here and WILL NOT
+ * cause the parent query to terminate.
+ */
+ throw new RuntimeException(ControllerTask.this.context
+ .getRunningQuery().halt(runningSubquery.getCause()));
+ }
+
+ return runningSubquery;
} finally {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -27,7 +27,6 @@
package com.bigdata.bop.controller;
-import java.nio.channels.ClosedByInterruptException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -43,10 +42,8 @@
import com.bigdata.bop.engine.IRunningQuery;
import com.bigdata.bop.engine.LocalChunkMessage;
import com.bigdata.bop.engine.QueryEngine;
-import com.bigdata.relation.accesspath.BufferClosedException;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
-import com.bigdata.util.InnerCause;
/**
* For each binding set presented, this operator executes a subquery. Any
@@ -425,7 +422,7 @@
throw ex;
}
-
+
if (ncopied == 0L && optional) {
/*
@@ -442,28 +439,47 @@
} catch (Throwable t) {
- /*
- * Note: SliceOp will cause other operators to be
- * interrupted during normal evaluation. Therefore, while
- * these exceptions should cause the subquery to terminate,
- * they should not be reported as errors to the parent
- * query.
- */
- if (!InnerCause.isInnerCause(t, InterruptedException.class)
- && !InnerCause.isInnerCause(t, BufferClosedException.class)
- && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)) {
-
- /*
- * If a subquery fails, then propagate the error to the
- * parent and rethrow the first cause error out of the
- * subquery.
- */
- throw new RuntimeException(ControllerTask.this.context
- .getRunningQuery().halt(t));
+// /*
+// * Note: SliceOp will cause other operators to be
+// * interrupted during normal evaluation. Therefore, while
+// * these exceptions should cause the subquery to terminate,
+// * they should not be reported as errors to the parent
+// * query.
+// */
+// if (!InnerCause.isInnerCause(t, InterruptedException.class)
+// && !InnerCause.isInnerCause(t, BufferClosedException.class)
+// && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)) {
+//
+// /*
+// * If a subquery fails, then propagate the error to the
+// * parent and rethrow the first cause error out of the
+// * subquery.
+// */
+// throw new RuntimeException(ControllerTask.this.context
+// .getRunningQuery().halt(t));
+//
+// }
+//
+// return runningSubquery;
- }
-
- return runningSubquery;
+ if (runningSubquery == null
+ || runningSubquery.getCause() != null) {
+ /*
+ * If things fail before we start the subquery, or if a
+ * subquery fails (due to abnormal termination), then
+ * propagate the error to the parent and rethrow the
+ * first cause error out of the subquery.
+ *
+ * Note: IHaltable#getCause() considers exceptions
+ * triggered by an interrupt to be normal termination.
+ * Such exceptions are NOT propagated here and WILL NOT
+ * cause the parent query to terminate.
+ */
+ throw new RuntimeException(ControllerTask.this.context
+ .getRunningQuery().halt(runningSubquery.getCause()));
+ }
+
+ return runningSubquery;
} finally {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -28,7 +28,6 @@
package com.bigdata.bop.engine;
import java.nio.ByteBuffer;
-import java.nio.channels.ClosedByInterruptException;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
@@ -51,12 +50,11 @@
import com.bigdata.bop.solutions.SliceOp;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.ITx;
-import com.bigdata.relation.accesspath.BufferClosedException;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.IBlockingBuffer;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.Haltable;
+import com.bigdata.util.concurrent.IHaltable;
/**
* Abstract base class for various {@link IRunningQuery} implementations. The
@@ -184,7 +182,7 @@
* Note: This is exposed to the {@link QueryEngine} to let it cache the
* {@link Future} for recently finished queries.
*/
- final protected Future<Void> getFuture() {
+ final protected IHaltable<Void> getFuture() {
return future;
@@ -665,7 +663,7 @@
if (runState.isAllDone()) {
// Normal termination.
- halt();
+ halt((Void)null);
}
@@ -771,14 +769,14 @@
}
- public void halt() {
+ final public void halt(final Void v) {
- lock.lock();
+ lock.lock();
try {
// signal normal completion.
- future.halt((Void) null);
+ future.halt((Void) v);
// interrupt anything which is running.
cancel(true/* mayInterruptIfRunning */);
@@ -791,7 +789,7 @@
}
- public Throwable halt(final Throwable t) {
+ final public <T extends Throwable> T halt(final T t) {
if (t == null)
throw new IllegalArgumentException();
@@ -802,23 +800,8 @@
try {
- /*
- * Note: SliceOp will cause other operators to be interrupted
- * during normal evaluation so it is not useful to log an
- * InterruptedException @ ERROR.
- */
- if (!InnerCause.isInnerCause(t, InterruptedException.class)
- && !InnerCause.isInnerCause(t, BufferClosedException.class)
- && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)) {
- log.error(toString(), t);
- // signal error condition.
- return future.halt(t);
- } else {
- // normal termination.
- future.halt((Void)null/* result */);
- // the caller's cause.
- return t;
- }
+ // halt the query, return [t].
+ return future.halt(t);
} finally {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -27,7 +27,6 @@
*/
package com.bigdata.bop.engine;
-import java.nio.channels.ClosedByInterruptException;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -861,20 +860,12 @@
try {
t.call();
} catch(Throwable t) {
- /*
- * Note: SliceOp will cause other operators to be
- * interrupted during normal evaluation. Therefore, while
- * these exceptions should cause the query to terminate,
- * they should not be reported as errors to the query
- * controller.
- */
- if (!InnerCause.isInnerCause(t, InterruptedException.class)
- && !InnerCause.isInnerCause(t, BufferClosedException.class)
- && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)
- ) {
- // Not an error that we should ignore.
- throw t;
- }
+ halt(t);
+ if (getCause() != null) {
+ // Abnormal termination.
+ throw new RuntimeException(getCause());
+ }
+ // normal termination - swallow the exception.
} finally {
t.context.getStats().elapsed.add(System.currentTimeMillis()
- begin);
@@ -911,10 +902,12 @@
* error message is necessary in order to catch errors in
* clientProxy.haltOp() (above and below).
*/
- final Throwable firstCause = halt(ex1);
+ // ensure halted.
+ halt(ex1);
+
final HaltOpMessage msg = new HaltOpMessage(getQueryId(), t.bopId,
- t.partitionId, serviceId, firstCause, t.sinkId,
+ t.partitionId, serviceId, getCause()/*firstCauseIfError*/, t.sinkId,
t.sinkMessagesOut.get(), t.altSinkId,
t.altSinkMessagesOut.get(), t.context.getStats());
try {
@@ -1484,7 +1477,7 @@
// return sink.flush();
}
- public void abort(Throwable cause) {
+ public void abort(final Throwable cause) {
open = false;
q.halt(cause);
// sink.abort(cause);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -29,7 +29,6 @@
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.Future;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
@@ -40,6 +39,7 @@
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.striterator.ICloseableIterator;
+import com.bigdata.util.concurrent.IHaltable;
/**
* Non-Remote interface exposing a limited set of the state of an executing
@@ -48,7 +48,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface IRunningQuery extends Future<Void>{
+public interface IRunningQuery extends IHaltable<Void> {
/**
* The query.
@@ -134,35 +134,35 @@
public long getElapsed();
/**
- * Cancel the running query (normal termination).
- * <p>
- * Note: This method provides a means for an operator to indicate that the
- * query should halt immediately for reasons other than abnormal
- * termination.
- * <p>
- * Note: For abnormal termination of a query, just throw an exception out of
- * the query operator implementation.
- */
- void halt();
-
- /**
- * Cancel the query (abnormal termination).
- *
- * @param t
- * The cause.
- *
- * @return The first cause.
- *
- * @throws IllegalArgumentException
- * if the argument is <code>null</code>.
- */
- Throwable halt(final Throwable t);
-
- /**
- * Return the cause if the query was terminated by an exception.
- * @return
- */
- Throwable getCause();
+// * Cancel the running query (normal termination).
+// * <p>
+// * Note: This method provides a means for an operator to indicate that the
+// * query should halt immediately for reasons other than abnormal
+// * termination.
+// * <p>
+// * Note: For abnormal termination of a query, just throw an exception out of
+// * the query operator implementation.
+// */
+// void halt();
+//
+// /**
+// * Cancel the query (abnormal termination).
+// *
+// * @param t
+// * The cause.
+// *
+// * @return The argument.
+// *
+// * @throws IllegalArgumentException
+// * if the argument is <code>null</code>.
+// */
+// Throwable halt(final Throwable t);
+//
+// /**
+// * Return the cause if the query was terminated by an exception.
+// * @return
+// */
+// Throwable getCause();
/**
* Return an iterator which will drain the solutions from the query. The
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -58,13 +58,14 @@
import com.bigdata.btree.IndexSegment;
import com.bigdata.btree.view.FusedView;
import com.bigdata.journal.IIndexManager;
-import com.bigdata.rdf.sail.bench.NanoSparqlClient;
+import com.bigdata.rdf.sail.bench.NanoSparqlServer;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
import com.bigdata.resources.IndexManager;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
import com.bigdata.util.concurrent.DaemonThreadFactory;
+import com.bigdata.util.concurrent.IHaltable;
/**
* A class managing execution of concurrent queries against a local
@@ -367,13 +368,13 @@
* enough that we can not have a false cache miss on a system which is
* heavily loaded by a bunch of light queries.
*/
- private LinkedHashMap<UUID, Future<Void>> doneQueries = new LinkedHashMap<UUID,Future<Void>>(
+ private LinkedHashMap<UUID, IHaltable<Void>> doneQueries = new LinkedHashMap<UUID,IHaltable<Void>>(
16/* initialCapacity */, .75f/* loadFactor */, true/* accessOrder */) {
private static final long serialVersionUID = 1L;
@Override
- protected boolean removeEldestEntry(Map.Entry<UUID, Future<Void>> eldest) {
+ protected boolean removeEldestEntry(Map.Entry<UUID, IHaltable<Void>> eldest) {
return size() > 100/* maximumCacheCapacity */;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -442,20 +442,12 @@
try {
t.call();
} catch(Throwable t) {
- /*
- * Note: SliceOp will cause other operators to be
- * interrupted during normal evaluation. Therefore, while
- * these exceptions should cause the query to terminate,
- * they should not be reported as errors to the query
- * controller.
- */
- if (!InnerCause.isInnerCause(t, InterruptedException.class)
- && !InnerCause.isInnerCause(t, BufferClosedException.class)
- && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)
- ) {
- // Not an error that we should ignore.
- throw t;
- }
+ halt(t);
+ if (getCause() != null) {
+ // Abnormal termination.
+ throw getCause();
+ }
+ // normal termination - swallow the exception.
} finally {
t.context.getStats().elapsed.add(System.currentTimeMillis()
- begin);
@@ -486,10 +478,12 @@
* error message is necessary in order to catch errors in
* clientProxy.haltOp() (above and below).
*/
- final Throwable firstCause = halt(ex1);
+
+ // ensure halted.
+ halt(ex1);
final HaltOpMessage msg = new HaltOpMessage(getQueryId(), t.bopId,
- -1/*partitionId*/, serviceId, firstCause, t.sinkId,
+ -1/*partitionId*/, serviceId, getCause()/*firstCauseIfError*/, t.sinkId,
0/*t.sinkMessagesOut.get()*/, t.altSinkId,
0/*t.altSinkMessagesOut.get()*/, t.context.getStats());
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/CancelQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/CancelQuery.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/CancelQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -34,7 +34,7 @@
public void run() {
if (cause == null)
- q.halt();
+ q.halt((Void)null);
else
q.halt(cause);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -27,6 +27,7 @@
package com.bigdata.bop.join;
+import java.nio.channels.ClosedByInterruptException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -77,6 +78,7 @@
import com.bigdata.service.DataService;
import com.bigdata.striterator.IChunkedOrderedIterator;
import com.bigdata.striterator.IKeyOrder;
+import com.bigdata.util.InnerCause;
import com.bigdata.util.concurrent.Haltable;
import com.bigdata.util.concurrent.LatchedExecutor;
@@ -743,9 +745,8 @@
* This is used for processing errors and also if this task is
* interrupted (because the sink has been closed).
*/
-
halt(t);
-
+
// reset the unsync buffers.
try {
// resetUnsyncBuffers();
@@ -1056,11 +1057,9 @@
return null;
} catch (Throwable t) {
+
+ throw new RuntimeException(halt(t));
- halt(t);
-
- throw new RuntimeException(t);
-
}
}
@@ -1692,10 +1691,8 @@
} catch (Throwable t) {
- halt(t);
+ throw new RuntimeException(halt(t));
- throw new RuntimeException(t);
-
} finally {
itr.close();
@@ -1900,10 +1897,8 @@
} catch (Throwable t) {
- halt(t);
+ throw new RuntimeException(halt(t));
- throw new RuntimeException(t);
-
} finally {
itr.close();
@@ -2092,10 +2087,8 @@
} catch (Throwable t) {
- halt(t);
+ throw new RuntimeException(halt(t));
- throw new RuntimeException(t);
-
}
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SliceOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SliceOp.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SliceOp.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -336,7 +336,7 @@
if (log.isInfoEnabled())
log.info("Slice will interrupt query.");
- context.getRunningQuery().halt();
+ context.getRunningQuery().halt((Void) null);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/Haltable.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -58,13 +58,15 @@
* <p>
* This class embeds certain knowledge about which exceptions may be observed
* during normal termination of asynchronous processes using I/O, thread pools,
- * and {@link IBlockingBuffer}s.
+ * and {@link IBlockingBuffer}s. See
+ * {@link #isNormalTerminationCause(Throwable)} for a list of the
+ * {@link Throwable} causes which are treated as normal termination.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id: AbstractHaltableProcess.java 2265 2009-10-26 12:51:06Z
* thompsonbry $
*/
-public class Haltable<V> implements Future<V> {
+public class Haltable<V> implements IHaltable<V> {
private final transient static Logger log = Logger
.getLogger(Haltable.class);
@@ -140,25 +142,26 @@
* the cause out of their own context.</strong>
*
* @param cause
- * The cause.
+ * The cause (required).
*
* @return The argument.
- *
- * @throws IllegalArgumentException
- * if the cause is <code>null</code>.
*/
final public <T extends Throwable> T halt(final T cause) {
final boolean didHalt;
lock.lock();
try {
if (didHalt = !halt) {
+ /*
+ * This is the first cause.
+ */
+ // note the first cause (and handle an illegal null if found).
firstCause = (cause != null ? cause
: new IllegalArgumentException());
+ // note if abnormal termination (firstCause only)
+ error = !isNormalTerminationCause(firstCause);
try {
// signal *all* listeners.
halted.signalAll();
- // note if abnormal termination (firstCause only)
- error = !isNormalTerminationCause(cause);
} finally {
halt = true; // volatile write.
}
@@ -178,18 +181,20 @@
return cause;
}
- /**
- * Return unless processing has been halted. The method should be invoked
- * from within the execution of the process itself so that it may notice
- * asynchronous termination. It will throw out the wrapper first cause if
- * the process is halted. External processes waiting on the {@link Future}
- * interface should use {@link #isDone()} which does not have the semantics
- * of asserting that the process should still be running.
- *
- * @throws RuntimeException
- * wrapping the {@link #firstCause} iff processing has been
- * halted.
- */
+ /**
+ * Return unless processing has been halted. The method should be invoked
+ * from within the execution of the process itself so that it may notice
+ * asynchronous termination. It will throw out the wrapped first cause if
+ * the process is halted.
+ * <p>
+ * Note: External processes waiting on the {@link Future} interface should
+ * use {@link #isDone()} which does not have the semantics of asserting that
+ * the process should still be running.
+ *
+ * @throws RuntimeException
+ * wrapping the {@link #firstCause} iff processing has been
+ * halted.
+ */
final public void halted() {
if (halt) {
@@ -313,24 +318,25 @@
}
- /**
- * Return the first {@link Throwable} which caused this process to halt, but
- * only for abnormal termination.
- *
- * @return The first {@link Throwable} which caused this process to halt and
- * <code>null</code> if the process has not halted or if it halted
- * through normal termination.
- */
- final public Throwable getCause() {
+ final public Throwable getCause() {
- if (!halt)
- return null;
+ lock.lock();
+ try {
+
+ if (!halt)
+ return null;
- if (!error)
- return null;
+ if (!error)
+ return null;
- return firstCause;
+ return firstCause;
+ } finally {
+
+ lock.unlock();
+
+ }
+
}
/**
@@ -344,50 +350,51 @@
}
- /**
- * Return <code>true</code> if the {@link Throwable} is a known normal
- * termination cause for the process. The method inspects the stack trace,
- * examining both the outer and {@link InnerCause}s. The following causes
- * are interpreted as normal termination:
- * <dl>
- * <dt>{@link InterruptedException}</dt>
- * <dd>The process was terminated by an interrupt. Interrupts are typically
- * used to terminate asynchronous processes when their production limit has
- * been satisfied or the consumer otherwise chooses to
- * {@link IAsynchronousIterator#close()} the iterator through which they are
- * consuming results from the process.</dd>
- * <dt>{@link CancellationException}</dt>
- * <dd>A process has been canceled using its {@link Future}.</dd>
- * <dt>{@link ClosedByInterruptException}</dt>
- * <dd>A process was interrupted during an IO operation.</dd>
- * <dt>{@link RejectedExecutionException}</dt>
- * <dd>A process was not executed because the pool against which it was
- * submitted had been shutdown (this of course implies that the work queue
- * was unbounded).</dd>
- * <dt>{@link BufferClosedException}</dt>
- * <dd>The {@link IBlockingBuffer} on which the process was writing was
- * asynchronously closed.</dd>
- * </dl>
- *
- * @param cause
- * The {@link Throwable}.
- *
- * @return <code>true</code> if the {@link Throwable} indicates normal
- * termination.
- */
+ /**
+ * Return <code>true</code> if the {@link Throwable} is a known normal
+ * termination cause for the process. The method inspects the stack trace,
+ * examining both the outer and {@link InnerCause}s. The following causes
+ * are interpreted as normal termination:
+ * <dl>
+ * <dt>{@link InterruptedException}</dt>
+ * <dd>The process was terminated by an interrupt. Interrupts are typically
+ * used to terminate asynchronous processes when their production limit has
+ * been satisfied or the consumer otherwise chooses to
+ * {@link IAsynchronousIterator#close()} the iterator through which they are
+ * consuming results from the process.</dd>
+ * <dt>{@link CancellationException}</dt>
+ * <dd>A process has been canceled using its {@link Future}.</dd>
+ * <dt>{@link ClosedByInterruptException}</dt>
+ * <dd>A process was interrupted during an IO operation.</dd>
+ * <dt>{@link BufferClosedException}</dt>
+ * <dd>The {@link IBlockingBuffer} on which the process was writing was
+ * asynchronously closed.</dd>
+ * <dt>{@link RejectedExecutionException}</dt>
+ * <dd>A process was not executed because the pool against which it was
+ * submitted had been shutdown (this of course implies that the work queue
+ * was unbounded as a bounded pool will throw this exception if the work
+ * queue is full).</dd>
+ * </dl>
+ *
+ * @param cause
+ * The {@link Throwable}.
+ *
+ * @return <code>true</code> if the {@link Throwable} indicates normal
+ * termination.
+ *
+ * @see #getCause()
+ */
protected boolean isNormalTerminationCause(final Throwable cause) {
-// if (InnerCause.isInnerCause(cause, CancelledException.class))
-// return true;
if (InnerCause.isInnerCause(cause, InterruptedException.class))
return true;
if (InnerCause.isInnerCause(cause, CancellationException.class))
return true;
if (InnerCause.isInnerCause(cause, ClosedByInterruptException.class))
return true;
+ if (InnerCause.isInnerCause(cause, BufferClosedException.class))
+ return true;
if (InnerCause.isInnerCause(cause, RejectedExecutionException.class))
return true;
- if (InnerCause.isInnerCause(cause, BufferClosedException.class))
- return true;
return false;
}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/IHaltable.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/IHaltable.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/concurrent/IHaltable.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -0,0 +1,46 @@
+package com.bigdata.util.concurrent;
+
+import java.util.concurrent.Future;
+
+/**
+ * Interface extends {@link Future} and provides an interface for managing the
+ * termination of a process from within that process.
+ *
+ * @param <V>
+ * The generic type of the computation to which the {@link Future}
+ * evaluates.
+ */
+public interface IHaltable<V> extends Future<V> {
+
+ /**
+ * Halt (normal termination).
+ */
+ void halt(V v);
+
+ /**
+ * Halt (exception thrown). <strong>The caller is responsible for throwing
+ * their given <i>cause</i> out of their own context.</strong> As a
+ * convenience, this method returns the given <i>cause</>.
+ *
+ * @param cause
+ * The cause (required).
+ *
+ * @return The argument.
+ */
+ <T extends Throwable> T halt(T cause);
+
+ /**
+ * Return the first {@link Throwable} which caused this process to halt, but
+ * only for abnormal termination.
+ * <p>
+ * {@link IHaltable} considers exceptions triggered by an interrupt to be
+ * normal termination of the process and will return <code>null</code> for
+ * such exceptions.
+ *
+ * @return The first {@link Throwable} which caused this process to halt and
+ * <code>null</code> if the process has not halted or if it halted
+ * through normal termination.
+ */
+ Throwable getCause();
+
+}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -80,17 +80,16 @@
return indexManager;
}
- /**
- * NOP (you have to test things like slices with a full integration).
- */
- public void halt() {
- log.warn("Mock object does not implement halt()");
- }
+ @Override
+ public void halt(Void v) {
+ log.warn("Mock object does not implement halt(Void)");
+ }
- public Throwable halt(Throwable t) {
+ @Override
+ public <T extends Throwable> T halt(T cause) {
log.warn("Mock object does not implement halt(Throwable)");
- return t;
- }
+ return cause;
+ }
public QueryEngine getQueryEngine() {
throw new UnsupportedOperationException();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/solutions/TestSliceOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/solutions/TestSliceOp.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/solutions/TestSliceOp.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -814,7 +814,7 @@
* Overridden to close the sink so the slice will terminate.
*/
@Override
- public void halt() {
+ public void halt(Void v) {
sink.close();
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsLiteral.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsLiteral.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/IsLiteral.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -55,7 +55,7 @@
/**
* Required deep copy constructor.
*/
- public IsLiteral(final IsInline op) {
+ public IsLiteral(final IsLiteral op) {
super(op);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -23,12 +23,13 @@
*/
package com.bigdata.rdf.internal.constraints;
+import java.util.Map;
+
import org.openrdf.query.algebra.MathExpr.MathOp;
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
-import com.bigdata.bop.IVariable;
import com.bigdata.bop.ImmutableBOp;
import com.bigdata.bop.NV;
import com.bigdata.rdf.internal.IV;
@@ -71,6 +72,27 @@
}
+ /**
+ * Required shallow copy constructor.
+ *
+ * @param args
+ * The operands.
+ * @param op
+ * The operation.
+ */
+ public MathBOp(final BOp[] args, Map<String,Object> anns) {
+
+ super(args,anns);
+
+ if (args.length != 2 || args[0] == null || args[1] == null
+ || getProperty(Annotations.OP) == null) {
+
+ throw new IllegalArgumentException();
+
+ }
+
+ }
+
/**
*
* @param left
@@ -84,11 +106,8 @@
public MathBOp(final IValueExpression<IV> left,
final IValueExpression<IV> right, final MathOp op) {
- super(new BOp[] { left, right }, NV.asMap(new NV(Annotations.OP, op)));
+ this(new BOp[] { left, right }, NV.asMap(new NV(Annotations.OP, op)));
- if (left == null || right == null || op == null)
- throw new IllegalArgumentException();
-
}
// /**
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -24,13 +24,11 @@
import org.openrdf.query.algebra.And;
import org.openrdf.query.algebra.Bound;
import org.openrdf.query.algebra.Compare;
-import org.openrdf.query.algebra.Compare.CompareOp;
import org.openrdf.query.algebra.Filter;
import org.openrdf.query.algebra.Group;
import org.openrdf.query.algebra.Join;
import org.openrdf.query.algebra.LeftJoin;
import org.openrdf.query.algebra.MathExpr;
-import org.openrdf.query.algebra.MathExpr.MathOp;
import org.openrdf.query.algebra.MultiProjection;
import org.openrdf.query.algebra.Not;
import org.openrdf.query.algebra.Or;
@@ -43,17 +41,18 @@
import org.openrdf.query.algebra.Regex;
import org.openrdf.query.algebra.SameTerm;
import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.StatementPattern.Scope;
import org.openrdf.query.algebra.TupleExpr;
import org.openrdf.query.algebra.UnaryTupleOperator;
import org.openrdf.query.algebra.Union;
import org.openrdf.query.algebra.ValueConstant;
import org.openrdf.query.algebra.ValueExpr;
import org.openrdf.query.algebra.Var;
+import org.openrdf.query.algebra.Compare.CompareOp;
+import org.openrdf.query.algebra.MathExpr.MathOp;
+import org.openrdf.query.algebra.StatementPattern.Scope;
import org.openrdf.query.algebra.evaluation.impl.EvaluationStrategyImpl;
import org.openrdf.query.algebra.evaluation.iterator.FilterIterator;
import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-import org.openrdf.query.parser.serql.AnonymousVarGenerator;
import com.bigdata.bop.BOp;
import com.bigdata.bop.BOpUtility;
@@ -62,12 +61,12 @@
import com.bigdata.bop.IConstant;
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
-import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.IValueExpression;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.ap.Predicate;
import com.bigdata.bop.constraint.AND;
import com.bigdata.bop.constraint.BOUND;
@@ -91,9 +90,9 @@
import com.bigdata.rdf.sail.sop.SOp;
import com.bigdata.rdf.sail.sop.SOp2BOpUtility;
import com.bigdata.rdf.sail.sop.SOpTree;
-import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
import com.bigdata.rdf.sail.sop.SOpTreeBuilder;
import com.bigdata.rdf.sail.sop.UnsupportedOperatorException;
+import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
import com.bigdata.rdf.spo.DefaultGraphSolutionExpander;
import com.bigdata.rdf.spo.ExplicitSPOFilter;
import com.bigdata.rdf.spo.ISPO;
@@ -526,9 +525,8 @@
log.info("unrecognized value in query: " + ex.getValue());
}
return new EmptyIteration<BindingSet, QueryEvaluationException>();
- } catch (QueryEvaluationException ex) {
- throw ex;
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+// log.error("Remove log stmt:"+ex,ex);// FIXME remove this - I am just looking for the root cause of something in the SAIL.
throw new QueryEvaluationException(ex);
}
}
@@ -716,50 +714,74 @@
// Submit query for evaluation.
runningQuery = queryEngine.eval(query);
-
- // Iterator draining the query results.
- final IAsynchronousIterator<IBindingSet[]> it1 =
- runningQuery.iterator();
-
- // De-chunk the IBindingSet[] visited by that iterator.
- final IChunkedOrderedIterator<IBindingSet> it2 =
- new ChunkedWrappedIterator<IBindingSet>(
- new Dechunkerator<IBindingSet>(it1));
- // Materialize IVs as RDF Values.
- CloseableIteration<BindingSet, QueryEvaluationException> result =
- // Monitor IRunningQuery and cancel if Sesame iterator is closed.
- new RunningQueryCloseableIteration<BindingSet, QueryEvaluationException>(runningQuery,
- // Convert bigdata binding sets to Sesame binding sets.
- new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
- // Materialize IVs as RDF Values.
- new BigdataBindingSetResolverator(database, it2).start(
- database.getExecutorService())));
-
-// No - will deadlock if buffer fills up
-// // Wait for the Future (checks for errors).
-// runningQuery.get();
-
- // use the basic filter iterator for remaining filters
- if (sesameFilters != null) {
- for (Filter f : sesameFilters) {
- if (log.isDebugEnabled()) {
- log.debug("attaching sesame filter: " + f);
- }
- result = new FilterIterator(f, result, this);
- }
- }
+ /*
+ * Wrap up the native bigdata query solution iterator as Sesame
+ * compatible iteration w/ any filters to be interpreted by Sesame.
+ */
+ return wrapQuery(runningQuery, sesameFilters);
- return result;
-
} catch (Throwable t) {
- if (runningQuery != null)
+ if (runningQuery != null) {
+ // ensure query is halted.
runningQuery.cancel(true/* mayInterruptIfRunning */);
+ }
+// log.error("Remove log stmt"+t,t);// FIXME remove this - I am just looking for the root cause of something in the SAIL.
throw new QueryEvaluationException(t);
}
}
+ /**
+ * Wrap the {@link IRunningQuery#iterator()}, returning a Sesame compatible
+ * iteration which will visit the materialized binding sets.
+ *
+ * @param runningQuery
+ * The query.
+ * @param sesameFilters
+ * Any filters to be applied by Sesame.
+ *
+ * @return The iterator.
+ *
+ * @throws QueryEvaluationException
+ */
+ private CloseableIteration<BindingSet, QueryEvaluationException> wrapQuery(
+ final IRunningQuery runningQuery,
+ final Collection<Filter> sesameFilters) throws QueryEvaluationException {
+
+ // The iterator draining the query solutions.
+ final IAsynchronousIterator<IBindingSet[]> it1 = runningQuery
+ .iterator();
+
+ // De-chunk the IBindingSet[] visited by that iterator.
+ final IChunkedOrderedIterator<IBindingSet> it2 =
+ new ChunkedWrappedIterator<IBindingSet>(
+ new Dechunkerator<IBindingSet>(it1));
+
+ // Materialize IVs as RDF Values.
+ CloseableIteration<BindingSet, QueryEvaluationException> result =
+ // Monitor IRunningQuery and cancel if Sesame iterator is closed.
+ new RunningQueryCloseableIteration<BindingSet, QueryEvaluationException>(runningQuery,
+ // Convert bigdata binding sets to Sesame binding sets.
+ new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
+ // Materialize IVs as RDF Values.
+ new BigdataBindingSetResolverator(database, it2).start(
+ database.getExecutorService())));
+
+ // use the basic filter iterator for remaining filters
+ if (sesameFilters != null) {
+ for (Filter f : sesameFilters) {
+ if (log.isDebugEnabled()) {
+ log.debug("attaching sesame filter: " + f);
+ }
+ result = new FilterIterator(f, result, this);
+ }
+ }
+
+ return result;
+
+ }
+
// /**
// * This is the method that will attempt to take a top-level join or left
// * join and turn it into a native bigdata rule. The Sesame operators Join
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -3437,6 +3437,8 @@
} catch (QueryEvaluationException e) {
+// log.error("Remove log stmt"+e,e);// FIXME remove this - I am just looking for the root cause of something in the SAIL.
+
throw new SailException(e);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java 2011-01-16 00:59:18 UTC (rev 4107)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java 2011-01-16 16:38:37 UTC (rev 4108)
@@ -2,7 +2,7 @@
import info.aduna.iteration.CloseableIteration;
-import java.util.concurrent.ExecutionException;
+import java.util.NoSuchElementException;
import org.openrdf.query.BindingSet;
import org.openrdf.query.QueryEvaluationException;
@@ -24,6 +24,12 @@
private final IRunningQuery runningQuery;
private final CloseableIteration<E, X> src;
private boolean checkedFuture = false;
+ /**
+ * The next element is buffered so we can always return it if the
+ * {@link #runningQuery} was not aborted at the time that {@link #hasNext()}
+ * return <code>true</code>.
+ */
+ private E current = null;
public RunningQueryCloseableIteration(final IRunningQuery runningQuery,
final CloseableIteration<E, X> src) {
@@ -39,25 +45,69 @@
}
public boolean hasNext() throws X {
- return src.hasNext();
- }
- public E next() throws X {
+ if (current != null) {
+ // Already buffered.
+ return true;
+ }
+
+ if (!src.hasNext()) {
+ // Source is exhausted.
+ return false;
+ }
+
+ // buffer the next element.
+ current = src.next();
+
+ // test for abnormal completion of the runningQuery.
if (!checkedFuture && runningQuery.isDone()) {
try {
runningQuery.get();
} catch (InterruptedException e) {
+ /*
+ * Interrupted while waiting on the Future (should not happen
+ * since the Future is already done).
+ */
throw (X) new QueryEvaluationException(e);
- } catch (ExecutionException e) {
- throw (X) new QueryEvaluationException(e);
+ } catch (Throwable e) {
+ /*
+ * Exception thrown by the runningQuery.
+ */
+ if (runningQuery.getCause() != null) {
+ // abnormal termination.
+ throw (X) new QueryEvaluationException(runningQuery.getCause());
+ }
+ // otherwise this is normal termination.
}
checkedFuture = true;
}
- return src.next();
+
+ // the next element is now buffered.
+ return true;
+
}
+ public E next() throws X {
+
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ final E tmp = current;
+
+ current = null;
+
+ return tmp;
+
+ }
+
+ /**
+ * Operation is not supported.
+ */
public void remove() throws X {
- src.remove();
+
+ // Not supported since we are buffering ahead.
+ throw new UnsupportedOperationException();
+
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-16 00:59:24
|
Revision: 4107
http://bigdata.svn.sourceforge.net/bigdata/?rev=4107&view=rev
Author: thompsonbry
Date: 2011-01-16 00:59:18 +0000 (Sun, 16 Jan 2011)
Log Message:
-----------
Change to JoinGraph to let it work with predicates which do not share any variables with other predicates in the join graph. This raises the exploration cost significantly for such queries (e.g., BSBM Q5) but it now produces a good answer (2x faster join path). I've updated the RTO issue (#64) to reflect some additional questions about how to handle predicates which do not share variables and how to handle variables which are "shared" only in the sense that they appear in a constraint on another predicate, but not as part of the access path pattern.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-15 23:53:24 UTC (rev 4106)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-16 00:59:18 UTC (rev 4107)
@@ -779,8 +779,9 @@
throw new IllegalArgumentException();
if (shared == null)
throw new IllegalArgumentException();
- if (shared.isEmpty())
- throw new IllegalArgumentException();
+ // Note: We need to allow edges which do not share variables
+// if (shared.isEmpty())
+// throw new IllegalArgumentException();
this.v1 = v1;
this.v2 = v2;
this.shared = shared;
@@ -1782,6 +1783,11 @@
/*
* Identify the edges by looking for shared variables among the
* predicates.
+ *
+ * Note: If a vertex does not share ANY variables then it is paired
+ * with every other vertex. Such joins will always produce a full
+ * cross product and they can be taken paired with any of the other
+ * vertices.
*/
{
@@ -1789,10 +1795,15 @@
for (int i = 0; i < v.length; i++) {
+ // consider a source vertex.
final IPredicate<?> p1 = v[i];
+ // #of vertices which share a variable with source vertex.
+ int nmatched = 0;
+
for (int j = i + 1; j < v.length; j++) {
+ // consider a possible target vertex.
final IPredicate<?> p2 = v[j];
final Set<IVariable<?>> shared = Rule.getSharedVars(p1,
@@ -1800,12 +1811,34 @@
if (shared != null && !shared.isEmpty()) {
+ // the source and target vertices share var(s).
tmp.add(new Edge(V[i], V[j], shared));
+
+ nmatched++;
}
}
+ if (nmatched == 0) {
+
+ /*
+ * The source vertex does not share any variables. In
+ * order to explore join paths which include that vertex
+ * we therefore pair it with each of the other vertices.
+ */
+ for (int j = 0; j < v.length; j++) {
+
+ if (j == i)
+ continue;
+
+ tmp.add(new Edge(V[i], V[j],
+ Collections.EMPTY_SET));
+
+ }
+
+ }
+
}
E = tmp.toArray(new Edge[0]);
@@ -2698,8 +2731,10 @@
this.context = context;
+ // The initial cutoff sampling limit.
limit = getLimit();
+ // The initial number of edges (1 step paths) to explore.
nedges = getNEdges();
if (limit <= 0)
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-15 23:53:24 UTC (rev 4106)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-16 00:59:18 UTC (rev 4107)
@@ -100,7 +100,7 @@
private AbstractTripleStore database;
/** The initial sampling limit. */
- private final int limit = 1000;
+ private final int limit = 100;
/** The #of edges considered for the initial paths. */
private final int nedges = 2;
@@ -113,15 +113,20 @@
* When true, do a warm up run of the plan generated by the static query
* optimizer.
*/
- private final boolean warmUp = true;
+ private final boolean warmUp = false;
/**
* The #of times to run each query. Use N GT ONE (1) if you want to converge
* onto the hot query performance.
*/
- private final int ntrials = 1;
+ private final int ntrials = 3;
/**
+ * When <code>true</code> runs the query in the given order.
+ */
+ private final boolean runGivenOrder = false;
+
+ /**
* When <code>true</code> runs the dynamic query optimizer and then evaluates
* the generated query plan.
*/
@@ -501,7 +506,7 @@
final String GIVEN = getName() + " : given ["+i+"] :";
- if (true/* originalOrder */) {
+ if (runGivenOrder) {
runQuery(GIVEN, queryEngine, preds);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-15 23:53:30
|
Revision: 4106
http://bigdata.svn.sourceforge.net/bigdata/?rev=4106&view=rev
Author: thompsonbry
Date: 2011-01-15 23:53:24 +0000 (Sat, 15 Jan 2011)
Log Message:
-----------
Modified JoinGraph to propagate the PipelineJoin.Annotations.CONSTRAINTS from the _predicate_ to the join. This allows us to annotate the predicate with constraints which will be imposed by the join. That makes it possible for the runtime query optimizer to generate the joins dynamically when they include IConstraint[]s.
See trac issue #64 (Runtime Query Optimizer).
Fixed some problems in the BSBM Q5 setup as hand coded for the runtime query optimizer.
Identified a problem with the runtime query optimizer where it will never include a vertex if it does not share any join variables with the other vertices in the graph (BSBM Q5 does this). I have not fixed this yet.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-15 22:49:40 UTC (rev 4105)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-15 23:53:24 UTC (rev 4106)
@@ -198,6 +198,10 @@
* approach should be able to handle queries without perfect / covering
* automatically. Then experiment with carrying fewer statement indices
* for quads.
+ *
+ * @todo Unit test when there are no solutions to the query. In this case there
+ * will be no paths identified by the optimizer and the final path length
+ * becomes zero.
*/
public class JoinGraph extends PipelineOp {
@@ -1032,10 +1036,10 @@
* path).
*/
final int joinId = 1;
- final PipelineJoin joinOp = new PipelineJoin(new BOp[] {}, //
+ final Map<String,Object> anns = NV.asMap(//
new NV(BOp.Annotations.BOP_ID, joinId),//
- new NV(PipelineJoin.Annotations.PREDICATE, vTarget.pred
- .setBOpId(3)),
+ // @todo Why not use a factory which avoids bopIds already in use?
+ new NV(PipelineJoin.Annotations.PREDICATE, vTarget.pred.setBOpId(3)),
// disallow parallel evaluation.
new NV(PipelineJoin.Annotations.MAX_PARALLEL,0),
// disable access path coalescing
@@ -1056,6 +1060,12 @@
new NV(PipelineJoin.Annotations.SHARED_STATE,true),
new NV(PipelineJoin.Annotations.EVALUATION_CONTEXT,BOpEvaluationContext.CONTROLLER)
);
+ if (vTarget.pred.getProperty(PipelineJoin.Annotations.CONSTRAINTS) != null) {
+ // Copy constraints from the predicate onto the join, which will apply them.
+ anns.put(PipelineJoin.Annotations.CONSTRAINTS, vTarget.pred
+ .getProperty(PipelineJoin.Annotations.CONSTRAINTS));
+ }
+ final PipelineJoin joinOp = new PipelineJoin(new BOp[] {}, anns);
final PipelineOp queryOp = joinOp;
@@ -1805,8 +1815,13 @@
}
/**
+ * Find a good join path in the data given the join graph. The join path
+ * is not guaranteed to be the best join path (the search performed by
+ * the runtime optimizer is not exhaustive) but it should always be a
+ * "good" join path and may often be the "best" join path.
*
* @param queryEngine
+ * The query engine.
* @param limit
* The limit for sampling a vertex and the initial limit for
* cutoff join evaluation.
@@ -1818,10 +1833,25 @@
* a join path, the starting vertex will be the vertex of
* that edge having the lower cardinality.
*
+ * @return The join path identified by the runtime query optimizer as
+ * the best path given the join graph and the data.
+ *
+ * @throws NoSolutionsException
+ * If there are no solutions for the join graph in the data
+ * (the query does not have any results).
+ *
* @throws Exception
+ *
+ * @todo It is possible that this could throw a
+ * {@link NoSolutionsException} if the cutoff joins do not use a
+ * large enough sample to find a join path which produces at least
+ * one solution. We need to automatically increase the depth of
+ * search for queries where we have cardinality estimation
+ * underflows or punt to another method to decide the join order.
*/
public Path runtimeOptimizer(final QueryEngine queryEngine,
- final int limit, final int nedges) throws Exception {
+ final int limit, final int nedges) throws Exception,
+ NoSolutionsException {
// Setup the join graph.
Path[] paths = round0(queryEngine, limit, nedges);
@@ -1838,12 +1868,19 @@
int round = 1;
- while (round < nvertices - 1) {
+ while (paths.length > 0 && round < nvertices - 1) {
paths = expand(queryEngine, limit, round++, paths);
}
+ if (paths.length == 0) {
+
+ // There are no solutions for the join graph in the data.
+ throw new NoSolutionsException();
+
+ }
+
// Should be one winner.
assert paths.length == 1;
@@ -2257,6 +2294,10 @@
final boolean v1Found = x.contains(edgeInGraph.v1);
final boolean v2Found = x.contains(edgeInGraph.v2);
+ if (log.isTraceEnabled())
+ log.trace("Edge: " + edgeInGraph + ", v1Found="
+ + v1Found + ", v2Found=" + v2Found);
+
if (!v1Found && !v2Found) {
// Edge is not connected to this path.
continue;
@@ -2277,6 +2318,9 @@
if (used.contains(tVertex)) {
// Vertex already used to extend this path.
+ if (log.isTraceEnabled())
+ log.trace("Edge: " + edgeInGraph
+ + " - already used to extend this path.");
continue;
}
@@ -2292,6 +2336,10 @@
// Add to the set of paths for this round.
tmp.add(p);
+ if (log.isTraceEnabled())
+ log.trace("Extended path with edge: " + edgeInGraph
+ + ", new path=" + p);
+
}
}
@@ -2806,6 +2854,13 @@
//
// anns.add(new NV(PipelineJoin.Annotations.SELECT, vars.toArray(new IVariable[vars.size()])));
+ if (p.getProperty(PipelineJoin.Annotations.CONSTRAINTS) != null) {
+ // Copy constraints from the predicate onto the join, which will
+ // apply them.
+ anns.add(new NV(PipelineJoin.Annotations.CONSTRAINTS, p
+ .getProperty(PipelineJoin.Annotations.CONSTRAINTS)));
+ }
+
final PipelineJoin joinOp = new PipelineJoin(
lastOp == null ? new BOp[0] : new BOp[] { lastOp },
anns.toArray(new NV[anns.size()]));
@@ -2935,4 +2990,34 @@
}
+ /**
+ * Exception thrown when the join graph does not have any solutions in the
+ * data (running the query does not produce any results).
+ */
+ public static class NoSolutionsException extends RuntimeException
+ {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public NoSolutionsException() {
+ super();
+ }
+
+ public NoSolutionsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public NoSolutionsException(String message) {
+ super(message);
+ }
+
+ public NoSolutionsException(Throwable cause) {
+ super(cause);
+ }
+
+ }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-15 22:49:40 UTC (rev 4105)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-15 23:53:24 UTC (rev 4106)
@@ -10,7 +10,6 @@
import org.apache.log4j.Logger;
import org.openrdf.query.algebra.Compare.CompareOp;
import org.openrdf.query.algebra.MathExpr.MathOp;
-import org.semanticweb.yars.nx.dt.numeric.XSDInt;
import com.bigdata.bop.BOp;
import com.bigdata.bop.BOpContextBase;
@@ -35,7 +34,7 @@
import com.bigdata.bop.fed.QueryEngineFactory;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.journal.Journal;
-import com.bigdata.rdf.internal.IVUtility;
+import com.bigdata.rdf.internal.XSDIntIV;
import com.bigdata.rdf.internal.constraints.CompareBOp;
import com.bigdata.rdf.internal.constraints.MathBOp;
import com.bigdata.rdf.model.BigdataURI;
@@ -101,7 +100,7 @@
private AbstractTripleStore database;
/** The initial sampling limit. */
- private final int limit = 100;
+ private final int limit = 1000;
/** The #of edges considered for the initial paths. */
private final int nedges = 2;
@@ -114,7 +113,7 @@
* When true, do a warm up run of the plan generated by the static query
* optimizer.
*/
- private final boolean warmUp = false;
+ private final boolean warmUp = true;
/**
* The #of times to run each query. Use N GT ONE (1) if you want to converge
@@ -153,8 +152,10 @@
/*
* Use a specific file generated by some external process.
*/
- file = new File("/data/bsbm/bsbm_284826/bigdata-bsbm.RW.jnl");
- namespace = "BSBM_284826";
+ final long pc = 284826; // BSBM 100M
+// final long pc = 566496; // BSBM 200M
+ file = new File("/data/bsbm/bsbm_"+pc+"/bigdata-bsbm.RW.jnl");
+ namespace = "BSBM_"+pc;
}
properties.setProperty(Journal.Options.FILE, file.toString());
@@ -260,7 +261,7 @@
.getValueFactory();
final String rdfs = "http://www.w3.org/2000/01/rdf-schema#";
- final String rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+// final String rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
final String bsbm = "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/";
// final BigdataURI rdfType = valueFactory.createURI(rdf + "type");
@@ -309,8 +310,8 @@
// The name space for the SPO relation.
final String[] spoRelation = new String[] { namespace + ".spo" };
- // The name space for the Lexicon relation.
- final String[] lexRelation = new String[] { namespace + ".lex" };
+// // The name space for the Lexicon relation.
+// final String[] lexRelation = new String[] { namespace + ".lex" };
final long timestamp = jnl.getLastCommitTime();
@@ -334,8 +335,6 @@
* predicates). The RTO knows to look for the CONSTRAINTS on
* the IPredicate and apply them to the constructed join
* operator.
- *
- * FIXME JOinGraph needs to do this ^^^^^^^
*/
new NV(PipelineJoin.Annotations.CONSTRAINTS,
new IConstraint[] {//
@@ -354,6 +353,7 @@
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
);
+
// ?product bsbm:productFeature ?prodFeature .
p2 = new SPOPredicate(new BOp[] { //
product,//
@@ -364,6 +364,7 @@
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
);
+
// <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productPropertyNumeric1 ?origProperty1 .
p3 = new SPOPredicate(new BOp[] { //
new Constant(product53999.getIV()),//
@@ -374,6 +375,7 @@
new NV(Annotations.TIMESTAMP, timestamp),//
new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
);
+
// ?product bsbm:productPropertyNumeric1 ?simProperty1 .
// FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
p4 = new SPOPredicate(new BOp[] { //
@@ -389,8 +391,8 @@
new CompareBOp(new BOp[] {
simProperty1,
new MathBOp(origProperty1,
- new Constant(new XSDInt(
- "120")),
+ new Constant(new XSDIntIV(
+ 120)),
MathOp.PLUS) }, NV
.asMap(new NV[] { new NV(
CompareBOp.Annotations.OP,
@@ -398,8 +400,8 @@
new CompareBOp(new BOp[] {
simProperty1,
new MathBOp(origProperty1,
- new Constant(new XSDInt(
- "120")),
+ new Constant(new XSDIntIV(
+ 120)),
MathOp.MINUS) }, NV
.asMap(new NV[] { new NV(
CompareBOp.Annotations.OP,
@@ -407,13 +409,6 @@
})//
);
- /*
- * com.bigdata.rdf.internal.constraints.CompareBOp(Var,MathBOp)[
- * com.bigdata.rdf.internal.constraints.CompareBOp.op=GT],
- * com.bigdata.rdf.internal.constraints.CompareBOp(Var,MathBOp)[
- * com.bigdata.rdf.internal.constraints.CompareBOp.op=LT]],
- */
-
// <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productPropertyNumeric2 ?origProperty2 .
p5 = new SPOPredicate(new BOp[] { //
new Constant(product53999.getIV()),//
@@ -440,17 +435,17 @@
new CompareBOp(new BOp[] {
simProperty2,
new MathBOp(origProperty2,
- new Constant(new XSDInt(
- "170")),
+ new Constant(new XSDIntIV(
+ 170)),
MathOp.PLUS) }, NV
.asMap(new NV[] { new NV(
CompareBOp.Annotations.OP,
CompareOp.LT) })),//
new CompareBOp(new BOp[] {
simProperty2,
- new MathBOp(origProperty1,
- new Constant(new XSDInt(
- "170")),
+ new MathBOp(origProperty2,
+ new Constant(new XSDIntIV(
+ 170)),
MathOp.MINUS) }, NV
.asMap(new NV[] { new NV(
CompareBOp.Annotations.OP,
@@ -473,7 +468,7 @@
* @throws Exception
*
* @todo To actually test anything this needs to compare the results (or at
- * least the #of result). We could also test for known good join
+ * least the #of results). We could also test for known good join
* orders as generated by the runtime optimizer, but that requires a
* known data set (e.g., U1 or U50) and non-random sampling.
*
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-15 22:49:46
|
Revision: 4105
http://bigdata.svn.sourceforge.net/bigdata/?rev=4105&view=rev
Author: thompsonbry
Date: 2011-01-15 22:49:40 +0000 (Sat, 15 Jan 2011)
Log Message:
-----------
Modified the thrown exception from IllegalArgumentException in RWStore#readData() to RuntimeException. This reflects the fact that the cause could be anything, including a ClosedByInterruptException which is most certainly not the result of an illegal argument.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2011-01-15 22:21:02 UTC (rev 4104)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2011-01-15 22:49:40 UTC (rev 4105)
@@ -1435,12 +1435,14 @@
/*
* Note: ClosedByInterruptException can be thrown out of
* FileChannelUtility.readAll(), typically because the LIMIT on
- * a query was satisified, but we do not want to log that as an
+ * a query was satisfied, but we do not want to log that as an
* error.
*/
// log.error(e,e);
- throw new IllegalArgumentException("Unable to read data", e);
+// throw new IllegalArgumentException("Unable to read data: "+e, e);
+ throw new RuntimeException(e);
+
}
} finally {
readLock.unlock();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-15 22:21:10
|
Revision: 4104
http://bigdata.svn.sourceforge.net/bigdata/?rev=4104&view=rev
Author: thompsonbry
Date: 2011-01-15 22:21:02 +0000 (Sat, 15 Jan 2011)
Log Message:
-----------
javadoc on IValueExpression, MathBOp, and CompareBOp.
formatting on /status requests for NanoSparqlServer.
added a test to run BSBM Q5 using the runtime query optimizer (I still have to modify JoinGraph to interpret the PipelineJoin.Annotation.CONSTRAINTS from the IPredicate before this can be run).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IValueExpression.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IValueExpression.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IValueExpression.java 2011-01-15 21:23:23 UTC (rev 4103)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IValueExpression.java 2011-01-15 22:21:02 UTC (rev 4104)
@@ -5,16 +5,16 @@
public interface IValueExpression<E> extends BOp, Serializable {
/**
- * Return the <i>as bound</i> value of the variable or constant. The <i>as
- * bound</i> value of an {@link IConstant} is the contant's value. The <i>as
- * bound</i> value of an {@link IVariable} is the bound value in the given
- * {@link IBindingSet} -or- <code>null</code> if the variable is not bound
- * in the {@link IBindingSet}.
+ * Return the <i>as bound</i> value of the variable, constant, or
+ * expression. The <i>as bound</i> value of an {@link IConstant} is the
+ * contant's value. The <i>as bound</i> value of an {@link IVariable} is the
+ * bound value in the given {@link IBindingSet} -or- <code>null</code> if
+ * the variable is not bound in the {@link IBindingSet}.
*
* @param bindingSet
* The binding set.
*
- * @return The as bound value of the constant or variable.
+ * @return The as bound value of the constant, variable, or expression.
*
* @throws IllegalArgumentException
* if this is an {@link IVariable} and the <i>bindingSet</i> is
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-15 21:23:23 UTC (rev 4103)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-15 22:21:02 UTC (rev 4104)
@@ -31,7 +31,6 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IValueExpression;
-import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
import com.bigdata.bop.constraint.BOpConstraint;
@@ -53,7 +52,7 @@
public interface Annotations extends PipelineOp.Annotations {
/**
- * The compare operator
+ * The compare operator, which is a {@link CompareOp} enum value.
*/
String OP = CompareBOp.class.getName() + ".op";
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2011-01-15 21:23:23 UTC (rev 4103)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2011-01-15 22:21:02 UTC (rev 4104)
@@ -35,7 +35,9 @@
import com.bigdata.rdf.internal.IVUtility;
/**
- * A math expression involving a left and right IValueExpression operand.
+ * A math expression involving a left and right IValueExpression operand. The
+ * operation to be applied to the operands is specified by the
+ * {@link Annotations#OP} annotation.
*/
final public class MathBOp extends ImmutableBOp
implements IValueExpression<IV> {
@@ -48,8 +50,11 @@
public interface Annotations extends ImmutableBOp.Annotations {
/**
- * The {@link IVariable} which is bound to that constant value
- * (optional).
+ * The operation to be applied to the left and right operands
+ * (required). The value of this annotation is a {@link MathOp}, such as
+ * {@link MathOp#PLUS}.
+ *
+ * @see MathOp
*/
String OP = MathBOp.class.getName() + ".op";
@@ -66,6 +71,16 @@
}
+ /**
+ *
+ * @param left
+ * The left operand.
+ * @param right
+ * The right operand.
+ * @param op
+ * The annotation specifying the operation to be performed on
+ * those operands.
+ */
public MathBOp(final IValueExpression<IV> left,
final IValueExpression<IV> right, final MathOp op) {
Added: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java 2011-01-15 22:21:02 UTC (rev 4104)
@@ -0,0 +1,687 @@
+package com.bigdata.bop.rdf.joinGraph;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Properties;
+
+import junit.framework.TestCase2;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.openrdf.query.algebra.Compare.CompareOp;
+import org.openrdf.query.algebra.MathExpr.MathOp;
+import org.semanticweb.yars.nx.dt.numeric.XSDInt;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.BOpContextBase;
+import com.bigdata.bop.BOpIdFactory;
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IConstraint;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.Var;
+import com.bigdata.bop.IPredicate.Annotations;
+import com.bigdata.bop.constraint.NEConstant;
+import com.bigdata.bop.controller.JoinGraph;
+import com.bigdata.bop.controller.JoinGraph.JGraph;
+import com.bigdata.bop.controller.JoinGraph.Path;
+import com.bigdata.bop.engine.BOpStats;
+import com.bigdata.bop.engine.IRunningQuery;
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.bop.engine.QueryLog;
+import com.bigdata.bop.fed.QueryEngineFactory;
+import com.bigdata.bop.join.PipelineJoin;
+import com.bigdata.journal.Journal;
+import com.bigdata.rdf.internal.IVUtility;
+import com.bigdata.rdf.internal.constraints.CompareBOp;
+import com.bigdata.rdf.internal.constraints.MathBOp;
+import com.bigdata.rdf.model.BigdataURI;
+import com.bigdata.rdf.model.BigdataValue;
+import com.bigdata.rdf.model.BigdataValueFactory;
+import com.bigdata.rdf.spo.SPOPredicate;
+import com.bigdata.rdf.store.AbstractTripleStore;
+import com.bigdata.rdf.store.DataLoader;
+import com.bigdata.rdf.store.DataLoader.ClosureEnum;
+import com.bigdata.relation.accesspath.IAsynchronousIterator;
+import com.bigdata.relation.rule.IRule;
+import com.bigdata.relation.rule.Rule;
+import com.bigdata.relation.rule.eval.DefaultEvaluationPlan2;
+import com.bigdata.relation.rule.eval.IRangeCountFactory;
+
+/**
+ * Unit tests for runtime query optimization using {@link JoinGraph} and the
+ * "BSBM" test set.
+ * <p>
+ * Note: When running large queries, be sure to provide a sufficient heap, set
+ * the -server flag, etc.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class TestJoinGraphOnBSBMData extends TestCase2 {
+
+ /**
+ *
+ */
+ public TestJoinGraphOnBSBMData() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestJoinGraphOnBSBMData(String name) {
+ super(name);
+ }
+
+ @Override
+ public Properties getProperties() {
+
+ final Properties p = new Properties(super.getProperties());
+
+// p.setProperty(Journal.Options.BUFFER_MODE, BufferMode.Transient
+// .toString());
+
+// p.setProperty(AbstractTripleStore.Options.QUADS_MODE, "true");
+
+ /*
+ * Don't compute closure in the data loader since it does TM, not
+ * database at once closure.
+ */
+ p.setProperty(DataLoader.Options.CLOSURE, ClosureEnum.None.toString());
+
+ return p;
+
+ }
+
+ private Journal jnl;
+
+ private AbstractTripleStore database;
+
+ /** The initial sampling limit. */
+ private final int limit = 100;
+
+ /** The #of edges considered for the initial paths. */
+ private final int nedges = 2;
+
+ private QueryEngine queryEngine;
+
+ private String namespace;
+
+ /**
+ * When true, do a warm up run of the plan generated by the static query
+ * optimizer.
+ */
+ private final boolean warmUp = false;
+
+ /**
+ * The #of times to run each query. Use N GT ONE (1) if you want to converge
+ * onto the hot query performance.
+ */
+ private final int ntrials = 1;
+
+ /**
+ * When <code>true</code> runs the dynamic query optimizer and then evaluates
+ * the generated query plan.
+ */
+ private final boolean runRuntimeQueryOptimizer = true;
+
+ /**
+ * When <code>true</code> runs the static query optimizer and then evaluates
+ * the generated query plan.
+ */
+ private final boolean runStaticQueryOptimizer = true;
+
+ /**
+ * Loads LUBM U1 into a triple store.
+ */
+ protected void setUp() throws Exception {
+
+// QueryLog.logTableHeader();
+
+ super.setUp();
+
+// System.err.println(UUID.randomUUID().toString());
+// System.exit(0);
+
+ final Properties properties = getProperties();
+
+ final File file;
+ {
+ /*
+ * Use a specific file generated by some external process.
+ */
+ file = new File("/data/bsbm/bsbm_284826/bigdata-bsbm.RW.jnl");
+ namespace = "BSBM_284826";
+ }
+
+ properties.setProperty(Journal.Options.FILE, file.toString());
+
+// properties.setProperty(Journal.Options.BUFFER_MODE,BufferMode.DiskRW.toString());
+
+// file.delete();
+
+ if (!file.exists()) {
+
+ fail("File not found: " + file);
+
+// jnl = new Journal(properties);
+//
+// final AbstractTripleStore tripleStore = new LocalTripleStore(jnl,
+// namespace, ITx.UNISOLATED, properties);
+//
+// // Create the KB instance.
+// tripleStore.create();
+//
+// tripleStore.getDataLoader().loadFiles(
+// new File("/root/Desktop/Downloads/barData/barData.trig"),
+// null/* baseURI */, RDFFormat.TRIG, null/* defaultGraph */,
+// null/* filter */);
+//
+// // Truncate the journal (trim its size).
+// jnl.truncate();
+//
+// // Commit the journal.
+// jnl.commit();
+//
+// // Close the journal.
+// jnl.close();
+
+ }
+
+ // Open the test resource.
+ jnl = new Journal(properties);
+
+ queryEngine = QueryEngineFactory
+ .getQueryController(jnl/* indexManager */);
+
+ database = (AbstractTripleStore) jnl.getResourceLocator().locate(
+ namespace, jnl.getLastCommitTime());
+
+ if (database == null)
+ throw new RuntimeException("Not found: " + namespace);
+
+ }
+
+ protected void tearDown() throws Exception {
+
+ if (database != null) {
+ database = null;
+ }
+
+ if (queryEngine != null) {
+ queryEngine.shutdownNow();
+ queryEngine = null;
+ }
+
+ if(jnl != null) {
+ jnl.close();
+ jnl = null;
+ }
+
+ super.tearDown();
+
+ }
+
+ /**
+ * BSBM Q5
+ *
+ * <pre>
+ * PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+ * PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+ * PREFIX bsbm: <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/>
+ *
+ * SELECT DISTINCT ?product ?productLabel
+ * WHERE {
+ * ?product rdfs:label ?productLabel .
+ * FILTER (<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> != ?product)
+ * <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productFeature ?prodFeature .
+ * ?product bsbm:productFeature ?prodFeature .
+ * <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productPropertyNumeric1 ?origProperty1 .
+ * ?product bsbm:productPropertyNumeric1 ?simProperty1 .
+ * FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
+ * <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productPropertyNumeric2 ?origProperty2 .
+ * ?product bsbm:productPropertyNumeric2 ?simProperty2 .
+ * FILTER (?simProperty2 < (?origProperty2 + 170) && ?simProperty2 > (?origProperty2 - 170))
+ * }
+ * ORDER BY ?productLabel
+ * LIMIT 5
+ * </pre>
+ * @throws Exception
+ */
+ public void test_bsbm_q5() throws Exception {
+
+ /*
+ * Resolve terms against the lexicon.
+ */
+ final BigdataValueFactory valueFactory = database.getLexiconRelation()
+ .getValueFactory();
+
+ final String rdfs = "http://www.w3.org/2000/01/rdf-schema#";
+ final String rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+ final String bsbm = "http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/vocabulary/";
+
+// final BigdataURI rdfType = valueFactory.createURI(rdf + "type");
+
+ final BigdataURI rdfsLabel = valueFactory.createURI(rdfs + "label");
+
+ final BigdataURI productFeature = valueFactory.createURI(bsbm
+ + "productFeature");
+
+ final BigdataURI productPropertyNumeric1 = valueFactory.createURI(bsbm
+ + "productPropertyNumeric1");
+
+ final BigdataURI productPropertyNumeric2 = valueFactory.createURI(bsbm
+ + "productPropertyNumeric2");
+
+ final BigdataURI product53999 = valueFactory
+ .createURI("http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999");
+
+ final BigdataValue[] terms = new BigdataValue[] { rdfsLabel,
+ productFeature, productPropertyNumeric1,
+ productPropertyNumeric2, product53999 };
+
+ // resolve terms.
+ database.getLexiconRelation()
+ .addTerms(terms, terms.length, true/* readOnly */);
+
+ {
+ for (BigdataValue tmp : terms) {
+ System.out.println(tmp + " : " + tmp.getIV());
+ if (tmp.getIV() == null)
+ throw new RuntimeException("Not defined: " + tmp);
+ }
+ }
+
+ final IPredicate[] preds;
+ final IPredicate p0, p1, p2, p3, p4, p5, p6;
+ {
+ final IVariable product = Var.var("product");
+ final IVariable productLabel = Var.var("productLabel");
+ final IVariable prodFeature= Var.var("prodFeature");
+ final IVariable simProperty1 = Var.var("simProperty1");
+ final IVariable simProperty2 = Var.var("simProperty2");
+ final IVariable origProperty1 = Var.var("origProperty1");
+ final IVariable origProperty2 = Var.var("origProperty2");
+
+ // The name space for the SPO relation.
+ final String[] spoRelation = new String[] { namespace + ".spo" };
+
+ // The name space for the Lexicon relation.
+ final String[] lexRelation = new String[] { namespace + ".lex" };
+
+ final long timestamp = jnl.getLastCommitTime();
+
+ int nextId = 0;
+
+// ?product rdfs:label ?productLabel .
+// FILTER (<http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> != ?product)
+ p0 = new SPOPredicate(new BOp[] {//
+ product,
+ new Constant(rdfsLabel.getIV()),
+ productLabel//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation),//
+ /*
+ * Note: In order to code up this query for the runtime
+ * query optimizer we need to attach the constraint
+ * (product53999 != ?product) to the access path rather than
+ * the join (the RTO does not accept join operators, just
+ * predicates). The RTO knows to look for the CONSTRAINTS on
+ * the IPredicate and apply them to the constructed join
+ * operator.
+ *
+ * FIXME JOinGraph needs to do this ^^^^^^^
+ */
+ new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new IConstraint[] {//
+ new NEConstant(product, new Constant(product53999
+ .getIV())) //
+ })//
+ );
+
+// <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productFeature ?prodFeature .
+ p1 = new SPOPredicate(new BOp[] { //
+ new Constant(product53999.getIV()),//
+ new Constant(productFeature.getIV()),//
+ prodFeature//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
+ );
+// ?product bsbm:productFeature ?prodFeature .
+ p2 = new SPOPredicate(new BOp[] { //
+ product,//
+ new Constant(productFeature.getIV()),//
+ prodFeature//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
+ );
+// <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productPropertyNumeric1 ?origProperty1 .
+ p3 = new SPOPredicate(new BOp[] { //
+ new Constant(product53999.getIV()),//
+ new Constant(productPropertyNumeric1.getIV()),//
+ origProperty1//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
+ );
+// ?product bsbm:productPropertyNumeric1 ?simProperty1 .
+// FILTER (?simProperty1 < (?origProperty1 + 120) && ?simProperty1 > (?origProperty1 - 120))
+ p4 = new SPOPredicate(new BOp[] { //
+ product,//
+ new Constant(productPropertyNumeric1.getIV()),//
+ simProperty1//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation),
+ new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new IConstraint[] {//
+ new CompareBOp(new BOp[] {
+ simProperty1,
+ new MathBOp(origProperty1,
+ new Constant(new XSDInt(
+ "120")),
+ MathOp.PLUS) }, NV
+ .asMap(new NV[] { new NV(
+ CompareBOp.Annotations.OP,
+ CompareOp.LT) })),//
+ new CompareBOp(new BOp[] {
+ simProperty1,
+ new MathBOp(origProperty1,
+ new Constant(new XSDInt(
+ "120")),
+ MathOp.MINUS) }, NV
+ .asMap(new NV[] { new NV(
+ CompareBOp.Annotations.OP,
+ CompareOp.GT) })),//
+ })//
+ );
+
+ /*
+ * com.bigdata.rdf.internal.constraints.CompareBOp(Var,MathBOp)[
+ * com.bigdata.rdf.internal.constraints.CompareBOp.op=GT],
+ * com.bigdata.rdf.internal.constraints.CompareBOp(Var,MathBOp)[
+ * com.bigdata.rdf.internal.constraints.CompareBOp.op=LT]],
+ */
+
+// <http://www4.wiwiss.fu-berlin.de/bizer/bsbm/v01/instances/dataFromProducer1092/Product53999> bsbm:productPropertyNumeric2 ?origProperty2 .
+ p5 = new SPOPredicate(new BOp[] { //
+ new Constant(product53999.getIV()),//
+ new Constant(productPropertyNumeric2.getIV()),//
+ origProperty2//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation)
+ );
+
+// ?product bsbm:productPropertyNumeric2 ?simProperty2 .
+// FILTER (?simProperty2 < (?origProperty2 + 170) && ?simProperty2 > (?origProperty2 - 170))
+ p6 = new SPOPredicate(new BOp[] { //
+ product,//
+ new Constant(productPropertyNumeric2.getIV()),//
+ simProperty2//
+ },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, spoRelation),
+ new NV(PipelineJoin.Annotations.CONSTRAINTS,
+ new IConstraint[] {//
+ new CompareBOp(new BOp[] {
+ simProperty2,
+ new MathBOp(origProperty2,
+ new Constant(new XSDInt(
+ "170")),
+ MathOp.PLUS) }, NV
+ .asMap(new NV[] { new NV(
+ CompareBOp.Annotations.OP,
+ CompareOp.LT) })),//
+ new CompareBOp(new BOp[] {
+ simProperty2,
+ new MathBOp(origProperty1,
+ new Constant(new XSDInt(
+ "170")),
+ MathOp.MINUS) }, NV
+ .asMap(new NV[] { new NV(
+ CompareBOp.Annotations.OP,
+ CompareOp.GT) })),//
+ })//
+ );
+
+ // the vertices of the join graph (the predicates).
+ preds = new IPredicate[] { p0, p1, p2, p3, p4, p5, p6 };
+
+ }
+
+ doTest(preds);
+
+ }
+
+ /**
+ *
+ * @param preds
+ * @throws Exception
+ *
+ * @todo To actually test anything this needs to compare the results (or at
+ * least the #of result). We could also test for known good join
+ * orders as generated by the runtime optimizer, but that requires a
+ * known data set (e.g., U1 or U50) and non-random sampling.
+ *
+ * @todo This is currently providing a "hot run" comparison by a series of
+ * trials. This means that the IO costs are effectively being wiped
+ * away, assuming that the file system cache is larger than the data
+ * set. The other way to compare performance is a cold cache / cold
+ * JVM run using the known solutions produced by the runtime versus
+ * static query optimizers.
+ */
+ private void doTest(final IPredicate[] preds) throws Exception {
+
+ if (warmUp)
+ runQuery("Warmup", queryEngine, runStaticQueryOptimizer(preds));
+
+ /*
+ * Run the runtime query optimizer once (its cost is not counted
+ * thereafter).
+ */
+ final IPredicate[] runtimePredOrder = runRuntimeQueryOptimizer(preds);
+
+ long totalRuntimeTime = 0;
+ long totalStaticTime = 0;
+
+ for (int i = 0; i < ntrials; i++) {
+
+ final String RUNTIME = getName() + " : runtime["+i+"] :";
+
+ final String STATIC = getName() + " : static ["+i+"] :";
+
+ final String GIVEN = getName() + " : given ["+i+"] :";
+
+ if (true/* originalOrder */) {
+
+ runQuery(GIVEN, queryEngine, preds);
+
+ }
+
+ if (runStaticQueryOptimizer) {
+
+ totalStaticTime += runQuery(STATIC, queryEngine,
+ runStaticQueryOptimizer(preds));
+
+ }
+
+ if (runRuntimeQueryOptimizer) {
+
+ /*
+ * Run the runtime query optimizer each time (its overhead is
+ * factored into the running comparison of the two query
+ * optimizers).
+ */
+// final IPredicate[] runtimePredOrder = runRuntimeQueryOptimizer(new JGraph(
+// preds));
+
+ // Evaluate the query using the selected join order.
+ totalRuntimeTime += runQuery(RUNTIME, queryEngine,
+ runtimePredOrder);
+
+ }
+
+ }
+
+ if(runStaticQueryOptimizer&&runRuntimeQueryOptimizer) {
+ System.err.println(getName() + " : Total times" + //
+ ": static=" + totalStaticTime + //
+ ", runtime=" + totalRuntimeTime + //
+ ", delta(static-runtime)=" + (totalStaticTime - totalRuntimeTime));
+ }
+
+ }
+
+ /**
+ * Apply the runtime query optimizer.
+ * <p>
+ * Note: This temporarily raises the {@link QueryLog} log level during
+ * sampling to make the log files cleaner (this can not be done for a
+ * deployed system since the logger level is global and there are concurrent
+ * query mixes).
+ *
+ * @return The predicates in order as recommended by the runtime query
+ * optimizer.
+ *
+ * @throws Exception
+ */
+ private IPredicate[] runRuntimeQueryOptimizer(final IPredicate[] preds) throws Exception {
+
+ final Logger tmp = Logger.getLogger(QueryLog.class);
+ final Level oldLevel = tmp.getEffectiveLevel();
+ tmp.setLevel(Level.WARN);
+
+ try {
+
+ final JGraph g = new JGraph(preds);
+
+ final Path p = g.runtimeOptimizer(queryEngine, limit, nedges);
+
+// System.err.println(getName() + " : runtime optimizer join order "
+// + Arrays.toString(Path.getVertexIds(p.edges)));
+
+ return p.getPredicates();
+
+ } finally {
+
+ tmp.setLevel(oldLevel);
+
+ }
+
+ }
+
+ /**
+ * Apply the static query optimizer.
+ *
+ * @return The predicates in order as recommended by the static query
+ * optimizer.
+ */
+ private IPredicate[] runStaticQueryOptimizer(final IPredicate[] preds) {
+
+ final BOpContextBase context = new BOpContextBase(queryEngine);
+
+ final IRule rule = new Rule("tmp", null/* head */, preds, null/* constraints */);
+
+ final DefaultEvaluationPlan2 plan = new DefaultEvaluationPlan2(
+ new IRangeCountFactory() {
+
+ public long rangeCount(final IPredicate pred) {
+ return context.getRelation(pred).getAccessPath(pred)
+ .rangeCount(false);
+ }
+
+ }, rule);
+
+ // evaluation plan order.
+ final int[] order = plan.getOrder();
+
+ final int[] ids = new int[order.length];
+
+ final IPredicate[] out = new IPredicate[order.length];
+
+ for (int i = 0; i < order.length; i++) {
+
+ out[i] = preds[order[i]];
+
+ ids[i] = out[i].getId();
+
+ }
+
+// System.err.println(getName() + " : static optimizer join order "
+// + Arrays.toString(ids));
+
+ return out;
+
+ }
+
+ /**
+ * Run a query joining a set of {@link IPredicate}s in the given join order.
+ *
+ * @return The elapsed query time (ms).
+ */
+ private static long runQuery(final String msg,
+ final QueryEngine queryEngine, final IPredicate[] predOrder)
+ throws Exception {
+
+ final BOpIdFactory idFactory = new BOpIdFactory();
+
+ final int[] ids = new int[predOrder.length];
+
+ for(int i=0; i<ids.length; i++) {
+
+ final IPredicate<?> p = predOrder[i];
+
+ idFactory.reserve(p.getId());
+
+ ids[i] = p.getId();
+
+ }
+
+ final PipelineOp queryOp = JoinGraph.getQuery(idFactory, predOrder);
+
+ // submit query to runtime optimizer.
+ final IRunningQuery q = queryEngine.eval(queryOp);
+
+ // drain the query results.
+ long nout = 0;
+ long nchunks = 0;
+ final IAsynchronousIterator<IBindingSet[]> itr = q.iterator();
+ try {
+ while (itr.hasNext()) {
+ final IBindingSet[] chunk = itr.next();
+ nout += chunk.length;
+ nchunks++;
+ }
+ } finally {
+ itr.close();
+ }
+
+ // check the Future for the query.
+ q.get();
+
+ // show the results.
+ final BOpStats stats = q.getStats().get(queryOp.getId());
+
+ System.err.println(msg + " : ids=" + Arrays.toString(ids)
+ + ", elapsed=" + q.getElapsed() + ", nout=" + nout
+ + ", nchunks=" + nchunks + ", stats=" + stats);
+
+ return q.getElapsed();
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java 2011-01-15 21:23:23 UTC (rev 4103)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java 2011-01-15 22:21:02 UTC (rev 4104)
@@ -858,10 +858,11 @@
final long age = now - query.begin;
- sb.append("age="
- + java.util.concurrent.TimeUnit.NANOSECONDS
- .toMillis(age) + "ms, queryId="
- + query.queryId + ", query=" + query.query+"\n");
+ sb.append("age="
+ + java.util.concurrent.TimeUnit.NANOSECONDS
+ .toMillis(age) + "ms, queryId="
+ + query.queryId + "\n");
+ sb.append(query.query + "\n");
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-15 21:23:29
|
Revision: 4103
http://bigdata.svn.sourceforge.net/bigdata/?rev=4103&view=rev
Author: thompsonbry
Date: 2011-01-15 21:23:23 +0000 (Sat, 15 Jan 2011)
Log Message:
-----------
Javadoc on QueryHints.
Bugfix to the BigdataEvaluationStrategyImpl to address problems where a smaller buffer backing the BigdataBindingSetResolverator could result in a deadlock during query (e.g., LUBM U50 Q8).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2011-01-15 14:59:52 UTC (rev 4102)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2011-01-15 21:23:23 UTC (rev 4103)
@@ -1710,106 +1710,167 @@
// startId, -1/* partitionId */,
// newBindingSetIterator(new HashBindingSet())));
- final IRunningQuery runningQuery = queryEngine.eval(query);
+ IRunningQuery runningQuery = null;
+ try {
+
+ // Submit query for evaluation.
+ runningQuery = queryEngine.eval(query);
+
+ // Iterator draining the query results.
+ final IAsynchronousIterator<IBindingSet[]> it1 =
+ runningQuery.iterator();
+
+ // De-chunk the IBindingSet[] visited by that iterator.
+ final IChunkedOrderedIterator<IBindingSet> it2 =
+ new ChunkedWrappedIterator<IBindingSet>(
+ new Dechunkerator<IBindingSet>(it1));
- final IAsynchronousIterator<IBindingSet[]> it1 =
- runningQuery.iterator();
-
-// final IChunkedOrderedIterator<IBindingSet> it2 =
-// new ChunkedArraysIterator<IBindingSet>(it1);
- final IChunkedOrderedIterator<IBindingSet> it2 = new ChunkedWrappedIterator<IBindingSet>(
- new Dechunkerator<IBindingSet>(it1));
-
- CloseableIteration<BindingSet, QueryEvaluationException> result =
- new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
- new BigdataBindingSetResolverator(database, it2).start(database
- .getExecutorService()));
+ // Materialize IVs as RDF Values.
+ CloseableIteration<BindingSet, QueryEvaluationException> result =
+ // Monitor IRunningQuery and cancel if Sesame iterator is closed.
+ new RunningQueryCloseableIteration<BindingSet, QueryEvaluationException>(runningQuery,
+ // Convert bigdata binding sets to Sesame binding sets.
+ new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
+ // Materialize IVs as RDF Values.
+ new BigdataBindingSetResolverator(database, it2).start(
+ database.getExecutorService())));
+
+// No - will deadlock if buffer fills up
+// // Wait for the Future (checks for errors).
+// runningQuery.get();
+
+ // use the basic filter iterator for remaining filters
+ if (step instanceof ProxyRuleWithSesameFilters) {
+ Collection<Filter> filters =
+ ((ProxyRuleWithSesameFilters) step).getSesameFilters();
+ if (log.isInfoEnabled() && filters.size() > 0) {
+ log.info("could not translate " + filters.size()
+ + " filters into native constraints:");
+ }
+ for (Filter filter : filters) {
+ if (log.isInfoEnabled())
+ log.info("\n" + filter.getCondition());
+ result = new FilterIterator(filter, result, this);
+ }
+ }
+// // use the basic filter iterator for remaining filters
+// if (sesameFilters != null) {
+// for (Filter f : sesameFilters) {
+// if (log.isDebugEnabled()) {
+// log.debug("attaching sesame filter: " + f);
+// }
+// result = new FilterIterator(f, result, this);
+// }
+// }
- /*
- * FIXME This will deadlock in the buffer fills - see
- * BigdataEvaluationStrategyImpl3 which contains a new code pattern for
- * this.
- */
- try {
- // Wait for the Future (checks for errors).
- runningQuery.get();
- } catch (Exception ex) {
- throw new QueryEvaluationException(ex);
- }
-
-// final boolean backchain = //
-// tripleSource.getDatabase().getAxioms().isRdfSchema()
-// && tripleSource.includeInferred
-// && tripleSource.conn.isQueryTimeExpander();
+ return result;
+
+ } catch (Throwable t) {
+ if (runningQuery != null)
+ runningQuery.cancel(true/* mayInterruptIfRunning */);
+ throw new QueryEvaluationException(t);
+ }
+
+// final IRunningQuery runningQuery = queryEngine.eval(query);
+//
+// final IAsynchronousIterator<IBindingSet[]> it1 =
+// runningQuery.iterator();
//
-// if (log.isDebugEnabled()) {
-// log.debug("Running tupleExpr as native rule:\n" + step);
-// log.debug("backchain: " + backchain);
-// }
+//// final IChunkedOrderedIterator<IBindingSet> it2 =
+//// new ChunkedArraysIterator<IBindingSet>(it1);
+// final IChunkedOrderedIterator<IBindingSet> it2 = new ChunkedWrappedIterator<IBindingSet>(
+// new Dechunkerator<IBindingSet>(it1));
//
-// // run the query as a native rule.
-// final IChunkedOrderedIterator<ISolution> itr1;
+// CloseableIteration<BindingSet, QueryEvaluationException> result =
+// new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
+// new BigdataBindingSetResolverator(database, it2).start(database
+// .getExecutorService()));
+//
+// /*
+// * FIXME This will deadlock in the buffer fills - see
+// * BigdataEvaluationStrategyImpl3 which contains a new code pattern for
+// * this.
+// */
// try {
-// final IEvaluationPlanFactory planFactory =
-// DefaultEvaluationPlanFactory2.INSTANCE;
-//
-// /*
-// * alternative evaluation orders for LUBM Q9 (default is 1 4, 2, 3,
-// * 0, 5). All three evaluation orders are roughly as good as one
-// * another. Note that tail[2] (z rdf:type ...) is entailed by the
-// * ontology and could be dropped from evaluation.
-// */
-// // final IEvaluationPlanFactory planFactory = new
-// // FixedEvaluationPlanFactory(
-// // // new int[] { 1, 4, 3, 0, 5, 2 } good
-// // // new int[] { 1, 3, 0, 4, 5, 2 } good
-// // );
-//
-// final IJoinNexusFactory joinNexusFactory = database
-// .newJoinNexusFactory(RuleContextEnum.HighLevelQuery,
-// ActionEnum.Query, IJoinNexus.BINDINGS,
-// null, // filter
-// false, // justify
-// backchain, //
-// planFactory, //
-// queryHints
-// );
-//
-// final IJoinNexus joinNexus = joinNexusFactory.newInstance(database
-// .getIndexManager());
-// itr1 = joinNexus.runQuery(step);
-//
+// // Wait for the Future (checks for errors).
+// runningQuery.get();
// } catch (Exception ex) {
// throw new QueryEvaluationException(ex);
// }
//
-// /*
-// * Efficiently resolve term identifiers in Bigdata ISolutions to RDF
-// * Values in Sesame 2 BindingSets and align the resulting iterator with
-// * the Sesame 2 API.
-// */
-// CloseableIteration<BindingSet, QueryEvaluationException> result =
-// new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
-// new BigdataSolutionResolverator(database, itr1).start(database
-// .getExecutorService()));
+//// final boolean backchain = //
+//// tripleSource.getDatabase().getAxioms().isRdfSchema()
+//// && tripleSource.includeInferred
+//// && tripleSource.conn.isQueryTimeExpander();
+////
+//// if (log.isDebugEnabled()) {
+//// log.debug("Running tupleExpr as native rule:\n" + step);
+//// log.debug("backchain: " + backchain);
+//// }
+////
+//// // run the query as a native rule.
+//// final IChunkedOrderedIterator<ISolution> itr1;
+//// try {
+//// final IEvaluationPlanFactory planFactory =
+//// DefaultEvaluationPlanFactory2.INSTANCE;
+////
+//// /*
+//// * alternative evaluation orders for LUBM Q9 (default is 1 4, 2, 3,
+//// * 0, 5). All three evaluation orders are roughly as good as one
+//// * another. Note that tail[2] (z rdf:type ...) is entailed by the
+//// * ontology and could be dropped from evaluation.
+//// */
+//// // final IEvaluationPlanFactory planFactory = new
+//// // FixedEvaluationPlanFactory(
+//// // // new int[] { 1, 4, 3, 0, 5, 2 } good
+//// // // new int[] { 1, 3, 0, 4, 5, 2 } good
+//// // );
+////
+//// final IJoinNexusFactory joinNexusFactory = database
+//// .newJoinNexusFactory(RuleContextEnum.HighLevelQuery,
+//// ActionEnum.Query, IJoinNexus.BINDINGS,
+//// null, // filter
+//// false, // justify
+//// backchain, //
+//// planFactory, //
+//// queryHints
+//// );
+////
+//// final IJoinNexus joinNexus = joinNexusFactory.newInstance(database
+//// .getIndexManager());
+//// itr1 = joinNexus.runQuery(step);
+////
+//// } catch (Exception ex) {
+//// throw new QueryEvaluationException(ex);
+//// }
+////
+//// /*
+//// * Efficiently resolve term identifiers in Bigdata ISolutions to RDF
+//// * Values in Sesame 2 BindingSets and align the resulting iterator with
+//// * the Sesame 2 API.
+//// */
+//// CloseableIteration<BindingSet, QueryEvaluationException> result =
+//// new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
+//// new BigdataSolutionResolverator(database, itr1).start(database
+//// .getExecutorService()));
+//
+// // use the basic filter iterator for remaining filters
+// if (step instanceof ProxyRuleWithSesameFilters) {
+// Collection<Filter> filters =
+// ((ProxyRuleWithSesameFilters) step).getSesameFilters();
+// if (log.isInfoEnabled() && filters.size() > 0) {
+// log.info("could not translate " + filters.size()
+// + " filters into native constraints:");
+// }
+// for (Filter filter : filters) {
+// if (log.isInfoEnabled())
+// log.info("\n" + filter.getCondition());
+// result = new FilterIterator(filter, result, this);
+// }
+// }
+//
+// return result;
- // use the basic filter iterator for remaining filters
- if (step instanceof ProxyRuleWithSesameFilters) {
- Collection<Filter> filters =
- ((ProxyRuleWithSesameFilters) step).getSesameFilters();
- if (log.isInfoEnabled() && filters.size() > 0) {
- log.info("could not translate " + filters.size()
- + " filters into native constraints:");
- }
- for (Filter filter : filters) {
- if (log.isInfoEnabled())
- log.info("\n" + filter.getCondition());
- result = new FilterIterator(filter, result, this);
- }
- }
-
- return result;
-
}
// /**
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java 2011-01-15 14:59:52 UTC (rev 4102)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/QueryHints.java 2011-01-15 21:23:23 UTC (rev 4103)
@@ -56,11 +56,22 @@
*/
String NAMESPACE = "BIGDATA_QUERY_HINTS";
- /**
- * Specify the query optimizer.
- *
- * @see QueryOptimizerEnum
- */
+ /**
+ * Specify the query optimizer. For example, you can disable the query
+ * optimizer using
+ *
+ * <pre>
+ * PREFIX BIGDATA_QUERY_HINTS: <http://www.bigdata.com/queryHints#com.bigdata.rdf.sail.QueryHints.optimizer=None>
+ * </pre>
+ *
+ * Disabling the query optimizer can be useful if you have a query for which
+ * the static query optimizer is producing a inefficient join ordering. With
+ * the query optimizer disabled for that query, the joins will be run in the
+ * order given. This makes it possible for you to decide on the right join
+ * ordering for that query.
+ *
+ * @see QueryOptimizerEnum
+ */
String OPTIMIZER = QueryHints.class.getName() + ".optimizer";
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-15 14:59:58
|
Revision: 4102
http://bigdata.svn.sourceforge.net/bigdata/?rev=4102&view=rev
Author: thompsonbry
Date: 2011-01-15 14:59:52 +0000 (Sat, 15 Jan 2011)
Log Message:
-----------
Modified to use the reduced query mix by default.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt 2011-01-15 14:53:26 UTC (rev 4101)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/ignoreQueries.txt 2011-01-15 14:59:52 UTC (rev 4102)
@@ -1 +1 @@
-3 5 6
\ No newline at end of file
+5 6
\ No newline at end of file
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-15 14:53:34
|
Revision: 4101
http://bigdata.svn.sourceforge.net/bigdata/?rev=4101&view=rev
Author: thompsonbry
Date: 2011-01-15 14:53:26 +0000 (Sat, 15 Jan 2011)
Log Message:
-----------
Working through a problem with SubqueryOp leading to non-termination of some queries in BSBM. This appears to be fixed at this checkpoint. I am not sure yet if the fix was the -XX:+UseMembar JVM argument (I was running with JDK 1.6.0_17 which can loose monitor wake up signals) or the modifications to SubqueryOp.
I have made several changes to improve the trapping of "normal" query termination exceptions (InterruptedException, BufferClosedException, and ClosedByInterruptException). All of these can arise in response to an interrupt triggered when a LIMIT is satisfied on the query.
The RWStore and WORMStore properties files were modified to turn on the new query evaluation strategy impl which handles optional join groups. The data need to be loaded with that option enabled and also with dataTime inlining enabled.
Added more reporting capabilities to the NanoSparqlServer, including reporting of the BOPs for the currently executing queries. This required adding a public method to IQueryClient to report on the UUIDs of the active IRunningQuery instances and making getRunningQuery(UUID) public so we can inspect the query by its UUID.
BlockingBuffer was generating a log message even when it would not log the message. This is fixed.
RWStore was logging an error in getData() when there are several non-error reasons why that method could throw an exception (an interrupt leading to a ClosedByInterrupt exception is the main one).
Modified SOp2BOpUtility to layer the slice over the subquery op and not the other way around.
The current BSBM performance status is excellent for the reduced query mix (without Q5 or Q6). Q5 appears to have a bad join plan which causes high CPU utilization. Q6 is visiting much more data than is otherwise required in order to satisfy a regex without a prefix anchor.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/AbstractChunkedResolverator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestRunState.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestNIOChunkMessage.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestThickChunkMessage.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/RWStore.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/WORMStore.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/build.xml
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/logging/log4j.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/log4j.properties
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -27,6 +27,7 @@
package com.bigdata.bop.controller;
+import java.nio.channels.ClosedByInterruptException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -46,7 +47,6 @@
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
import com.bigdata.util.InnerCause;
-import com.bigdata.util.concurrent.LatchedExecutor;
/**
* For each binding set presented, this operator executes a subquery. Any
@@ -56,7 +56,7 @@
* semantics). Each subquery is run as a separate query but will be cancelled if
* the parent query is cancelled.
*
- * FIXME Parallel evaluation of subqueries is not implemented. What is the
+ * @todo Parallel evaluation of subqueries is not implemented. What is the
* appropriate parallelism for this operator? More parallelism should reduce
* latency but could increase the memory burden. Review this decision once we
* have the RWStore operating as a binding set buffer on the Java process heap.
@@ -83,30 +83,30 @@
/**
* When <code>true</code> the subquery has optional semantics (if the
* subquery fails, the original binding set will be passed along to the
- * downstream sink anyway).
+ * downstream sink anyway) (default {@value #DEFAULT_OPTIONAL}).
*/
String OPTIONAL = SubqueryOp.class.getName() + ".optional";
boolean DEFAULT_OPTIONAL = false;
- /**
- * The maximum parallelism with which the subqueries will be evaluated
- * (default {@value #DEFAULT_MAX_PARALLEL}).
- */
- String MAX_PARALLEL = SubqueryOp.class.getName()
- + ".maxParallel";
+// /**
+// * The maximum parallelism with which the subqueries will be evaluated
+// * (default {@value #DEFAULT_MAX_PARALLEL}).
+// */
+// String MAX_PARALLEL = SubqueryOp.class.getName()
+// + ".maxParallel";
+//
+// int DEFAULT_MAX_PARALLEL = 1;
- int DEFAULT_MAX_PARALLEL = 1;
-
}
- /**
- * @see Annotations#MAX_PARALLEL
- */
- public int getMaxParallel() {
- return getProperty(Annotations.MAX_PARALLEL,
- Annotations.DEFAULT_MAX_PARALLEL);
- }
+// /**
+// * @see Annotations#MAX_PARALLEL
+// */
+// public int getMaxParallel() {
+// return getProperty(Annotations.MAX_PARALLEL,
+// Annotations.DEFAULT_MAX_PARALLEL);
+// }
/**
* Deep copy constructor.
@@ -171,14 +171,14 @@
*/
private static class ControllerTask implements Callable<Void> {
- private final SubqueryOp controllerOp;
+// private final SubqueryOp controllerOp;
private final BOpContext<IBindingSet> context;
// private final List<FutureTask<IRunningQuery>> tasks = new LinkedList<FutureTask<IRunningQuery>>();
// private final CountDownLatch latch;
private final boolean optional;
- private final int nparallel;
+// private final int nparallel;
private final PipelineOp subquery;
- private final Executor executor;
+// private final Executor executor;
public ControllerTask(final SubqueryOp controllerOp, final BOpContext<IBindingSet> context) {
@@ -188,21 +188,21 @@
if (context == null)
throw new IllegalArgumentException();
- this.controllerOp = controllerOp;
+// this.controllerOp = controllerOp;
this.context = context;
this.optional = controllerOp.getProperty(Annotations.OPTIONAL,
Annotations.DEFAULT_OPTIONAL);
- this.nparallel = controllerOp.getProperty(Annotations.MAX_PARALLEL,
- Annotations.DEFAULT_MAX_PARALLEL);
+// this.nparallel = controllerOp.getProperty(Annotations.MAX_PARALLEL,
+// Annotations.DEFAULT_MAX_PARALLEL);
this.subquery = (PipelineOp) controllerOp
.getRequiredProperty(Annotations.SUBQUERY);
- this.executor = new LatchedExecutor(context.getIndexManager()
- .getExecutorService(), nparallel);
+// this.executor = new LatchedExecutor(context.getIndexManager()
+// .getExecutorService(), nparallel);
// this.latch = new CountDownLatch(controllerOp.arity());
@@ -258,26 +258,37 @@
for(IBindingSet bset : chunk) {
- FutureTask<IRunningQuery> ft = new FutureTask<IRunningQuery>(
- new SubqueryTask(bset, subquery, context));
+ final IRunningQuery runningSubquery = new SubqueryTask(
+ bset, subquery, context).call();
- // run the subquery.
- executor.execute(ft);
+ if (!runningSubquery.isDone()) {
- try {
+ throw new AssertionError("Future not done: "
+ + runningSubquery.toString());
+
+ }
- // wait for the outcome.
- ft.get();
-
- } finally {
-
- /*
- * Ensure that the inner task is cancelled if the
- * outer task is interrupted.
- */
- ft.cancel(true/* mayInterruptIfRunning */);
-
- }
+// Note: Variant using executor, but still does not support parallel evaluation of subqueries.
+// final FutureTask<IRunningQuery> ft = new FutureTask<IRunningQuery>(
+// new SubqueryTask(bset, subquery, context));
+//
+// try {
+//
+// // run the subquery.
+// executor.execute(ft);
+//
+// // wait for the outcome.
+// ft.get();
+//
+// } finally {
+//
+// /*
+// * Ensure that the inner task is cancelled if the
+// * outer task is interrupted.
+// */
+// ft.cancel(true/* mayInterruptIfRunning */);
+//
+// }
}
@@ -362,16 +373,15 @@
public IRunningQuery call() throws Exception {
+ // The subquery
+ IRunningQuery runningSubquery = null;
+ // The iterator draining the subquery
IAsynchronousIterator<IBindingSet[]> subquerySolutionItr = null;
- IRunningQuery runningQuery = null;
try {
final QueryEngine queryEngine = parentContext.getRunningQuery()
.getQueryEngine();
-// final IRunningQuery runningQuery = queryEngine
-// .eval(subQueryOp);
-
final BOp startOp = BOpUtility.getPipelineStart(subQueryOp);
final int startId = startOp.getId();
@@ -379,7 +389,7 @@
final UUID queryId = UUID.randomUUID();
// execute the subquery, passing in the source binding set.
- runningQuery = queryEngine
+ runningSubquery = queryEngine
.eval(
queryId,
(PipelineOp) subQueryOp,
@@ -391,40 +401,58 @@
new ThickAsynchronousIterator<IBindingSet[]>(
new IBindingSet[][] { new IBindingSet[] { bset } })));
- // Iterator visiting the subquery solutions.
- subquerySolutionItr = runningQuery.iterator();
+ long ncopied = 0L;
+ try {
+
+ // Iterator visiting the subquery solutions.
+ subquerySolutionItr = runningSubquery.iterator();
- // Copy solutions from the subquery to the query.
- final long ncopied = BOpUtility.copy(subquerySolutionItr,
- parentContext.getSink(), null/* sink2 */,
- null/* constraints */, null/* stats */);
-
- // wait for the subquery.
- runningQuery.get();
+ // Copy solutions from the subquery to the query.
+ ncopied = BOpUtility.copy(subquerySolutionItr,
+ parentContext.getSink(), null/* sink2 */,
+ null/* constraints */, null/* stats */);
+ // wait for the subquery to halt / test for errors.
+ runningSubquery.get();
+
+ } catch (InterruptedException ex) {
+
+ // this thread was interrupted, so cancel the subquery.
+ runningSubquery
+ .cancel(true/* mayInterruptIfRunning */);
+
+ // rethrow the exception.
+ throw ex;
+
+ }
+
if (ncopied == 0L && optional) {
/*
* Since there were no solutions for the subquery, copy
* the original binding set to the default sink.
*/
- parentContext.getSink().add(new IBindingSet[]{bset});
+
+ parentContext.getSink().add(new IBindingSet[]{bset});
}
// done.
- return runningQuery;
+ return runningSubquery;
} catch (Throwable t) {
- /*
- * Note: SliceOp will cause other operators to be
- * interrupted during normal evaluation but we do not want
- * to terminate the parent query when this occurs.
- */
- if (!InnerCause.isInnerCause(t, InterruptedException.class)
- && !InnerCause.isInnerCause(t, BufferClosedException.class)) {
-
+ /*
+ * Note: SliceOp will cause other operators to be
+ * interrupted during normal evaluation. Therefore, while
+ * these exceptions should cause the subquery to terminate,
+ * they should not be reported as errors to the parent
+ * query.
+ */
+ if (!InnerCause.isInnerCause(t, InterruptedException.class)
+ && !InnerCause.isInnerCause(t, BufferClosedException.class)
+ && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)) {
+
/*
* If a subquery fails, then propagate the error to the
* parent and rethrow the first cause error out of the
@@ -435,13 +463,25 @@
}
- return runningQuery;
+ return runningSubquery;
} finally {
- if (subquerySolutionItr != null)
- subquerySolutionItr.close();
+ try {
+ // ensure subquery is halted.
+ if (runningSubquery != null)
+ runningSubquery
+ .cancel(true/* mayInterruptIfRunning */);
+
+ } finally {
+
+ // ensure the subquery solution iterator is closed.
+ if (subquerySolutionItr != null)
+ subquerySolutionItr.close();
+
+ }
+
}
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/AbstractRunningQuery.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -28,6 +28,7 @@
package com.bigdata.bop.engine;
import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
import java.util.Collections;
import java.util.Map;
import java.util.UUID;
@@ -799,19 +800,25 @@
try {
- /*
- * Note: SliceOp will cause other operators to be interrupted
- * during normal evaluation so it is not useful to log an
- * InterruptedException @ ERROR.
- */
- if (!InnerCause.isInnerCause(t, InterruptedException.class)
- && !InnerCause.isInnerCause(t, BufferClosedException.class))
- log.error(toString(), t);
-
try {
- // signal error condition.
- return future.halt(t);
+ /*
+ * Note: SliceOp will cause other operators to be interrupted
+ * during normal evaluation so it is not useful to log an
+ * InterruptedException @ ERROR.
+ */
+ if (!InnerCause.isInnerCause(t, InterruptedException.class)
+ && !InnerCause.isInnerCause(t, BufferClosedException.class)
+ && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)) {
+ log.error(toString(), t);
+ // signal error condition.
+ return future.halt(t);
+ } else {
+ // normal termination.
+ future.halt((Void)null/* result */);
+ // the caller's cause.
+ return t;
+ }
} finally {
@@ -990,16 +997,28 @@
public String toString() {
final StringBuilder sb = new StringBuilder(getClass().getName());
- sb.append("{queryId=" + queryId);
- sb.append(",deadline=" + deadline.get());
- sb.append(",isDone=" + isDone());
- sb.append(",isCancelled=" + isCancelled());
- sb.append(",runState=" + runState);
- sb.append(",controller=" + controller);
- sb.append(",clientProxy=" + clientProxy);
- sb.append(",query=" + query);
- sb.append("}");
- return sb.toString();
+ sb.append("{queryId=" + queryId);
+ /*
+ * Note: Obtaining the lock here is required to avoid concurrent
+ * modification exception in RunState's toString() when there is a
+ * concurrent change in the RunState. It also makes the isDone() and
+ * isCancelled() reporting atomic.
+ */
+ lock.lock();
+ try {
+ sb.append(",elapsed=" + getElapsed());
+ sb.append(",deadline=" + deadline.get());
+ sb.append(",isDone=" + isDone());
+ sb.append(",isCancelled=" + isCancelled());
+ sb.append(",runState=" + runState);
+ } finally {
+ lock.unlock();
+ }
+ sb.append(",controller=" + controller);
+ sb.append(",clientProxy=" + clientProxy);
+ sb.append(",query=" + query);
+ sb.append("}");
+ return sb.toString();
}
// abstract protected IChunkHandler getChunkHandler();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/ChunkedRunningQuery.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -27,6 +27,7 @@
*/
package com.bigdata.bop.engine;
+import java.nio.channels.ClosedByInterruptException;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -783,26 +784,33 @@
public void run() {
- super.run();
+ try {
- /*
- * This task is done executing so remove its Future before we
- * attempt to schedule another task for the same
- * (bopId,partitionId).
- */
- final ConcurrentHashMap<ChunkFutureTask, ChunkFutureTask> map = operatorFutures
- .get(new BSBundle(t.bopId, t.partitionId));
+ super.run();
+
+ } finally {
- if (map != null) {
+ /*
+ * This task is done executing so remove its Future before we
+ * attempt to schedule another task for the same
+ * (bopId,partitionId).
+ */
- map.remove(this, this);
+ final ConcurrentHashMap<ChunkFutureTask, ChunkFutureTask> map = operatorFutures
+ .get(new BSBundle(t.bopId, t.partitionId));
- }
+ if (map != null) {
- // Schedule another task if any messages are waiting.
- ChunkedRunningQuery.this.scheduleNext(new BSBundle(
- t.bopId, t.partitionId));
-
+ map.remove(this, this);
+
+ }
+
+ }
+
+ // Schedule another task if any messages are waiting.
+ ChunkedRunningQuery.this.scheduleNext(new BSBundle(t.bopId,
+ t.partitionId));
+
}
}
@@ -852,6 +860,21 @@
final long begin = System.currentTimeMillis();
try {
t.call();
+ } catch(Throwable t) {
+ /*
+ * Note: SliceOp will cause other operators to be
+ * interrupted during normal evaluation. Therefore, while
+ * these exceptions should cause the query to terminate,
+ * they should not be reported as errors to the query
+ * controller.
+ */
+ if (!InnerCause.isInnerCause(t, InterruptedException.class)
+ && !InnerCause.isInnerCause(t, BufferClosedException.class)
+ && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)
+ ) {
+ // Not an error that we should ignore.
+ throw t;
+ }
} finally {
t.context.getStats().elapsed.add(System.currentTimeMillis()
- begin);
@@ -876,19 +899,10 @@
} catch (Throwable ex1) {
- /*
- * Note: SliceOp will cause other operators to be interrupted
- * during normal evaluation so it is not useful to log an
- * InterruptedException @ ERROR.
- */
- if (!InnerCause.isInnerCause(ex1, InterruptedException.class)
- && !InnerCause.isInnerCause(ex1, BufferClosedException.class)
- ) {
- // Log an error.
- log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
- + ", bop=" + t.bop, ex1);
- }
-
+ // Log an error.
+ log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
+ + ", bop=" + t.bop, ex1);
+
/*
* Mark the query as halted on this node regardless of whether
* we are able to communicate with the query controller.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryClient.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryClient.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryClient.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -10,6 +10,13 @@
*/
public interface IQueryClient extends IQueryPeer {
+ /**
+ * Return the set of queries which are running as of the moment when the
+ * request was processed. Queries reported in the returned array may
+ * terminate at any time.
+ */
+ UUID[] getRunningQueries();
+
/**
* Return the query.
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -58,6 +58,7 @@
import com.bigdata.btree.IndexSegment;
import com.bigdata.btree.view.FusedView;
import com.bigdata.journal.IIndexManager;
+import com.bigdata.rdf.sail.bench.NanoSparqlClient;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
import com.bigdata.resources.IndexManager;
@@ -967,17 +968,20 @@
}
}
-
+
/**
- * Return the {@link AbstractRunningQuery} associated with that query identifier.
+ * Return the {@link AbstractRunningQuery} associated with that query
+ * identifier.
*
* @param queryId
* The query identifier.
*
- * @return The {@link AbstractRunningQuery} -or- <code>null</code> if there is no
- * query associated with that query identifier.
+ * @return The {@link AbstractRunningQuery} -or- <code>null</code> if there
+ * is no query associated with that query identifier.
+ *
+ * @todo Exposed to {@link NanoSparqlServer}
*/
- protected AbstractRunningQuery getRunningQuery(final UUID queryId) {
+ public /*protected*/ AbstractRunningQuery getRunningQuery(final UUID queryId) {
if(queryId == null)
throw new IllegalArgumentException();
@@ -1164,4 +1168,10 @@
}
+ public UUID[] getRunningQueries() {
+
+ return runningQueries.keySet().toArray(new UUID[0]);
+
+ }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/StandaloneChainedRunningQuery.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -27,6 +27,7 @@
package com.bigdata.bop.engine;
+import java.nio.channels.ClosedByInterruptException;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
@@ -440,6 +441,21 @@
final long begin = System.currentTimeMillis();
try {
t.call();
+ } catch(Throwable t) {
+ /*
+ * Note: SliceOp will cause other operators to be
+ * interrupted during normal evaluation. Therefore, while
+ * these exceptions should cause the query to terminate,
+ * they should not be reported as errors to the query
+ * controller.
+ */
+ if (!InnerCause.isInnerCause(t, InterruptedException.class)
+ && !InnerCause.isInnerCause(t, BufferClosedException.class)
+ && !InnerCause.isInnerCause(t, ClosedByInterruptException.class)
+ ) {
+ // Not an error that we should ignore.
+ throw t;
+ }
} finally {
t.context.getStats().elapsed.add(System.currentTimeMillis()
- begin);
@@ -457,20 +473,11 @@
StandaloneChainedRunningQuery.this.haltOp(msg);
} catch (Throwable ex1) {
-log.fatal(ex1,ex1); // FIXME remove log stmt.
- /*
- * Note: SliceOp will cause other operators to be interrupted
- * during normal evaluation so it is not useful to log an
- * InterruptedException @ ERROR.
- */
- if (!InnerCause.isInnerCause(ex1, InterruptedException.class)
- && !InnerCause.isInnerCause(ex1, BufferClosedException.class)
- ) {
- // Log an error.
- log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
- + ", bop=" + t.bop, ex1);
- }
+ // Log an error.
+ log.error("queryId=" + getQueryId() + ", bopId=" + t.bopId
+ + ", bop=" + t.bop, ex1);
+
/*
* Mark the query as halted on this node regardless of whether
* we are able to communicate with the query controller.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -163,7 +163,7 @@
* {@inheritDoc}
*/
@Override
- protected FederatedRunningQuery getRunningQuery(final UUID queryId) {
+ public /*protected*/ FederatedRunningQuery getRunningQuery(final UUID queryId) {
return (FederatedRunningQuery) super.getRunningQuery(queryId);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -1074,6 +1074,8 @@
logTimeout += Math.min(maxLogTimeout, logTimeout);
+ if(log.isInfoEnabled()) {
+
final String msg = "blocked: ntries="
+ ntries
+ ", elapsed="
@@ -1096,6 +1098,8 @@
// issue warning.
log.warn(msg);
}
+
+ }
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -1432,7 +1432,13 @@
}
}
} catch (Throwable e) {
- log.error(e,e);
+ /*
+ * Note: ClosedByInterruptException can be thrown out of
+ * FileChannelUtility.readAll(), typically because the LIMIT on
+ * a query was satisified, but we do not want to log that as an
+ * error.
+ */
+// log.error(e,e);
throw new IllegalArgumentException("Unable to read data", e);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/AbstractChunkedResolverator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/AbstractChunkedResolverator.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/AbstractChunkedResolverator.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -239,9 +239,11 @@
} finally {
- src.close();
-
- buffer.close();
+ try {
+ src.close();
+ } finally {
+ buffer.close();
+ }
}
@@ -336,7 +338,11 @@
log.info("lastIndex=" + lastIndex + ", chunkSize="
+ (chunk != null ? "" + chunk.length : "N/A"));
- // asynchronous close by the consumer of the producer's buffer.
+ /*
+ * Asynchronous close by the consumer of the producer's buffer. This
+ * will cause the ChunkConsumerTask to abort if it is still running and
+ * that will cause the [src] to be closed.
+ */
buffer.close();
chunk = null;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestRunState.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestRunState.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestRunState.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -952,6 +952,10 @@
throws RemoteException {
}
+ public UUID[] getRunningQueries() {
+ return null;
+ }
+
}
/**
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestNIOChunkMessage.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestNIOChunkMessage.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestNIOChunkMessage.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -259,6 +259,10 @@
throws RemoteException {
}
+ public UUID[] getRunningQueries() {
+ return null;
+ }
+
}
private static class MyNIOChunkMessage<E> extends NIOChunkMessage<E> {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestThickChunkMessage.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestThickChunkMessage.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestThickChunkMessage.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -232,6 +232,10 @@
throws RemoteException {
}
+ public UUID[] getRunningQueries() {
+ return null;
+ }
+
}
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/RWStore.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/RWStore.properties 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/RWStore.properties 2011-01-15 14:53:26 UTC (rev 4101)
@@ -24,6 +24,17 @@
com.bigdata.namespace.BSBM_284826.spo.SPO.com.bigdata.btree.BTree.branchingFactor=512
com.bigdata.namespace.BSBM_284826.spo.OSP.com.bigdata.btree.BTree.branchingFactor=470
+# Reduce the branching factor for the lexicon since BSBM uses a lot of long
+# literals. Note that you have to edit this override to specify the namespace
+# into which the BSBM data will be loaded.
+com.bigdata.namespace.BSBM_566496.lex.TERM2ID.com.bigdata.btree.BTree.branchingFactor=32
+com.bigdata.namespace.BSBM_566496.lex.ID2TERM.com.bigdata.btree.BTree.branchingFactor=32
+
+# 4k pages.
+com.bigdata.namespace.BSBM_566496.spo.POS.com.bigdata.btree.BTree.branchingFactor=970
+com.bigdata.namespace.BSBM_566496.spo.SPO.com.bigdata.btree.BTree.branchingFactor=512
+com.bigdata.namespace.BSBM_566496.spo.OSP.com.bigdata.btree.BTree.branchingFactor=470
+
# Override the #of write cache buffers.
com.bigdata.journal.AbstractJournal.writeCacheBufferCount=12
@@ -59,3 +70,6 @@
# 10000 is default.
com.bigdata.rdf.sail.bufferCapacity=100000
+
+# direct sesame to bop translation.
+com.bigdata.rdf.sail.newEvalStrategy=true
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/WORMStore.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/WORMStore.properties 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/WORMStore.properties 2011-01-15 14:53:26 UTC (rev 4101)
@@ -38,3 +38,6 @@
# 10000 is default.
com.bigdata.rdf.sail.bufferCapacity=100000
+
+# direct sesame to bop translation.
+com.bigdata.rdf.sail.newEvalStrategy=true
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/build.xml 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/build.xml 2011-01-15 14:53:26 UTC (rev 4101)
@@ -51,14 +51,14 @@
<exclude name="**/*.java" />
<exclude name="**/package.html" />
</fileset>
- <!-- copy log4j configuration file. -->
- <fileset dir="${bsbm.dir}/src/resources/logging" />
</copy>
<copy toDir="${build.dir}/bin">
<!-- copy benchmark data and queries. -->
<fileset dir="${bsbm.dir}/src/resources/bsbm-data" />
<!-- copy the journal configuration file. -->
<fileset file="${bsbm.dir}/*.properties" />
+ <!-- copy log4j configuration file. -->
+ <fileset dir="${bsbm.dir}/src/resources/logging" />
</copy>
</target>
@@ -143,7 +143,7 @@
<!-- delete file if it exists so we load into a new journal. -->
<delete file="${bsbm.journalFile}" />
<java classname="com.bigdata.rdf.store.DataLoader" fork="true" failonerror="true" dir="${build.dir}/bin">
- <arg line="-namespace ${bsbm.namespace} ${bsbm.journalPropertyFile} ${bsbm.outputFile}.${bsbm.outputType}${bsbm.compressType}" />
+ <arg line="-verbose -namespace ${bsbm.namespace} ${bsbm.journalPropertyFile} ${bsbm.outputFile}.${bsbm.outputType}${bsbm.compressType}" />
<!-- specify/override the journal file name. -->
<jvmarg line="${queryJvmArgs} -Dcom.bigdata.journal.AbstractJournal.file=${bsbm.journalFile}
-Dcom.bigdata.rdf.store.DataLoader.bufferCapacity=1000000
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/logging/log4j.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/logging/log4j.properties 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/logging/log4j.properties 2011-01-15 14:53:26 UTC (rev 4101)
@@ -244,7 +244,7 @@
log4j.additivity.com.bigdata.bop.engine.RunState$TableLog=false
log4j.appender.queryRunStateLog=org.apache.log4j.FileAppender
log4j.appender.queryRunStateLog.Threshold=ALL
-log4j.appender.queryRunStateLog.File=queryRunState.log
+log4j.appender.queryRunStateLog.File=queryRunState.csv
log4j.appender.queryRunStateLog.Append=true
# I find that it is nicer to have this unbuffered since you can see what
# is going on and to make sure that I have complete rule evaluation logs
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/log4j.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/log4j.properties 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/log4j.properties 2011-01-15 14:53:26 UTC (rev 4101)
@@ -156,6 +156,8 @@
#log4j.logger.com.bigdata.relation.accesspath.IAccessPath=DEBUG
#log4j.logger.com.bigdata.rdf.sail.BigdataSail=DEBUG
+#log4j.logger.com.bigdata.rdf.sail.Rule2BOpUtility=INFO
+log4j.logger.com.bigdata.bop.controller.JoinGraph=INFO
#log4j.logger.com.bigdata.rdf.sail.TestNamedGraphs=DEBUG
log4j.logger.com.bigdata.rdf.sail.QuadsTestCase=DEBUG
#log4j.logger.com.bigdata.relation.rule.eval.NestedSubqueryWithJoinThreadsTask=DEBUG
@@ -178,7 +180,7 @@
log4j.logger.com.bigdata.rdf.store.DataLoader=INFO
# Test suite logger.
-log4j.logger.junit=INFO
+#log4j.logger.junit=INFO
#log4j.logger.junit=DEBUG
log4j.logger.com.bigdata.btree.AbstractBTreeTestCase=INFO
@@ -202,6 +204,7 @@
##
# Summary query evaluation log (tab delimited file).
+# Uncomment the next line to enable.
#log4j.logger.com.bigdata.bop.engine.QueryLog=INFO,queryLog
log4j.additivity.com.bigdata.bop.engine.QueryLog=false
log4j.appender.queryLog=org.apache.log4j.FileAppender
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -1725,6 +1725,11 @@
new BigdataBindingSetResolverator(database, it2).start(database
.getExecutorService()));
+ /*
+ * FIXME This will deadlock in the buffer fills - see
+ * BigdataEvaluationStrategyImpl3 which contains a new code pattern for
+ * this.
+ */
try {
// Wait for the Future (checks for errors).
runningQuery.get();
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -711,24 +711,34 @@
final Collection<Filter> sesameFilters)
throws QueryEvaluationException {
+ IRunningQuery runningQuery = null;
try {
- final IRunningQuery runningQuery = queryEngine.eval(query);
-
+ // Submit query for evaluation.
+ runningQuery = queryEngine.eval(query);
+
+ // Iterator draining the query results.
final IAsynchronousIterator<IBindingSet[]> it1 =
runningQuery.iterator();
+ // De-chunk the IBindingSet[] visited by that iterator.
final IChunkedOrderedIterator<IBindingSet> it2 =
new ChunkedWrappedIterator<IBindingSet>(
new Dechunkerator<IBindingSet>(it1));
-
- CloseableIteration<BindingSet, QueryEvaluationException> result =
+
+ // Materialize IVs as RDF Values.
+ CloseableIteration<BindingSet, QueryEvaluationException> result =
+ // Monitor IRunningQuery and cancel if Sesame iterator is closed.
+ new RunningQueryCloseableIteration<BindingSet, QueryEvaluationException>(runningQuery,
+ // Convert bigdata binding sets to Sesame binding sets.
new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
+ // Materialize IVs as RDF Values.
new BigdataBindingSetResolverator(database, it2).start(
- database.getExecutorService()));
+ database.getExecutorService())));
- // Wait for the Future (checks for errors).
- runningQuery.get();
+// No - will deadlock if buffer fills up
+// // Wait for the Future (checks for errors).
+// runningQuery.get();
// use the basic filter iterator for remaining filters
if (sesameFilters != null) {
@@ -740,13 +750,13 @@
}
}
- return result;
-
- } catch (QueryEvaluationException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new QueryEvaluationException(ex);
- }
+ return result;
+
+ } catch (Throwable t) {
+ if (runningQuery != null)
+ runningQuery.cancel(true/* mayInterruptIfRunning */);
+ throw new QueryEvaluationException(t);
+ }
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailGraphQuery.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -228,8 +228,8 @@
try {
- TupleExpr tupleExpr = getParsedQuery().getTupleExpr();
- BigdataSailConnection sailCon =
+ final TupleExpr tupleExpr = getParsedQuery().getTupleExpr();
+ final BigdataSailConnection sailCon =
(BigdataSailConnection) getConnection().getSailConnection();
CloseableIteration<? extends BindingSet, QueryEvaluationException> bindingsIter =
sailCon.evaluate(
@@ -242,7 +242,7 @@
bindingsIter) {
@Override
protected boolean accept(BindingSet bindingSet) {
- Value context = bindingSet.getValue("context");
+ final Value context = bindingSet.getValue("context");
return bindingSet.getValue("subject") instanceof Resource
&& bindingSet.getValue("predicate") instanceof URI
&& bindingSet.getValue("object") instanceof Value
@@ -254,15 +254,15 @@
// Convert the BindingSet objects to actual RDF statements
final ValueFactory vf = getConnection().getRepository().getValueFactory();
- CloseableIteration<Statement, QueryEvaluationException> stIter;
+ final CloseableIteration<Statement, QueryEvaluationException> stIter;
stIter = new ConvertingIteration<BindingSet, Statement, QueryEvaluationException>(bindingsIter) {
@Override
protected Statement convert(BindingSet bindingSet) {
- Resource subject = (Resource)bindingSet.getValue("subject");
- URI predicate = (URI)bindingSet.getValue("predicate");
- Value object = bindingSet.getValue("object");
- Resource context = (Resource)bindingSet.getValue("context");
+ final Resource subject = (Resource)bindingSet.getValue("subject");
+ final URI predicate = (URI)bindingSet.getValue("predicate");
+ final Value object = bindingSet.getValue("object");
+ final Resource context = (Resource)bindingSet.getValue("context");
if (context == null) {
return vf.createStatement(subject, predicate, object);
@@ -277,10 +277,11 @@
return new GraphQueryResultImpl(getParsedQuery().getQueryNamespaces(), stIter);
} else {
+ // native construct.
// Convert the BindingSet objects to actual RDF statements
final ValueFactory vf = getConnection().getRepository().getValueFactory();
- CloseableIteration<? extends Statement, QueryEvaluationException> stIter;
+ final CloseableIteration<? extends Statement, QueryEvaluationException> stIter;
stIter = new BigdataConstructIterator(sailCon.getTripleStore(), bindingsIter, vf);
return new GraphQueryResultImpl(getParsedQuery()
.getQueryNamespaces(), stIter);
Added: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/RunningQueryCloseableIteration.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -0,0 +1,63 @@
+package com.bigdata.rdf.sail;
+
+import info.aduna.iteration.CloseableIteration;
+
+import java.util.concurrent.ExecutionException;
+
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.QueryEvaluationException;
+
+import com.bigdata.bop.engine.IRunningQuery;
+
+/**
+ * Iteration construct wraps an {@link IRunningQuery} with logic to (a) verify
+ * that the {@link IRunningQuery} has not encountered an error; and (b) to cancel
+ * the {@link IRunningQuery} when the iteration is {@link #close() closed}.
+ * @author thompsonbry
+ *
+ * @param <E>
+ * @param <X>
+ */
+public class RunningQueryCloseableIteration<E extends BindingSet, X extends QueryEvaluationException>
+ implements CloseableIteration<E, X> {
+
+ private final IRunningQuery runningQuery;
+ private final CloseableIteration<E, X> src;
+ private boolean checkedFuture = false;
+
+ public RunningQueryCloseableIteration(final IRunningQuery runningQuery,
+ final CloseableIteration<E, X> src) {
+
+ this.runningQuery = runningQuery;
+ this.src = src;
+
+ }
+
+ public void close() throws X {
+ runningQuery.cancel(true/* mayInterruptIfRunning */);
+ src.close();
+ }
+
+ public boolean hasNext() throws X {
+ return src.hasNext();
+ }
+
+ public E next() throws X {
+ if (!checkedFuture && runningQuery.isDone()) {
+ try {
+ runningQuery.get();
+ } catch (InterruptedException e) {
+ throw (X) new QueryEvaluationException(e);
+ } catch (ExecutionException e) {
+ throw (X) new QueryEvaluationException(e);
+ }
+ checkedFuture = true;
+ }
+ return src.next();
+ }
+
+ public void remove() throws X {
+ src.remove();
+ }
+
+}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -49,6 +49,7 @@
import java.util.Map;
import java.util.Properties;
import java.util.TreeMap;
+import java.util.UUID;
import java.util.Vector;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
@@ -74,8 +75,12 @@
import org.openrdf.sail.SailException;
import com.bigdata.LRUNexus;
+import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.BufferAnnotations;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.engine.IRunningQuery;
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.bop.fed.QueryEngineFactory;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IndexMetadata;
import com.bigdata.journal.AbstractJournal;
@@ -709,7 +714,7 @@
return config.timestamp;
}
-
+
/**
* Respond to a status request.
*
@@ -719,15 +724,27 @@
* @param params
* @return
* @throws Exception
+ *
+ * @todo add statistics for top-N queries based on query template
+ * identifiers, which can be communicated using query hints. See //
+ * wait for the subquery.
+ * @todo Report on the average query latency, average concurrency of query
+ * evaluation, etc.
*/
public Response doStatus(final String uri, final String method,
final Properties header,
final LinkedHashMap<String, Vector<String>> params) throws Exception {
+ // SPARQL queries accepted by the SPARQL end point.
final boolean showQueries = params.get("showQueries") != null;
+ // IRunningQuery objects currently running on the query controller.
+ final boolean showRunningQueries = params.get("showRunningQueries") != null;
+
+ // Information about the KB (stats, properties).
final boolean showKBInfo = params.get("showKBInfo") != null;
+ // bigdata namespaces known to the index manager.
final boolean showNamespaces = params.get("showNamespaces") != null;
final StringBuilder sb = new StringBuilder();
@@ -789,15 +806,17 @@
}
// show the disk access details.
- sb.append(jnl.getBufferStrategy().getCounters().toString()+"\n\n");
+ sb.append(jnl.getBufferStrategy().getCounters().toString()+"\n");
}
if(showQueries) {
/*
- * Show the queries which are currently executing.
+ * Show the queries which are currently executing (accepted by the NanoSparqlServer).
*/
+
+ sb.append("\n");
final long now = System.nanoTime();
@@ -850,6 +869,86 @@
}
+ if(showRunningQueries) {
+
+ /*
+ * Show the queries which are currently executing (actually running
+ * on the QueryEngine).
+ */
+
+ sb.append("\n");
+
+ final QueryEngine queryEngine = (QueryEngine) QueryEngineFactory
+ .getQueryController(indexManager);
+
+ final UUID[] queryIds = queryEngine.getRunningQueries();
+
+// final long now = System.nanoTime();
+
+ final TreeMap<Long, IRunningQuery> ages = new TreeMap<Long, IRunningQuery>(new Comparator<Long>() {
+ /**
+ * Comparator puts the entries into descending order by the query
+ * execution time (longest running queries are first).
+ */
+ public int compare(final Long o1, final Long o2) {
+ if(o1.longValue()<o2.longValue()) return 1;
+ if(o1.longValue()>o2.longValue()) return -1;
+ return 0;
+ }
+ });
+
+ for(UUID queryId : queryIds) {
+
+ final IRunningQuery query = queryEngine
+ .getRunningQuery(queryId);
+
+ if (query == null) {
+ // Already terminated.
+ continue;
+ }
+
+ ages.put(query.getElapsed(), query);
+
+ }
+
+ {
+
+ final Iterator<IRunningQuery> itr = ages.values().iterator();
+
+ while (itr.hasNext()) {
+
+ final IRunningQuery query = itr.next();
+
+ if (query.isDone() && query.getCause() != null) {
+ // Already terminated (normal completion).
+ continue;
+ }
+
+ /*
+ * @todo The runstate and stats could be formatted into an
+ * HTML table ala QueryLog or RunState.
+ */
+ sb.append("age=" + query.getElapsed() + "ms\n");
+ sb.append("queryId=" + query.getQueryId() + "\n");
+ sb.append(query.toString());
+ sb.append("\n");
+ sb.append(BOpUtility.toString(query.getQuery()));
+ sb.append("\n");
+ sb.append("\n");
+
+// final long age = query.getElapsed();
+// sb.append("age="
+// + java.util.concurrent.TimeUnit.NANOSECONDS
+// .toMillis(age) + "ms, queryId="
+// + query.getQueryId() + "\nquery="
+// + BOpUtility.toString(query.getQuery()) + "\n");
+
+ }
+
+ }
+
+ }
+
return new Response(HTTP_OK, MIME_TEXT_PLAIN, sb.toString());
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java 2011-01-14 18:39:44 UTC (rev 4100)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOp2BOpUtility.java 2011-01-15 14:53:26 UTC (rev 4101)
@@ -169,20 +169,6 @@
PipelineOp left = Rule2BOpUtility.convert(
rule, conditionals, idFactory, db, queryEngine, queryHints);
- if (!left.getEvaluationContext().equals(
- BOpEvaluationContext.CONTROLLER)) {
- /*
- * Wrap with an operator which will be evaluated on the query
- * controller so the results will be streamed back to the query
- * controller in scale-out.
- */
- left = new SliceOp(new BOp[] { left }, NV.asMap(//
- new NV(BOp.Annotations.BOP_ID, idFactory
- .incrementAndGet()), //
- new NV(BOp.Annotations.EVALUATION_CONTEXT,
- BOpEvaluationContext.CONTROLLER)));
- }
-
/*
* Start with left=<this join group> and add a SubqueryOp for each
* sub group.
@@ -207,6 +193,33 @@
}
}
+ if (!left.getEvaluationContext()
+ .equals(BOpEvaluationContext.CONTROLLER)
+ && !(left instanceof SubqueryOp)) {
+ /*
+ * Wrap with an operator which will be evaluated on the query
+ * controller so the results will be streamed back to the query
+ * controller in scale-out.
+ *
+ * @todo For scale-out, we probably need to stream the results back
+ * to the node from which the subquery was issued. If the subquery
+ * is issued against the local query engine where the IBindingSet
+ * was produced, then the that query engine is the query controller
+ * for the subquery and a SliceOp on the subquery would bring the
+ * results for the subquery back to that query controller. There is
+ * no requirement that the query controller for the subquery and the
+ * query controller for the parent query be the same node. [I am not
+ * doing this currently in order to test whether there is a problem
+ * with SliceOp which interactions with SubqueryOp to allow
+ * incorrect termination under some circumstances.
+ */
+ left = new SliceOp(new BOp[] { left }, NV.asMap(//
+ new NV(BOp.Annotations.BOP_ID, idFactory
+ .incrementAndGet()), //
+ new NV(BOp.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER)));
+ }
+
return left;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
