Menu
▾
▴
bigdata-commit — Commit traffic.
This list is closed, nobody may subscribe to it.
| 2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
| 2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
| 2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
| 2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <tho...@us...> - 2011-01-20 22:07:57
|
Revision: 4150
http://bigdata.svn.sourceforge.net/bigdata/?rev=4150&view=rev
Author: thompsonbry
Date: 2011-01-20 22:07:47 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
Added some things to the svn ignore list.
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools/
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/bsbmtools
___________________________________________________________________
Added: svn:ignore
+ ant-build
rampup.tsv
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 22:06:01
|
Revision: 4149
http://bigdata.svn.sourceforge.net/bigdata/?rev=4149&view=rev
Author: thompsonbry
Date: 2011-01-20 22:05:55 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
Fixed build broken by commit of the memory manager with a missing interface (I just commented out the references to the interface -- see the detailed commit log).
Added the addr for which a problem was reported to the runtime exception thrown by the RWStore.
Added the TestMemoryManager into the automated test suite runs (TestAll).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestAll.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestAll.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2011-01-20 21:54:50 UTC (rev 4148)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2011-01-20 22:05:55 UTC (rev 4149)
@@ -1441,7 +1441,7 @@
// log.error(e,e);
// throw new IllegalArgumentException("Unable to read data: "+e, e);
- throw new RuntimeException(e);
+ throw new RuntimeException("addr=" + addr + " : cause=" + e, e);
}
} finally {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java 2011-01-20 21:54:50 UTC (rev 4148)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java 2011-01-20 22:05:55 UTC (rev 4149)
@@ -82,7 +82,7 @@
if ((m_allocation + m_sectorSize) > m_maxResource) {
throw new MemoryManagerResourceError();
}
- SectorAllocator sector = new SectorAllocator(this, null);
+ SectorAllocator sector = new SectorAllocator(this);//, null);
sector.setSectorAddress(m_allocation, m_sectorSize);
sector.setIndex(m_sectors.size());
m_sectors.add(sector);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java 2011-01-20 21:54:50 UTC (rev 4148)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java 2011-01-20 22:05:55 UTC (rev 4149)
@@ -33,7 +33,7 @@
import com.bigdata.io.DirectBufferPool;
import com.bigdata.rwstore.FixedOutputStream;
-import com.bigdata.rwstore.IWriteCacheManager;
+//import com.bigdata.rwstore.IWriteCacheManager;
/**
* The SectorAllocator is designed as an alternative the the standard RWStore
@@ -126,11 +126,11 @@
final ISectorManager m_store;
boolean m_onFreeList = false;
private long m_diskAddr;
- private final IWriteCacheManager m_writes;
+// private final IWriteCacheManager m_writes;
- public SectorAllocator(ISectorManager store, IWriteCacheManager writes) {
+ public SectorAllocator(ISectorManager store) {//, IWriteCacheManager writes) {
m_store = store;
- m_writes = writes;
+// m_writes = writes;
}
/**
@@ -276,10 +276,10 @@
m_store.addToFreeList(this);
}
- if (m_writes != null && m_writes.removeWriteToAddr(getPhysicalAddress(bit))) {
- if (log.isTraceEnabled())
- log.trace("Removed potential DUPLICATE");
- }
+// if (m_writes != null && m_writes.removeWriteToAddr(getPhysicalAddress(bit))) {
+// if (log.isTraceEnabled())
+// log.trace("Removed potential DUPLICATE");
+// }
}
return false;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestAll.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestAll.java 2011-01-20 21:54:50 UTC (rev 4148)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/TestAll.java 2011-01-20 22:05:55 UTC (rev 4149)
@@ -63,6 +63,8 @@
suite.addTest(com.bigdata.rwstore.TestRWJournal.suite());
+ suite.addTest(com.bigdata.rwstore.sector.TestAll.suite());
+
return suite;
}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestAll.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestAll.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestAll.java 2011-01-20 22:05:55 UTC (rev 4149)
@@ -0,0 +1,70 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Oct 14, 2006
+ */
+
+package com.bigdata.rwstore.sector;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Runs all tests for all journal implementations.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: TestAll.java 4069 2011-01-09 20:58:02Z thompsonbry $
+ */
+public class TestAll extends TestCase {
+
+ /**
+ *
+ */
+ public TestAll() {
+ }
+
+ /**
+ * @param arg0
+ */
+ public TestAll(String arg0) {
+ super(arg0);
+ }
+
+ /**
+ * Returns a test that will run each of the implementation specific test
+ * suites in turn.
+ */
+ public static Test suite()
+ {
+
+ final TestSuite suite = new TestSuite("memory manager");
+
+ suite.addTestSuite(com.bigdata.rwstore.sector.TestMemoryManager.class);
+
+ return suite;
+
+ }
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 21:54:56
|
Revision: 4148
http://bigdata.svn.sourceforge.net/bigdata/?rev=4148&view=rev
Author: thompsonbry
Date: 2011-01-20 21:54:50 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
The rampup target needs to run in the bsbmtools directory.
Removed the print queries target. It requires a custom integration with bsbmtools which has not been ported yet.
Modified Paths:
--------------
tags/BSBM3/bigdata-perf/bsbm3/build.xml
Modified: tags/BSBM3/bigdata-perf/bsbm3/build.xml
===================================================================
--- tags/BSBM3/bigdata-perf/bsbm3/build.xml 2011-01-20 21:51:50 UTC (rev 4147)
+++ tags/BSBM3/bigdata-perf/bsbm3/build.xml 2011-01-20 21:54:50 UTC (rev 4148)
@@ -184,7 +184,7 @@
</target>
<target name="rampup" depends="compile" description="Runs the benchmark queries against the loaded data until system performance reaches a steady state as defined by the benchmark.">
- <java classname="benchmark.testdriver.TestDriver" fork="true" failonerror="true" dir="${build.dir}/bin">
+ <java classname="benchmark.testdriver.TestDriver" fork="true" failonerror="true" dir="${bsbmtools.dir}">
<arg value="-rampup" />
@@ -210,80 +210,4 @@
</java>
</target>
- <target name="run-query" depends="compile" description="Runs the benchmark queries against the loaded data.">
- <java classname="benchmark.testdriver.TestDriver" fork="true" failonerror="true" dir="${bsbmtools.dir}">
- <!-- -runs # is the #of query mix runs (default is 500). -->
- <arg value="-runs" />
- <arg value="${bsbm.runs}" />
-
- <!-- -w # is the #of warmup query mixes (default is 50). -->
- <arg value="-w" />
- <arg value="${bsbm.w}" />
-
- <!-- -mt # is the #of concurrent clients. -->
- <arg value="-mt" />
- <arg value="${bsbm.mt}" />
-
- <!-- -qdir dir is the query directory (default is queries). -->
- <!--<arg value="-qdir"/><arg value="src/resources/bsbm_data"/>-->
-
- <!-- -idir dir is the test data directory (default td_data). -->
- <arg value="-idir" />
- <arg value="${bsbm.dataDir}" />
-
- <!-- The randomizer seed. -->
- <arg value="-seed" />
- <arg value="${bsbm.seed}" />
-
- <!-- -o file is the name of the xml output file. -->
- <arg value="-o" />
- <arg value="${bsbm.resultsDir}/benchmark_result_pc${bsbm.pc}_runs${bsbm.runs}_mt${bsbm.mt}.xml" />
-
- <!-- The SPARQL endpoint. -->
- <arg value="http://localhost:${bsbm.nanoServerPort}" />
-
- <classpath>
- <path refid="runtime.classpath" />
- </classpath>
- </java>
- </target>
-
- <target name="run-print-queries" depends="compile" description="Prints some sample benchmark queries to be run against the loaded data.">
- <java classname="benchmark.bigdata.TestDriver" fork="true" failonerror="true" dir="${build.dir}/bin">
- <!-- -runs # is the #of query mix runs (default is 500). -->
- <arg value="-runs" />
- <arg value="${bsbm.runs}" />
-
- <!-- -w # is the #of warmup query mixes (default is 50). -->
- <arg value="-w" />
- <arg value="${bsbm.w}" />
-
- <!-- -mt # is the #of concurrent clients. -->
- <arg value="-mt" />
- <arg value="${bsbm.mt}" />
-
- <!-- -qdir dir is the query directory (default is queries). -->
- <!--<arg value="-qdir"/><arg value="src/resources/bsbm_data"/>-->
-
- <!-- -idir dir is the test data directory (default td_data). -->
- <arg value="-idir" />
- <arg value="${bsbm.dataDir}" />
-
- <!-- The randomizer seed. -->
- <arg value="-seed" />
- <arg value="${bsbm.seed}" />
-
- <!-- -o file is the name of the xml output file. -->
- <arg value="-o" />
- <arg value="${bsbm.resultsDir}/benchmark_result_pc${bsbm.pc}_runs${bsbm.runs}_mt${bsbm.mt}.xml" />
-
- <!-- The SPARQL endpoint. -->
- <arg value="http://localhost:${bsbm.nanoServerPort}" />
-
- <classpath>
- <path refid="runtime.classpath" />
- </classpath>
- </java>
- </target>
-
</project>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 21:51:58
|
Revision: 4147
http://bigdata.svn.sourceforge.net/bigdata/?rev=4147&view=rev
Author: thompsonbry
Date: 2011-01-20 21:51:50 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
The rampup target needs to run in the bsbmtools directory.
Removed the print queries target. It requires a custom integration with bsbmtools which has not been ported yet.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.xml
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.xml 2011-01-20 21:40:07 UTC (rev 4146)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm3/build.xml 2011-01-20 21:51:50 UTC (rev 4147)
@@ -184,7 +184,7 @@
</target>
<target name="rampup" depends="compile" description="Runs the benchmark queries against the loaded data until system performance reaches a steady state as defined by the benchmark.">
- <java classname="benchmark.testdriver.TestDriver" fork="true" failonerror="true" dir="${build.dir}/bin">
+ <java classname="benchmark.testdriver.TestDriver" fork="true" failonerror="true" dir="${bsbmtools.dir}">
<arg value="-rampup" />
@@ -248,42 +248,4 @@
</java>
</target>
- <target name="run-print-queries" depends="compile" description="Prints some sample benchmark queries to be run against the loaded data.">
- <java classname="benchmark.bigdata.TestDriver" fork="true" failonerror="true" dir="${build.dir}/bin">
- <!-- -runs # is the #of query mix runs (default is 500). -->
- <arg value="-runs" />
- <arg value="${bsbm.runs}" />
-
- <!-- -w # is the #of warmup query mixes (default is 50). -->
- <arg value="-w" />
- <arg value="${bsbm.w}" />
-
- <!-- -mt # is the #of concurrent clients. -->
- <arg value="-mt" />
- <arg value="${bsbm.mt}" />
-
- <!-- -qdir dir is the query directory (default is queries). -->
- <!--<arg value="-qdir"/><arg value="src/resources/bsbm_data"/>-->
-
- <!-- -idir dir is the test data directory (default td_data). -->
- <arg value="-idir" />
- <arg value="${bsbm.dataDir}" />
-
- <!-- The randomizer seed. -->
- <arg value="-seed" />
- <arg value="${bsbm.seed}" />
-
- <!-- -o file is the name of the xml output file. -->
- <arg value="-o" />
- <arg value="${bsbm.resultsDir}/benchmark_result_pc${bsbm.pc}_runs${bsbm.runs}_mt${bsbm.mt}.xml" />
-
- <!-- The SPARQL endpoint. -->
- <arg value="http://localhost:${bsbm.nanoServerPort}" />
-
- <classpath>
- <path refid="runtime.classpath" />
- </classpath>
- </java>
- </target>
-
</project>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 21:40:13
|
Revision: 4146
http://bigdata.svn.sourceforge.net/bigdata/?rev=4146&view=rev
Author: thompsonbry
Date: 2011-01-20 21:40:07 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
Counting solutions which join before applying the filter fixed the 1st and 3rd of these queries. Also applying the filter to the original solution in the case where we output the original solution if the optional join failed fixes the middle query, so all three are good now.
http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bound/manifest#dawg-bound-query-001
http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-002
http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-003
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-20 19:49:12 UTC (rev 4145)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-20 21:40:07 UTC (rev 4146)
@@ -1684,16 +1684,11 @@
final IBindingSet bs = bindingSets[bindex];
- if (constraints != null) {
-
- // verify constraint.
- if(!BOpUtility.isConsistent(constraints, bs)) {
- // skip solutions which fail the constraint.
- continue;
- }
-
- }
-
+ if(!BOpUtility.isConsistent(constraints, bs)) {
+ // Failed by the constraint on the join.
+ continue;
+ }
+
if (log.isTraceEnabled())
log
.trace("Passing on solution which fails an optional join: "
@@ -2077,6 +2072,21 @@
int bindex = 0;
for (IBindingSet bset : bindingSets) {
+ // #of binding sets accepted.
+ naccepted++;
+
+ /* #of elements accepted for this binding set.
+ *
+ * Note: We count binding sets as accepted before we
+ * apply the constraints. This has the effect that
+ * an optional join which produces solutions that
+ * are then rejected by a FILTER associated with the
+ * optional predicate WILL NOT pass on the original
+ * solution even if ALL solutions produced by the
+ * join are rejected by the filter.
+ */
+ this.naccepted[bindex]++;
+
/*
* Clone the binding set since it is tested for each
* element visited.
@@ -2098,15 +2108,12 @@
// Accept this binding set.
unsyncBuffer.add(bset);
- // #of binding sets accepted.
- naccepted++;
+// // #of binding sets accepted.
+// naccepted++;
// #of output solutions generated.
stats.outputSolutions.increment();
- // #of elements accepted for this binding set.
- this.naccepted[bindex]++;
-
}
bindex++;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 19:49:18
|
Revision: 4145
http://bigdata.svn.sourceforge.net/bigdata/?rev=4145&view=rev
Author: thompsonbry
Date: 2011-01-20 19:49:12 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
javadoc.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/NOT.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/NOT.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/NOT.java 2011-01-20 19:26:49 UTC (rev 4144)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/NOT.java 2011-01-20 19:49:12 UTC (rev 4145)
@@ -57,7 +57,7 @@
public NOT(final IConstraint x) {
- super(new BOp[] { x }, null/*annocations*/);
+ super(new BOp[] { x }, null/*Annotations*/);
if (x == null)
throw new IllegalArgumentException();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-20 19:26:55
|
Revision: 4144
http://bigdata.svn.sourceforge.net/bigdata/?rev=4144&view=rev
Author: mrpersonick
Date: 2011-01-20 19:26:49 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
unit test for nested unions
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java
Added: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedUnions.java 2011-01-20 19:26:49 UTC (rev 4144)
@@ -0,0 +1,229 @@
+/**
+Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 16, 2009
+ */
+
+package com.bigdata.rdf.sail;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.log4j.Logger;
+import org.openrdf.model.Literal;
+import org.openrdf.model.URI;
+import org.openrdf.model.ValueFactory;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.model.vocabulary.RDFS;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQueryResult;
+import org.openrdf.query.algebra.Projection;
+import org.openrdf.query.algebra.QueryRoot;
+import org.openrdf.query.algebra.TupleExpr;
+import org.openrdf.query.impl.BindingImpl;
+import org.openrdf.repository.RepositoryConnection;
+import org.openrdf.repository.sail.SailTupleQuery;
+
+import com.bigdata.bop.BOpUtility;
+import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.rdf.axioms.NoAxioms;
+import com.bigdata.rdf.sail.sop.SOp;
+import com.bigdata.rdf.sail.sop.SOp2BOpUtility;
+import com.bigdata.rdf.sail.sop.SOpTree;
+import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
+import com.bigdata.rdf.sail.sop.SOpTree.SOpGroups;
+import com.bigdata.rdf.sail.sop.SOpTreeBuilder;
+import com.bigdata.rdf.store.AbstractTripleStore;
+import com.bigdata.rdf.store.BD;
+import com.bigdata.rdf.vocab.NoVocabulary;
+
+/**
+ * Unit tests the optionals aspects of the {@link BigdataSail} implementation.
+ *
+ *
+ * @author <a href="mailto:mrp...@us...">Mike Personick</a>
+ * @version $Id$
+ */
+public class TestNestedUnions extends ProxyBigdataSailTestCase {
+
+ /*
+ * TODO
+ *
+ * 1. arrange sesame operator tree into bigdata predicates with optional
+ * group specified.
+ *
+ * 2. re-work how constraints are attached - need to be attached to the
+ * first appearance of a variable within its group, not globally across all
+ * predicates (constraints are group-local)
+ *
+ * 3. some constraints will be conditional routing op instead of an
+ * actual constraint on a predicate. this occurs when the variables in
+ * the constraint do not appear anywhere in the optional group
+ *
+ * 4. need to punt the query if we can't evaluate a filter inside an
+ * optional group natively, need to recognize this
+ *
+ * 5.
+ */
+
+ protected static final Logger log = Logger.getLogger(TestNestedUnions.class);
+
+ protected static final boolean INFO = log.isInfoEnabled();
+
+ @Override
+ public Properties getProperties() {
+
+ Properties props = super.getProperties();
+
+ props.setProperty(BigdataSail.Options.AXIOMS_CLASS, NoAxioms.class.getName());
+ props.setProperty(BigdataSail.Options.VOCABULARY_CLASS, NoVocabulary.class.getName());
+ props.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false");
+ props.setProperty(BigdataSail.Options.JUSTIFY, "false");
+ props.setProperty(BigdataSail.Options.TEXT_INDEX, "false");
+
+ return props;
+
+ }
+
+ /**
+ *
+ */
+ public TestNestedUnions() {
+ }
+
+ /**
+ * @param arg0
+ */
+ public TestNestedUnions(String arg0) {
+ super(arg0);
+ }
+
+ public void testSimplestNestedUnion() throws Exception {
+
+// final Sail sail = new MemoryStore();
+// sail.initialize();
+// final Repository repo = new SailRepository(sail);
+
+ final BigdataSail sail = getSail();
+ sail.initialize();
+ final BigdataSailRepository repo = new BigdataSailRepository(sail);
+
+ final RepositoryConnection cxn = repo.getConnection();
+ cxn.setAutoCommit(false);
+
+ try {
+
+ final ValueFactory vf = sail.getValueFactory();
+
+ /*
+ * Create some terms.
+ */
+ final URI john = vf.createURI(BD.NAMESPACE + "john");
+ final URI mary = vf.createURI(BD.NAMESPACE + "mary");
+ final URI leon = vf.createURI(BD.NAMESPACE + "leon");
+ final URI paul = vf.createURI(BD.NAMESPACE + "paul");
+ final URI brad = vf.createURI(BD.NAMESPACE + "brad");
+ final URI fred = vf.createURI(BD.NAMESPACE + "fred");
+ final URI knows = vf.createURI(BD.NAMESPACE + "knows");
+
+ /*
+ * Create some statements.
+ */
+ cxn.add(paul, knows, mary);
+ cxn.add(brad, knows, john);
+ cxn.add(mary, knows, fred);
+ cxn.add(john, knows, leon);
+
+ /*
+ * Note: The either flush() or commit() is required to flush the
+ * statement buffers to the database before executing any operations
+ * that go around the sail.
+ */
+ cxn.commit();
+
+ {
+
+ String query =
+ "prefix bd: <"+BD.NAMESPACE+"> " +
+ "select * " +
+ "where { " +
+ " ?a bd:knows ?b . " +
+ " { " +
+ " ?b bd:knows bd:fred . " +
+ " } UNION { " +
+ " ?b bd:knows bd:leon . " +
+ " } " +
+ "}";
+
+ final SailTupleQuery tupleQuery = (SailTupleQuery)
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(false /* includeInferred */);
+
+ if (log.isInfoEnabled()) {
+
+ final BigdataSailTupleQuery bdTupleQuery =
+ (BigdataSailTupleQuery) tupleQuery;
+ final QueryRoot root = (QueryRoot) bdTupleQuery.getTupleExpr();
+ final Projection p = (Projection) root.getArg();
+ final TupleExpr tupleExpr = p.getArg();
+ final SOpTreeBuilder stb = new SOpTreeBuilder();
+ final SOpTree tree = stb.collectSOps(tupleExpr);
+
+ log.info(tree);
+ log.info(query);
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ while (result.hasNext()) {
+ log.info(result.next());
+ }
+
+ }
+
+ final Collection<BindingSet> answer = new LinkedList<BindingSet>();
+ answer.add(createBindingSet(
+ new BindingImpl("a", paul),
+ new BindingImpl("b", mary)
+ ));
+ answer.add(createBindingSet(
+ new BindingImpl("a", brad),
+ new BindingImpl("b", john)
+ ));
+
+ final TupleQueryResult result = tupleQuery.evaluate();
+ compare(result, answer);
+
+ }
+
+ } finally {
+ cxn.close();
+ sail.shutDown();
+ }
+
+ }
+
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-20 18:58:16
|
Revision: 4142
http://bigdata.svn.sourceforge.net/bigdata/?rev=4142&view=rev
Author: mrpersonick
Date: 2011-01-20 18:58:10 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
turned constraints into conditional routing ops instead of annotations
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-20 17:49:16 UTC (rev 4141)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-20 18:58:10 UTC (rev 4142)
@@ -89,7 +89,8 @@
final IV right = ((IValueExpression<IV>) get(1)).get(s);
if (left == null || right == null)
- return true; // not yet bound.
+// return true; // not yet bound.
+ return false; // no longer allow unbound values
final CompareOp op = (CompareOp) getProperty(Annotations.OP);
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-20 17:49:16 UTC (rev 4141)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-20 18:58:10 UTC (rev 4142)
@@ -620,12 +620,12 @@
// }
// just add all the constraints to the very last tail for now
- if (i == (order.length-1) && rule.getConstraintCount() > 0) {
- final Iterator<IConstraint> it = rule.getConstraints();
- while (it.hasNext()) {
- constraints.add(it.next());
- }
- }
+// if (i == (order.length-1) && rule.getConstraintCount() > 0) {
+// final Iterator<IConstraint> it = rule.getConstraints();
+// while (it.hasNext()) {
+// constraints.add(it.next());
+// }
+// }
// annotations for this join.
final List<NV> anns = new LinkedList<NV>();
@@ -730,6 +730,24 @@
}
+ if (rule.getConstraintCount() > 0) {
+ final Iterator<IConstraint> it = rule.getConstraints();
+ while (it.hasNext()) {
+ final IConstraint c = it.next();
+ final int condId = idFactory.incrementAndGet();
+ final PipelineOp condOp = applyQueryHints(
+ new ConditionalRoutingOp(new BOp[]{left},
+ NV.asMap(new NV[]{//
+ new NV(BOp.Annotations.BOP_ID,condId),
+ new NV(ConditionalRoutingOp.Annotations.CONDITION, c),
+ })), queryHints);
+ left = condOp;
+ if (log.isDebugEnabled()) {
+ log.debug("adding conditional routing op: " + condOp);
+ }
+ }
+ }
+
if (log.isInfoEnabled()) {
// just for now while i'm debugging
log.info("rule=" + rule + ":::query="
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-20 17:49:23
|
Revision: 4141
http://bigdata.svn.sourceforge.net/bigdata/?rev=4141&view=rev
Author: mrpersonick
Date: 2011-01-20 17:49:16 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
working through tck failures
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2011-01-20 17:48:21 UTC (rev 4140)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2011-01-20 17:49:16 UTC (rev 4141)
@@ -745,14 +745,11 @@
s = s.replaceAll("com.bigdata.bop.join.", "");
s = s.replaceAll("com.bigdata.bop.solutions.", "");
s = s.replaceAll("com.bigdata.bop.rdf.filter.", "");
+ s = s.replaceAll("com.bigdata.bop.bset", "");
s = s.replaceAll("com.bigdata.bop.", "");
s = s.replaceAll("com.bigdata.rdf.sail.", "");
s = s.replaceAll("com.bigdata.rdf.spo.", "");
-// s = s.replaceAll("com.bigdata.bop..", "");
-// s = s.replaceAll("com.bigdata.bop..", "");
-// s = s.replaceAll("com.bigdata.bop..", "");
-// s = s.replaceAll("com.bigdata.bop..", "");
-// s = s.replaceAll("com.bigdata.bop..", "");
+ s = s.replaceAll("com.bigdata.rdf.internal.constraints.", "");
return s;
}
@@ -1027,3 +1024,4 @@
}
}
+
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java 2011-01-20 17:48:21 UTC (rev 4140)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java 2011-01-20 17:49:16 UTC (rev 4141)
@@ -236,6 +236,8 @@
} else if (arg instanceof LeftJoin) {
// collectSOps(sops, (LeftJoin) arg, rslj, groupId.incrementAndGet(), g);
collectSOps(sops, (LeftJoin) arg, rslj, g, pg);
+ } else if (arg instanceof SingletonSet) {
+ // do nothing
} else {
throw new UnsupportedOperatorException(arg);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2011-01-20 17:48:21 UTC (rev 4140)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2011-01-20 17:49:16 UTC (rev 4141)
@@ -41,11 +41,18 @@
import junit.framework.TestSuite;
import org.apache.log4j.Logger;
+import org.openrdf.model.Statement;
import org.openrdf.query.Dataset;
+import org.openrdf.query.Query;
+import org.openrdf.query.QueryLanguage;
+import org.openrdf.query.TupleQuery;
+import org.openrdf.query.TupleQueryResult;
import org.openrdf.query.parser.sparql.ManifestTest;
import org.openrdf.query.parser.sparql.SPARQLQueryTest;
import org.openrdf.repository.Repository;
+import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.RepositoryException;
+import org.openrdf.repository.RepositoryResult;
import org.openrdf.repository.dataset.DatasetRepository;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.sail.memory.MemoryStore;
@@ -55,8 +62,8 @@
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.IIndexManager;
import com.bigdata.rdf.sail.BigdataSail;
+import com.bigdata.rdf.sail.BigdataSail.Options;
import com.bigdata.rdf.sail.BigdataSailRepository;
-import com.bigdata.rdf.sail.BigdataSail.Options;
/**
* Test harness for running the SPARQL test suites.
@@ -173,68 +180,71 @@
*/
static final Collection<String> testURIs = Arrays.asList(new String[] {
-// busted with EvalStrategy1
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-2",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#filter-scope-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#join-scope-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-4",
-
-// busted with EvalStrategy2 with LeftJoin enabled
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-12",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#opt-filter-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#opt-filter-2",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-3",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-001",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-004",
-
-// Dataset crap
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/i18n/manifest#normalization-1",
-
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset/manifest#dawg-dataset-01",
-
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2//manifest#",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-str-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-str-2",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-datatype-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#sameTerm-simple",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#sameTerm-eq",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#sameTerm-not-eq",
-//
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-1",
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-2",
-
- /*
- * Tests which fail with 2 data services.
- */
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#bgp-no-match",//Ok
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#prefix-name-1",//OK
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#spoo-1",//BOOM
-
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-05",
+ /*
+ * working through the new query engine failures: 0 errors, 19 failures
+ */
+ /*
+ * Basically we are having a lot of problems with our compare
+ * operator, which is supposed to do fuzzy comparisons that
+ * sometimes requires materialized RDF values. These I feel I can
+ * handle on my own.
+ */
+
+ // "a" and "a"^^xsd:string have different term ids? also bnodes are different
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-07",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-08",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-10",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-11",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-12",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-cmp-01",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-cmp-02",
+
+ /*
+ * These tests have to do with that that weird "well designed"
+ * optional nesting P = A OPT (B OPT C) where A and C share
+ * variables not in B. I think I can handle these on my own.
+ */
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-1",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#join-scope-1",
+
/*
- * working through the new query engine failures
+ * Everything below this point I need help with.
*/
- // please someone explain this shit to me
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-12"
+ /*
+ * These failures have to do with nested UNIONs - we don't seem to
+ * be handling them correctly at all.
+ */
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#join-combo-1",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#join-combo-2",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-2",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-4",
- // this is that weird "well designed" optional shit P = A OPT (B OPT C) where A and C share variables not in B
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-1"
-
- // where do we put the !bound(?e) constraint???
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bound/manifest#dawg-bound-query-001"
+ /*
+ * This one is truly bizarre - involving a non-optional subquuery
+ * plus an optional subquery. Don't even know where to start on this
+ * guy.
+ */
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#filter-scope-1",
- // "a" and "a"^^xsd:string have different term ids? also bnodes are different
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-07"
+ /*
+ * Sometimes, a filter is the entire join group, and it should not
+ * be able to see variables outside the group. Frankly I do not
+ * understand this one.
+ */
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#filter-nested-2",
+
+ /*
+ * These demonstrate the problem of where to put non-optional
+ * filters that need to be evaluated after optional tails and
+ * optional join groups.
+ */
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bound/manifest#dawg-bound-query-001",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-1",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-002",
+ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-003",
- // help, non-optional subquery?? wtf
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#filter-scope-1"
-
- // this uncovers an obvious bug in our SubqueryOp
-// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-001"
-
});
/**
@@ -471,42 +481,42 @@
}
-// @Override
-// protected void runTest()
-// throws Exception
-// {
-// RepositoryConnection con = getQueryConnection(dataRep);
-// try {
-//
+ @Override
+ protected void runTest()
+ throws Exception
+ {
+ RepositoryConnection con = getQueryConnection(dataRep);
+ try {
+
// log.info("database dump:");
// RepositoryResult<Statement> stmts = con.getStatements(null, null, null, false);
// while (stmts.hasNext()) {
// log.info(stmts.next());
// }
-// log.info("dataset:\n" + dataset);
-//
-// String queryString = readQueryString();
-// log.info("query:\n" + getQueryString());
-//
-// Query query = con.prepareQuery(QueryLanguage.SPARQL, queryString, queryFileURL);
-// if (dataset != null) {
-// query.setDataset(dataset);
-// }
-//
-// if (query instanceof TupleQuery) {
-// TupleQueryResult queryResult = ((TupleQuery)query).evaluate();
-// while (queryResult.hasNext()) {
-// log.info("query result:\n" + queryResult.next());
-// }
-// }
-//
-// }
-// finally {
-// con.close();
-// }
-//
-// super.runTest();
-// }
+ log.info("dataset:\n" + dataset);
+
+ String queryString = readQueryString();
+ log.info("query:\n" + getQueryString());
+
+ Query query = con.prepareQuery(QueryLanguage.SPARQL, queryString, queryFileURL);
+ if (dataset != null) {
+ query.setDataset(dataset);
+ }
+
+ if (query instanceof TupleQuery) {
+ TupleQueryResult queryResult = ((TupleQuery)query).evaluate();
+ while (queryResult.hasNext()) {
+ log.info("query result:\n" + queryResult.next());
+ }
+ }
+
+ }
+ finally {
+ con.close();
+ }
+
+ super.runTest();
+ }
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-20 17:48:27
|
Revision: 4140
http://bigdata.svn.sourceforge.net/bigdata/?rev=4140&view=rev
Author: mrpersonick
Date: 2011-01-20 17:48:21 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
took out SELECT annotation for now - it's broken
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-20 16:54:51 UTC (rev 4139)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-20 17:48:21 UTC (rev 4140)
@@ -632,8 +632,8 @@
anns.add(new NV(BOp.Annotations.BOP_ID, joinId));
- anns.add(new NV(PipelineJoin.Annotations.SELECT,
- selectVars[order[i]]));
+// anns.add(new NV(PipelineJoin.Annotations.SELECT,
+// selectVars[order[i]]));
// No. The join just looks at the Predicate's optional annotation.
// if (pred.isOptional())
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2011-01-20 16:54:57
|
Revision: 4139
http://bigdata.svn.sourceforge.net/bigdata/?rev=4139&view=rev
Author: martyncutcher
Date: 2011-01-20 16:54:51 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
Initial SectorAllocator unit tests
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestMemoryManager.java
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestMemoryManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestMemoryManager.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestMemoryManager.java 2011-01-20 16:54:51 UTC (rev 4139)
@@ -0,0 +1,168 @@
+package com.bigdata.rwstore.sector;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import com.bigdata.rwstore.sector.MemoryManager.MemoryManagerResourceError;
+import com.bigdata.util.concurrent.DaemonThreadFactory;
+
+import junit.framework.TestCase;
+
+public class TestMemoryManager extends TestCase {
+
+ final int sectorSize = 10 * 1024 * 1024; // 10M
+ MemoryManager manager = null;
+
+ final char[] c_testData = genTestData();
+
+ char[] genTestData() {
+ String src = "The quick brown fox jumped over the lazy dog";
+
+ StringBuffer buf = new StringBuffer();
+ while (buf.length() < (20 * 1024))
+ buf.append(src);
+
+ return buf.toString().toCharArray();
+ }
+
+ Random r = new Random();
+ private String genString(int min, int i) {
+ return new String(c_testData, 0, min + r.nextInt(i-min));
+ // return new String(c_testData, 0, i - 20);
+ }
+
+ public void testSimpleAllocations() {
+ installMemoryManager();
+
+ String helloWorld = "Hello World";
+
+ final long saddr = allocate(manager, helloWorld);
+
+ String retstr = getString(saddr);
+
+ assertTrue(helloWorld.equals(retstr));
+
+ System.out.println(helloWorld + " allocated address: " + saddr + " returned: " + retstr);
+ }
+
+ private void installMemoryManager() {
+ manager = new MemoryManager(5 * sectorSize, sectorSize);
+ }
+
+ /**
+ * The address mappings are between the integer allocation address and
+ * the sector index (16 signed bits) and offset (16 unsigned bits).
+ */
+ public void testAddressMappings() {
+ int i = 10000000;
+ while (--i > 0) {
+ // final int sector = r.nextInt(12);
+ final int sector = r.nextInt(32 * 1024);
+ final int bit = r.nextInt(64 * 1024);
+ final int rwaddr = SectorAllocator.makeAddr(sector, bit);
+ final int rsector = SectorAllocator.getSectorIndex(rwaddr);
+ final int rbit = SectorAllocator.getSectorOffset(rwaddr);
+ assertTrue("Address Error " + i + " , sector: " + sector + " != " + rsector + " or " + bit + " != " + rbit, (sector == rsector) && (bit == rbit));
+ }
+ }
+
+ public void testStressAllocations() {
+ installMemoryManager();
+
+ for (int i = 0; i < 20; i++) {
+ doStressAllocations(manager, true, 80000, 5 + r.nextInt(200));
+ }
+ }
+
+ public void testAllocationContexts() {
+ installMemoryManager();
+
+ AllocationContext context = manager.createAllocationContext();
+ for (int i = 0; i < 500; i++) {
+ doStressAllocations(context, false, 5000, 5 + r.nextInt(3000));
+ context.clear();
+ }
+ }
+
+ public void testStressConcurrent() throws InterruptedException {
+ installMemoryManager();
+
+ final int nclients = 20;
+
+ final ExecutorService executorService = Executors.newFixedThreadPool(
+ nclients, DaemonThreadFactory.defaultThreadFactory());
+
+ final Collection<Callable<Long>> tasks = new HashSet<Callable<Long>>();
+ for (int i = 0; i < nclients; i++) {
+ tasks.add(new Callable<Long>() {
+ public Long call() throws Exception {
+ try {
+ doStressAllocations(manager, false, 50000, 5 + r.nextInt(600));
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+ return null;
+ }
+ });
+ }
+
+ executorService.invokeAll(tasks);
+ executorService.awaitTermination(5, TimeUnit.SECONDS);
+ }
+
+ public void doStressAllocations(final IMemoryManager mm, final boolean clear, final int tests, final int maxString) {
+ if (clear)
+ mm.clear();
+
+ int allocs = 0;
+ int frees = 0;
+ ArrayList<Long> addrs = new ArrayList<Long>();
+ try {
+ for (int i = 0; i < tests; i++) {
+ long addr1 = allocate(mm, genString(1, maxString));
+ allocs++;
+ if ((i % 2) == 0) {
+ addrs.add(Long.valueOf(addr1));
+ } else if (i > 1000) {
+ int f = r.nextInt(addrs.size());
+ long faddr = ((Long) addrs.remove(f)).longValue();
+ mm.free(faddr);
+ // System.out.println("freeing: " + faddr);
+ frees++;
+ }
+ }
+ } catch (MemoryManagerResourceError err) {
+ // all okay
+ }
+
+ System.out.println("Committed " + allocs + " allocations, and " + frees + " frees");
+ }
+
+ private String getString(long saddr) {
+ final ByteBuffer ret = manager.get(saddr)[0];
+ final byte[] data;
+ if (ret.isDirect()) {
+ ByteBuffer indbuf = ByteBuffer.allocate(ret.remaining());
+ data = indbuf.array();
+ indbuf.put(ret);
+ indbuf.flip();
+ } else {
+ data = ret.array();
+ }
+
+ return new String(data);
+ }
+
+ private long allocate(final IMemoryManager mm, String val) {
+ ByteBuffer bb = ByteBuffer.wrap(val.getBytes());
+
+ return mm.allocate(bb);
+ }
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/rwstore/sector/TestMemoryManager.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2011-01-20 16:54:16
|
Revision: 4138
http://bigdata.svn.sourceforge.net/bigdata/?rev=4138&view=rev
Author: martyncutcher
Date: 2011-01-20 16:54:09 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
First commit of SectorAllocator and MemoryManager
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/ISectorManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java 2011-01-20 16:54:09 UTC (rev 4138)
@@ -0,0 +1,150 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+package com.bigdata.rwstore.sector;
+
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+
+/**
+ * The AllocationContext is used to maintain a handle on allocations made
+ * within some specific environment (context).
+ *
+ * In this way, clearing a context will return all allocations to the more
+ * general pool.
+ *
+ * There are two obvious implementaiton strategies:
+ * 1) Retaining set of addresses allocated
+ * 2) Retaining a copy of the allocated bits
+ *
+ * If it was not for the BLOB implementations which require length
+ * data to manage the freeing of an allocation, it would be efficient to
+ * maintain copies of the allocation bits. This remoains an option for the
+ * future but requires a relatively complex callback protocol.
+ *
+ * For this reason, the initial implementation maintains a set of allocated
+ * addresses.
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class AllocationContext implements IMemoryManager {
+
+ final IMemoryManager m_parent;
+
+ SectorAllocation m_head = null;
+
+ HashSet<Long> m_addresses = new HashSet<Long>();
+
+ public AllocationContext(IMemoryManager parent) {
+ m_parent = parent;
+ }
+
+ public long allocate(final ByteBuffer data) {
+ final long addr = m_parent.allocate(data);
+
+ // getSectorAllocation(addr).allocate(addr);
+ m_addresses.add(Long.valueOf(addr));
+
+ return addr;
+ }
+
+ /**
+ * The main reason for the AllocationContext is to be
+ * able to atomically release the associated allocations
+ */
+ public void clear() {
+ for (Long addr : m_addresses) {
+ m_parent.free(addr);
+ }
+
+ m_addresses.clear();
+ }
+
+ public void free(final long addr) {
+ // getSectorAllocation(addr).free(addr);
+ m_addresses.remove(Long.valueOf(addr));
+
+ m_parent.free(addr);
+ }
+
+ public ByteBuffer[] get(long addr) {
+ return m_parent.get(addr);
+ }
+
+ public AllocationContext createAllocationContext() {
+ return new AllocationContext(this);
+ }
+
+ private int segmentID(final long addr) {
+ final int rwaddr = MemoryManager.getAllocationAddress(addr);
+
+ return SectorAllocator.getSectorIndex(rwaddr);
+ }
+
+ private int segmentOffset(final long addr) {
+ final int rwaddr = MemoryManager.getAllocationAddress(addr);
+
+ return SectorAllocator.getSectorOffset(rwaddr);
+ }
+
+ SectorAllocation getSectorAllocation(final long addr) {
+ final int index = segmentID(addr);
+ if (m_head == null) {
+ m_head = new SectorAllocation(index);
+ }
+ SectorAllocation sa = m_head;
+ while (sa.m_index != index) {
+ if (sa.m_next == null) {
+ sa.m_next = new SectorAllocation(index);
+ }
+ sa = sa.m_next;
+ }
+
+ return sa;
+ }
+
+ class SectorAllocation {
+ final int m_index;
+ final int[] m_bits = new int[SectorAllocator.NUM_ENTRIES];
+ SectorAllocation m_next = null;
+
+ SectorAllocation(final int index) {
+ m_index = index;
+ }
+
+ public void allocate(long addr) {
+ assert !SectorAllocator.tstBit(m_bits, segmentOffset(addr));
+
+ SectorAllocator.setBit(m_bits, segmentOffset(addr));
+ }
+
+ public void free(long addr) {
+ assert SectorAllocator.tstBit(m_bits, segmentOffset(addr));
+
+ SectorAllocator.clrBit(m_bits, segmentOffset(addr));
+ }
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/AllocationContext.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java 2011-01-20 16:54:09 UTC (rev 4138)
@@ -0,0 +1,75 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+package com.bigdata.rwstore.sector;
+
+import java.nio.ByteBuffer;
+
+public interface IMemoryManager {
+ /**
+ * Allocates space on the backing resource and copies the provided data.
+ *
+ * @param data - will be copied to the backing resource
+ * @return the address to be passed to the get method to retrieve the data
+ */
+ public long allocate(ByteBuffer data);
+
+ /**
+ * The ByteBuffer[] return enables the handling of blobs that span more
+ * than a single slot, without the need to create an intermediate ByteBuffer.
+ *
+ * This will support transfers directly to other direct ByteBuffers, for
+ * example for network IO.
+ *
+ * Using ByteBuffer:put the returned array can be efficiently copied to
+ * another ByteBuffer:
+ *
+ * ByteBuffer mybb;
+ * ByteBuffer[] bufs = get(addr);
+ * for (ByteBuffer b : bufs) {
+ * mybb.put(b);
+ * }
+ *
+ * @param addr previouslt returned by allocate
+ * @return array of ByteBuffers
+ */
+ public ByteBuffer[] get(long addr);
+
+ /**
+ * Frees the address and makes available for recycling
+ *
+ * @param addr to be freed
+ */
+ public void free(long addr);
+
+ /**
+ * Clears all current allocations
+ */
+ public void clear();
+
+ /**
+ * Clears all current allocations
+ */
+ public AllocationContext createAllocationContext();
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/IMemoryManager.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/ISectorManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/ISectorManager.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/ISectorManager.java 2011-01-20 16:54:09 UTC (rev 4138)
@@ -0,0 +1,80 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rwstore.sector;
+
+/**
+ * The SectorManager defines the contract required to manage a set of
+ * SectorAllocators.
+ *
+ * The SectorManager is passed to the SectorAllocator constructors and they
+ * will callback to manage their free list availability, and to trim the
+ * allocated storage if required.
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public interface ISectorManager {
+
+ /**
+ * This request is made when the sectorAllocator no longer has a full set
+ * of block allocations available.
+ *
+ * The allocator will issue this callback to help the SectorManager manage
+ * an effective freelist of available allocators.
+ *
+ * @param sectorAllocator to be removed
+ */
+ void removeFromFreeList(SectorAllocator sectorAllocator);
+
+ /**
+ * When suficient alocations have been freed for recycling that a threshold
+ * of availability of reached for all block sizes, then the allocator
+ * calls back to the SectorManager to signal it is available to be returned
+ * to the free list.
+ *
+ * @param sectorAllocator to be added
+ */
+ void addToFreeList(SectorAllocator sectorAllocator);
+
+ /**
+ * When a sector is first created, it will remain at the head of the free
+ * list until one of two conditions has been reached:
+ *
+ * 1) The allocation has been saturated
+ * 2) The bit space has been filled
+ *
+ * In the case of (2), then it is possible that significant allocation
+ * space cannot be utilised - which will happen if the average allocation
+ * is less than 1K. In this situation, the sector can be trimmed and the
+ * space made available to the next sector.
+ *
+ * trimSector will only be called in this condition - on the first occasion
+ * that the allocator is removed from the freeList.
+ *
+ * @param trim - the amount by which the sector allocation can be reduced
+ */
+ void trimSector(long trim, SectorAllocator sector);
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/ISectorManager.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java 2011-01-20 16:54:09 UTC (rev 4138)
@@ -0,0 +1,313 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+package com.bigdata.rwstore.sector;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.log4j.Logger;
+
+import com.bigdata.io.DirectBufferPool;
+import com.bigdata.io.IByteArraySlice;
+
+/**
+ * The MemoryManager manages an off-heap Direct Buffer. It uses the new
+ * SectorAllocator to allocate slots within the address range.
+ *
+ * The interface is designed to support efficient transfer between NIO
+ * buffers.
+ *
+ * The most complex aspect of the implementation is the BLOB representation,
+ * requiring a mapping across multiple allocation slots. This is managed
+ * using recursive calls in the main three methods: allocate, free and get.
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class MemoryManager implements IMemoryManager, ISectorManager {
+
+ protected static final Logger log = Logger
+ .getLogger(MemoryManager.class);
+
+ final ByteBuffer m_resource;
+
+ final private ReentrantLock m_allocationLock = new ReentrantLock();
+
+ int m_allocation = 0;
+ final int m_sectorSize;
+ final int m_maxResource;
+
+ final ArrayList<SectorAllocator> m_sectors = new ArrayList<SectorAllocator>();
+ final ArrayList<SectorAllocator> m_free = new ArrayList<SectorAllocator>();
+
+ public MemoryManager(final int maxResource, final int sectorSize) {
+ m_resource = ByteBuffer.allocateDirect(maxResource);
+ m_sectorSize = sectorSize;
+ m_maxResource = maxResource;
+ }
+
+ public class MemoryManagerResourceError extends RuntimeException {
+ protected MemoryManagerResourceError() {}
+ }
+
+ public long allocate(final ByteBuffer data) {
+ m_allocationLock.lock();
+ try {
+ final int size = data.remaining();
+ if (size <= SectorAllocator.BLOB_SIZE) {
+ if (m_free.size() == 0) {
+ if ((m_allocation + m_sectorSize) > m_maxResource) {
+ throw new MemoryManagerResourceError();
+ }
+ SectorAllocator sector = new SectorAllocator(this, null);
+ sector.setSectorAddress(m_allocation, m_sectorSize);
+ sector.setIndex(m_sectors.size());
+ m_sectors.add(sector);
+
+ m_allocation += m_sectorSize;
+ }
+
+ final SectorAllocator sector = m_free.get(0);
+
+ final int rwaddr = sector.alloc(size);
+
+ if (SectorAllocator.getSectorIndex(rwaddr) >= m_sectors.size()) {
+ throw new IllegalStateException("Address: " + rwaddr + " yields index: " + SectorAllocator.getSectorIndex(rwaddr));
+ }
+
+ if (log.isTraceEnabled())
+ log.trace("allocating bit: " + SectorAllocator.getSectorOffset(rwaddr));
+
+ // Now copy the data to the backing resource
+ final long paddr = sector.getPhysicalAddress(SectorAllocator.getSectorOffset(rwaddr));
+ final ByteBuffer dest = m_resource.duplicate();
+ dest.position((int) paddr);
+ dest.limit((int) (paddr + size));
+ dest.put(data);
+
+ return makeAddr(rwaddr, size);
+ } else {
+ /**
+ * For Blob allocation call the normal allocate and retrieve
+ * the allocation address to store in the blob header.
+ */
+ final int nblocks = SectorAllocator.getBlobBlockCount(size);
+ final ByteBuffer hdrbuf = ByteBuffer.allocate(nblocks * 4);
+ for (int i = 0; i < nblocks; i++) {
+ final ByteBuffer src = data.duplicate();
+ final int pos = SectorAllocator.BLOB_SIZE * i;
+ src.position(pos);
+ final int bsize = i < (nblocks-1) ? SectorAllocator.BLOB_SIZE : size - pos;
+ src.limit(pos + bsize);
+
+ /*
+ * BLOB RECURSION
+ */
+ final long bpaddr = allocate(src);
+ hdrbuf.putInt(getAllocationAddress(bpaddr));
+ }
+
+ // now allocate the blob header and fix the return address size
+ hdrbuf.flip();
+ final int retaddr = getAllocationAddress(allocate(hdrbuf));
+
+ return makeAddr(retaddr, size);
+ }
+
+ } finally {
+ m_allocationLock.unlock();
+ }
+ }
+
+ /**
+ * The ByteBuffer[] return enables the handling of blobs that span more
+ * than a single slot, without the need to create an intermediate ByteBuffer.
+ *
+ * This will support transfers directly to other direct ByteBuffers, for
+ * example for network IO.
+ *
+ * Using ByteBuffer:put the returned array can be efficiently copied to
+ * another ByteBuffer:
+ *
+ * ByteBuffer mybb;
+ * ByteBuffer[] bufs = get(addr);
+ * for (ByteBuffer b : bufs) {
+ * mybb.put(b);
+ * }
+ *
+ * @param addr
+ * @return
+ */
+ public ByteBuffer[] get(final long addr) {
+ int rwaddr = getAllocationAddress(addr);
+ int size = getAllocationSize(addr);
+
+ if (size <= SectorAllocator.BLOB_SIZE) {
+ return new ByteBuffer[] { getBuffer(rwaddr, size) };
+ } else {
+ // This will be a BLOB, so retrieve the header, then parse
+ // to retrieve components and assign to ByteBuffer[]
+ final ByteBuffer hdrbuf = getBlobHdr(addr);
+
+ final int nblocks = hdrbuf.remaining() / 4;
+
+ final ByteBuffer[] blobbufs = new ByteBuffer[nblocks];
+ int remaining = size;
+ for (int i = 0; i < nblocks; i++) {
+ int blockSize = remaining <= SectorAllocator.BLOB_SIZE ? remaining : SectorAllocator.BLOB_SIZE;
+ blobbufs[i] = getBuffer(hdrbuf.getInt(), blockSize);
+ }
+
+ return blobbufs;
+ }
+ }
+
+ /**
+ * Given an address of a blob, determine the size of the header and
+ * create an address to support direct retrieval of the header.
+ *
+ * This is required to support get and free.
+ *
+ * @param addr of blob header
+ * @return the ByteBuffer containing the header
+ */
+ private ByteBuffer getBlobHdr(final long addr) {
+ int size = getAllocationSize(addr);
+
+ final int nblocks = SectorAllocator.getBlobBlockCount(size);
+ final int hdrsize = 4*nblocks;
+
+ // Mockup hdraddr with header size to retrieve the ByteBuffer
+ final long hdraddr = (addr & 0xFFFFFFFF00000000L) | hdrsize;
+
+ return get(hdraddr)[0];
+ }
+
+ private ByteBuffer getBuffer(final int rwaddr, final int size) {
+ final SectorAllocator sector = getSector(rwaddr);
+ final int offset = SectorAllocator.getSectorOffset(rwaddr);
+
+ int resAddr = (int) sector.getPhysicalAddress(offset);
+
+ final ByteBuffer ret = m_resource.duplicate();
+ ret.position(resAddr);
+ ret.limit(resAddr + size);
+
+ return ret;
+ }
+
+ private SectorAllocator getSector(int rwaddr) {
+ final int index = SectorAllocator.getSectorIndex(rwaddr);
+ if (index >= m_sectors.size())
+ throw new IllegalStateException("Address: " + rwaddr + " yields index: " + index + " >= sector:size(): " + m_sectors.size());
+
+ return m_sectors.get(index);
+ }
+
+ static int getAllocationAddress(final long addr) {
+ return (int) (addr >> 32L);
+ }
+
+ static int getAllocationSize(long addr) {
+ return (int) (addr & 0xFFFFL);
+ }
+
+ public void free(final long addr) {
+ m_allocationLock.lock();
+ try {
+ final int rwaddr = getAllocationAddress(addr);
+ final int size = getAllocationSize(addr);
+
+ if (size <= SectorAllocator.BLOB_SIZE) {
+
+ getSector(rwaddr).free(SectorAllocator.getSectorOffset(rwaddr));
+ } else {
+ final ByteBuffer hdrbuf = getBlobHdr(addr);
+ final int spos = hdrbuf.position();
+ final int hdrsize = hdrbuf.limit() - spos;
+ final int nblocks = hdrsize / 4;
+
+ // free each block
+ int remaining = size;
+ for (int i = 0; i < nblocks; i++) {
+ int blockSize = remaining <= SectorAllocator.BLOB_SIZE ? remaining : SectorAllocator.BLOB_SIZE;
+ final long mkaddr = makeAddr(hdrbuf.getInt(), blockSize);
+ /*
+ * BLOB RECURSION
+ */
+ free(mkaddr);
+ }
+ hdrbuf.position(spos);
+ // now free the header
+ free(makeAddr(rwaddr, hdrsize));
+ }
+ } finally {
+ m_allocationLock.unlock();
+ }
+ }
+
+ private long makeAddr(final int rwaddr, final int size) {
+
+ long addr = rwaddr;
+ addr <<= 32;
+ addr += size;
+
+ assert rwaddr == getAllocationAddress(addr);
+
+ return addr;
+ }
+
+ public void clear() {
+ m_sectors.clear();
+ m_free.clear();
+ m_allocation = 0;
+ }
+
+ public void releaseResources() throws InterruptedException {
+ DirectBufferPool.INSTANCE.release(m_resource);
+ }
+
+ public void addToFreeList(final SectorAllocator sector) {
+ m_free.add(sector);
+ }
+
+ public void removeFromFreeList(SectorAllocator sector) {
+ assert m_free.get(0) == sector;
+
+ m_free.remove(sector);
+ }
+
+ public void trimSector(final long trim, final SectorAllocator sector) {
+ assert m_free.get(0) == sector;
+
+ m_allocation -= trim;
+ }
+
+ public AllocationContext createAllocationContext() {
+ return new AllocationContext(this);
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/MemoryManager.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java 2011-01-20 16:54:09 UTC (rev 4138)
@@ -0,0 +1,533 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+package com.bigdata.rwstore.sector;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.log4j.Logger;
+
+import com.bigdata.io.DirectBufferPool;
+import com.bigdata.rwstore.FixedOutputStream;
+import com.bigdata.rwstore.IWriteCacheManager;
+
+/**
+ * The SectorAllocator is designed as an alternative the the standard RWStore
+ * FixedAllocators.
+ *
+ * The idea of the SectorAllocator is to efficiently contain within a single
+ * region as dense a usage as possible. Since a SectorAllocator is able to
+ * allocate a full range of slot sizes, it should be able to service several
+ * thousand allocations and maximise disk locality on write.
+ *
+ * Furthermore, it presents an option to be synced with the backing store -
+ * similarly to a MappedFile, in which case a single write for the entire
+ * sector could be made for update.
+ *
+ * What we do not want is to run out of bits and to leave significant unused
+ * space in the sector. This could happen if we primarily allocated small
+ * slots - say on average 512 bytes. In this case, the maximum 1636 entries
+ * would map 1636 * 32 * 512 bytes => only just over 26M, so a 64M sector
+ * is massively wasteful. The solution is to increment the sector reserve
+ * as required for each tab, say by a minimum of 256K while ensuring always less
+ * than 64M. Bit waste in the allocator - where the sector memory is allocated
+ * with far fewer bits than available is less of an issue, although it does
+ * impact on teh total number of allocations available. The absolute maximum
+ * size of an RWStore is derived from the maximum sector size * the number
+ * of sectors. An 8K sector allows for 32K bits, which, which only requires
+ * 15 unsigned bits, leaving a signed17bits for the sector index = 64K of
+ * sectors. Implying a maximum addressable store file of 64M * 64K,
+ * = 4TB of full sectors. If the average sector only requires 32M, then the
+ * total store would be reduced appropriately.
+ *
+ * The maximum theoretical storage is yielded by MAX_INT * AVG_SLOT_SIZE, so
+ * 2GB * 2K (avg) would equate to the optimal maximum addressable allocations
+ * and file size. An AVG of > 2K yields fewer allocations and an AVG of < 2K
+ * a reduced file size.
+ *
+ * TODO: add parameterisation of META_SIZE for exploitation by MemoryManager.
+ * TODO: cache block starts in m_addresses to simplify/optimise bit2Offset
+ *
+ * @author Martyn Cutcher
+ *
+ */
+public class SectorAllocator {
+ protected static final Logger log = Logger
+ .getLogger(SectorAllocator.class);
+
+ static final int getBitMask(int bits) {
+ int ret = 0;
+
+ for (int i = 0; i < bits; i++) ret += 1 << i;
+
+ return ret;
+ }
+ static final int SECTOR_INDEX_BITS = 16;
+ static final int SECTOR_OFFSET_BITS = 32-SECTOR_INDEX_BITS;
+ static final int SECTOR_OFFSET_MASK = getBitMask(SECTOR_OFFSET_BITS);
+
+ static final int META_SIZE = 8192; // 8K
+
+ final static int SECTOR_SIZE = 64 * 1024 * 1024; // 10M
+ final static int NUM_ENTRIES = (META_SIZE - 12) / (4 + 1); // 8K - index - address / (4 + 1) bits plus tag
+ final int[] BIT_MASKS = {0x1, 0x3, 0x7, 0xF, 0xFF, 0xFFFF, 0xFFFFFFFF};
+ final static int BLOB_SIZE = 4096;
+ final static int BLOB_CHAIN_OFFSET = BLOB_SIZE - 4;
+ final static int[] ALLOC_SIZES = {64, 128, 256, 512, 1024, 2048, BLOB_SIZE};
+ final static int[] ALLOC_BITS = {32, 32, 32, 32, 32, 32, 32};
+ int m_index;
+ long m_sectorAddress;
+ int m_maxSectorSize;
+ byte[] m_tags = new byte[NUM_ENTRIES];
+ int[] m_bits = new int[NUM_ENTRIES]; // 128 - sectorAddress(1) - m_tags(4)
+
+ int[] m_transientbits = new int[NUM_ENTRIES];
+ int[] m_commitbits = new int[NUM_ENTRIES];
+ int[] m_addresses = new int[NUM_ENTRIES];
+
+ // maintain count against each alloc size, this provides ready access to be
+ // able to check the minimum number of bits for all tag sizes. No
+ // SectorAllocator should be on the free list unless there are free bits
+ // available for all tags.
+ //
+ // In order to return a SectorAllocator to the free list we can check not
+ // only the total number of bits, but the average number of bits for the
+ // tag, dividing the numebr of free bits by the total (number of blocks)
+ // for each tag.
+ int[] m_free = new int[ALLOC_SIZES.length];
+ int[] m_total = new int[ALLOC_SIZES.length];
+ int[] m_allocations = new int[ALLOC_SIZES.length];
+ int[] m_recycles = new int[ALLOC_SIZES.length];
+
+ final ISectorManager m_store;
+ boolean m_onFreeList = false;
+ private long m_diskAddr;
+ private final IWriteCacheManager m_writes;
+
+ public SectorAllocator(ISectorManager store, IWriteCacheManager writes) {
+ m_store = store;
+ m_writes = writes;
+ }
+
+ /**
+ * Must find tag type that size fits in (or BLOB) and then find
+ * block of type into which an allocation can be made.
+ *
+ *
+ */
+ public int alloc(int size) {
+ if (size > BLOB_SIZE) {
+ throw new IllegalArgumentException("Cannot directly allocate a BLOB, use PSOutputStream");
+ }
+
+ if (!m_onFreeList)
+ throw new IllegalStateException("Allocation request to allocator " + m_index + " not on the free list");
+
+ byte tag = 0;
+ while (size > ALLOC_SIZES[tag]) tag++;
+
+ assert m_free[tag] > 0;
+
+ // now find allocated tag areas..
+ int sbit = 0;
+ int lbits = 0;
+ for (int i = 0; i < NUM_ENTRIES; i++) {
+ final int ttag = m_tags[i];
+
+ if (ttag == -1) {
+ throw new IllegalStateException("Allocator should not be on the FreeList for tag: " + ALLOC_SIZES[tag]);
+ }
+
+ lbits = ALLOC_BITS[ttag];
+
+ if (ttag == tag) {
+ final int bits = m_transientbits[i];
+ int bit = fndBit(bits);
+
+ if (bit != -1) {
+ sbit += bit;
+
+ if (log.isTraceEnabled())
+ log.trace("Setting bit: " + sbit);
+
+ setBit(m_bits, sbit);
+ setBit(m_transientbits, sbit);
+
+ if (!tstBit(m_bits, sbit)) {
+ throw new IllegalStateException("WTF with bit:" + sbit);
+ }
+ m_free[tag]--;
+
+ m_allocations[tag]++;
+
+ if (m_free[tag] == 0) {
+ if (!addNewTag(tag)) {
+ m_store.removeFromFreeList(this);
+ m_onFreeList = false;
+ }
+ }
+
+ int raddr = makeAddr(m_index, sbit);
+
+ if (log.isTraceEnabled())
+ log.trace("Allocating " + m_index + ":" + sbit + " as " + raddr + " for " + size);
+
+ if (getSectorIndex(raddr) != m_index) {
+ throw new IllegalStateException("Address: " + raddr + " does not yield index: " + m_index);
+ }
+
+ return raddr;
+ }
+ }
+ sbit += lbits; // bump over current tag's bits
+ }
+
+ return 0;
+ }
+
+ public static int makeAddr(final int index, final int bit) {
+ return -((index << SECTOR_OFFSET_BITS) + bit);
+ }
+
+ private boolean addNewTag(byte tag) {
+ int allocated = 0;
+ for (int i = 0; i < m_tags.length; i++) {
+ if (m_tags[i] == -1) {
+ int block = this.ALLOC_SIZES[tag] * 32;
+ if ((allocated + block) <= m_maxSectorSize) {
+ m_tags[i] = tag;
+ m_free[tag] += 32;
+ m_total[tag]++;
+ if (log.isTraceEnabled())
+ log.trace("addnewTag block for: " + ALLOC_SIZES[tag]);
+ if ((i+1) == m_tags.length) {
+ int trim = m_maxSectorSize - (allocated + block);
+
+ m_store.trimSector(trim, this);
+ }
+ return true;
+ } else {
+ if (log.isDebugEnabled())
+ log.debug("addnewTag FALSE due to Sector SIZE");
+
+ return false;
+ }
+ } else {
+ allocated += ALLOC_SIZES[m_tags[i]] * 32;
+ }
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("addnewTag FALSE due to Sector BITS");
+ return false;
+ }
+
+ /**
+ *
+ * @param bit
+ * @return
+ */
+ public boolean free(final int bit) {
+ if (!tstBit(m_bits, bit)) {
+ throw new IllegalStateException("Request to free bit not set: " + bit);
+ }
+
+ clrBit(m_bits, bit);
+ if (!tstBit(m_commitbits, bit)) {
+ if (!tstBit(m_transientbits, bit)) {
+ throw new IllegalStateException("Request to free transient bit not set" + bit);
+ }
+
+ clrBit(m_transientbits, bit);
+ int tag = bit2tag(bit);
+ m_free[tag]++;
+
+ m_recycles[tag]++;
+
+ // The hasFree test is too coarse, ideally we should test for
+ // percentage of free bits - say 10% PLUS a minimum of say 10
+ // for each tag type.
+ if ((!m_onFreeList) && hasFree(5)) { // minimum of 10 bits for all tags
+ m_onFreeList = true;
+ m_store.addToFreeList(this);
+ }
+
+ if (m_writes != null && m_writes.removeWriteToAddr(getPhysicalAddress(bit))) {
+ if (log.isTraceEnabled())
+ log.trace("Removed potential DUPLICATE");
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ *
+ * @param bit
+ * @return the block size
+ */
+ int bit2Size(int bit) {
+ for (int t = 0; t < NUM_ENTRIES; t++) {
+ int tag = m_tags[t];
+ if (tag == -1) {
+ throw new IllegalStateException("bit offset too large");
+ }
+ int bits = ALLOC_BITS[tag];
+ if (bit < bits) {
+ return ALLOC_SIZES[tag];
+ }
+ bit -= bits;
+ }
+
+ return 0;
+ }
+ /**
+ *
+ * @param bit
+ * @return the offset in the sector
+ */
+ int bit2Offset(int bit) {
+ int offset = 0;
+ for (int t = 0; t < NUM_ENTRIES; t++) {
+ int tag = m_tags[t];
+ if (tag == -1) {
+ throw new IllegalStateException("bit offset too large");
+ }
+ int bits = ALLOC_BITS[tag];
+ if (bit < bits) {
+ offset += ALLOC_SIZES[tag] * bit;
+ return offset;
+ } else {
+ offset += ALLOC_SIZES[tag] * bits;
+ bit -= bits;
+ }
+ }
+
+ return 0;
+ }
+ /**
+ * Since we know that all allocations are 32 bits each, there is no need to
+ * san through the array.
+ *
+ * @param bit
+ * @return the tag of the bit
+ */
+ public int bit2tag(int bit) {
+ return m_tags[bit/32];
+ }
+ /**
+ *
+ */
+ public long getPhysicalAddress(int offset) {
+ return m_sectorAddress + bit2Offset(offset);
+ }
+
+ public int getPhysicalSize(int offset) {
+ return bit2Size(offset);
+ }
+
+ public long getStartAddr() {
+ return m_sectorAddress;
+ }
+
+ public String getStats() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ public boolean hasFree(int threshold) {
+ for (int i = 0; i < m_free.length; i++) {
+ if ((m_free[i]/m_total[i]) < threshold)
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Checks
+ * @return if there is a positive free count for all tags
+ */
+ public boolean hasFree() {
+ return hasFree(1);
+ }
+
+ public void preserveSessionData() {
+ // TODO Auto-generated method stub
+
+ }
+
+ public void read(DataInputStream str) {
+ try {
+ m_index = str.readInt();
+ m_sectorAddress = str.readLong();
+ int taglen = str.read(m_tags);
+ assert taglen == m_tags.length;
+
+ for (int i = 0; i < NUM_ENTRIES; i++) {
+ m_bits[i] = str.readInt();
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ public long getDiskAddr() {
+ return m_diskAddr;
+ }
+
+ public void setDiskAddr(long addr) {
+ m_diskAddr = addr;
+ }
+
+ public boolean verify(int addr) {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ public byte[] write() {
+ byte[] buf = new byte[META_SIZE];
+ DataOutputStream str = new DataOutputStream(new FixedOutputStream(buf));
+ try {
+ str.writeInt(m_index);
+ str.writeLong(m_sectorAddress);
+ str.write(m_tags);
+ for (int i = 0; i < NUM_ENTRIES; i++) {
+ str.writeInt(m_bits[i]);
+ }
+
+ m_transientbits = (int[]) m_bits.clone();
+ m_commitbits = (int[]) m_bits.clone();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ str.close();
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+
+ return buf;
+ }
+
+ public int addressSize(int offset) {
+ return bit2Size(offset);
+ }
+
+ public void setIndex(int index) {
+ assert m_index == 0;
+
+ m_index = index;
+ }
+
+ public void addAddresses(final ArrayList<Long> addrs) {
+ addrs.add(Long.valueOf(m_sectorAddress));
+ }
+
+ static void clrBit(int[] bits, int bitnum) {
+ int index = bitnum / 32;
+ int bit = bitnum % 32;
+
+ int val = bits[index];
+
+ val &= ~(1 << bit);
+
+ bits[index] = val;
+ }
+
+ static void setBit(int[] bits, int bitnum) {
+ int index = bitnum / 32;
+ int bit = bitnum % 32;
+
+ bits[index] |= 1 << bit;
+ }
+
+ static boolean tstBit(int[] bits, int bitnum) {
+ int index = bitnum / 32;
+ int bit = bitnum % 32;
+
+ return (bits[index] & 1 << bit) != 0;
+ }
+
+ /*
+ * use divide and conquer rather than shifting through
+ */
+ int fndBit(int bits) {
+ for (int n = 0; n < 8; n++) { // check nibbles
+ if ((bits & 0x0F) != 0xF) {
+ for (int b = 0; b < 4; b++) {
+ if ((bits & (1 << b)) == 0) {
+ return b + (n * 4);
+ }
+ }
+ }
+ bits >>>= 4; //right shift a nibble
+ }
+
+ return -1;
+ }
+
+ /**
+ * As well as setting the address, this is the point when the
+ * allocator can pre-allocate the first set of tags.
+ *
+ * @param sectorAddress managed by this Allocator
+ */
+ public void setSectorAddress(final long sectorAddress, final int maxsize) {
+ if (log.isInfoEnabled())
+ log.info("setting sector address: " + sectorAddress);
+
+ m_sectorAddress = sectorAddress;
+ m_maxSectorSize = maxsize;
+ for (int i = 0; i < ALLOC_SIZES.length; i++) {
+ m_tags[i] = (byte) i;
+ m_free[i] = 32;
+ m_total[i] = 1;
+ }
+ for (int i = ALLOC_SIZES.length; i < NUM_ENTRIES; i++) {
+ m_tags[i] = (byte) -1;
+ }
+
+ m_onFreeList = true;
+ m_store.addToFreeList(this);
+ }
+
+ public static int getSectorIndex(int rwaddr) {
+ return (-rwaddr) >> SECTOR_OFFSET_BITS;
+ }
+
+ public static int getSectorOffset(int rwaddr) {
+ return (-rwaddr) & SECTOR_OFFSET_MASK;
+ }
+
+ public static int getBlobBlockCount(int size) {
+ final int nblocks = (size + BLOB_SIZE - 1) / BLOB_SIZE;
+
+ return nblocks;
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/rwstore/sector/SectorAllocator.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 15:53:55
|
Revision: 4137
http://bigdata.svn.sourceforge.net/bigdata/?rev=4137&view=rev
Author: thompsonbry
Date: 2011-01-20 15:53:49 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
Making the CONSTRAINT on a join apply to the original binding set when the predicate is optional.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-20 13:38:30 UTC (rev 4136)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-20 15:53:49 UTC (rev 4137)
@@ -48,6 +48,7 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.BOpContext;
import com.bigdata.bop.BOpEvaluationContext;
+import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
@@ -1683,6 +1684,16 @@
final IBindingSet bs = bindingSets[bindex];
+ if (constraints != null) {
+
+ // verify constraint.
+ if(!BOpUtility.isConsistent(constraints, bs)) {
+ // skip solutions which fail the constraint.
+ continue;
+ }
+
+ }
+
if (log.isTraceEnabled())
log
.trace("Passing on solution which fails an optional join: "
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-20 13:38:36
|
Revision: 4136
http://bigdata.svn.sourceforge.net/bigdata/?rev=4136&view=rev
Author: thompsonbry
Date: 2011-01-20 13:38:30 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
Turned off the new evaluation strategy by default again.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-20 02:05:21 UTC (rev 4135)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-20 13:38:30 UTC (rev 4136)
@@ -382,7 +382,7 @@
public static final String NEW_EVAL_STRATEGY = BigdataSail.class.getPackage()
.getName()+ ".newEvalStrategy";
- public static final String DEFAULT_NEW_EVAL_STRATEGY = "true";
+ public static final String DEFAULT_NEW_EVAL_STRATEGY = "false";
/**
* Option as to whether or not to allow Sesame evaluation of queries
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-20 02:05:28
|
Revision: 4135
http://bigdata.svn.sourceforge.net/bigdata/?rev=4135&view=rev
Author: mrpersonick
Date: 2011-01-20 02:05:21 +0000 (Thu, 20 Jan 2011)
Log Message:
-----------
working through tck failures
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/SameTermBOp.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -737,7 +737,26 @@
return sb.toString();
}
+
+ public static String toString2(final BOp bop) {
+ String s = toString(bop);
+ s = s.replaceAll("com.bigdata.bop.controller.", "");
+ s = s.replaceAll("com.bigdata.bop.join.", "");
+ s = s.replaceAll("com.bigdata.bop.solutions.", "");
+ s = s.replaceAll("com.bigdata.bop.rdf.filter.", "");
+ s = s.replaceAll("com.bigdata.bop.", "");
+ s = s.replaceAll("com.bigdata.rdf.sail.", "");
+ s = s.replaceAll("com.bigdata.rdf.spo.", "");
+// s = s.replaceAll("com.bigdata.bop..", "");
+// s = s.replaceAll("com.bigdata.bop..", "");
+// s = s.replaceAll("com.bigdata.bop..", "");
+// s = s.replaceAll("com.bigdata.bop..", "");
+// s = s.replaceAll("com.bigdata.bop..", "");
+ return s;
+
+ }
+
private static void toString(final BOp bop, final StringBuilder sb,
final int indent) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/SubqueryOp.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -158,6 +158,18 @@
return new FutureTask<Void>(new ControllerTask(this, context));
}
+
+ public String toString() {
+
+ final StringBuilder sb = new StringBuilder(super.toString());
+ sb.append("\n{\n");
+ final PipelineOp subquery = (PipelineOp)
+ getRequiredProperty(Annotations.SUBQUERY);
+ sb.append(BOpUtility.toString(subquery));
+ sb.append("\n}");
+ return sb.toString();
+
+ }
/**
* Evaluates the arguments of the operator as subqueries. The arguments are
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -98,12 +98,18 @@
return _accept(left.compareTo(right));
}
- if (!IVUtility.canNumericalCompare(left))
- throw new NotNumericalException("cannot numerical compare: " + left);
+ if (!IVUtility.canNumericalCompare(left) ||
+ !IVUtility.canNumericalCompare(right)) {
+ if (op == CompareOp.EQ) {
+ return false;
+ } else if (op == CompareOp.NE) {
+ return true;
+ } else {
+ throw new NotNumericalException("cannot numerical compare: "
+ + left + " " + op + " " + right);
+ }
+ }
- if (!IVUtility.canNumericalCompare(right))
- throw new NotNumericalException("cannot numerical compare: " + right);
-
return _accept(IVUtility.numericalCompare(left, right));
}
Added: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/SameTermBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/SameTermBOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/SameTermBOp.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -0,0 +1,87 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.internal.constraints;
+
+import java.util.Map;
+
+import org.openrdf.query.algebra.Compare.CompareOp;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IValueExpression;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.constraint.BOpConstraint;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.IVUtility;
+
+/**
+ * Compare two terms for exact equality.
+ */
+public class SameTermBOp extends BOpConstraint {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Required shallow copy constructor.
+ */
+ public SameTermBOp(final BOp[] values,
+ final Map<String, Object> annotations) {
+ super(values, annotations);
+ }
+
+ /**
+ * Required deep copy constructor.
+ */
+ public SameTermBOp(final SameTermBOp op) {
+ super(op);
+ }
+
+ public SameTermBOp(final IValueExpression<IV> left,
+ final IValueExpression<IV> right) {
+
+ super(new BOp[] { left, right }, null);
+
+ if (left == null || right == null)
+ throw new IllegalArgumentException();
+
+ }
+
+ public boolean accept(final IBindingSet s) {
+
+ final IV left = ((IValueExpression<IV>) get(0)).get(s);
+ final IV right = ((IValueExpression<IV>) get(1)).get(s);
+
+ if (left == null || right == null)
+ return true; // not yet bound.
+
+ return left.equals(right);
+
+ }
+
+}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl3.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -29,6 +29,7 @@
import org.openrdf.query.algebra.Join;
import org.openrdf.query.algebra.LeftJoin;
import org.openrdf.query.algebra.MathExpr;
+import org.openrdf.query.algebra.MathExpr.MathOp;
import org.openrdf.query.algebra.MultiProjection;
import org.openrdf.query.algebra.Not;
import org.openrdf.query.algebra.Or;
@@ -41,15 +42,13 @@
import org.openrdf.query.algebra.Regex;
import org.openrdf.query.algebra.SameTerm;
import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.StatementPattern.Scope;
import org.openrdf.query.algebra.TupleExpr;
import org.openrdf.query.algebra.UnaryTupleOperator;
import org.openrdf.query.algebra.Union;
import org.openrdf.query.algebra.ValueConstant;
import org.openrdf.query.algebra.ValueExpr;
import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.Compare.CompareOp;
-import org.openrdf.query.algebra.MathExpr.MathOp;
-import org.openrdf.query.algebra.StatementPattern.Scope;
import org.openrdf.query.algebra.evaluation.impl.EvaluationStrategyImpl;
import org.openrdf.query.algebra.evaluation.iterator.FilterIterator;
import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
@@ -61,12 +60,12 @@
import com.bigdata.bop.IConstant;
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.IValueExpression;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
-import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.ap.Predicate;
import com.bigdata.bop.constraint.AND;
import com.bigdata.bop.constraint.BOUND;
@@ -84,15 +83,16 @@
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.constraints.CompareBOp;
import com.bigdata.rdf.internal.constraints.MathBOp;
+import com.bigdata.rdf.internal.constraints.SameTermBOp;
import com.bigdata.rdf.lexicon.LexiconRelation;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.sail.BigdataSail.Options;
import com.bigdata.rdf.sail.sop.SOp;
import com.bigdata.rdf.sail.sop.SOp2BOpUtility;
import com.bigdata.rdf.sail.sop.SOpTree;
+import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
import com.bigdata.rdf.sail.sop.SOpTreeBuilder;
import com.bigdata.rdf.sail.sop.UnsupportedOperatorException;
-import com.bigdata.rdf.sail.sop.SOpTree.SOpGroup;
import com.bigdata.rdf.spo.DefaultGraphSolutionExpander;
import com.bigdata.rdf.spo.ExplicitSPOFilter;
import com.bigdata.rdf.spo.ISPO;
@@ -118,7 +118,6 @@
import com.bigdata.striterator.Dechunkerator;
import com.bigdata.striterator.DistinctFilter;
import com.bigdata.striterator.IChunkedOrderedIterator;
-import com.bigdata.util.concurrent.Haltable;
/**
* Extended to rewrite Sesame {@link TupleExpr}s onto native {@link Rule}s and
@@ -516,6 +515,59 @@
}
+ @Override
+ public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(
+ final Filter filter, final BindingSet bs)
+ throws QueryEvaluationException {
+
+ if (!nativeJoins) {
+ // Use Sesame 2 evaluation
+ return super.evaluate(filter, bs);
+ }
+
+ if (filter.getArg() instanceof StatementPattern) {
+ // no need to run a query for this, a simple access path scan will do
+ return super.evaluate(filter, bs);
+ }
+
+ if (log.isInfoEnabled()) {
+ log.info("evaluating top-level Filter operator");
+ }
+
+ try {
+
+ return evaluateNatively(filter, bs);
+
+ } catch (UnsupportedOperatorException ex) {
+
+ if (allowSesameQueryEvaluation) {
+
+ // Use Sesame 2 evaluation
+
+ log.warn("could not evaluate natively, using Sesame evaluation");
+
+ if (log.isInfoEnabled()) {
+ log.info(ex.getOperator());
+ }
+
+ // turn off native joins for the remainder, we can't do
+ // partial execution
+ nativeJoins = false;
+
+ // defer to Sesame
+ return super.evaluate(filter, bs);
+
+ } else {
+
+ // allow the query to fail
+ throw new UnsupportedOperatorException(ex);
+
+ }
+
+ }
+
+ }
+
CloseableIteration<BindingSet, QueryEvaluationException>
evaluateNatively(final TupleExpr tupleExpr, final BindingSet bs)
throws QueryEvaluationException, UnsupportedOperatorException {
@@ -706,7 +758,7 @@
queryEngine, queryHints);
if (log.isInfoEnabled())
- log.info(BOpUtility.toString(query));
+ log.info("\n"+BOpUtility.toString2(query));
}
@@ -737,6 +789,11 @@
result = new FilterIterator(f, result, this);
}
}
+
+// System.err.println("results");
+// while (result.hasNext()) {
+// System.err.println(result.next());
+// }
return result;
@@ -1832,10 +1889,13 @@
toValueExpression(sameTerm.getLeftArg());
final IValueExpression<IV> iv2 =
toValueExpression(sameTerm.getRightArg());
- return new CompareBOp(iv1, iv2, CompareOp.EQ);
+ return new SameTermBOp(iv1, iv2);
}
- private IConstraint toConstraint(Compare compare) {
+ private IConstraint toConstraint(final Compare compare) {
+ if (!database.isInlineLiterals()) {
+ throw new UnsupportedOperatorException(compare);
+ }
final IValueExpression<IV> iv1 =
toValueExpression(compare.getLeftArg());
final IValueExpression<IV> iv2 =
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -382,7 +382,7 @@
public static final String NEW_EVAL_STRATEGY = BigdataSail.class.getPackage()
.getName()+ ".newEvalStrategy";
- public static final String DEFAULT_NEW_EVAL_STRATEGY = "false";
+ public static final String DEFAULT_NEW_EVAL_STRATEGY = "true";
/**
* Option as to whether or not to allow Sesame evaluation of queries
@@ -396,8 +396,6 @@
public static final String DEFAULT_ALLOW_SESAME_QUERY_EVALUATION = "false";
-
-
}
/**
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -598,27 +598,35 @@
final Collection<IConstraint> constraints =
new LinkedList<IConstraint>();
- /*
- * Peek through the predicate's args to find its variables. Use
- * these to attach constraints to the join based on the variables
- * that make their first appearance in this tail.
- */
- for (BOp arg : pred.args()) {
- if (arg instanceof IVariable<?>) {
- final IVariable<?> v = (IVariable<?>) arg;
- /*
- * We do a remove because we don't ever need to run these
- * constraints again during subsequent joins once they have
- * been run once at the initial appearance of the variable.
- *
- * @todo revisit this when we dynamically re-order running
- * joins
- */
- if (constraintsByVar.containsKey(v))
- constraints.addAll(constraintsByVar.remove(v));
- }
+// /*
+// * Peek through the predicate's args to find its variables. Use
+// * these to attach constraints to the join based on the variables
+// * that make their first appearance in this tail.
+// */
+// for (BOp arg : pred.args()) {
+// if (arg instanceof IVariable<?>) {
+// final IVariable<?> v = (IVariable<?>) arg;
+// /*
+// * We do a remove because we don't ever need to run these
+// * constraints again during subsequent joins once they have
+// * been run once at the initial appearance of the variable.
+// *
+// * @todo revisit this when we dynamically re-order running
+// * joins
+// */
+// if (constraintsByVar.containsKey(v))
+// constraints.addAll(constraintsByVar.remove(v));
+// }
+// }
+
+ // just add all the constraints to the very last tail for now
+ if (i == (order.length-1) && rule.getConstraintCount() > 0) {
+ final Iterator<IConstraint> it = rule.getConstraints();
+ while (it.hasNext()) {
+ constraints.add(it.next());
+ }
}
-
+
// annotations for this join.
final List<NV> anns = new LinkedList<NV>();
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTree.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -153,6 +153,10 @@
return SOpTree.this;
}
+ public boolean isRoot() {
+ return group == 0;
+ }
+
}
public class SOpGroups implements Iterable<SOpGroup> {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/sop/SOpTreeBuilder.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -234,7 +234,8 @@
} else if (arg instanceof Join) {
collectSOps(sops, (Join) arg, rslj, g, pg);
} else if (arg instanceof LeftJoin) {
- collectSOps(sops, (LeftJoin) arg, rslj, groupId.incrementAndGet(), g);
+// collectSOps(sops, (LeftJoin) arg, rslj, groupId.incrementAndGet(), g);
+ collectSOps(sops, (LeftJoin) arg, rslj, g, pg);
} else {
throw new UnsupportedOperatorException(arg);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2011-01-19 22:42:44 UTC (rev 4134)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2011-01-20 02:05:21 UTC (rev 4135)
@@ -212,6 +212,29 @@
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/basic/manifest#spoo-1",//BOOM
// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-05",
+
+ /*
+ * working through the new query engine failures
+ */
+
+ // please someone explain this shit to me
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-12"
+
+ // this is that weird "well designed" optional shit P = A OPT (B OPT C) where A and C share variables not in B
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-1"
+
+ // where do we put the !bound(?e) constraint???
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/bound/manifest#dawg-bound-query-001"
+
+ // "a" and "a"^^xsd:string have different term ids? also bnodes are different
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-07"
+
+ // help, non-optional subquery?? wtf
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#filter-scope-1"
+
+ // this uncovers an obvious bug in our SubqueryOp
+// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-001"
+
});
/**
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-19 22:42:50
|
Revision: 4134
http://bigdata.svn.sourceforge.net/bigdata/?rev=4134&view=rev
Author: thompsonbry
Date: 2011-01-19 22:42:44 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
Removed an intervening call to getRandomAccess() in favor of getChannel().
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java 2011-01-19 22:41:51 UTC (rev 4133)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractBufferStrategy.java 2011-01-19 22:42:44 UTC (rev 4134)
@@ -474,7 +474,7 @@
final long count = src.getNextOffset();
// the source channel.
- final FileChannel srcChannel = src.getRandomAccessFile().getChannel();
+ final FileChannel srcChannel = src.getChannel();
// the output channel.
final FileChannel outChannel = out.getChannel();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-19 22:41:58
|
Revision: 4133
http://bigdata.svn.sourceforge.net/bigdata/?rev=4133&view=rev
Author: thompsonbry
Date: 2011-01-19 22:41:51 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
More work on the PartitionedJoinGroup utility and its test suite.
Some javadoc and minor edits related to [1].
[1] https://sourceforge.net/apps/trac/bigdata/ticket/123
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskBackedBufferStrategy.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java 2011-01-19 21:22:01 UTC (rev 4132)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java 2011-01-19 22:41:51 UTC (rev 4133)
@@ -30,12 +30,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*
- * @todo However, how do we manage when there are things like conditional
- * routing operators? [Answer - the CONDITION is raised onto the subquery
- * such that we only conditionally run the subquery rather than routing
- * out of the subquery if the condition is not satisfied - MikeP is making
- * this change.]
- *
* @todo The order of the {@link IPredicate}s in the tail plan is currently
* unchanged from their given order (optional joins without constraints
* can not reduce the selectivity of the query). However, it could be
@@ -75,39 +69,26 @@
* The set of variables bound by the non-optional predicates.
*/
private final Set<IVariable<?>> joinGraphVars = new LinkedHashSet<IVariable<?>>();
-
- /**
- * An unordered list of constraints which do not involve ANY variables.
- * These constraints should be run first, before the join graph.
- *
- * @todo integrate into evaluation.
- */
- private final List<IConstraint> runFirstConstraints = new LinkedList<IConstraint>();
/**
- * The set of the {@link IPredicate}s which have been flagged as
- * "run first". These must all be non-optional predicates. They are usually
- * special access paths created using an expander which replaces a mock
- * access path. For example, free text search.
+ * The set of non-optional predicates which have been flagged as
+ * "run first". These are usually special access paths created using an
+ * expander which replaces a mock access path. For example, free text
+ * search.
*/
private final List<IPredicate<?>> headPlan = new LinkedList<IPredicate<?>>();
/**
- * The set of constraints which can be evaluated with the head plan
- * predicates because the variables appearing in those constraints are known
- * to become bound within the head plan.
- */
- private final List<IConstraint> headPlanConstraints = new LinkedList<IConstraint>();
-
- /**
* The set of non-optional predicates which represent the join graph.
*/
- private final List<IPredicate<?>> joinGraphPredicates = new LinkedList<IPredicate<?>>();
+ private final List<IPredicate<?>> joinGraph = new LinkedList<IPredicate<?>>();
/**
- * The set of constraints which can be evaluated with the join graph
- * predicates because the variables appearing in those constraints are known
- * to become bound within the join graph.
+ * The set of constraints which can be evaluated with the head plan and/or
+ * join graph predicates because the variables appearing in those
+ * constraints are known to become bound within the join graph. ( The
+ * {@link #headPlan} and the {@link #joinGraph} share the same
+ * pool of constraints.)
*/
private final List<IConstraint> joinGraphConstraints = new LinkedList<IConstraint>();
@@ -128,9 +109,10 @@
* The values are the sets of constraints to run for that tail.
*/
private final Map<Integer/* predId */, List<IConstraint>> tailPlanConstraintMap = new LinkedHashMap<Integer, List<IConstraint>>();
-
+
/**
- * The set of variables bound by the non-optional predicates.
+ * The set of variables bound by the non-optional predicates (either the
+ * head plan or the join graph).
*/
public Set<IVariable<?>> getJoinGraphVars() {
return joinGraphVars;
@@ -139,9 +121,8 @@
/**
* The {@link IPredicate}s in the join graph (required joins).
*/
- public IPredicate<?>[] getJoinGraphPredicates() {
- return joinGraphPredicates.toArray(new IPredicate[joinGraphPredicates
- .size()]);
+ public IPredicate<?>[] getJoinGraph() {
+ return joinGraph.toArray(new IPredicate[joinGraph.size()]);
}
/**
@@ -275,7 +256,7 @@
headPlan.add(p);
} else {
// part of the join graph.
- joinGraphPredicates.add(p);
+ joinGraph.add(p);
}
/*
* Add to the set of variables which will be bound by the time
@@ -300,12 +281,9 @@
.getSpannedVariables(c);
if (!vitr.hasNext()) {
/*
- * This is a constraint which does not involve any variable so
- * we should evaluate it as soon as possible. I.e., before the
- * join graph.
+ * All constraints should have at least one variable.
*/
- runFirstConstraints.add(c); // @todo unit test.
- continue;
+ throw new RuntimeException("No variables in constraint: " + c);
}
while (vitr.hasNext()) {
final IVariable<?> var = vitr.next();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskBackedBufferStrategy.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskBackedBufferStrategy.java 2011-01-19 21:22:01 UTC (rev 4132)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskBackedBufferStrategy.java 2011-01-19 22:41:51 UTC (rev 4133)
@@ -109,7 +109,7 @@
/**
* Forces the data to disk.
*/
- public void force( boolean metadata ) {
+ public void force(final boolean metadata) {
try {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2011-01-19 21:22:01 UTC (rev 4132)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2011-01-19 22:41:51 UTC (rev 4133)
@@ -1363,10 +1363,10 @@
}
/**
- * {@link #flushWriteCache() flushs} the {@link #writeCache} before syncing
+ * {@link #flushWriteCache() flushes} the {@link #writeCache} before syncing
* the disk.
*/
- public void force(boolean metadata) {
+ public void force(final boolean metadata) {
assertOpen();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java 2011-01-19 21:22:01 UTC (rev 4132)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java 2011-01-19 22:41:51 UTC (rev 4133)
@@ -194,11 +194,11 @@
// verify all predicates were placed into the join graph.
assertSameIteratorAnyOrder("joinGraph", preds, Arrays.asList(
- fixture.getJoinGraphPredicates()).iterator());
+ fixture.getJoinGraph()).iterator());
// there are no constraints.
- assertEquals("joinGraphConstraints.size", 0, fixture
- .getJoinGraphConstraints().length);
+ assertEquals("joinGraphConstraints.size", new IConstraint[] {},
+ fixture.getJoinGraphConstraints());
// there is no tail plan.
assertEquals("tailPlan", new IPredicate[] {}, fixture.getTailPlan());
@@ -225,7 +225,7 @@
// verify all predicates were placed into the join graph.
assertSameIteratorAnyOrder("joinGraph", preds, Arrays.asList(
- fixture.getJoinGraphPredicates()).iterator());
+ fixture.getJoinGraph()).iterator());
// verify all constraints were place on the join graph.
assertSameIteratorAnyOrder("joinGraphConstraints", constraints,
@@ -343,7 +343,7 @@
// verify predicates placed into the join graph.
assertSameIteratorAnyOrder("joinGraph", new IPredicate[] { p0, p1,
- p5 }, Arrays.asList(fixture.getJoinGraphPredicates())
+ p5 }, Arrays.asList(fixture.getJoinGraph())
.iterator());
// there are no constraints on the join graph predicates.
@@ -392,7 +392,7 @@
// verify predicates placed into the join graph.
assertSameIteratorAnyOrder("joinGraph", new IPredicate[] { p0, p1,
- p5 }, Arrays.asList(fixture.getJoinGraphPredicates())
+ p5 }, Arrays.asList(fixture.getJoinGraph())
.iterator());
// verify constraints on the join graph.
@@ -423,8 +423,6 @@
/**
* @todo test with headPlan.
*
- * @todo test for runFirst constraints.
- *
* @todo test logic to attach constraints to non-optional joins based on a
* given join path (not yet written).
*/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-19 21:22:08
|
Revision: 4132
http://bigdata.svn.sourceforge.net/bigdata/?rev=4132&view=rev
Author: thompsonbry
Date: 2011-01-19 21:22:01 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
More unit tests for the PartitionedJoinGroup.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java 2011-01-19 21:21:03 UTC (rev 4131)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java 2011-01-19 21:22:01 UTC (rev 4132)
@@ -31,7 +31,10 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
*
* @todo However, how do we manage when there are things like conditional
- * routing operators?
+ * routing operators? [Answer - the CONDITION is raised onto the subquery
+ * such that we only conditionally run the subquery rather than routing
+ * out of the subquery if the condition is not satisfied - MikeP is making
+ * this change.]
*
* @todo The order of the {@link IPredicate}s in the tail plan is currently
* unchanged from their given order (optional joins without constraints
@@ -48,38 +51,97 @@
* leave a variable unbound).
*
* @todo runFirst flag on the expander (for free text search). this should be an
- * annotation. this can be a headPlan. [There can be constraints which are
- * evaluated against the head plan. They need to get attached to the joins
- * generated for the head plan. MikeP writes: There is a free text search
- * access path that replaces the actual access path for the predicate,
- * which is meaningless in an of itself because the P is magical.]
+ * annotation. this can be a [headPlan]. [There can be constraints which
+ * are evaluated against the head plan. They need to get attached to the
+ * joins generated for the head plan. MikeP writes: There is a free text
+ * search access path that replaces the actual access path for the
+ * predicate, which is meaningless in an of itself because the P is
+ * magical.]
*
- * @todo inline APs and get rid of DataSetJoin. Rewrite NG and DG first.
- *
* @todo write a method which returns the set of constraints which should be run
* for the last predicate in a given join path (a join path is just an
* ordered array of predicates).
+ *
+ * FIXME Add a method to generate a runnable query plan from a collection
+ * of predicates and constraints. This is a bit different for the join
+ * graph and the optionals in the tail plan. The join graph itself should
+ * either be a {@link JoinGraph} operator which gets evaluated at run time
+ * or reordered by whichever optimizer is selected for the query (query
+ * hints).
*/
public class PartitionedJoinGroup {
-// private final IPredicate<?>[] headPlan;
-//
-// private final IConstraint[] headPlanConstraints;
+ /**
+ * The set of variables bound by the non-optional predicates.
+ */
+ private final Set<IVariable<?>> joinGraphVars = new LinkedHashSet<IVariable<?>>();
- private final IPredicate<?>[] joinGraphPredicates;
+ /**
+ * An unordered list of constraints which do not involve ANY variables.
+ * These constraints should be run first, before the join graph.
+ *
+ * @todo integrate into evaluation.
+ */
+ private final List<IConstraint> runFirstConstraints = new LinkedList<IConstraint>();
- private final IConstraint[] joinGraphConstraints;
+ /**
+ * The set of the {@link IPredicate}s which have been flagged as
+ * "run first". These must all be non-optional predicates. They are usually
+ * special access paths created using an expander which replaces a mock
+ * access path. For example, free text search.
+ */
+ private final List<IPredicate<?>> headPlan = new LinkedList<IPredicate<?>>();
- private final IPredicate<?>[] tailPlan;
+ /**
+ * The set of constraints which can be evaluated with the head plan
+ * predicates because the variables appearing in those constraints are known
+ * to become bound within the head plan.
+ */
+ private final List<IConstraint> headPlanConstraints = new LinkedList<IConstraint>();
-// private final IConstraint[] tailPlanConstraints;
-
+ /**
+ * The set of non-optional predicates which represent the join graph.
+ */
+ private final List<IPredicate<?>> joinGraphPredicates = new LinkedList<IPredicate<?>>();
/**
+ * The set of constraints which can be evaluated with the join graph
+ * predicates because the variables appearing in those constraints are known
+ * to become bound within the join graph.
+ */
+ private final List<IConstraint> joinGraphConstraints = new LinkedList<IConstraint>();
+
+ /**
+ * A set of optional predicates which will be run after the join graph.
+ */
+ private final List<IPredicate<?>> tailPlan = new LinkedList<IPredicate<?>>();
+
+ /**
+ * An unordered list of those constraints containing at least one variable
+ * known be bound (and optionally) bound within the tail plan.
+ */
+ private final List<IConstraint> tailPlanConstraints = new LinkedList<IConstraint>();
+
+ /**
+ * A map indicating which constraints are run for which predicate in the
+ * tail plan. The keys are the bopIds of the predicates in the tail plan.
+ * The values are the sets of constraints to run for that tail.
+ */
+ private final Map<Integer/* predId */, List<IConstraint>> tailPlanConstraintMap = new LinkedHashMap<Integer, List<IConstraint>>();
+
+ /**
+ * The set of variables bound by the non-optional predicates.
+ */
+ public Set<IVariable<?>> getJoinGraphVars() {
+ return joinGraphVars;
+ }
+
+ /**
* The {@link IPredicate}s in the join graph (required joins).
*/
public IPredicate<?>[] getJoinGraphPredicates() {
- return joinGraphPredicates;
+ return joinGraphPredicates.toArray(new IPredicate[joinGraphPredicates
+ .size()]);
}
/**
@@ -91,7 +153,8 @@
* query optimizer when it decides on an evaluation order for those joins).
*/
public IConstraint[] getJoinGraphConstraints() {
- return joinGraphConstraints;
+ return joinGraphConstraints
+ .toArray(new IConstraint[joinGraphConstraints.size()]);
}
/**
@@ -101,25 +164,60 @@
* tail plan in which their variable(S) MIGHT have been bound.
*/
public IPredicate<?>[] getTailPlan() {
- return tailPlan;
+ return tailPlan.toArray(new IPredicate[tailPlan.size()]);
}
-// private PartitionedJoinGroup(//
-// final IPredicate<?>[] headPlan,//
-// final IConstraint[] headPlanConstraints,//
-// final IPredicate<?>[] joinGraphPredicates,//
-// final IConstraint[] joinGraphConstraints,//
-// final IPredicate<?>[] tailPlan,//
-// final IConstraint[] tailPlanConstraints//
-// ) {
-// this.headPlan = headPlan;
-// this.headPlanConstraints = headPlanConstraints;
-// this.joinGraphPredicates = joinGraphPredicates;
-// this.joinGraphConstraints = joinGraphConstraints;
-// this.tailPlan = tailPlan;
-// this.tailPlanConstraints = tailPlanConstraints;
-// }
+ /**
+ * Return the set of {@link IConstraint}s which should be evaluated when an
+ * identified predicate having SPARQL optional semantics is evaluated. For
+ * constraints whose variables are not known to be bound when entering the
+ * tail plan, the constraint should be evaluated at the last predicate for
+ * which its variables MIGHT become bound.
+ *
+ * @param bopId
+ * The identifier for an {@link IPredicate} appearing in the tail
+ * plan.
+ *
+ * @return The set of constraints to be imposed by the join which evaluates
+ * that predicate. This will be an empty array if there are no
+ * constraints which can be imposed when that predicate is
+ * evaluated.
+ *
+ * @throws IllegalArgumentException
+ * if there is no such predicate in the tail plan.
+ */
+ public IConstraint[] getTailPlanConstraints(final int bopId) {
+ boolean found = false;
+
+ for (IPredicate<?> p : tailPlan) {
+
+ if (p.getId() == bopId) {
+
+ found = true;
+
+ break;
+
+ }
+
+ }
+
+ if (!found)
+ throw new IllegalArgumentException(
+ "No such predicate in tail plan: bopId=" + bopId);
+
+ final List<IConstraint> constraints = tailPlanConstraintMap.get(bopId);
+
+ if (constraints == null) {
+
+ return new IConstraint[0];
+
+ }
+
+ return constraints.toArray(new IConstraint[constraints.size()]);
+
+ }
+
/**
* Analyze a set of {@link IPredicate}s representing optional and
* non-optional joins and a collection of {@link IConstraint}s, partitioning
@@ -156,52 +254,6 @@
}
/*
- * An unordered list of constraints which do not involve ANY variables.
- * These constraints should be run first, before the join graph.
- *
- * @todo add to the class instance fields.
- */
- final List<IConstraint> runFirstConstraints = new LinkedList<IConstraint>();
-
-// final List<IPredicate<?>> headPlan = new LinkedList<IPredicate<?>>();
-//
-// final List<IConstraint> headPlanConstraints = new LinkedList<IConstraint>();
-
- /*
- * The non-optional predicates.
- */
- final List<IPredicate<?>> joinGraphPredicates = new LinkedList<IPredicate<?>>();
-
- /*
- * The set of variables bound by the non-optional predicates.
- */
- final Set<IVariable<?>> joinGraphVars = new LinkedHashSet<IVariable<?>>();
-
- /*
- * An unordered list of those constraints whose variables are known to
- * be bound by the non-optional predicates.
- */
- final List<IConstraint> joinGraphConstraints = new LinkedList<IConstraint>();
-
- /*
- * The predicates representing the optional joins.
- */
- final List<IPredicate<?>> tailPlan = new LinkedList<IPredicate<?>>();
-
- /*
- * An unordered list of those constraints containing at least one
- * variable known to NOT be bound by the non-optional predicates.
- */
- final List<IConstraint> tailPlanConstraints = new LinkedList<IConstraint>();
-
- /*
- * Map indicating which constraints are run for which predicate in the
- * tail plan. The keys are the bopIds of the predicates in the tail
- * plan. The values are the sets of constraints to run for that tail.
- */
- final Map<Integer/* predId */, List<IConstraint>> tailPlanConstraintMap = new LinkedHashMap<Integer, List<IConstraint>>();
-
- /*
* First identify the predicates which correspond to non-optional joins.
* All other pipeline operators are inserted into the tail plan in the
* order in which they are given.
@@ -209,25 +261,38 @@
for (IPredicate<?> p : sourcePreds) {
if (p == null)
throw new IllegalArgumentException();
- if (!p.isOptional()) {
+ if (p.isOptional()) {
+ if (p.getAccessPathExpander() != null
+ && p.getAccessPathExpander().runFirst())
+ throw new IllegalStateException(
+ "runFirst is not compatible with optional: " + p);
+ // an optional predicate
+ tailPlan.add(p);
+ } else {
// non-optional predicate.
- joinGraphPredicates.add(p);
- // variables which will be bound by the join graph.
+ if (p.getAccessPathExpander() != null
+ && p.getAccessPathExpander().runFirst()) {
+ headPlan.add(p);
+ } else {
+ // part of the join graph.
+ joinGraphPredicates.add(p);
+ }
+ /*
+ * Add to the set of variables which will be bound by the time
+ * the join graph is done executing.
+ */
final Iterator<IVariable<?>> vitr = BOpUtility
.getArgumentVariables(p);
while (vitr.hasNext()) {
joinGraphVars.add(vitr.next());
}
- } else {
- // an optional predicate
- tailPlan.add(p);
}
}
/*
- * Now break the constraints into two groups - those whose variables are
- * bound by the predicates in the join graph (required joins) and those
- * having at least one variable bound by an optional join.
+ * Now break the constraints into different groups based on their
+ * variables and when those variables are known to be bound (required
+ * joins) or might be bound (optionals).
*/
for (IConstraint c : constraints) {
boolean allFound = true;
@@ -239,7 +304,7 @@
* we should evaluate it as soon as possible. I.e., before the
* join graph.
*/
- runFirstConstraints.add(c);
+ runFirstConstraints.add(c); // @todo unit test.
continue;
}
while (vitr.hasNext()) {
@@ -342,19 +407,6 @@
}
- /*
- * Assign to instance fields.
- */
- // @todo headPlan
-// this.headPlan = null;
-// this.headPlanConstraints = null;
- this.joinGraphPredicates = joinGraphPredicates
- .toArray(new IPredicate[joinGraphPredicates.size()]);
- this.joinGraphConstraints = joinGraphConstraints
- .toArray(new IConstraint[joinGraphConstraints.size()]);
- this.tailPlan = tailPlan.toArray(new IPredicate[tailPlan.size()]);
- // @todo tailPlanConstraintMap
-
}
-
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java 2011-01-19 21:21:03 UTC (rev 4131)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java 2011-01-19 21:22:01 UTC (rev 4132)
@@ -27,9 +27,21 @@
package com.bigdata.bop.controller;
+import java.util.Arrays;
+import java.util.Iterator;
+
import junit.framework.TestCase2;
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.Var;
+import com.bigdata.bop.IPredicate.Annotations;
+import com.bigdata.bop.ap.Predicate;
+import com.bigdata.bop.constraint.NEConstant;
/**
* Unit tests for {@link PartitionedJoinGroup}.
@@ -84,13 +96,407 @@
}
/**
- * @todo test with headPlan, tailPlan.
- * @todo test association of constraints to optional joins.
+ * A test based loosely on LUBM Q2. There are no RDF specific constructions
+ * used here.
+ */
+ public void test_requiredJoins() {
+
+ final String rdfType = "rdfType";
+ final String graduateStudent = "graduateStudent";
+ final String university = "university";
+ final String department = "department";
+ final String memberOf = "memberOf";
+ final String subOrganizationOf = "subOrganizationOf";
+ final String undergraduateDegreeFrom = "undergraduateDegreeFrom";
+
+ final IPredicate<?>[] preds;
+ final IPredicate<?> p0, p1, p2, p3, p4, p5;
+ final IVariable<?> x = Var.var("x");
+ final IVariable<?> y = Var.var("y");
+ final IVariable<?> z = Var.var("z");
+ {
+
+ // The name space for the SPO relation.
+ final String[] relation = new String[] { "spo" };
+
+ final long timestamp = System.currentTimeMillis();
+
+ int nextId = 0;
+
+ // ?x a ub:GraduateStudent .
+ p0 = new Predicate(new BOp[] { x,
+ new Constant<String>(rdfType),
+ new Constant<String>(graduateStudent) },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?y a ub:University .
+ p1 = new Predicate(new BOp[] { y,
+ new Constant<String>(rdfType),
+ new Constant<String>(university) },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?z a ub:Department .
+ p2 = new Predicate(new BOp[] { z,
+ new Constant<String>(rdfType),
+ new Constant<String>(department) },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?x ub:memberOf ?z .
+ p3 = new Predicate(new BOp[] { x,
+ new Constant<String>(memberOf), z },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?z ub:subOrganizationOf ?y .
+ p4 = new Predicate(new BOp[] { z,
+ new Constant<String>(subOrganizationOf), y },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?x ub:undergraduateDegreeFrom ?y
+ p5 = new Predicate(new BOp[] { x,
+ new Constant<String>(undergraduateDegreeFrom), y },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // the vertices of the join graph (the predicates).
+ preds = new IPredicate[] { p0, p1, p2, p3, p4, p5 };
+ }
+
+ // Test w/o any constraints.
+ {
+
+ final IConstraint[] constraints = new IConstraint[] {
+
+ };
+
+ final PartitionedJoinGroup fixture = new PartitionedJoinGroup(
+ preds, constraints);
+
+ // all variables are bound within the join graph.
+ assertSameIteratorAnyOrder("joinGraphVars", new IVariable[] { x, y,
+ z }, fixture.getJoinGraphVars().iterator());
+
+ // verify all predicates were placed into the join graph.
+ assertSameIteratorAnyOrder("joinGraph", preds, Arrays.asList(
+ fixture.getJoinGraphPredicates()).iterator());
+
+ // there are no constraints.
+ assertEquals("joinGraphConstraints.size", 0, fixture
+ .getJoinGraphConstraints().length);
+
+ // there is no tail plan.
+ assertEquals("tailPlan", new IPredicate[] {}, fixture.getTailPlan());
+
+ }
+
+ // Test w/ constraint(s) on the join graph.
+ {
+
+ final IConstraint c1 = new NEConstant(x,
+ new Constant<String>("Bob"));
+
+ final IConstraint c2 = new NEConstant(y,
+ new Constant<String>("UNCG"));
+
+ final IConstraint[] constraints = new IConstraint[] { c1, c2 };
+
+ final PartitionedJoinGroup fixture = new PartitionedJoinGroup(
+ preds, constraints);
+
+ // all variables are bound within the join graph.
+ assertSameIteratorAnyOrder("joinGraphVars", new IVariable[] { x, y,
+ z }, fixture.getJoinGraphVars().iterator());
+
+ // verify all predicates were placed into the join graph.
+ assertSameIteratorAnyOrder("joinGraph", preds, Arrays.asList(
+ fixture.getJoinGraphPredicates()).iterator());
+
+ // verify all constraints were place on the join graph.
+ assertSameIteratorAnyOrder("joinGraphConstraints", constraints,
+ Arrays.asList(fixture.getJoinGraphConstraints()).iterator());
+
+ // there is no tail plan.
+ assertEquals("tailPlan", new IPredicate[] {}, fixture.getTailPlan());
+
+ }
+
+ }
+
+ /**
+ * A test when there are optional joins involved. In this test, we again
+ * start with LUBM Q2, but the predicates which would bind <code>z</code>
+ * are both marked as optional. This should shift the constraint on [z] into
+ * the tail plan as well.
+ */
+ public void test_withOptionalJoins() {
+
+ final String rdfType = "rdfType";
+ final String graduateStudent = "graduateStudent";
+ final String university = "university";
+ final String department = "department";
+ final String memberOf = "memberOf";
+ final String subOrganizationOf = "subOrganizationOf";
+ final String undergraduateDegreeFrom = "undergraduateDegreeFrom";
+
+ final IPredicate<?>[] preds;
+ final IPredicate<?> p0, p1, p2, p3, p4, p5;
+ final IVariable<?> x = Var.var("x");
+ final IVariable<?> y = Var.var("y");
+ final IVariable<?> z = Var.var("z");
+ {
+
+ // The name space for the SPO relation.
+ final String[] relation = new String[] { "spo" };
+
+ final long timestamp = System.currentTimeMillis();
+
+ int nextId = 0;
+
+ // ?x a ub:GraduateStudent .
+ p0 = new Predicate(new BOp[] { x,
+ new Constant<String>(rdfType),
+ new Constant<String>(graduateStudent) },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?y a ub:University .
+ p1 = new Predicate(new BOp[] { y,
+ new Constant<String>(rdfType),
+ new Constant<String>(university) },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?z a ub:Department . (optional)
+ p2 = new Predicate(new BOp[] { z,
+ new Constant<String>(rdfType),
+ new Constant<String>(department) },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.OPTIONAL, true),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?x ub:memberOf ?z . (optional).
+ p3 = new Predicate(new BOp[] { x,
+ new Constant<String>(memberOf), z },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.OPTIONAL, true),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?z ub:subOrganizationOf ?y . (optional).
+ p4 = new Predicate(new BOp[] { z,
+ new Constant<String>(subOrganizationOf), y },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.OPTIONAL, true),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // ?x ub:undergraduateDegreeFrom ?y
+ p5 = new Predicate(new BOp[] { x,
+ new Constant<String>(undergraduateDegreeFrom), y },//
+ new NV(BOp.Annotations.BOP_ID, nextId++),//
+ new NV(Annotations.TIMESTAMP, timestamp),//
+ new NV(IPredicate.Annotations.RELATION_NAME, relation)//
+ );
+
+ // the vertices of the join graph (the predicates).
+ preds = new IPredicate[] { p0, p1, p2, p3, p4, p5 };
+ }
+
+ // Test w/o any constraints.
+ {
+
+ final IConstraint[] constraints = new IConstraint[] {
+
+ };
+
+ final PartitionedJoinGroup fixture = new PartitionedJoinGroup(
+ preds, constraints);
+
+ // only {x,y} are bound within the join graph.
+ assertSameIteratorAnyOrder("joinGraphVars",
+ new IVariable[] { x, y }, fixture.getJoinGraphVars()
+ .iterator());
+
+ // verify predicates placed into the join graph.
+ assertSameIteratorAnyOrder("joinGraph", new IPredicate[] { p0, p1,
+ p5 }, Arrays.asList(fixture.getJoinGraphPredicates())
+ .iterator());
+
+ // there are no constraints on the join graph predicates.
+ assertEquals("joinGraphConstraints.size", 0, fixture
+ .getJoinGraphConstraints().length);
+
+ // {p2, p3,p4} are in the tail plan.
+ assertEquals("tailPlan", new IPredicate[] { p2, p3, p4 }, fixture
+ .getTailPlan());
+
+ // no constraints were assigned to optional predicate [p2].
+ assertEquals("", 0,
+ fixture.getTailPlanConstraints(p2.getId()).length);
+
+ // no constraints were assigned to optional predicate [p3].
+ assertEquals("", 0,
+ fixture.getTailPlanConstraints(p3.getId()).length);
+
+ // no constraints were assigned to optional predicate [p4].
+ assertEquals("", 0,
+ fixture.getTailPlanConstraints(p4.getId()).length);
+
+ }
+
+ // Test w/ constraint(s) on the join graph.
+ {
+
+ final IConstraint c1 = new NEConstant(x,
+ new Constant<String>("Bob"));
+
+ final IConstraint c2 = new NEConstant(y,
+ new Constant<String>("UNCG"));
+
+ final IConstraint c3 = new NEConstant(z,
+ new Constant<String>("Physics"));
+
+ final IConstraint[] constraints = new IConstraint[] { c1, c2, c3 };
+
+ final PartitionedJoinGroup fixture = new PartitionedJoinGroup(
+ preds, constraints);
+
+ // only {x,y} are bound within the join graph.
+ assertSameIteratorAnyOrder("joinGraphVars",
+ new IVariable[] { x, y }, fixture.getJoinGraphVars()
+ .iterator());
+
+ // verify predicates placed into the join graph.
+ assertSameIteratorAnyOrder("joinGraph", new IPredicate[] { p0, p1,
+ p5 }, Arrays.asList(fixture.getJoinGraphPredicates())
+ .iterator());
+
+ // verify constraints on the join graph.
+ assertSameIteratorAnyOrder("joinGraphConstraints",
+ new IConstraint[] { c1, c2 }, Arrays.asList(
+ fixture.getJoinGraphConstraints()).iterator());
+
+ // {p2,p3,p4} are in the tail plan.
+ assertEquals("tailPlan", new IPredicate[] { p2, p3, p4 }, fixture
+ .getTailPlan());
+
+ // no constraints were assigned to optional predicate [p2].
+ assertEquals("", new IConstraint[] {}, fixture
+ .getTailPlanConstraints(p2.getId()));
+
+ // no constraints were assigned to optional predicate [p3].
+ assertEquals("", new IConstraint[] {}, fixture
+ .getTailPlanConstraints(p3.getId()));
+
+ // the constraint on [z] was assigned to optional predicate [p4].
+ assertEquals("", new IConstraint[] { c3 }, fixture
+ .getTailPlanConstraints(p4.getId()));
+
+ }
+
+ }
+
+ /**
+ * @todo test with headPlan.
+ *
+ * @todo test for runFirst constraints.
+ *
* @todo test logic to attach constraints to non-optional joins based on a
* given join path (not yet written).
*/
public void test_something() {
fail("write tests");
}
-
+
+ /**
+ * Verifies that the iterator visits the specified objects in some arbitrary
+ * ordering and that the iterator is exhausted once all expected objects
+ * have been visited. The implementation uses a selection without
+ * replacement "pattern".
+ */
+ @SuppressWarnings("unchecked")
+ static public void assertSameIteratorAnyOrder(final Object[] expected,
+ final Iterator actual) {
+
+ assertSameIteratorAnyOrder("", expected, actual);
+
+ }
+
+ /**
+ * Verifies that the iterator visits the specified objects in some arbitrary
+ * ordering and that the iterator is exhausted once all expected objects
+ * have been visited. The implementation uses a selection without
+ * replacement "pattern".
+ */
+ @SuppressWarnings("unchecked")
+ static public void assertSameIteratorAnyOrder(final String msg,
+ final Object[] expected, final Iterator actual) {
+
+ // Populate a map that we will use to realize the match and
+ // selection without replacement logic.
+
+ final int nrange = expected.length;
+
+ java.util.Map range = new java.util.HashMap();
+
+ for (int j = 0; j < nrange; j++) {
+
+ range.put(expected[j], expected[j]);
+
+ }
+
+ // Do selection without replacement for the objects visited by
+ // iterator.
+
+ for (int j = 0; j < nrange; j++) {
+
+ if (!actual.hasNext()) {
+
+ fail(msg + ": Index exhausted while expecting more object(s)"
+ + ": index=" + j);
+
+ }
+
+ Object actualObject = actual.next();
+
+ if (range.remove(actualObject) == null) {
+
+ fail("Object not expected" + ": index=" + j + ", object="
+ + actualObject);
+
+ }
+
+ }
+
+ if (actual.hasNext()) {
+
+ fail("Iterator will deliver too many objects.");
+
+ }
+
+ }
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-19 21:21:10
|
Revision: 4131
http://bigdata.svn.sourceforge.net/bigdata/?rev=4131&view=rev
Author: thompsonbry
Date: 2011-01-19 21:21:03 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
Turned off backchain() for the named and default graph expanders since inference is not compatible with quads and those expanders were only ever used in quads mode.
Backchain access path support has not made it into the new query engine yet. I've filed an issue [1] for this and also added some javadoc notes on IAccessPathExpander and RDFJoinNexus concerning
[1] https://sourceforge.net/apps/trac/bigdata/ticket/231
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IAccessPathExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IAccessPathExpander.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IAccessPathExpander.java 2011-01-19 18:34:43 UTC (rev 4130)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IAccessPathExpander.java 2011-01-19 21:21:03 UTC (rev 4131)
@@ -54,19 +54,24 @@
* onto or otherwise override the given {@link IAccessPath}.
*/
IAccessPath<E> getAccessPath(IAccessPath<E> accessPath);
-
+
/**
* Add the backchainer on top of the expander.
*
* @return true if the backchainer should run
+ *
+ * @deprecated Never <code>true</code>. The backchainer is only run for
+ * normal predicates in triples mode at this time. If it is to
+ * be layer, it should be layered as an annotation. See
+ * https://sourceforge.net/apps/trac/bigdata/ticket/231.
*/
boolean backchain();
-
+
/**
* If true, the predicate for this expander will be given priority in the
* join order.
*
- * @return true if the predicate should be run first
+ * @return true if the predicate should be run first
*/
boolean runFirst();
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java 2011-01-19 18:34:43 UTC (rev 4130)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java 2011-01-19 21:21:03 UTC (rev 4131)
@@ -391,6 +391,7 @@
// }
// @todo raise into SPORelation#getAccessPath/3?
+ // @see https://sourceforge.net/apps/trac/bigdata/ticket/231
if(backchain && relation instanceof SPORelation) {
if (expander == null || expander.backchain()) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java 2011-01-19 18:34:43 UTC (rev 4130)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java 2011-01-19 21:21:03 UTC (rev 4131)
@@ -247,7 +247,7 @@
public boolean backchain() {
- return true;
+ return false;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java 2011-01-19 18:34:43 UTC (rev 4130)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java 2011-01-19 21:21:03 UTC (rev 4131)
@@ -180,7 +180,7 @@
public boolean backchain() {
- return true;
+ return false;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-19 18:34:52
|
Revision: 4130
http://bigdata.svn.sourceforge.net/bigdata/?rev=4130&view=rev
Author: thompsonbry
Date: 2011-01-19 18:34:43 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
- Got rid of PipelineJoin.Annotations.OPTIONAL and just test
IPredicate.isOptional() instead of propagating the annotation to the
join.
- Moved CONSTRAINTS back to the PipelineJoin.
- Changed the namespaces of some annotations (IPredicate mainly).
- Modified the various runtime query optimizer test suites to run against small data sets bundled within SVN by default.
- Added the "bar" data set from MRoy and the data generator for that data set.
- Working on a PartitionedJoinGroup utility class which will break down a set of predicates and constraints into a join graph, head plan, and tail plan.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestSubqueryOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBSBMData.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnBarData.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphOnLubm.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestPartitionedJoinGroup.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/resources/data/barData/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/resources/data/barData/barData.trig.gz
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/AbstractJoinGraphTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/GenerateBarData.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/joinGraph/TestJoinGraphWithRDF.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -31,8 +31,6 @@
import java.util.List;
import java.util.Map;
-import com.bigdata.bop.IPredicate.Annotations;
-
import cutthecrap.utils.striterators.IPropertySet;
/**
@@ -182,6 +180,13 @@
*/
BOpEvaluationContext getEvaluationContext();
+ /**
+ * Return <code>true</code> iff this operator is a controller.
+ *
+ * @see Annotations#CONTROLLER
+ */
+ boolean isController();
+
// /**
// * Return <code>true</code> iff this operator is an access path which writes
// * on the database.
@@ -248,6 +253,8 @@
/**
* This annotation determines where an operator will be evaluated
* (default {@value #DEFAULT_EVALUATION_CONTEXT}).
+ *
+ * @see BOpEvaluationContext
*/
String EVALUATION_CONTEXT = BOp.class.getName() + ".evaluationContext";
@@ -260,6 +267,8 @@
* arguments. Thus control operators mark a boundary in pipelined
* evaluation. Some examples of control operators include UNION, STEPS,
* and STAR (aka transitive closure).
+ *
+ * @see BOp#isController()
*/
String CONTROLLER = BOp.class.getName()+".controller";
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -424,6 +424,12 @@
}
+ public boolean isController() {
+
+ return getProperty(Annotations.CONTROLLER, false);
+
+ }
+
/**
* Set an annotation.
* <p>
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -34,7 +34,6 @@
import com.bigdata.bop.ap.filter.BOpFilterBase;
import com.bigdata.bop.ap.filter.BOpTupleFilter;
import com.bigdata.bop.ap.filter.DistinctFilter;
-import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleCursor;
@@ -85,7 +84,7 @@
* @see https://sourceforge.net/apps/trac/bigdata/ticket/180 (Migrate
* the RDFS inference and truth maintenance logic to BOPs)
*/
- String RELATION_NAME = "relationName";
+ String RELATION_NAME = IPredicate.class.getName() + ".relationName";
// /**
// * The {@link IKeyOrder} which will be used to read on the relation.
@@ -98,15 +97,9 @@
// String KEY_ORDER = "keyOrder";
/**
- * <code>true</code> iff the predicate is optional (the right operand of
- * a left join).
- *
- * @deprecated This flag is being moved to the join operator (or should
- * it stay with the predicate so we can work with join
- * graphs more easily, but note that join graphs never
- * contain optional joins).
+ * <code>true</code> iff the predicate has SPARQL optional semantics.
*/
- String OPTIONAL = "optional";
+ String OPTIONAL = IPredicate.class.getName() + ".optional";
// /**
// * Constraints on the elements read from the relation.
@@ -146,7 +139,7 @@
*
* @see IRangeQuery#rangeIterator(byte[], byte[], int, int, IFilter)
*/
- String INDEX_LOCAL_FILTER = "indexLocalFilter";
+ String INDEX_LOCAL_FILTER = IPredicate.class.getName() + ".indexLocalFilter";
/**
* An optional {@link BOpFilterBase} to be applied to the elements of
@@ -163,7 +156,7 @@
* one another. You can chain {@link FilterBase} filters together as
* well.
*/
- String ACCESS_PATH_FILTER = "accessPathFilter";
+ String ACCESS_PATH_FILTER = IPredicate.class.getName() + ".accessPathFilter";
/**
* Access path expander pattern. This allows you to wrap or replace the
@@ -192,13 +185,13 @@
*
* @see IAccessPathExpander
*/
- String ACCESS_PATH_EXPANDER = "accessPathExpander";
+ String ACCESS_PATH_EXPANDER = IPredicate.class.getName() + ".accessPathExpander";
/**
* The partition identifier -or- <code>-1</code> if the predicate does
* not address a specific shard.
*/
- String PARTITION_ID = "partitionId";
+ String PARTITION_ID = IPredicate.class.getName() + ".partitionId";
int DEFAULT_PARTITION_ID = -1;
@@ -240,7 +233,7 @@
*
* @see BOpEvaluationContext
*/
- String REMOTE_ACCESS_PATH = "remoteAccessPath";
+ String REMOTE_ACCESS_PATH = IPredicate.class.getName() + ".remoteAccessPath";
boolean DEFAULT_REMOTE_ACCESS_PATH = true;
@@ -307,9 +300,9 @@
* Operators which read or write on the database must declare the
* {@link Annotations#TIMESTAMP} associated with that operation.
*
- * @see Annotations#TIMESTAMP
+ * @see #TIMESTAMP
*/
- String MUTATION = BOp.class.getName() + ".mutation";
+ String MUTATION = IPredicate.class.getName() + ".mutation";
boolean DEFAULT_MUTATION = false;
@@ -317,17 +310,15 @@
* The timestamp (or transaction identifier) used by this operator if it
* reads or writes on the database (no default).
*
- * @see com.bigdata.bop.IPredicate.Annotations#MUTATION
- *
- * @todo Move to {@link IPredicate}?
+ * @see #MUTATION
*/
- String TIMESTAMP = BOp.class.getName() + ".timestamp";
+ String TIMESTAMP = IPredicate.class.getName() + ".timestamp";
- /**
- * An optional {@link IConstraint}[] which places restrictions on the
- * legal patterns in the variable bindings.
- */
- String CONSTRAINTS = PipelineJoin.class.getName() + ".constraints";
+// /**
+// * An optional {@link IConstraint}[] which places restrictions on the
+// * legal patterns in the variable bindings.
+// */
+// String CONSTRAINTS = PipelineJoin.class.getName() + ".constraints";
}
@@ -404,14 +395,9 @@
* For mutation, some {@link IRelation}s may require that all variables
* appearing in the head are bound. This and similar constraints can be
* enforced using {@link IConstraint}s on the {@link IRule}.
- * <p>
- * More control over the behavior of optionals may be gained through the use
- * of an {@link IAccessPathExpander} pattern.
*
* @return <code>true</code> iff this predicate is optional when evaluating
* a JOIN.
- *
- * @deprecated By {@link PipelineJoin.Annotations#OPTIONAL}
*/
public boolean isOptional();
@@ -441,21 +427,21 @@
// */
// public IElementFilter<E> getConstraint();
- /**
- * Return the optional {@link IConstraint}[] to be applied by a join which
- * evaluates this {@link IPredicate}.
- * <p>
- * Note: The {@link Annotations#CONSTRAINTS} are annotated on the
- * {@link IPredicate} rather than the join operators so they may be used
- * with join graphs, which are expressed solely as an unordered set of
- * {@link IPredicate}s. Using join graphs, we are able to do nifty things
- * such as runtime query optimization which would not be possible if the
- * annotations were decorating the joins since we would be unable to
- * dynamically generate the join operators with the necessary annotations.
- *
- * @see Annotations#CONSTRAINTS
- */
- public IConstraint[] constraints();
+// /**
+// * Return the optional {@link IConstraint}[] to be applied by a join which
+// * evaluates this {@link IPredicate}.
+// * <p>
+// * Note: The {@link Annotations#CONSTRAINTS} are annotated on the
+// * {@link IPredicate} rather than the join operators so they may be used
+// * with join graphs, which are expressed solely as an unordered set of
+// * {@link IPredicate}s. Using join graphs, we are able to do nifty things
+// * such as runtime query optimization which would not be possible if the
+// * annotations were decorating the joins since we would be unable to
+// * dynamically generate the join operators with the necessary annotations.
+// *
+// * @see Annotations#CONSTRAINTS
+// */
+// public IConstraint[] constraints();
/**
* Return the optional filter to be evaluated local to the index.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -91,19 +91,19 @@
boolean DEFAULT_SHARED_STATE = false;
- /**
- * Annotation used to mark a set of (non-optional) joins which may be
- * freely reordered by the query optimizer in order to minimize the
- * amount of work required to compute the solutions.
- * <p>
- * Note: Optional joins MAY NOT appear within a join graph. Optional
- * joins SHOULD be evaluated as part of the "tail plan" following the
- * join graph, but before operations such as SORT, DISTINCT, etc. When
- * the query plan includes {@link #CONDITIONAL_GROUP}s, those groups
- * include a leading {@link #JOIN_GRAPH} (required joins) followed by
- * zero or more optional joins.
- */
- String JOIN_GRAPH = PipelineOp.class.getName() + ".joinGraph";
+// /**
+// * Annotation used to mark a set of (non-optional) joins which may be
+// * freely reordered by the query optimizer in order to minimize the
+// * amount of work required to compute the solutions.
+// * <p>
+// * Note: Optional joins MAY NOT appear within a join graph. Optional
+// * joins SHOULD be evaluated as part of the "tail plan" following the
+// * join graph, but before operations such as SORT, DISTINCT, etc. When
+// * the query plan includes {@link #CONDITIONAL_GROUP}s, those groups
+// * include a leading {@link #JOIN_GRAPH} (required joins) followed by
+// * zero or more optional joins.
+// */
+// String JOIN_GRAPH = PipelineOp.class.getName() + ".joinGraph";
// /**
// * Annotation used to mark a set of operators belonging to a conditional
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -245,12 +245,12 @@
//
// }
- public IConstraint[] constraints() {
+// public IConstraint[] constraints() {
+//
+// return getProperty(IPredicate.Annotations.CONSTRAINTS, null/* defaultValue */);
+//
+// }
- return getProperty(IPredicate.Annotations.CONSTRAINTS, null/* defaultValue */);
-
- }
-
final public IFilter getIndexLocalFilter() {
return (IFilter) getProperty(Annotations.INDEX_LOCAL_FILTER);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/JoinGraph.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -47,13 +47,13 @@
import org.apache.log4j.Logger;
import com.bigdata.bop.BOp;
-import com.bigdata.bop.BOpBase;
import com.bigdata.bop.BOpContext;
import com.bigdata.bop.BOpContextBase;
import com.bigdata.bop.BOpEvaluationContext;
import com.bigdata.bop.BOpIdFactory;
import com.bigdata.bop.BOpUtility;
import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IElement;
import com.bigdata.bop.IPredicate;
import com.bigdata.bop.IVariable;
@@ -80,10 +80,10 @@
* A join graph with annotations for estimated cardinality and other details in
* support of runtime query optimization. A join graph is a collection of
* relations and joins which connect those relations. This boils down to a
- * collection of {@link IPredicate}s (selects on relations) and shared variables
- * (which identify joins). Operators other than standard joins (including
- * optional joins, sort, order by, etc.) must be handled downstream from the
- * join graph in a "tail plan".
+ * collection of {@link IPredicate}s (selects on relations), shared variables
+ * (which identify joins), and {@link IConstraint}s (which limit solutions).
+ * Operators other than standard joins (including optional joins, sort, order
+ * by, etc.) must be handled downstream from the join graph in a "tail plan".
*
* @see http://arxiv.org/PS_cache/arxiv/pdf/0810/0810.4809v1.pdf, XQuery Join
* Graph Isolation.
@@ -197,10 +197,10 @@
* approach should be able to handle queries without perfect / covering
* automatically. Then experiment with carrying fewer statement indices
* for quads.
- *
+ *
* @todo Unit test when there are no solutions to the query. In this case there
* will be no paths identified by the optimizer and the final path length
- * becomes zero.
+ * becomes zero.
*/
public class JoinGraph extends PipelineOp {
@@ -214,17 +214,24 @@
*/
public interface Annotations extends PipelineOp.Annotations {
- /**
- * The vertices of the join graph expressed an an {@link IPredicate}[].
- */
- String VERTICES = JoinGraph.class.getName() + ".vertices";
+ /**
+ * The vertices of the join graph, expressed an an {@link IPredicate}[]
+ * (required).
+ */
+ String VERTICES = JoinGraph.class.getName() + ".vertices";
- /**
- * The initial limit for cutoff sampling (default
- * {@value #DEFAULT_LIMIT}).
- */
- String LIMIT = JoinGraph.class.getName() + ".limit";
+ /**
+ * The constraints on the join graph, expressed an an
+ * {@link IConstraint}[] (optional, defaults to no constraints).
+ */
+ String CONSTRAINTS = JoinGraph.class.getName() + ".constraints";
+ /**
+ * The initial limit for cutoff sampling (default
+ * {@value #DEFAULT_LIMIT}).
+ */
+ String LIMIT = JoinGraph.class.getName() + ".limit";
+
int DEFAULT_LIMIT = 100;
/**
@@ -240,12 +247,21 @@
/**
* @see Annotations#VERTICES
*/
- public IPredicate[] getVertices() {
+ public IPredicate<?>[] getVertices() {
return (IPredicate[]) getRequiredProperty(Annotations.VERTICES);
}
+ /**
+ * @see Annotations#CONSTRAINTS
+ */
+ public IConstraint[] getConstraints() {
+
+ return (IConstraint[]) getProperty(Annotations.CONSTRAINTS, null/* none */);
+
+ }
+
/**
* @see Annotations#LIMIT
*/
@@ -264,25 +280,42 @@
}
- public JoinGraph(final NV... anns) {
+ public JoinGraph(final BOp[] args, final NV... anns) {
- this(BOpBase.NOARGS, NV.asMap(anns));
+ this(args, NV.asMap(anns));
}
- public JoinGraph(final BOp[] args, final Map<String, Object> anns) {
+ public JoinGraph(final BOp[] args, final Map<String, Object> anns) {
- super(args, anns);
+ super(args, anns);
- switch (getEvaluationContext()) {
- case CONTROLLER:
- break;
- default:
- throw new UnsupportedOperationException(
- Annotations.EVALUATION_CONTEXT + "="
- + getEvaluationContext());
- }
+ // required property.
+ final IPredicate<?>[] vertices = (IPredicate[]) getProperty(Annotations.VERTICES);
+ if (vertices == null)
+ throw new IllegalArgumentException(Annotations.VERTICES);
+
+ if (vertices.length == 0)
+ throw new IllegalArgumentException(Annotations.VERTICES);
+
+ if (getLimit() <= 0)
+ throw new IllegalArgumentException(Annotations.LIMIT);
+
+ if (getNEdges() <= 0)
+ throw new IllegalArgumentException(Annotations.NEDGES);
+
+ if (!isController())
+ throw new IllegalArgumentException();
+
+ switch (getEvaluationContext()) {
+ case CONTROLLER:
+ break;
+ default:
+ throw new IllegalArgumentException(Annotations.EVALUATION_CONTEXT
+ + "=" + getEvaluationContext());
+ }
+
}
public FutureTask<Void> eval(final BOpContext<IBindingSet> context) {
@@ -383,7 +416,7 @@
/**
* The most recently taken sample of the {@link Vertex}.
*/
- VertexSample sample = null;
+ transient VertexSample sample = null;
Vertex(final IPredicate<?> pred) {
@@ -768,7 +801,7 @@
* the cutoff sample of a join path having this edge except for the
* degenerate case where the edge is the first edge in the join path.
*/
- public EdgeSample sample = null;
+ transient EdgeSample sample = null;
public Edge(final Vertex v1, final Vertex v2,
final Set<IVariable<?>> shared) {
@@ -1560,6 +1593,10 @@
* FIXME CONSTRAINT ORDERING : It is illegal to add a vertex to the
* path if any variable appearing in its CONSTRAINTS would not be
* bound.
+ *
+ * FIXME CONSTRAINT ORDERING : Rather than constraints imposing an
+ * ordering on joins, constraints need to be attached dynamically to
+ * the first join for which their variables are known to be bound.
*/
final EdgeSample edgeSample = e.estimateCardinality(queryEngine,
@@ -1787,7 +1824,31 @@
return sb.toString();
}
- public JGraph(final IPredicate[] v) {
+ /**
+ *
+ * @param v
+ * The vertices of the join graph. These are
+ * {@link IPredicate}s associated with required joins.
+ * @param constraints
+ * The constraints of the join graph (optional). Since all
+ * joins in the join graph are required, constraints are
+ * dynamically attached to the first join in which all of
+ * their variables are bound.
+ *
+ * @throws IllegalArgumentException
+ * if the vertices is <code>null</code>.
+ * @throws IllegalArgumentException
+ * if the vertices is an empty array.
+ * @throws IllegalArgumentException
+ * if any element of the vertices is <code>null</code>.
+ * @throws IllegalArgumentException
+ * if any constraint uses a variable which is never bound by
+ * the given predicates.
+ *
+ * @todo unit test for a constraint using a variable which is never
+ * bound.
+ */
+ public JGraph(final IPredicate[] v, final IConstraint[] constraints) {
if (v == null)
throw new IllegalArgumentException();
@@ -1797,8 +1858,11 @@
V = new Vertex[v.length];
- for (int i = 0; i < v.length; i++) {
+ for (int i = 0; i < v.length; i++) {
+ if (v[i] == null)
+ throw new IllegalArgumentException();
+
V[i] = new Vertex(v[i]);
}
@@ -1833,9 +1897,12 @@
// The set of identified edges for vertices which share vars.
final List<Edge> tmp = new LinkedList<Edge>();
-
+
+ // The set of vertices which share variables.
+ final Set<Vertex> sharedEdgeVertices = new LinkedHashSet<Vertex>();
+
// The set of vertices which do not share variables.
- final List<Vertex> unshared = new LinkedList<Vertex>();
+ final List<Vertex> unsharedEdgeVertices = new LinkedList<Vertex>();
for (int i = 0; i < v.length; i++) {
@@ -1865,31 +1932,44 @@
* or not the join path would bind the variable(s)
* appearing in the CONSTRAINT.
*/
+
+ if (log.isDebugEnabled())
+ log.debug("vertices shared variable(s): vars="
+ + shared + ", v1=" + p1 + ", v2=" + p2);
+
tmp.add(new Edge(V[i], V[j], shared));
+ sharedEdgeVertices.add(V[i]);
+
+ sharedEdgeVertices.add(V[j]);
+
nmatched++;
}
}
- if (nmatched == 0) {
+ if (nmatched == 0 && !sharedEdgeVertices.contains(V[i])) {
/*
* The source vertex does not share any variables.
*/
-
- unshared.add(V[i]);
+ log
+ .warn("Vertex does not share any variables: "
+ + V[i]);
+
+ unsharedEdgeVertices.add(V[i]);
+
}
}
E = tmp.toArray(new Edge[0]);
- this.unshared = unshared.toArray(new Vertex[0]);
+ this.unshared = unsharedEdgeVertices.toArray(new Vertex[0]);
- if(!unshared.isEmpty()) {
+ if(!unsharedEdgeVertices.isEmpty()) {
/*
* FIXME This needs to be supported. We should explore and
@@ -1902,8 +1982,9 @@
* the evaluation).
*/
- throw new UnsupportedOperationException(
- "Some predicates do not share any variables with other predicates.");
+ throw new UnsupportedOperationException(
+ "Some predicates do not share any variables with other predicates: unshared="
+ + unsharedEdgeVertices);
}
@@ -2794,7 +2875,7 @@
private final BOpContext<IBindingSet> context;
- private final JGraph g;
+// private final JGraph g;
private int limit;
@@ -2813,22 +2894,20 @@
// The initial number of edges (1 step paths) to explore.
nedges = getNEdges();
- if (limit <= 0)
- throw new IllegalArgumentException();
+// if (limit <= 0)
+// throw new IllegalArgumentException();
+//
+// if (nedges <= 0)
+// throw new IllegalArgumentException();
- if (nedges <= 0)
- throw new IllegalArgumentException();
+// g = new JGraph(getVertices(), getConstraints());
- final IPredicate[] v = getVertices();
-
- g = new JGraph(v);
-
}
public Void call() throws Exception {
// Create the join graph.
- final JGraph g = new JGraph(getVertices());
+ final JGraph g = new JGraph(getVertices(), getConstraints());
// Find the best join path.
final Path p = g.runtimeOptimizer(context.getRunningQuery()
@@ -2839,7 +2918,7 @@
// Generate the query from the join path.
final PipelineOp queryOp = JoinGraph.getQuery(idFactory, p
- .getPredicates());
+ .getPredicates(), getConstraints());
// Run the query, blocking until it is done.
JoinGraph.runSubquery(context, queryOp);
@@ -2850,27 +2929,27 @@
} // class JoinGraphTask
- /**
- * Places vertices into order by the {@link BOp#getId()} associated with
- * their {@link IPredicate}.
- */
- private static class BOpIdComparator implements Comparator<Vertex> {
+// /**
+// * Places vertices into order by the {@link BOp#getId()} associated with
+// * their {@link IPredicate}.
+// */
+// private static class BOpIdComparator implements Comparator<Vertex> {
+//
+// private static final transient Comparator<Vertex> INSTANCE = new BOpIdComparator();
+//
+//// @Override
+// public int compare(final Vertex o1, final Vertex o2) {
+// final int id1 = o1.pred.getId();
+// final int id2 = o2.pred.getId();
+// if (id1 < id2)
+// return -1;
+// if (id2 > id1)
+// return 1;
+// return 0;
+// }
+//
+// }
- private static final transient Comparator<Vertex> INSTANCE = new BOpIdComparator();
-
-// @Override
- public int compare(final Vertex o1, final Vertex o2) {
- final int id1 = o1.pred.getId();
- final int id2 = o2.pred.getId();
- if (id1 < id2)
- return -1;
- if (id2 > id1)
- return 1;
- return 0;
- }
-
- }
-
/**
* Places edges into order by ascending estimated cardinality. Edges which
* are not weighted are ordered to the end.
@@ -2915,18 +2994,27 @@
* we are not yet handling anything except standard joins in the runtime
* optimizer.
*/
-
- /**
- * Generate a query plan from an ordered collection of predicates.
- *
- * @param p
- * The join path.
- *
- * @return The query plan.
- */
- static public PipelineOp getQuery(final BOpIdFactory idFactory,
- final IPredicate[] preds) {
+ /**
+ * Generate a query plan from an ordered collection of predicates.
+ *
+ * @param p
+ * The join path.
+ *
+ * @return The query plan.
+ *
+ * FIXME Verify that constraints are attached correctly to the
+ * returned query.
+ */
+ static public PipelineOp getQuery(final BOpIdFactory idFactory,
+ final IPredicate[] preds, final IConstraint[] constraints) {
+
+ if (constraints != null && constraints.length != 0) {
+ // FIXME Constraints must be attached to joins.
+ throw new UnsupportedOperationException(
+ "Constraints must be attached to joins!");
+ }
+
final PipelineJoin[] joins = new PipelineJoin[preds.length];
// final PipelineOp startOp = new StartOp(new BOp[] {},
@@ -3191,7 +3279,7 @@
return sharedVars;
}
-
+
/**
* Exception thrown when the join graph does not have any solutions in the
* data (running the query does not produce any results).
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -0,0 +1,360 @@
+package com.bigdata.bop.controller;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.bigdata.bop.BOpUtility;
+import com.bigdata.bop.IConstraint;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.controller.JoinGraph.NoSolutionsException;
+
+/**
+ * Class accepts a join group and partitions it into a join graph and a tail
+ * plan.
+ * <p>
+ * A join group consists of an ordered collection of {@link IPredicate}s and an
+ * unordered collection of {@link IConstraint}s. {@link IPredicate} representing
+ * non-optional joins are extracted into a {@link JoinGraph} along with any
+ * {@link IConstraint}s whose variables are guaranteed to be bound by the
+ * implied joins.
+ * <p>
+ * The remainder of the {@link IPredicate}s and {@link IConstraint}s form a
+ * "tail plan". {@link IConstraint}s in the tail plan are attached to the last
+ * {@link IPredicate} at which their variable(s) MIGHT have become bound.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ *
+ * @todo However, how do we manage when there are things like conditional
+ * routing operators?
+ *
+ * @todo The order of the {@link IPredicate}s in the tail plan is currently
+ * unchanged from their given order (optional joins without constraints
+ * can not reduce the selectivity of the query). However, it could be
+ * worthwhile to run optionals with constraints before those without
+ * constraints since the constraints can reduce the selectivity of the
+ * query. If we do this, then we need to reorder the optionals based on
+ * the partial order imposed what variables they MIGHT bind (which are not
+ * bound by the join graph).
+ *
+ * @todo Things like LET can also bind variables. So can a subquery. Analysis of
+ * those will tell us whether the variable will definitely or
+ * conditionally become bound (I am assuming that a LET can conditionally
+ * leave a variable unbound).
+ *
+ * @todo runFirst flag on the expander (for free text search). this should be an
+ * annotation. this can be a headPlan. [There can be constraints which are
+ * evaluated against the head plan. They need to get attached to the joins
+ * generated for the head plan. MikeP writes: There is a free text search
+ * access path that replaces the actual access path for the predicate,
+ * which is meaningless in an of itself because the P is magical.]
+ *
+ * @todo inline APs and get rid of DataSetJoin. Rewrite NG and DG first.
+ *
+ * @todo write a method which returns the set of constraints which should be run
+ * for the last predicate in a given join path (a join path is just an
+ * ordered array of predicates).
+ */
+public class PartitionedJoinGroup {
+
+// private final IPredicate<?>[] headPlan;
+//
+// private final IConstraint[] headPlanConstraints;
+
+ private final IPredicate<?>[] joinGraphPredicates;
+
+ private final IConstraint[] joinGraphConstraints;
+
+ private final IPredicate<?>[] tailPlan;
+
+// private final IConstraint[] tailPlanConstraints;
+
+
+ /**
+ * The {@link IPredicate}s in the join graph (required joins).
+ */
+ public IPredicate<?>[] getJoinGraphPredicates() {
+ return joinGraphPredicates;
+ }
+
+ /**
+ * The {@link IConstraint}s to be applied to the {@link IPredicate}s in the
+ * join graph. Each {@link IConstraint} should be applied as soon as all of
+ * its variable(s) are known to be bound. The constraints are not attached
+ * to the {@link IPredicate}s in the join graph because the evaluation order
+ * of those {@link IPredicate}s is not yet known (it will be determined by a
+ * query optimizer when it decides on an evaluation order for those joins).
+ */
+ public IConstraint[] getJoinGraphConstraints() {
+ return joinGraphConstraints;
+ }
+
+ /**
+ * The {@link IPredicate}s representing optional joins. Any
+ * {@link IConstraint}s having variable(s) NOT bound by the required joins
+ * will already have been attached to the last {@link IPredicate} in the
+ * tail plan in which their variable(S) MIGHT have been bound.
+ */
+ public IPredicate<?>[] getTailPlan() {
+ return tailPlan;
+ }
+
+// private PartitionedJoinGroup(//
+// final IPredicate<?>[] headPlan,//
+// final IConstraint[] headPlanConstraints,//
+// final IPredicate<?>[] joinGraphPredicates,//
+// final IConstraint[] joinGraphConstraints,//
+// final IPredicate<?>[] tailPlan,//
+// final IConstraint[] tailPlanConstraints//
+// ) {
+// this.headPlan = headPlan;
+// this.headPlanConstraints = headPlanConstraints;
+// this.joinGraphPredicates = joinGraphPredicates;
+// this.joinGraphConstraints = joinGraphConstraints;
+// this.tailPlan = tailPlan;
+// this.tailPlanConstraints = tailPlanConstraints;
+// }
+
+ /**
+ * Analyze a set of {@link IPredicate}s representing optional and
+ * non-optional joins and a collection of {@link IConstraint}s, partitioning
+ * them into a join graph and a tail plan.
+ *
+ * @param sourcePreds
+ * The predicates.
+ * @param constraints
+ * The constraints.
+ *
+ * @return A data structure representing both the join graph and the tail
+ * plan.
+ *
+ * @throws IllegalArgumentException
+ * if the source predicates array is <code>null</code>.
+ * @throws IllegalArgumentException
+ * if the source predicates array is empty.
+ * @throws IllegalArgumentException
+ * if any element of the source predicates array is
+ * <code>null</code>.
+ */
+ public PartitionedJoinGroup(//
+ final IPredicate<?>[] sourcePreds,//
+ IConstraint[] constraints) {
+
+ if (sourcePreds == null)
+ throw new IllegalArgumentException();
+ if (sourcePreds.length == 0)
+ throw new IllegalArgumentException();
+
+ if (constraints == null) {
+ // replace with an empty array.
+ constraints = new IConstraint[0];
+ }
+
+ /*
+ * An unordered list of constraints which do not involve ANY variables.
+ * These constraints should be run first, before the join graph.
+ *
+ * @todo add to the class instance fields.
+ */
+ final List<IConstraint> runFirstConstraints = new LinkedList<IConstraint>();
+
+// final List<IPredicate<?>> headPlan = new LinkedList<IPredicate<?>>();
+//
+// final List<IConstraint> headPlanConstraints = new LinkedList<IConstraint>();
+
+ /*
+ * The non-optional predicates.
+ */
+ final List<IPredicate<?>> joinGraphPredicates = new LinkedList<IPredicate<?>>();
+
+ /*
+ * The set of variables bound by the non-optional predicates.
+ */
+ final Set<IVariable<?>> joinGraphVars = new LinkedHashSet<IVariable<?>>();
+
+ /*
+ * An unordered list of those constraints whose variables are known to
+ * be bound by the non-optional predicates.
+ */
+ final List<IConstraint> joinGraphConstraints = new LinkedList<IConstraint>();
+
+ /*
+ * The predicates representing the optional joins.
+ */
+ final List<IPredicate<?>> tailPlan = new LinkedList<IPredicate<?>>();
+
+ /*
+ * An unordered list of those constraints containing at least one
+ * variable known to NOT be bound by the non-optional predicates.
+ */
+ final List<IConstraint> tailPlanConstraints = new LinkedList<IConstraint>();
+
+ /*
+ * Map indicating which constraints are run for which predicate in the
+ * tail plan. The keys are the bopIds of the predicates in the tail
+ * plan. The values are the sets of constraints to run for that tail.
+ */
+ final Map<Integer/* predId */, List<IConstraint>> tailPlanConstraintMap = new LinkedHashMap<Integer, List<IConstraint>>();
+
+ /*
+ * First identify the predicates which correspond to non-optional joins.
+ * All other pipeline operators are inserted into the tail plan in the
+ * order in which they are given.
+ */
+ for (IPredicate<?> p : sourcePreds) {
+ if (p == null)
+ throw new IllegalArgumentException();
+ if (!p.isOptional()) {
+ // non-optional predicate.
+ joinGraphPredicates.add(p);
+ // variables which will be bound by the join graph.
+ final Iterator<IVariable<?>> vitr = BOpUtility
+ .getArgumentVariables(p);
+ while (vitr.hasNext()) {
+ joinGraphVars.add(vitr.next());
+ }
+ } else {
+ // an optional predicate
+ tailPlan.add(p);
+ }
+ }
+
+ /*
+ * Now break the constraints into two groups - those whose variables are
+ * bound by the predicates in the join graph (required joins) and those
+ * having at least one variable bound by an optional join.
+ */
+ for (IConstraint c : constraints) {
+ boolean allFound = true;
+ final Iterator<IVariable<?>> vitr = BOpUtility
+ .getSpannedVariables(c);
+ if (!vitr.hasNext()) {
+ /*
+ * This is a constraint which does not involve any variable so
+ * we should evaluate it as soon as possible. I.e., before the
+ * join graph.
+ */
+ runFirstConstraints.add(c);
+ continue;
+ }
+ while (vitr.hasNext()) {
+ final IVariable<?> var = vitr.next();
+ if (!joinGraphVars.contains(var)) {
+ /*
+ * This constraint will be evaluated against the tail plan.
+ */
+ allFound = false;
+ tailPlanConstraints.add(c);
+ break;
+ }
+ }
+ if (allFound) {
+ /*
+ * This constraint will be evaluated by the join graph for the
+ * first join in in which all of the variables used by the
+ * constraint are known to be bound.
+ */
+ joinGraphConstraints.add(c);
+ }
+ }
+
+ /*
+ * If a variable is not bound by a required predicate, then we attach
+ * any constraint using that variable to the last optional predicate in
+ * which that variable MIGHT become bound.
+ */
+ {
+ /*
+ * Populate a map from each variable not bound in the join graph to
+ * the last index in the tail plan at which it MIGHT become bound.
+ */
+ final Map<IVariable<?>, Integer/* lastIndexOf */> lastIndexOf = new LinkedHashMap<IVariable<?>, Integer>();
+ int indexOf = 0;
+ for (IPredicate<?> p : tailPlan) {
+ final Iterator<IVariable<?>> vitr = BOpUtility
+ .getArgumentVariables(p);
+ while (vitr.hasNext()) {
+ final IVariable<?> var = vitr.next();
+ lastIndexOf.put(var, Integer.valueOf(indexOf));
+ }
+ indexOf++;
+ }
+ /*
+ * For each constraint using at least one variable NOT bound by the
+ * join graph, find the maximum value of lastIndexOf for the
+ * variable(s) in that constraint. That is the index of the operator
+ * in the tail plan to which the constraint should be attached.
+ */
+ for (IConstraint c : tailPlanConstraints) {
+ final Iterator<IVariable<?>> vitr = BOpUtility
+ .getSpannedVariables(c);
+ Integer maxLastIndexOf = null;
+ while (vitr.hasNext()) {
+ final IVariable<?> var = vitr.next();
+ if (joinGraphVars.contains(var)) {
+ // This variable is bound by the join graph.
+ continue;
+ }
+ final Integer tmp = lastIndexOf.get(var);
+ if (tmp == null) {
+ // This variable is never bound by the query.
+ throw new NoSolutionsException(
+ "Variable is never bound: " + var);
+ }
+ if (maxLastIndexOf == null
+ || tmp.intValue() > maxLastIndexOf.intValue()) {
+ maxLastIndexOf = tmp;
+ }
+ } // next variable.
+ if (maxLastIndexOf == null) {
+ // A logic error.
+ throw new AssertionError("maxLastIndexOf is undefined: "
+ + c);
+ }
+ /*
+ * Add the constraint to the last predicate at which any of its
+ * variables MIGHT have become bound.
+ */
+ {
+ /*
+ * The bopId for the predicate in the tail plan for which,
+ * when that predicate is evaluated, we will run this
+ * constraint.
+ */
+ final int predId = tailPlan.get(maxLastIndexOf).getId();
+ /*
+ * The constraint(s) (if any) already associated with that
+ * predicate.
+ */
+ List<IConstraint> tmp = tailPlanConstraintMap.get(predId);
+ if (tmp == null) {
+ tmp = new LinkedList<IConstraint>();
+ tailPlanConstraintMap.put(predId, tmp);
+ }
+ tmp.add(c);
+ }
+ } // next tail plan constraint.
+
+ }
+
+ /*
+ * Assign to instance fields.
+ */
+ // @todo headPlan
+// this.headPlan = null;
+// this.headPlanConstraints = null;
+ this.joinGraphPredicates = joinGraphPredicates
+ .toArray(new IPredicate[joinGraphPredicates.size()]);
+ this.joinGraphConstraints = joinGraphConstraints
+ .toArray(new IConstraint[joinGraphConstraints.size()]);
+ this.tailPlan = tailPlan.toArray(new IPredicate[tailPlan.size()]);
+ // @todo tailPlanConstraintMap
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/PartitionedJoinGroup.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -129,18 +129,27 @@
*/
String SELECT = PipelineJoin.class.getName() + ".select";
- /**
- * Marks the join as "optional" in the SPARQL sense. Binding sets which
- * fail the join will be routed to the alternative sink as specified by
- * either {@link PipelineOp.Annotations#ALT_SINK_REF} or
- * {@link PipelineOp.Annotations#ALT_SINK_GROUP}.
- *
- * @see #DEFAULT_OPTIONAL
- */
- String OPTIONAL = PipelineJoin.class.getName() + ".optional";
+// /**
+// * Marks the join as "optional" in the SPARQL sense. Binding sets which
+// * fail the join will be routed to the alternative sink as specified by
+// * either {@link PipelineOp.Annotations#ALT_SINK_REF} or
+// * {@link PipelineOp.Annotations#ALT_SINK_GROUP}.
+// *
+// * @see #DEFAULT_OPTIONAL
+// *
+// * @deprecated We should just inspect
+// * {@link IPredicate.Annotations#OPTIONAL}.
+// */
+// String OPTIONAL = PipelineJoin.class.getName() + ".optional";
+//
+// boolean DEFAULT_OPTIONAL = false;
- boolean DEFAULT_OPTIONAL = false;
-
+ /**
+ * An {@link IConstraint}[] which places restrictions on the legal
+ * patterns in the variable bindings (optional).
+ */
+ String CONSTRAINTS = PipelineJoin.class.getName() + ".constraints";
+
/**
* The maximum parallelism with which the pipeline will consume the
* source {@link IBindingSet}[] chunk.
@@ -435,16 +444,30 @@
}
- /**
- * @see Annotations#OPTIONAL
- */
+ /**
+ * Return the value of {@link IPredicate#isOptional()} for the
+ * {@link IPredicate} associated with this join.
+ *
+ * @see IPredicate.Annotations#OPTIONAL
+ */
public boolean isOptional() {
- return getProperty(Annotations.OPTIONAL, Annotations.DEFAULT_OPTIONAL);
+// return getProperty(Annotations.OPTIONAL, Annotations.DEFAULT_OPTIONAL);
+ return getPredicate().isOptional();
}
/**
+ *
+ * @see Annotations#CONSTRAINTS
+ */
+ public IConstraint[] constraints() {
+
+ return getProperty(Annotations.CONSTRAINTS, null/* defaultValue */);
+
+ }
+
+ /**
* @see Annotations#MAX_PARALLEL
*/
public int getMaxParallel() {
@@ -642,7 +665,7 @@
this.joinOp = joinOp;
this.predicate = joinOp.getPredicate();
- this.constraints = predicate.constraints();
+ this.constraints = joinOp.constraints();
this.maxParallel = joinOp.getMaxParallel();
if (maxParallel < 0)
throw new IllegalArgumentException(Annotations.MAX_PARALLEL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestAll.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestAll.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestAll.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -75,6 +75,12 @@
suite.addTestSuite(TestSubqueryOp.class);
+ suite.addTestSuite(TestPartitionedJoinGroup.class);
+
+ suite.addTestSuite(TestJoinGraph.class);
+
+ suite.addTestSuite(TestJGraph.class);
+
// @todo test STAR (transitive closure).
// suite.addTestSuite(TestStar.class);
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJGraph.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJGraph.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -0,0 +1,254 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Jan 18, 2011
+ */
+
+package com.bigdata.bop.controller;
+
+import com.bigdata.bop.controller.JoinGraph.JGraph;
+
+import junit.framework.TestCase2;
+
+/**
+ * Test suite for {@link JGraph}, which is the core implementation of the
+ * runtime query optimizer logic.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ *
+ * @todo There are some operations which depend on equality or hash code
+ * behavior for vertices and perhaps edges so those things should also be
+ * tested.
+ */
+public class TestJGraph extends TestCase2 {
+
+ /**
+ *
+ */
+ public TestJGraph() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestJGraph(String name) {
+ super(name);
+ }
+
+ public void test_something() {
+
+ fail("write tests");
+
+ }
+
+// /**
+// * Test ability to identify shared variables appearing either as predicate
+// * operands or as part of CONSTRAINTS or FILTERS.
+// */
+// public void test_getSharedVariables() {
+// fail("write test");
+// }
+//
+// /**
+// * Test ability to recognize when there is a predicate without any shared
+// * variables.
+// */
+// public void test_noSharedVariables() {
+// fail("write test");
+// }
+//
+// public void test_getMinimumCardinalityEdge() {
+// fail("write test");
+// }
+//
+// public void test_moreEdgesToExecute() {
+// fail("write test");
+// }
+//
+// // @todo also getEdgeCount()
+// public void test_getEdges() {
+// fail("write test");
+// }
+//
+// public void test_getSelectedJoinPath() {
+// fail("write test");
+// }
+//
+// public void test_getBestAlternativeJoinPath() {
+// fail("write test");
+// }
+//
+// public void test_getVertex() {
+// fail("write test");
+// }
+//
+// // getEdge(v1,v2)
+// public void test_getEdge() {
+// fail("write test");
+// }
+//
+// // test ability to obtain a Path which extends another path.
+// public void test_Path_addEdge() {
+// fail("write test");
+// }
+
+// @Override
+// public Properties getProperties() {
+//
+// final Properties p = new Properties(super.getProperties());
+//
+// p.setProperty(Journal.Options.BUFFER_MODE, BufferMode.Transient
+// .toString());
+//
+// return p;
+//
+// }
+//
+// static private final String namespace = "ns";
+//
+// Journal jnl;
+//
+// R rel;
+//
+// public void setUp() throws Exception {
+//
+// jnl = new Journal(getProperties());
+//
+// }
+//
+// /**
+// * Create and populate relation in the {@link #namespace}.
+// *
+// * @return The #of distinct entries.
+// */
+// private int loadData(final int scale) {
+//
+// final String[] names = new String[] { "John", "Mary", "Saul", "Paul",
+// "Leon", "Jane", "Mike", "Mark", "Jill", "Jake", "Alex", "Lucy" };
+//
+// final Random rnd = new Random();
+//
+// // #of distinct instances of each name.
+// final int populationSize = Math.max(10, (int) Math.ceil(scale / 10.));
+//
+// // #of trailing zeros for each name.
+// final int nzeros = 1 + (int) Math.ceil(Math.log10(populationSize));
+//
+//// System.out.println("scale=" + scale + ", populationSize="
+//// + populationSize + ", nzeros=" + nzeros);
+//
+// final NumberFormat fmt = NumberFormat.getIntegerInstance();
+// fmt.setMinimumIntegerDigits(nzeros);
+// fmt.setMaximumIntegerDigits(nzeros);
+// fmt.setGroupingUsed(false);
+//
+// // create the relation.
+// final R rel = new R(jnl, namespace, ITx.UNISOLATED, new Properties());
+// rel.create();
+//
+// // data to insert.
+// final E[] a = new E[scale];
+//
+// for (int i = 0; i < scale; i++) {
+//
+// final String n1 = names[rnd.nextInt(names.length)]
+// + fmt.format(rnd.nextInt(populationSize));
+//
+// final String n2 = names[rnd.nextInt(names.length)]
+// + fmt.format(rnd.nextInt(populationSize));
+//
+//// System.err.println("i=" + i + ", n1=" + n1 + ", n2=" + n2);
+//
+// a[i] = new E(n1, n2);
+//
+// }
+//
+// // sort before insert for efficiency.
+// Arrays.sort(a,R.primaryKeyOrder.getComparator());
+//
+// // insert data (the records are not pre-sorted).
+// final long ninserts = rel.insert(new ChunkedArrayIterator<E>(a.length, a, null/* keyOrder */));
+//
+// // Do commit since not scale-out.
+// jnl.commit();
+//
+// // should exist as of the last commit point.
+// this.rel = (R) jnl.getResourceLocator().locate(namespace,
+// ITx.READ_COMMITTED);
+//
+// assertNotNull(rel);
+//
+// return (int) ninserts;
+//
+// }
+//
+// public void tearDown() throws Exception {
+//
+// if (jnl != null) {
+// jnl.destroy();
+// jnl = null;
+// }
+//
+// // clear reference.
+// rel = null;
+//
+// }
+
+// public void test_something() {
+
+//// final int scale = 10000;
+////
+//// final int nrecords = loadData(scale);
+//
+// final IVariable<?> x = Var.var("x");
+//
+// final IVariable<?> y = Var.var("y");
+//
+// final IPredicate<E> p1 = new Predicate<E>(new BOp[] { x, y },
+// new NV(IPredicate.Annotations.RELATION_NAME,
+// new String[] { namespace }),//
+// new NV(IPredicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED)//
+// );
+//
+// final IPredicate<E> p2 = new Predicate<E>(new BOp[] { x, y },
+// new NV(IPredicate.Annotations.RELATION_NAME,
+// new String[] { namespace }),//
+// new NV(IPredicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED)//
+// );
+//
+// final IPredicate<E> p3 = new Predicate<E>(new BOp[] { x, y },
+// new NV(IPredicate.Annotations.RELATION_NAME,
+// new String[] { namespace }),//
+// new NV(IPredicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED)//
+// );
+//
+// new JoinGraph(//
+// new NV(BOp.Annotations.BOP_ID, 1),//
+// new NV(JoinGraph.Annotations.VERTICES,new IPredicate[]{}),//
+// new NV(JoinGraph.Annotations.SAMPLE_SIZE, 100)//
+// );
+//}
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJGraph.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java 2011-01-19 18:22:38 UTC (rev 4129)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/controller/TestJoinGraph.java 2011-01-19 18:34:43 UTC (rev 4130)
@@ -29,11 +29,24 @@
import junit.framework.TestCase2;
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.BOpEvaluationContext;
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IConstraint;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.Var;
+import com.bigdata.bop.ap.Predicate;
+import com.bigdata.bop.constraint.NEConstant;
+
/**
- * Unit tests for runtime query optimization using {@link JoinGraph}.
+ * Unit tests for the {@link JoinGraph} operator.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
+ *
+ * @todo Test evaluation of the operator as well. A lot of the guts of that are
+ * tested by {@link TestJGraph}.
*/
public class TestJoinGraph extends TestCase2 {
@@ -50,193 +63,224 @@
super(name);
}
- /*
- * Among other things, there are some operations which depend on equality or
- * hash code behavior for vertices and perhaps edges so those things should
- * also be tested.
- */
-
- public void test_getMinimumCardinalityEdge() {
- fail("write test");
- }
+ public void test_ctor() {
- public void test_moreEdgesToExecute() {
- fail("write test");
- }
+ // correct acceptance.
+ {
+ final IPredicate[] vertices = new IPredicate[] {
+ new Predicate(new BOp[]{Var.var("x"),Var.var("y")}),//
+ new Predicate(new BOp[]{Var.var("y"),Var.var("z")}),//
+ };
+ final IConstraint[] constraints = null;
+ final JoinGraph joinGraph = new JoinGraph(new BOp[0],//
+ new NV(JoinGraph.Annotations.VERTICES, vertices),//
+ new NV(JoinGraph.Annotations.CONTROLLER, true), //
+ new NV(JoinGraph.Annotations.EVALUATION_CONTEXT,
+ BOpEvaluationContext.CONTROLLER)//
+ );
+ assertEquals("vertices", vertices, joinGraph.getVertices());
+ assertEquals("cons...
[truncated message content] |
|
From: <tho...@us...> - 2011-01-19 18:22:44
|
Revision: 4129
http://bigdata.svn.sourceforge.net/bigdata/?rev=4129&view=rev
Author: thompsonbry
Date: 2011-01-19 18:22:38 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
I had the wrong default client port for zookeeper in the code in several places.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
branches/QUADS_QUERY_BRANCH/build.properties
branches/QUADS_QUERY_BRANCH/build.xml
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2011-01-19 16:13:24 UTC (rev 4128)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/jini/start/AbstractFedZooTestCase.java 2011-01-19 18:22:38 UTC (rev 4129)
@@ -126,7 +126,7 @@
// ZookeeperProcessHelper.startZookeeper(config, listener);
final int clientPort = Integer.valueOf(System
- .getProperty("test.zookeeper.clientPort","2181"));
+ .getProperty("test.zookeeper.clientPort","2081"));
// Verify zookeeper is running on the local host at the client port.
{
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2011-01-19 16:13:24 UTC (rev 4128)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/zookeeper/AbstractZooTestCase.java 2011-01-19 18:22:38 UTC (rev 4129)
@@ -30,9 +30,7 @@
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
-import java.net.BindException;
import java.net.InetAddress;
-import java.net.ServerSocket;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
@@ -73,33 +71,33 @@
super(name);
}
- /**
- * Return an open port on current machine. Try the suggested port first. If
- * suggestedPort is zero, just select a random port
- */
- protected static int getPort(final int suggestedPort) throws IOException {
-
- ServerSocket openSocket;
-
- try {
-
- openSocket = new ServerSocket(suggestedPort);
-
- } catch (BindException ex) {
-
- // the port is busy, so look for a random open port
- openSocket = new ServerSocket(0);
-
- }
+// /**
+// * Return an open port on current machine. Try the suggested port first. If
+// * suggestedPort is zero, just select a random port
+// */
+// protected static int getPort(final int suggestedPort) throws IOException {
+//
+// ServerSocket openSocket;
+//
+// try {
+//
+// openSocket = new ServerSocket(suggestedPort);
+//
+// } catch (BindException ex) {
+//
+// // the port is busy, so look for a random open port
+// openSocket = new ServerSocket(0);
+//
+// }
+//
+// final int port = openSocket.getLocalPort();
+//
+// openSocket.close();
+//
+// return port;
+//
+// }
- final int port = openSocket.getLocalPort();
-
- openSocket.close();
-
- return port;
-
- }
-
/**
* A configuration file used by some of the unit tests in this package. It
* contains a description of the zookeeper server instance in case we need
@@ -193,7 +191,7 @@
.getProperty("test.zookeeper.tickTime","2000"));
clientPort = Integer.valueOf(System.getProperty(
- "test.zookeeper.clientPort", "2181"));
+ "test.zookeeper.clientPort", "2081"));
/*
* Note: This MUST be the actual session timeout that the zookeeper
@@ -202,9 +200,9 @@
this.sessionTimeout = tickTime * 2;
// Verify zookeeper is running on the local host at the client port.
+ final InetAddress localIpAddr = NicUtil.getInetAddress(null, 0,
+ null, true);
{
- final InetAddress localIpAddr = NicUtil.getInetAddress(null, 0,
- null, true);
try {
ZooHelper.ruok(localIpAddr, clientPort);
} catch (Throwable t) {
@@ -216,7 +214,9 @@
// // if necessary, start zookeeper (a server instance).
// ZookeeperProcessHelper.startZookeeper(config, listener);
- zookeeperAccessor = new ZooKeeperAccessor("localhost:" + clientPort, sessionTimeout);
+ zookeeperAccessor = new ZooKeeperAccessor(localIpAddr
+ .getHostAddress()
+ + ":" + clientPort, sessionTimeout);
zookeeper = zookeeperAccessor.getZookeeper();
Modified: branches/QUADS_QUERY_BRANCH/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.properties 2011-01-19 16:13:24 UTC (rev 4128)
+++ branches/QUADS_QUERY_BRANCH/build.properties 2011-01-19 18:22:38 UTC (rev 4129)
@@ -381,4 +381,4 @@
# file for the zookeeper instance used to run CI.
test.zookeeper.installDir=/Users/bryan/zookeeper-3.2.1
test.zookeeper.tickTime=2000
-test.zookeeper.clientPort=2181
+test.zookeeper.clientPort=2081
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2011-01-19 16:13:24 UTC (rev 4128)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2011-01-19 18:22:38 UTC (rev 4129)
@@ -1604,8 +1604,8 @@
<!-- tests. These properties MUST be consistent with the actual -->
<!-- zookeeper configuration. Zookeeper is assumed (by the tests) to -->
<!-- be running on the localhost. -->
- <property name="test.zookeeper.tickTime" value="2000" />
- <property name="test.zookeeper.clientPort" value="2888" />
+ <property name="test.zookeeper.tickTime" value="${test.zookeeper.tickTime}" />
+ <property name="test.zookeeper.clientPort" value="${test.zookeeper.clientPort}" />
<!-- The zookeeper install directory. -->
<property name="test.zookeeper.installDir" value="${zookeeper.installDir}" />
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-19 16:13:31
|
Revision: 4128
http://bigdata.svn.sourceforge.net/bigdata/?rev=4128&view=rev
Author: thompsonbry
Date: 2011-01-19 16:13:24 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
Added comment that the CONDITION for the ConditionalRoutingOp should be lifted into the SubqueryOp. This way we can issue fewer subqueries.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-19 02:03:23 UTC (rev 4127)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2011-01-19 16:13:24 UTC (rev 4128)
@@ -550,7 +550,7 @@
PipelineOp left = startOp;
- if (conditionals != null) {
+ if (conditionals != null) { // @todo lift into CONDITION on SubqueryOp
for (IConstraint c : conditionals) {
final int condId = idFactory.incrementAndGet();
final PipelineOp condOp = applyQueryHints(
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-19 02:03:30
|
Revision: 4127
http://bigdata.svn.sourceforge.net/bigdata/?rev=4127&view=rev
Author: mrpersonick
Date: 2011-01-19 02:03:23 +0000 (Wed, 19 Jan 2011)
Log Message:
-----------
added support for exact match and prefix match from sparql
Modified Paths:
--------------
branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/search/FullTextIndex.java
branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java
branches/JOURNAL_HA_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java
Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/search/FullTextIndex.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/search/FullTextIndex.java 2011-01-18 21:59:35 UTC (rev 4126)
+++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/search/FullTextIndex.java 2011-01-19 02:03:23 UTC (rev 4127)
@@ -28,8 +28,6 @@
package com.bigdata.search;
-import info.aduna.i18n.languagetag.IanaLanguageTag;
-
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
Modified: branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java 2011-01-18 21:59:35 UTC (rev 4126)
+++ branches/JOURNAL_HA_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java 2011-01-19 02:03:23 UTC (rev 4127)
@@ -14,6 +14,7 @@
import com.bigdata.rdf.internal.TermId;
import com.bigdata.rdf.internal.VTE;
import com.bigdata.rdf.internal.XSDDoubleIV;
+import com.bigdata.rdf.lexicon.ITextIndexer;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.spo.SPO;
@@ -21,7 +22,6 @@
import com.bigdata.rdf.spo.SPOPredicate;
import com.bigdata.rdf.store.AbstractTripleStore;
import com.bigdata.rdf.store.BD;
-import com.bigdata.rdf.store.IRawTripleStore;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.rule.IPredicate;
import com.bigdata.relation.rule.ISolutionExpander;
@@ -139,14 +139,28 @@
if (hiterator == null) {
assert database!=null;
assert query != null;
- if (database.getLexiconRelation().getSearchEngine() == null)
+
+ final ITextIndexer textNdx =
+ database.getLexiconRelation().getSearchEngine();
+
+ if (textNdx == null)
throw new UnsupportedOperationException(
"No free text index?");
+
// final long begin = System.nanoTime();
- hiterator = database.getLexiconRelation()
- .getSearchEngine().search(query.getLabel(),
+
+ String s = query.getLabel();
+ final boolean prefixMatch;
+ if (s.indexOf('*') >= 0) {
+ prefixMatch = true;
+ s = s.replaceAll("\\*", "");
+ } else {
+ prefixMatch = false;
+ }
+
+ hiterator = textNdx.search(s,
query.getLanguage(),
- false/* prefixMatch */,
+ prefixMatch,
minRelevance == null ? 0d : minRelevance.doubleValue()/* minCosine */,
maxHits == null ? 10000 : maxHits.intValue()+1/* maxRank */,
1000L/* timeout */, TimeUnit.MILLISECONDS);
Modified: branches/JOURNAL_HA_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java
===================================================================
--- branches/JOURNAL_HA_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java 2011-01-18 21:59:35 UTC (rev 4126)
+++ branches/JOURNAL_HA_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSearchQuery.java 2011-01-19 02:03:23 UTC (rev 4127)
@@ -925,6 +925,138 @@
}
+ { // exact match
+
+ final String searchQuery = "brown cow";
+ final double minRelevance = 0.0d;
+
+ final String query =
+ "select ?s ?o ?score " +
+ "where " +
+ "{ " +
+ " ?s <"+RDFS.LABEL+"> ?o . " +
+ " ?o <"+BD.SEARCH+"> \""+searchQuery+"\" . " +
+ " ?o <"+BD.RELEVANCE+"> ?score . " +
+// " ?o <"+BD.MIN_RELEVANCE+"> \""+minRelevance+"\" . " +
+// " ?o <"+BD.MAX_HITS+"> \"5\" . " +
+ " filter regex(?o, \""+searchQuery+"\") " +
+ "} " +
+ "order by desc(?score)";
+
+ log.info("\n"+query);
+
+ final TupleQuery tupleQuery =
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+ TupleQueryResult result = tupleQuery.evaluate();
+
+ int i = 0;
+ while (result.hasNext()) {
+ log.info(i++ + ": " + result.next().toString());
+ }
+ assertTrue("wrong # of results: " + i, i == 2);
+
+ result = tupleQuery.evaluate();
+
+ Collection<BindingSet> answer = new LinkedList<BindingSet>();
+
+ final ITextIndexer search =
+ sail.getDatabase().getLexiconRelation().getSearchEngine();
+ final Hiterator<IHit> hits =
+ search.search(searchQuery,
+ null, // languageCode
+ false, // prefixMatch
+ minRelevance, // minCosine
+ 10000, // maxRank (=maxResults + 1)
+ 1000L, // timeout
+ TimeUnit.MILLISECONDS // unit
+ );
+
+ while (hits.hasNext()) {
+ final IHit hit = hits.next();
+ final IV id = new TermId(VTE.LITERAL, hit.getDocId());
+ final Literal score = vf.createLiteral(hit.getCosine());
+ final URI s = uris.get(id);
+ final Literal o = literals.get(id);
+ if (!o.getLabel().contains(searchQuery))
+ continue;
+ final BindingSet bs = createBindingSet(
+ new BindingImpl("s", s),
+ new BindingImpl("o", o),
+ new BindingImpl("score", score));
+ log.info(bs);
+ answer.add(bs);
+ }
+
+ compare(result, answer);
+
+ }
+
+ { // prefix match
+
+ final String searchQuery = "bro*";
+ final double minRelevance = 0.0d;
+
+ final String query =
+ "select ?s ?o ?score " +
+ "where " +
+ "{ " +
+ " ?s <"+RDFS.LABEL+"> ?o . " +
+ " ?o <"+BD.SEARCH+"> \""+searchQuery+"\" . " +
+ " ?o <"+BD.RELEVANCE+"> ?score . " +
+// " ?o <"+BD.MIN_RELEVANCE+"> \""+minRelevance+"\" . " +
+// " ?o <"+BD.MAX_HITS+"> \"5\" . " +
+// " filter regex(?o, \""+searchQuery+"\") " +
+ "} " +
+ "order by desc(?score)";
+
+ log.info("\n"+query);
+
+ final TupleQuery tupleQuery =
+ cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
+ tupleQuery.setIncludeInferred(true /* includeInferred */);
+ TupleQueryResult result = tupleQuery.evaluate();
+
+ int i = 0;
+ while (result.hasNext()) {
+ log.info(i++ + ": " + result.next().toString());
+ }
+ assertTrue("wrong # of results: " + i, i == 3);
+
+ result = tupleQuery.evaluate();
+
+ Collection<BindingSet> answer = new LinkedList<BindingSet>();
+
+ final ITextIndexer search =
+ sail.getDatabase().getLexiconRelation().getSearchEngine();
+ final Hiterator<IHit> hits =
+ search.search(searchQuery,
+ null, // languageCode
+ true, // prefixMatch
+ minRelevance, // minCosine
+ 10000, // maxRank (=maxResults + 1)
+ 1000L, // timeout
+ TimeUnit.MILLISECONDS // unit
+ );
+
+ while (hits.hasNext()) {
+ final IHit hit = hits.next();
+ final IV id = new TermId(VTE.LITERAL, hit.getDocId());
+ final Literal score = vf.createLiteral(hit.getCosine());
+ final URI s = uris.get(id);
+ final Literal o = literals.get(id);
+ final BindingSet bs = createBindingSet(
+ new BindingImpl("s", s),
+ new BindingImpl("o", o),
+ new BindingImpl("score", score));
+ log.info(bs);
+ answer.add(bs);
+ }
+
+ compare(result, answer);
+
+ }
+
} finally {
cxn.close();
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-18 21:59:41
|
Revision: 4126
http://bigdata.svn.sourceforge.net/bigdata/?rev=4126&view=rev
Author: thompsonbry
Date: 2011-01-18 21:59:35 +0000 (Tue, 18 Jan 2011)
Log Message:
-----------
echoing the tickTime and clientPort for zookeeper during the build.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/build.xml
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 21:57:47 UTC (rev 4125)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2011-01-18 21:59:35 UTC (rev 4126)
@@ -1783,6 +1783,10 @@
</echo>
<echo> federation: ${bigdata.fedname}
</echo>
+ <echo> zkClientPort: ${test.zookeeper.clientPort}
+</echo>
+ <echo> zkTickTime: ${test.zookeeper.tickTime}
+</echo>
<echo>
</echo>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
4 messages has been excluded from this view by a project administrator.
