This list is closed, nobody may subscribe to it.
2010 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
(139) |
Aug
(94) |
Sep
(232) |
Oct
(143) |
Nov
(138) |
Dec
(55) |
---|---|---|---|---|---|---|---|---|---|---|---|---|
2011 |
Jan
(127) |
Feb
(90) |
Mar
(101) |
Apr
(74) |
May
(148) |
Jun
(241) |
Jul
(169) |
Aug
(121) |
Sep
(157) |
Oct
(199) |
Nov
(281) |
Dec
(75) |
2012 |
Jan
(107) |
Feb
(122) |
Mar
(184) |
Apr
(73) |
May
(14) |
Jun
(49) |
Jul
(26) |
Aug
(103) |
Sep
(133) |
Oct
(61) |
Nov
(51) |
Dec
(55) |
2013 |
Jan
(59) |
Feb
(72) |
Mar
(99) |
Apr
(62) |
May
(92) |
Jun
(19) |
Jul
(31) |
Aug
(138) |
Sep
(47) |
Oct
(83) |
Nov
(95) |
Dec
(111) |
2014 |
Jan
(125) |
Feb
(60) |
Mar
(119) |
Apr
(136) |
May
(270) |
Jun
(83) |
Jul
(88) |
Aug
(30) |
Sep
(47) |
Oct
(27) |
Nov
(23) |
Dec
|
2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
2016 |
Jan
|
Feb
|
Mar
(4) |
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
From: <tho...@us...> - 2010-10-12 17:53:19
|
Revision: 3774 http://bigdata.svn.sourceforge.net/bigdata/?rev=3774&view=rev Author: thompsonbry Date: 2010-10-12 17:53:12 +0000 (Tue, 12 Oct 2010) Log Message: ----------- Added Serializable to several classes to address RMI errors. Added wrapper to return a proxy to the MetadataServer to address a Serialization error where the MetadataServer was attempting to return a FutureTask rather than its proxy. Modified the test harness to allow testing of individual tests per BigdataSparqlTest. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/DiskCostModel.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/IndexSegmentCostModel.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/BTreeCostModel.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -26,6 +26,7 @@ */ package com.bigdata.bop.cost; +import java.io.Serializable; import java.text.NumberFormat; import com.bigdata.btree.AbstractBTree; @@ -54,10 +55,16 @@ * focus on one branch of the {@link BTree} could cause nothing but the * root to be in the cache when probing a different branch. */ -public class BTreeCostModel { +public class BTreeCostModel implements Serializable { - private final DiskCostModel diskCostModel; + /** + * @todo should be either Externalizable and explicitly managed versioning + * or Serializable with a public interface for versioning. + */ + private static final long serialVersionUID = 1L; + private final DiskCostModel diskCostModel; + /** * * @param diskCostModel Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/DiskCostModel.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/DiskCostModel.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/DiskCostModel.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -26,6 +26,8 @@ */ package com.bigdata.bop.cost; +import java.io.Serializable; + /** * A cost model of the disk. * @@ -38,10 +40,16 @@ * transfer rate. However, SCSI does much better than SATA when it can * reorder the writes for improved locality. */ -public class DiskCostModel { +public class DiskCostModel implements Serializable { - public static final DiskCostModel DEFAULT = new DiskCostModel(10d, 41943040); + /** + * @todo should be either Externalizable and explicitly managed versioning + * or Serializable with a public interface for versioning. + */ + private static final long serialVersionUID = 1L; + public static final DiskCostModel DEFAULT = new DiskCostModel(10d, 41943040); + /** * The average disk seek time (milliseconds). */ Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/IndexSegmentCostModel.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/IndexSegmentCostModel.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/IndexSegmentCostModel.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -26,6 +26,8 @@ */ package com.bigdata.bop.cost; +import java.io.Serializable; + import com.bigdata.btree.IndexSegment; /** @@ -38,10 +40,16 @@ * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ -public class IndexSegmentCostModel { +public class IndexSegmentCostModel implements Serializable { - private final DiskCostModel diskCostModel; + /** + * @todo should be either Externalizable and explicitly managed versioning + * or Serializable with a public interface for versioning. + */ + private static final long serialVersionUID = 1L; + private final DiskCostModel diskCostModel; + /** * * @param diskCostModel Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -1,6 +1,8 @@ package com.bigdata.bop.cost; +import java.io.Serializable; + /** * A report on the expected cost of an index key range scan. * @@ -8,9 +10,15 @@ * Thompson</a> * @version $Id$ */ -public class ScanCostReport { +public class ScanCostReport implements Serializable { - /** + /** + * @todo should be either Externalizable and explicitly managed versioning + * or Serializable with a public interface for versioning. + */ + private static final long serialVersionUID = 1L; + + /** * The fast range count. */ public final long rangeCount; Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -8,7 +8,8 @@ public class SubqueryCostReport implements Serializable { /** - * + * @todo should be either Externalizable and explicitly managed versioning + * or Serializable with a public interface for versioning. */ private static final long serialVersionUID = 1L; Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties 2010-10-12 17:53:12 UTC (rev 3774) @@ -173,6 +173,7 @@ #log4j.logger.com.bigdata.relation.accesspath.IAccessPath=DEBUG #log4j.logger.com.bigdata.rdf.sail.BigdataSail=DEBUG +log4j.logger.com.bigdata.rdf.sail.Rule2BOpUtility=INFO #log4j.logger.com.bigdata.rdf.sail.TestNamedGraphs=DEBUG #log4j.logger.com.bigdata.rdf.sail.QuadsTestCase=DEBUG #log4j.logger.com.bigdata.relation.rule.eval.NestedSubqueryWithJoinThreadsTask=DEBUG Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/MetadataServer.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -43,6 +43,7 @@ import org.apache.log4j.MDC; +import com.bigdata.btree.proc.IIndexProcedure; import com.bigdata.service.IDataService; import com.bigdata.service.IMetadataService; import com.bigdata.service.MetadataService; @@ -300,6 +301,18 @@ } /** + * Extends the base behavior to return an RMI compatible proxy for the + * {@link Future}. + */ + @Override + public Future submit(final long tx, final String name, + final IIndexProcedure proc) { + + return getFederation().getProxy(super.submit(tx, name, proc)); + + } + + /** * Extends the base behavior to return a {@link Name} of the service * from the {@link Configuration}. If no name was specified in the * {@link Configuration} then the value returned by the base class is Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -485,9 +485,9 @@ } - if (true||log.isDebugEnabled()) { + if (true||log.isInfoEnabled()) { // just for now while i'm debugging - log.info ("rule=" + rule + "\nquery=" + log.info("rule=" + rule + ":::query=" + BOpUtility.toString(left)); } Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -28,6 +28,7 @@ import java.util.UUID; import junit.framework.Test; +import junit.framework.TestSuite; import org.apache.log4j.Logger; import org.openrdf.query.Dataset; @@ -36,22 +37,63 @@ import org.openrdf.repository.Repository; import org.openrdf.repository.dataset.DatasetRepository; +import com.bigdata.btree.keys.KeyBuilder; import com.bigdata.journal.ITx; import com.bigdata.rdf.sail.BigdataSail; import com.bigdata.rdf.sail.BigdataSailRepository; +import com.bigdata.rdf.sail.BigdataSail.Options; import com.bigdata.rdf.store.ScaleOutTripleStore; import com.bigdata.service.jini.JiniClient; import com.bigdata.service.jini.JiniFederation; /** + * Runs the SPARQL test suite against a {@link JiniFederation}, which must be + * already deployed. Each test in the suite is run against a distinct quad store + * in its own bigdata namespace. + * * @author <a href="mailto:dm...@us...">David MacMillan</a> * @version $Id$ */ public class BigdataFederationSparqlTest extends SPARQLQueryTest { - public static Test suite () - throws Exception - { + + /** + * Skip the dataset tests for now until we can figure out what is wrong with + * them. + * + * FIXME Fix the dataset tests. There is some problem in how the data to be + * loaded into the fixture is being resolved in these tests. + */ + public static Test suite() throws Exception { + + return suite(true /*hideDatasetTests*/); + + } + + public static Test suite(final boolean hideDatasetTests) throws Exception { + + TestSuite suite1 = fullSuite(); + + // Only run the specified tests? + if (!BigdataSparqlTest.testURIs.isEmpty()) { + final TestSuite suite = new TestSuite(); + for (String s : BigdataSparqlTest.testURIs) { + suite.addTest(BigdataSparqlTest.getSingleTest(suite1, s)); + } + return suite; + } + + if(hideDatasetTests) + suite1 = BigdataSparqlTest.filterOutDataSetTests(suite1); + + return suite1; + + } + + /** + * Return the entire test suite. + */ + public static TestSuite fullSuite() throws Exception { return ManifestTest.suite ( new Factory () @@ -124,32 +166,180 @@ return "SPARQLTest_" + UUID.randomUUID ().toString () ; } - private Properties getProperties () + /** + * Configuration options for the KB instances used to run the SPARQL + * compliance test suite. + */ + private Properties getProperties () throws Exception { +// Note: This approach does not work because we are using a different namespace for each test. +// /* +// * Pick up properties configured for the client as defaults. +// * +// * You can specify those properties using NV[] for the component. +// */ +// final String component = System.getProperty ( COMPONENT_PROPERTY, DEFAULT_COMPONENT_PROPERTY ) ; +// final Properties properties = getFederation().getClient().getProperties( +// component); +// return properties; if ( null == _properties ) { - // - // TODO What do we really need here? Don't some of these entail others? - // - _properties = new Properties () ; - _properties.put ( BigdataSail.Options.QUADS_MODE, "true" ) ; - _properties.put ( BigdataSail.Options.TRUTH_MAINTENANCE, "false" ) ; - _properties.put ( BigdataSail.Options.NATIVE_JOINS, "true" ) ; - _properties.put ( BigdataSail.Options.QUERY_TIME_EXPANDER, "true" ) ; + +// /* Multiplier for the scatter effect. +// */ +// final int scatterFactor = 1; +// final int scatterFactor_term2id = 1; +// final int dataServiceCount = 2; +// +// /* The #of index partitions to allocate on a scatter split. ZERO +// * (0) means that 2 index partitions will be allocated per +// * data service which partiticpates in the scatter split. +// * Non-zero values directly give the #of index partitions to +// * create. +// */ +// final int scatterSplitIndexPartitionCount = ConfigMath.multiply +// ( scatterFactor, +// dataServiceCount +// ); +// final int scatterSplitIndexPartitionCount_term2id = ConfigMath.multiply +// ( scatterFactor_term2id, +// dataServiceCount +// ); +// +// // Use all discovered data services when scattering an index. +// final int scatterSplitDataServiceCount = 0; +// +// /* Scatter split trigger point. The scatter split will not be +// * triggered until the initial index partition has reached +// * this percentage of a nominal index partition in size. +// */ +// final double scatterSplitPercentOfSplitThreshold = 0.5;//was .5 +// +// /* +// * Multipliers that compensate for the consumer/producer ratio for +// * the asynchronous index write API. These are empirical factors +// * based on observing the ratio (chunkWritingTime/chunkWaitingTime). +// * Assuming a constant chunk writing time, if the chunk size for each +// * index is adjusted by its multiplier then this ratio would be 1:1. +// * In practice, the chunk writing time is not a linear function of +// * the chunk size, which is one reason why we prefer larger chunks +// * and why the asynchronous write API is a win. +// * +// * Note: These factors were set relative to TERM2ID. However, when +// * I reduced the scatterFactor for TERM2ID by 1/2, I doubled its +// * chunk size to keep up the same throughput so it is now at 2.00 +// * rather than 1.00. +// */ +// final double chunkSizeFactor_id2term = 1.79; +// final double chunkSizeFactor_term2id = 2.00; +// final double chunkSizeFactor_stmts = 8.00; +// +// /* The nominal sink chunk size. For each index, this is adjusted +// * by the factor specified above. +// */ +//// static private sinkChunkSize = 10000; +// final int sinkChunkSize = 1000; + + /* + * Specify / override some triple store properties. + * + * Note: You must reference this object in the section for the + * component which will actually create the KB instance, e.g., + * either the RDFDataLoadMaster or the LubmGeneratorMaster. + */ + _properties = new Properties (); + + /* + * Setup for quads. + */ + _properties.put ( BigdataSail.Options.QUADS_MODE, "true" ); + _properties.put ( BigdataSail.Options.TRUTH_MAINTENANCE, "false" ); + _properties.put ( BigdataSail.Options.QUERY_TIME_EXPANDER, "false" ); + + if (BigdataSparqlTest.cannotInlineTests.contains(testURI)) + _properties.setProperty(Options.INLINE_LITERALS, "false"); + + /* + * The Sesame TCK forces statement level connection auto-commit so + * we set a flag to permit that here. However, auto-commit and this + * flag SHOULD NOT be used outside of the test suite as they provide + * an extreme performance penalty. + */ _properties.put ( BigdataSail.Options.ALLOW_AUTO_COMMIT, "true" ) ; - _properties.put ( BigdataSail.Options.ISOLATABLE_INDICES, "false" ) ; - _properties.put ( BigdataSail.Options.STAR_JOINS, "false" ) ; - _properties.put ( BigdataSail.Options.TEXT_INDEX, "false" ) ; + + /* + * Provide Unicode support for keys with locale-based string + * collation. This is more expensive in key creation during loading, + * but allows key comparison and sorting in the specified locale in + * queries. + * + * @see com.bigdata.btree.keys.CollatorEnum + */ + _properties.put(KeyBuilder.Options.COLLATOR,"ICU"); + _properties.put(KeyBuilder.Options.USER_LANGUAGE,"en"); + _properties.put(KeyBuilder.Options.USER_COUNTRY,"US"); + _properties.put(KeyBuilder.Options.USER_VARIANT,""); + + /* + * Turn off the full text index (search for literals by keyword). + */ + _properties.put(BigdataSail.Options.TEXT_INDEX, "false"); + + /* + * Turn on bloom filter for the SPO index (good up to ~2M index + * entries for scale-up -or- for any size index for scale-out). This + * is a big win for some queries on scale-out indices since we can + * avoid touching the disk if the bloom filter reports "false" for a + * key. + */ + _properties.put(BigdataSail.Options.BLOOM_FILTER, "true"); + + /* + * The #of low order bits from the TERM2ID index partition local + * counter that will be reversed and written into the high-order + * bits of the term identifier. This has a strong effect on the + * distribution of bulk index read/write operations for the triple + * store. For a given value of N, a bulk write will tend to touch + * 2^N index partitions. Therefore if this is even roughly on the + * order of the number of index partitions, each bulk write will + * tend to be scattered to all index partitions. + * + * Note: If this value is too large then the writes WITHIN the index + * partitions will become uniformly distributed, which will + * negatively impact index performance. + */ + _properties.put(BigdataSail.Options.TERMID_BITS_TO_REVERSE,"2"); + + /* + * Option may be enabled to store blank nodes such that they are + * stable (they are not stored by default). + */ + // new NV(BigdataSail.Options.STORE_BLANK_NODES,"true"); + } return _properties ; } - public static final String CONFIG_PROPERTY = "bigdata.configuration" ; - + /** + * The name of the jini configuration file for the federation. + */ + public static final String CONFIG_PROPERTY = "bigdata.configuration"; + +// /** +// * The name of the component in that configuration file whose "properties" +// * field will be used to initialize the KB. +// */ +// public static final String COMPONENT_PROPERTY = "bigdata.component" ; +// +// /** +// * The default value for {@link #COMPONENT_PROPERTY}. +// */ +// public static final String DEFAULT_COMPONENT_PROPERTY = "SparqlQuadsTestSuite"; + private static final Logger _logger = Logger.getLogger ( BigdataFederationSparqlTest.class ) ; private static JiniFederation<Object> _fed = null ; private static Properties _properties = null ; private ScaleOutTripleStore _ts = null ; -} \ No newline at end of file +} Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -70,7 +70,7 @@ * numeric values and these tests test for syntatic differences, i.e. * 01 != 1. */ - protected static Collection<String> cannotInlineTests = Arrays.asList(new String[] { + static final Collection<String> cannotInlineTests = Arrays.asList(new String[] { "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-01", "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-03", "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-04", @@ -131,7 +131,7 @@ * * @return The test suite without the data set tests. */ - static protected TestSuite filterOutDataSetTests(final TestSuite suite1) { + static TestSuite filterOutDataSetTests(final TestSuite suite1) { final TestSuite suite2 = new TestSuite(suite1.getName()); @@ -158,7 +158,7 @@ * suite is run. When specified, only the tests matching these test URIs are * run. */ - static final protected Collection<String> testURIs = Arrays.asList(new String[] { + static final Collection<String> testURIs = Arrays.asList(new String[] { /* // busted with EvalStrategy1 "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-2", @@ -200,6 +200,10 @@ // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-07", // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-11", // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-star-1" + + /* + * Problems with JiniFederation. + */ }); /** @@ -216,16 +220,16 @@ * if there is no test in the suite which is associated with * that testURI. */ - protected static BigdataSparqlTest getSingleTest(TestSuite suite, + static SPARQLQueryTest getSingleTest(TestSuite suite, final String testURI) throws RuntimeException { - BigdataSparqlTest test = null; + SPARQLQueryTest test = null; final Enumeration e1 = suite.tests(); while (e1.hasMoreElements()) { suite = (TestSuite) e1.nextElement(); final Enumeration e2 = suite.tests(); while (e2.hasMoreElements()) { - test = (BigdataSparqlTest) e2.nextElement(); + test = (SPARQLQueryTest) e2.nextElement(); if (testURI.equals(test.getTestURI())) { return test; } @@ -275,9 +279,9 @@ } - public String getTestURI() { - return testURI; - } +// public String getTestURI() { +// return testURI; +// } /** * Overridden to destroy the backend database and its files on the disk. Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java 2010-10-12 14:59:42 UTC (rev 3773) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java 2010-10-12 17:53:12 UTC (rev 3774) @@ -631,4 +631,12 @@ int secLastSlashIdx = manifestFileURL.lastIndexOf('/', lastSlashIdx - 1); return manifestFileURL.substring(secLastSlashIdx + 1, lastSlashIdx); } + + /** + * Made visible to the test suites so we can filter for specific tests. + */ + public String getTestURI() { + return testURI; + } + } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <dm...@us...> - 2010-10-12 14:59:49
|
Revision: 3773 http://bigdata.svn.sourceforge.net/bigdata/?rev=3773&view=rev Author: dmacgbr Date: 2010-10-12 14:59:42 +0000 (Tue, 12 Oct 2010) Log Message: ----------- Make JiniClient static in test class. Prefer logger over stdout/stderr in Rule2BOpUtility Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-12 14:33:04 UTC (rev 3772) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-12 14:59:42 UTC (rev 3773) @@ -487,7 +487,7 @@ if (true||log.isDebugEnabled()) { // just for now while i'm debugging - System.err.println("rule=" + rule + "\nquery=" + log.info ("rule=" + rule + "\nquery=" + BOpUtility.toString(left)); } Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-12 14:33:04 UTC (rev 3772) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-12 14:59:42 UTC (rev 3773) @@ -68,7 +68,15 @@ { super ( URI, name, query, results, dataSet, laxCardinality ) ; } - + + @Override public void runTest () + throws Exception + { + _logger.info ( String.format ( ">>>>> Running test: %s", testURI ) ) ; + super.runTest () ; + _logger.info ( String.format ( ">>>>> Completed test: %s", testURI ) ) ; + } + @Override public void tearDown () throws Exception { @@ -140,7 +148,8 @@ private static final Logger _logger = Logger.getLogger ( BigdataFederationSparqlTest.class ) ; - private JiniFederation<Object> _fed = null ; + private static JiniFederation<Object> _fed = null ; + private static Properties _properties = null ; + private ScaleOutTripleStore _ts = null ; - private Properties _properties = null ; } \ No newline at end of file This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ble...@us...> - 2010-10-12 14:33:11
|
Revision: 3772 http://bigdata.svn.sourceforge.net/bigdata/?rev=3772&view=rev Author: blevine218 Date: 2010-10-12 14:33:04 +0000 (Tue, 12 Oct 2010) Log Message: ----------- Initial support for conditional unit test execution Added Paths: ----------- branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalParameterized.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalRunner.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/IgnoreIf.java Added: branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalParameterized.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalParameterized.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalParameterized.java 2010-10-12 14:33:04 UTC (rev 3772) @@ -0,0 +1,127 @@ +package com.bigdata.test; + +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.junit.runner.Runner; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.Suite; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; +import org.junit.runners.model.TestClass; + +/** + * Similar to the Parameterized runner but uses a ConditionalRunner rather than the default + * runner when generating the parameterized test instances. + * + * @author blevine + * + */ +public class ConditionalParameterized extends Suite +{ + + private final ArrayList<Runner> runners = new ArrayList<Runner>(); + + /** + * Constructor + * + * @param klass + * @throws Throwable + */ + public ConditionalParameterized(Class<?> klass) throws Throwable + { + super(klass, Collections.<Runner> emptyList()); + List<Object[]> parametersList = getParametersList(getTestClass()); + + for (int i = 0; i < parametersList.size(); i++) + { + runners.add(new TestClassRunnerForParameters(getTestClass().getJavaClass(), parametersList, i)); + } + } + + @Override + protected List<Runner> getChildren() + { + return runners; + } + + @SuppressWarnings("unchecked") + private List<Object[]> getParametersList(TestClass klass) throws Throwable + { + return (List<Object[]>) getParametersMethod(klass).invokeExplosively(null); + } + + private FrameworkMethod getParametersMethod(TestClass testClass) throws Exception + { + List<FrameworkMethod> methods = testClass.getAnnotatedMethods(Parameters.class); + for (FrameworkMethod each : methods) + { + int modifiers = each.getMethod().getModifiers(); + if (Modifier.isStatic(modifiers) && Modifier.isPublic(modifiers)) + return each; + } + + throw new Exception("No public static parameters method on class " + testClass.getName()); + } + + + private class TestClassRunnerForParameters extends ConditionalRunner + { + private final int parmSetNumber; + private final List<Object[]> parmList; + + TestClassRunnerForParameters(Class<?> type, List<Object[]> parameterList, int i) throws InitializationError + { + super(type); + parmList = parameterList; + parmSetNumber = i; + } + + @Override + public Object createTest() throws Exception + { + return getTestClass().getOnlyConstructor().newInstance(computeParams()); + } + + private Object[] computeParams() throws Exception + { + try + { + return parmList.get(parmSetNumber); + } + catch (ClassCastException e) + { + throw new Exception(String.format("%s.%s() must return a Collection of arrays.", getTestClass().getName(), getParametersMethod(getTestClass()).getName())); + } + } + + @Override + protected String getName() + { + return String.format("[%s]", parmSetNumber); + } + + @Override + protected String testName(final FrameworkMethod method) + { + return String.format("%s[%s]", method.getName(), parmSetNumber); + } + + @Override + protected void validateConstructor(List<Throwable> errors) + { + validateOnlyOneConstructor(errors); + } + + @Override + protected Statement classBlock(RunNotifier notifier) + { + return childrenInvoker(notifier); + } + } + +} Added: branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalRunner.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalRunner.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/ConditionalRunner.java 2010-10-12 14:33:04 UTC (rev 3772) @@ -0,0 +1,100 @@ +/** + +Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved. + +Contact: + SYSTAP, LLC + 4501 Tower Road + Greensboro, NC 27410 + lic...@bi... + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package com.bigdata.test; + +import org.junit.internal.runners.model.EachTestNotifier; +import org.junit.runner.Description; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; + +/** + * A subclass of the default JUnit runner that checks for and process the @IgnoreIf conditional annotation + * + * @author blevine + * + */ +public class ConditionalRunner extends BlockJUnit4ClassRunner +{ + + /** + * Constructor + * + * @param klass + * @throws InitializationError + */ + public ConditionalRunner(Class<?> klass) throws InitializationError + { + super(klass); + } + + /* (non-Javadoc) + * @see org.junit.runners.BlockJUnit4ClassRunner#runChild(org.junit.runners.model.FrameworkMethod, org.junit.runner.notification.RunNotifier) + */ + @Override + protected void runChild(FrameworkMethod method, RunNotifier notifier) + { + // Is the annotation on the method? + IgnoreIf ann = method.getAnnotation(IgnoreIf.class); + + // Is the annotation on the class? + if (ann == null) + { + ann = method.getMethod().getDeclaringClass().getAnnotation(IgnoreIf.class); + } + + // If we found the annotation on the method or it's declaring class, check the condition + // and if true, skip this method (test case). + if (ann != null) + { + boolean condition = ann.value(); + + if (condition) + { + EachTestNotifier eachNotifier = makeNotifier(method, notifier); + eachNotifier.fireTestIgnored(); + return; + } + } + + // If we didn't find the annotation or if the annotation's condition evaluated to false process + // the test case normally. + + super.runChild(method, notifier); + } + + /** + * Unfortunately declared private in super-class. Cloned from there + * + * @param method + * @param notifier + * @return + */ + private EachTestNotifier makeNotifier(FrameworkMethod method, RunNotifier notifier) + { + Description description = describeChild(method); + return new EachTestNotifier(notifier, description); + } +} Added: branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/IgnoreIf.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/IgnoreIf.java (rev 0) +++ branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/test/IgnoreIf.java 2010-10-12 14:33:04 UTC (rev 3772) @@ -0,0 +1,41 @@ +/** + +Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved. + +Contact: + SYSTAP, LLC + 4501 Tower Road + Greensboro, NC 27410 + lic...@bi... + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ +package com.bigdata.test; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * A conditional implementation of Junit's @Ignore annotation + * @author blevine + * + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface IgnoreIf { + boolean value() default true; +} + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mar...@us...> - 2010-10-12 13:11:55
|
Revision: 3771 http://bigdata.svn.sourceforge.net/bigdata/?rev=3771&view=rev Author: martyncutcher Date: 2010-10-12 13:11:49 +0000 (Tue, 12 Oct 2010) Log Message: ----------- Fix URI encode Modified Paths: -------------- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java 2010-10-12 12:05:26 UTC (rev 3770) +++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java 2010-10-12 13:11:49 UTC (rev 3771) @@ -9,6 +9,7 @@ import java.io.FileNotFoundException; import java.io.FilenameFilter; import java.io.InputStream; +import java.net.URLEncoder; import java.util.Properties; import org.openrdf.model.Statement; @@ -620,18 +621,34 @@ try { - System.out.println("looking for " + resource); - is = new FileInputStream(new File(resource)); baseURI = new File(resource).toURI().toString(); } catch (FileNotFoundException ex) { + is = getClass().getResourceAsStream(resource); + java.net.URL resourceUrl = + getClass().getResource(resource); - System.out.println("no file, retrieving from resource"); - is = getClass().getResourceAsStream(resource); - baseURI = getClass().getResource(resource).toURI() + //if the resource couldn't be found in the file system + //and couldn't be found by searching from this class' + //package (com.bigdata.rdf.rules) as root, then use + //the class loader to try searching from the root of + //the JAR itself + if (resourceUrl == null) { + is = getClass().getClassLoader().getResourceAsStream(resource); + resourceUrl = + getClass().getClassLoader().getResource(resource); + } + + if (resourceUrl == null) { + log.warn("resource not found ["+resource+"]"); + throw new Exception("FAILURE: resource not found ["+resource+"]"); + } + + // must encode before new URI() + baseURI = new java.net.URI(URLEncoder.encode( + resourceUrl.toString(), "UTF-8")) .toString(); - } if (is == null) { This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-12 12:05:33
|
Revision: 3770 http://bigdata.svn.sourceforge.net/bigdata/?rev=3770&view=rev Author: thompsonbry Date: 2010-10-12 12:05:26 +0000 (Tue, 12 Oct 2010) Log Message: ----------- Fix to AbstractSubquery (UNION) to propagate an error thrown by the subquery to the parent query. Added halt(Throwable) to IRunningQuery to support this. Fixed the last four test failures for scale-out quads query. The problem was that queries which used a UNION were not wrapping the subqueries with an operator which ran on the query controller. I moved the logic to do this from BigdataEvaluationStrategyImpl into Rule2BOpUtility so that it is consistently applied each time we have to wrap a rule (as converted into a bigdata operator tree) with a SliceOp scoped to be evaluated on the query controller. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/controller/AbstractSubqueryOp.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -263,7 +263,8 @@ } finally { // Cancel any tasks which are still running. - cancelTasks(); + for (FutureTask<RunningQuery> ft : tasks) + ft.cancel(true/* mayInterruptIfRunning */); context.getSink().close(); @@ -275,16 +276,6 @@ } /** - * Cancel any running tasks. - */ - private void cancelTasks() { - - for (FutureTask<RunningQuery> ft : tasks) - ft.cancel(true/* mayInterruptIfRunning */); - - } - - /** * Run a subquery. * * @author <a href="mailto:tho...@us...">Bryan @@ -338,12 +329,14 @@ } catch (Throwable t) { - // If a subquery fails, then cancel all of the subqueries. - ControllerTask.this.cancelTasks(); + /* + * If a subquery fails, then propagate the error to the + * parent and rethrow the first cause error out of the + * subquery. + */ + throw new RuntimeException(ControllerTask.this.context + .getRunningQuery().halt(t)); - // rethrow the exception. - throw new RuntimeException(t); - } finally { if (subquerySolutionItr != null) Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IRunningQuery.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -72,4 +72,17 @@ */ void halt(); + /** + * Cancel the query (abnormal termination). + * + * @param t + * The cause. + * + * @return The first cause. + * + * @throws IllegalArgumentException + * if the argument is <code>null</code>. + */ + Throwable halt(final Throwable t); + } Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -1217,17 +1217,6 @@ } - /** - * Cancel the query (abnormal termination). - * - * @param t - * The cause. - * - * @return The first cause. - * - * @throws IllegalArgumentException - * if the argument is <code>null</code>. - */ public Throwable halt(final Throwable t) { if (t == null) Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/MockRunningQuery.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -78,6 +78,11 @@ log.warn("Mock object does not implement halt()"); } + public Throwable halt(Throwable t) { + log.warn("Mock object does not implement halt(Throwable)"); + return t; + } + public QueryEngine getQueryEngine() { throw new UnsupportedOperationException(); } Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -54,7 +54,6 @@ import com.bigdata.BigdataStatics; import com.bigdata.bop.BOp; -import com.bigdata.bop.BOpEvaluationContext; import com.bigdata.bop.Constant; import com.bigdata.bop.HashBindingSet; import com.bigdata.bop.IBindingSet; @@ -75,7 +74,6 @@ import com.bigdata.bop.engine.QueryEngine; import com.bigdata.bop.engine.RunningQuery; import com.bigdata.bop.solutions.ISortOrder; -import com.bigdata.bop.solutions.SliceOp; import com.bigdata.btree.IRangeQuery; import com.bigdata.btree.keys.IKeyBuilderFactory; import com.bigdata.rdf.internal.DummyIV; @@ -1711,28 +1709,10 @@ */ final AtomicInteger idFactory = new AtomicInteger(0); - /* - * Convert the step to a bigdata operator tree. - */ - PipelineOp tmp = Rule2BOpUtility.convert(step, idFactory, database, + // Convert the step to a bigdata operator tree. + query = Rule2BOpUtility.convert(step, idFactory, database, queryEngine); - if (!tmp.getEvaluationContext().equals( - BOpEvaluationContext.CONTROLLER)) { - /* - * Wrap with an operator which will be evaluated on the query - * controller. - */ - tmp = new SliceOp(new BOp[] { tmp }, NV.asMap(// - new NV(BOp.Annotations.BOP_ID, idFactory - .incrementAndGet()), // - new NV(BOp.Annotations.EVALUATION_CONTEXT, - BOpEvaluationContext.CONTROLLER))); - - } - - query = tmp; - if (log.isInfoEnabled()) log.info(query); Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -78,7 +78,6 @@ import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.InGraphHashSetFilter; import com.bigdata.rdf.spo.NamedGraphSolutionExpander; -import com.bigdata.rdf.spo.SPORelation; import com.bigdata.rdf.store.AbstractTripleStore; import com.bigdata.relation.IRelation; import com.bigdata.relation.accesspath.AccessPath; @@ -235,8 +234,29 @@ final AtomicInteger idFactory, final AbstractTripleStore db, final QueryEngine queryEngine) { - if (step instanceof IRule<?>) - return convert((IRule<?>) step, idFactory, db, queryEngine); + if (step instanceof IRule<?>) { + + // Convert the step to a bigdata operator tree. + PipelineOp tmp = convert((IRule<?>) step, idFactory, db, + queryEngine); + + if (!tmp.getEvaluationContext().equals( + BOpEvaluationContext.CONTROLLER)) { + /* + * Wrap with an operator which will be evaluated on the query + * controller. + */ + tmp = new SliceOp(new BOp[] { tmp }, NV.asMap(// + new NV(BOp.Annotations.BOP_ID, idFactory + .incrementAndGet()), // + new NV(BOp.Annotations.EVALUATION_CONTEXT, + BOpEvaluationContext.CONTROLLER))); + + } + + return tmp; + + } return convert((IProgram) step, idFactory, db, queryEngine); Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-11 19:55:40 UTC (rev 3769) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-12 12:05:26 UTC (rev 3770) @@ -35,11 +35,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; -import java.util.Iterator; import java.util.Properties; import junit.framework.Test; -import junit.framework.TestCase; import junit.framework.TestSuite; import org.openrdf.query.Dataset; @@ -59,10 +57,6 @@ import com.bigdata.rdf.sail.BigdataSailRepository; import com.bigdata.rdf.sail.BigdataSail.Options; -import cutthecrap.utils.striterators.Expander; -import cutthecrap.utils.striterators.SingleValueIterator; -import cutthecrap.utils.striterators.Striterator; - /** * Test harness for running the SPARQL test suites. * @@ -198,22 +192,38 @@ // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-2", /* - * busted with scale-out quads query. + * busted with scale-out quads query (problem was that the + * subqueries did not have a top-level operator which ran on + * the query controller). */ // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-union-001", // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-07", // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-11", // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-star-1" }); + + /** + * Return the sole test in the suite associated with the specified testURI. + * + * @param suite + * The test suite. + * @param testURI + * The test URI (these are defined by the DAWG). + * + * @return An instance of this class which will run just that one test. + * + * @throws RuntimeException + * if there is no test in the suite which is associated with + * that testURI. + */ + protected static BigdataSparqlTest getSingleTest(TestSuite suite, + final String testURI) throws RuntimeException { - protected static BigdataSparqlTest getSingleTest(TestSuite suite, String testURI) throws Exception { - BigdataSparqlTest test = null; -// TestSuite suite = (TestSuite) BigdataSparqlTest.suite(false); - Enumeration e1 = suite.tests(); + final Enumeration e1 = suite.tests(); while (e1.hasMoreElements()) { suite = (TestSuite) e1.nextElement(); - Enumeration e2 = suite.tests(); + final Enumeration e2 = suite.tests(); while (e2.hasMoreElements()) { test = (BigdataSparqlTest) e2.nextElement(); if (testURI.equals(test.getTestURI())) { This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-11 19:55:47
|
Revision: 3769 http://bigdata.svn.sourceforge.net/bigdata/?rev=3769&view=rev Author: thompsonbry Date: 2010-10-11 19:55:40 +0000 (Mon, 11 Oct 2010) Log Message: ----------- Change to the test suite harness per MikeP to allow us to run just those tests specified in an inline array. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataEmbeddedFederationSparqlTest.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-11 18:33:47 UTC (rev 3768) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-11 19:55:40 UTC (rev 3769) @@ -465,7 +465,7 @@ } - if (log.isDebugEnabled()) { + if (true||log.isDebugEnabled()) { // just for now while i'm debugging System.err.println("rule=" + rule + "\nquery=" + BOpUtility.toString(left)); Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataEmbeddedFederationSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataEmbeddedFederationSparqlTest.java 2010-10-11 18:33:47 UTC (rev 3768) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataEmbeddedFederationSparqlTest.java 2010-10-11 19:55:40 UTC (rev 3769) @@ -28,18 +28,15 @@ package com.bigdata.rdf.sail.tck; import java.io.File; -import java.util.Enumeration; import java.util.Properties; import junit.framework.Test; import junit.framework.TestSuite; - import net.jini.config.ConfigurationException; import org.openrdf.query.Dataset; import org.openrdf.query.parser.sparql.ManifestTest; import org.openrdf.query.parser.sparql.SPARQLQueryTest; -import org.openrdf.query.parser.sparql.SPARQLQueryTest.Factory; import org.openrdf.repository.Repository; import org.openrdf.repository.RepositoryException; import org.openrdf.repository.dataset.DatasetRepository; @@ -76,10 +73,11 @@ } /** - * Skip the dataset tests for now until we can figure out what is wrong - * with them. + * Skip the dataset tests for now until we can figure out what is wrong with + * them. * - * @todo FIXME fix the dataset tests + * FIXME Fix the dataset tests. There is some problem in how the data to be + * loaded into the fixture is being resolved in these tests. */ public static Test suite() throws Exception { @@ -91,14 +89,20 @@ TestSuite suite1 = suiteEmbeddedFederation(); - if (!hideDatasetTests) { - - return suite1; - + // Only run the specified tests? + if (!testURIs.isEmpty()) { + final TestSuite suite = new TestSuite(); + for (String s : testURIs) { + suite.addTest(getSingleTest(suite1, s)); + } + return suite; } - return filterOutDataSetTests(suite1); + if(hideDatasetTests) + suite1 = filterOutDataSetTests(suite1); + return suite1; + } /** Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-11 18:33:47 UTC (rev 3768) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java 2010-10-11 19:55:40 UTC (rev 3769) @@ -28,16 +28,20 @@ package com.bigdata.rdf.sail.tck; import info.aduna.io.IOUtil; -import java.io.File; + import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; +import java.util.Iterator; import java.util.Properties; + import junit.framework.Test; +import junit.framework.TestCase; import junit.framework.TestSuite; + import org.openrdf.query.Dataset; import org.openrdf.query.parser.sparql.ManifestTest; import org.openrdf.query.parser.sparql.SPARQLQueryTest; @@ -46,6 +50,7 @@ import org.openrdf.repository.dataset.DatasetRepository; import org.openrdf.repository.sail.SailRepository; import org.openrdf.sail.memory.MemoryStore; + import com.bigdata.btree.keys.CollatorEnum; import com.bigdata.btree.keys.StrengthEnum; import com.bigdata.journal.BufferMode; @@ -53,9 +58,11 @@ import com.bigdata.rdf.sail.BigdataSail; import com.bigdata.rdf.sail.BigdataSailRepository; import com.bigdata.rdf.sail.BigdataSail.Options; -import com.bigdata.rdf.store.LocalTripleStore; -import com.bigdata.relation.AbstractResource; +import cutthecrap.utils.striterators.Expander; +import cutthecrap.utils.striterators.SingleValueIterator; +import cutthecrap.utils.striterators.Striterator; + /** * Test harness for running the SPARQL test suites. * @@ -87,13 +94,14 @@ "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-9", }); - private static String datasetTests = "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset"; - +// private static String datasetTests = "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset"; + /** - * Skip the dataset tests for now until we can figure out what is wrong - * with them. + * Skip the dataset tests for now until we can figure out what is wrong with + * them. * - * @todo FIXME fix the dataset tests + * FIXME Fix the dataset tests. There is some problem in how the data to be + * loaded into the fixture is being resolved in these tests. */ public static Test suite() throws Exception { @@ -105,17 +113,31 @@ TestSuite suite1 = suiteLTSWithPipelineJoins(); - if (!hideDatasetTests) { - - return suite1; - + // Only run the specified tests? + if (!testURIs.isEmpty()) { + final TestSuite suite = new TestSuite(); + for (String s : testURIs) { + suite.addTest(getSingleTest(suite1, s)); + } + return suite; } - return filterOutDataSetTests(suite1); + if(hideDatasetTests) + suite1 = filterOutDataSetTests(suite1); + return suite1; + } - - static protected Test filterOutDataSetTests(TestSuite suite1) { + + /** + * Hack filters out the "dataset" tests. + * + * @param suite1 + * The test suite. + * + * @return The test suite without the data set tests. + */ + static protected TestSuite filterOutDataSetTests(final TestSuite suite1) { final TestSuite suite2 = new TestSuite(suite1.getName()); @@ -136,7 +158,74 @@ return suite2; } + + /** + * An array of URIs for tests to be run. When null or empty the default test + * suite is run. When specified, only the tests matching these test URIs are + * run. + */ + static final protected Collection<String> testURIs = Arrays.asList(new String[] { +/* +// busted with EvalStrategy1 + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-2", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#filter-scope-1", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#join-scope-1", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-4", + +// busted with EvalStrategy2 with LeftJoin enabled + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/open-world/manifest#open-eq-12", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#nested-opt-1", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#opt-filter-1", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/algebra/manifest#opt-filter-2", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-optional-complex-3", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-001", + "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional-filter/manifest#dawg-optional-filter-004", +*/ +// Dataset crap + // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/i18n/manifest#normalization-1" + + // "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/dataset/manifest#dawg-dataset-01" + +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2//manifest#", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-str-1", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-str-2", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#dawg-datatype-1", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#sameTerm-simple", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#sameTerm-eq", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-builtin/manifest#sameTerm-not-eq", +// +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-1", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/expr-equals/manifest#eq-graph-2", + + /* + * busted with scale-out quads query. + */ +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/optional/manifest#dawg-union-001", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-07", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/graph/manifest#dawg-graph-11", +// "http://www.w3.org/2001/sw/DataAccess/tests/data-r2/distinct/manifest#distinct-star-1" + }); + protected static BigdataSparqlTest getSingleTest(TestSuite suite, String testURI) throws Exception { + + BigdataSparqlTest test = null; +// TestSuite suite = (TestSuite) BigdataSparqlTest.suite(false); + Enumeration e1 = suite.tests(); + while (e1.hasMoreElements()) { + suite = (TestSuite) e1.nextElement(); + Enumeration e2 = suite.tests(); + while (e2.hasMoreElements()) { + test = (BigdataSparqlTest) e2.nextElement(); + if (testURI.equals(test.getTestURI())) { + return test; + } + } + } + + throw new RuntimeException("could not find a test with that URI"); + + } + /** * Return the test suite. */ This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <res...@us...> - 2010-10-11 18:33:56
|
Revision: 3768 http://bigdata.svn.sourceforge.net/bigdata/?rev=3768&view=rev Author: resendes Date: 2010-10-11 18:33:47 +0000 (Mon, 11 Oct 2010) Log Message: ----------- Fixes and unit tests for com.bigdata.util.config package. Modified Paths: -------------- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigurationUtil.java branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/Log4jLoggingHandler.java branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/LogUtil.java branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/NicUtil.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigDeployUtilTest.java Added Paths: ----------- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigurationUtilTest.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/Log4jLoggingHandlerTest.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/LogUtilTest.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/NicUtilTest.java Modified: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-10-11 17:26:00 UTC (rev 3767) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigDeployUtil.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -48,6 +48,12 @@ * invoked from within a Jini configuration. */ public class ConfigDeployUtil { + + private ConfigDeployUtil() { + //prevent instantiation + throw new AssertionError + ("ConfigDeployUtil cannot be instantiated"); + } private static Properties deploymentProps = null; @@ -63,6 +69,22 @@ //use current locale private static final NumberFormat numberFormat = NumberFormat.getInstance(); + /** + * Gets the <code>String</code> value associated with the given parameter + * in the configured properties file. The value, if any, returned will be: + * 1) the explicitly defined value for the property, or + * 2) the default value for that property, if a default exists. + * If the provided parameter has an associated set of valid entries in the + * properties files, then the value obtained, above, will be validated + * against those entries. If either the value (explicit or default) can't be + * found or fails the validation check, + * a <code>ConfigurationException</code> will be thrown. + * @param parameter The property name to lookup + * @return the <code>String</code> value associated the requested parameter, + * if available + * @throws ConfigurationException if the parameter value + * (explicit or default) was not defined. + */ public static String getString(String parameter) throws ConfigurationException { @@ -70,7 +92,23 @@ validateString(parameter, value); return value; } - + + /** + * Gets the <code>String[]</code> value associated with the given parameter + * in the configured properties file. The value, if any, returned will be: + * 1) the explicitly defined value for the property, or + * 2) the default value for that property, if a default exists. + * If the provided parameter has an associated set of valid entries in the + * properties files, then the component values obtained, above, will be validated + * against those entries. If either the value (explicit or default) can't be + * found or fails the validation check, + * a <code>ConfigurationException</code> will be thrown. + * @param parameter The property name to lookup + * @return the <code>String[]</code> value associated the requested parameter, + * if available + * @throws ConfigurationException if the parameter value + * (explicit or default) was not defined. + */ public static String[] getStringArray(String parameter) throws ConfigurationException { @@ -79,12 +117,46 @@ return value; } + /** + * Gets the <code>int</code> value associated with the given parameter + * in the configured properties file. The value, if any, returned will be: + * 1) the explicitly defined value for the property, or + * 2) the default value for that property, if a default exists. + * If the provided parameter has an associated set of valid entries in the + * properties files, then the value obtained, above, will be validated + * against those entries. If either the value (explicit or default) can't be + * found or fails the validation check, + * a <code>ConfigurationException</code> will be thrown. + * @param parameter The property name to lookup + * @return the <code>int</code> value associated the requested parameter, + * if available + * @throws ConfigurationException if the parameter value + * (explicit or default) was not defined. + */ + public static int getInt(String parameter) throws ConfigurationException { int value; value = validateInt(parameter, get(parameter)); return value; } + /** + * Gets the <code>long</code> value associated with the given parameter + * in the configured properties file. The value, if any, returned will be: + * 1) the explicitly defined value for the property, or + * 2) the default value for that property, if a default exists. + * If the provided parameter has an associated set of valid entries in the + * properties files, then the value obtained, above, will be validated + * against those entries. If either the value (explicit or default) can't be + * found or fails the validation check, + * a <code>ConfigurationException</code> will be thrown. + * @param parameter The property name to lookup + * @return the <code>long</code> value associated the requested parameter, + * if available + * @throws ConfigurationException if the parameter value + * (explicit or default) was not defined. + */ + public static long getLong(String parameter) throws ConfigurationException { long value; @@ -92,6 +164,22 @@ return value; } + /** + * Gets the <code>boolean</code> value associated with the given parameter + * in the configured properties file. The value, if any, returned will be: + * 1) the explicitly defined value for the property, or + * 2) the default value for that property, if a default exists. + * If the provided parameter has an associated set of valid entries in the + * properties files, then the value obtained, above, will be validated + * against those entries. If either the value (explicit or default) can't be + * found or fails the validation check, + * a <code>ConfigurationException</code> will be thrown. + * @param parameter The property name to lookup + * @return the <code>boolean</code> value associated the requested parameter, + * if available + * @throws ConfigurationException if the parameter value + * (explicit or default) was not defined. + */ public static boolean getBoolean(String parameter) throws ConfigurationException { boolean value; @@ -114,6 +202,17 @@ return boolValue; } + /** + * Gets the description value associated with the given parameter + * in the configured properties file. The method returns + * <code>null</code> if the property is not found. + * @param parameter The property name to lookup + * @return the <code>String</code> value associated the requested parameter, + * if available. Otherwise return <code>null</code> + * @throws ConfigurationException if there was a problem accessing + * the parameter value. + */ + public static String getDescription(String parameter) throws ConfigurationException { @@ -122,6 +221,18 @@ return value; } + /** + * Gets the type value associated with the given parameter + * in the configured properties file. The method returns + * <code>null</code> if the property is not found. + * @param parameter The property name to lookup + * @return the <code>String</code> value of the type + * associated the requested parameter, + * if available. Otherwise return <code>null</code> + * @throws ConfigurationException if there was a problem accessing + * the parameter value. + */ + public static String getType(String parameter) throws ConfigurationException { @@ -130,6 +241,15 @@ return value; } + /** + * Gets the default value associated with the given parameter + * in the configured properties file. + * @param parameter The property name to lookup + * @return the <code>String</code> value of the type + * associated the requested parameter, + * if available. Otherwise return <code>null</code> + * @throws ConfigurationException if no default value was specified + */ public static String getDefault(String parameter) throws ConfigurationException { @@ -445,10 +565,9 @@ * (defined in <code>loadDefaultProps</code> and <code>loadOverrideProps</code>). * This method is synchronized in order to ensure that the returned reference is * a singleton instance. - * Note: This method should be private, but is needed by the unit test in order - * to access and modify the <code>Properties</code> method. - * @return Properties instance containing the default and user-defined overrides - * for configuration properties. + * Note: This method should be private, but is needed by the unit tests in order + * to access and modify the underlying <code>Properties</code> object. + * @return Properties instance containing the configuration properties. */ synchronized static Properties getDeploymentProperties() { if(deploymentProps == null) { Modified: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigurationUtil.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigurationUtil.java 2010-10-11 17:26:00 UTC (rev 3767) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/ConfigurationUtil.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -28,10 +28,10 @@ import org.apache.log4j.Level; import org.apache.log4j.Logger; -import com.sun.jini.config.ConfigUtil; import net.jini.url.httpmd.HttpmdUtil; import java.io.IOException; +import java.net.UnknownHostException; /** * Utility class that provides a set of static convenience methods @@ -43,6 +43,11 @@ * This class cannot be instantiated. */ public class ConfigurationUtil { + + private ConfigurationUtil() { + //prevent instantiation + throw new AssertionError + ("ConfigurationUtil cannot be instantiated"); } private static final Logger logger = LogUtil.getLog4jLogger( ConfigurationUtil.class ); @@ -95,7 +100,7 @@ * <p> * Note that the <code>name</code> parameter is handled specially by * this method. The value input to that <code>String</code> parameter - * is interpretted in one of two ways by this method: + * is interpreted in one of two ways by this method: * <p><ul> * <li> The name of the local <i>network interface</i> over which the * class server will communicate with clients requesting the @@ -109,7 +114,7 @@ * This method first treats the value of the <code>name</code> parameter * as a network interface, attempting to determine that interface's * corresponding IP address. Should that attempt fail, then this method - * then assumes that the caller intended this parameter to be interpretted + * then assumes that the caller intended this parameter to be interpreted * as an IP address or host name; in which case, this method uses * the value of the parameter as the <i>address/host</i> component of * codebase being constructed. @@ -181,7 +186,7 @@ * any value, including <code>null</code>). * * @throws IllegalArgumentException if the value input for - * <code>port</code> is negtive. + * <code>port</code> is negative. */ public static String computeCodebase(String name, String jarFile, @@ -220,6 +225,10 @@ logger.log(Level.TRACE, name+" - not a valid " +"network interface, assuming host name"); } + if (ipAddr==null) { + throw new UnknownHostException( + "Could not determine IP address for given name:" + name); + } // Construct the codebase, either httpmd or http if(doHttpmd) { Modified: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/Log4jLoggingHandler.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/Log4jLoggingHandler.java 2010-10-11 17:26:00 UTC (rev 3767) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/Log4jLoggingHandler.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -60,11 +60,12 @@ this.log4jLogger = LogUtil.getLog4jRootLogger(); } - // Get the handler's configuration + // Set the handler's configuration this.setLevel(java.util.logging.Level.FINEST); this.setFormatter(new SimpleFormatter()); } + @Override public void publish(LogRecord record) { if(! isLoggable(record)) return; @@ -79,8 +80,10 @@ ( new XLoggingEvent(log4jLogger, record, formattedMessage) ); } + @Override public void close() { /* no-op */ } + @Override public void flush() { /* no-op */ } Modified: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/LogUtil.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/LogUtil.java 2010-10-11 17:26:00 UTC (rev 3767) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/LogUtil.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -25,11 +25,6 @@ package com.bigdata.util.config; -import java.io.File; -import java.util.Set; -import java.util.HashSet; - -import com.bigdata.DataFinder; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.xml.DOMConfigurator; @@ -78,14 +73,31 @@ } } + /** + * Return log4j logger for the provided <code>String<code> argument. + * @param componentName The name of the logger to return + * @return log4j logger associated with the provided name + */ public static Logger getLog4jLogger(String componentName) { return Logger.getLogger(componentName); } + /** + * Return log4j logger for the provided <code>Class<code> argument. + * @param componentClass The class whose name will be used to return the + * associated logger. + * @return log4j logger associated with the provided class' name + */ + public static Logger getLog4jLogger(Class componentClass) { return Logger.getLogger(componentClass); } + /** + * Return the root log4j logger. + * @return the root log4j logger + */ + public static Logger getLog4jRootLogger() { return Logger.getRootLogger(); } Modified: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/NicUtil.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/NicUtil.java 2010-10-11 17:26:00 UTC (rev 3767) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/config/NicUtil.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -78,7 +78,8 @@ * network interface to return (for example, typical * values for this parameter might be, "eth0", "eth1", * "hme01", "lo", etc., depending on how the underlying - * platform is configured). + * platform is configured). The empty string will + * return the loopback interface. * * @return an instance of <code>NetworkInterface</code> that represents * the network interface corresponding to the given @@ -154,13 +155,12 @@ * the desired network interface(s) correspond. * * @return an array whose elements are each instances of - * <code>NetworkInterface[]</code>, in which each such - * instance corresponds to the given <code>name</code>, - * or <code>null</code> if there is no network interface - * corresponding to that name value. + * <code>NetworkInterface</code>, in which each such + * instance corresponds to the given <code>name</code>. * - * Note that if the value given for the <code>name</code> - * parameter is the <code>String</code> "all", then this + * Note that if the special value "all" is given for the + * <code>name</code> + * parameter, then this * method will return an array containing all of the * network interfaces installed on the current node, * regardless of each interface's name. @@ -176,31 +176,24 @@ { NetworkInterface [] nics = null; if (name.equals("all")) { - Enumeration en = NetworkInterface.getNetworkInterfaces(); - List nicList = (en != null) ? - Collections.list(en) : Collections.EMPTY_LIST; - nics = (NetworkInterface[])(nicList.toArray - (new NetworkInterface[nicList.size()]) ); + Enumeration<NetworkInterface> en = NetworkInterface.getNetworkInterfaces(); + List<NetworkInterface> nicList = + ((en != null) + ? Collections.list(en) + : Collections.<NetworkInterface>emptyList()); + nics = (nicList.toArray(new NetworkInterface[nicList.size()]) ); } else { - nics = new NetworkInterface[1]; - nics[0] = NetworkInterface.getByName(name); - if (nics[0] == null) { - // try to lookup by IP address - InetAddress targetIp = null; - try { - targetIp = InetAddress.getByName(name); - nics[0] = NetworkInterface.getByInetAddress(targetIp); - } catch (UnknownHostException uhe) { - // ignore, return null - } - } + NetworkInterface nif = getNetworkInterface(name); + nics = ((nif != null) + ? new NetworkInterface[] { nif } + : new NetworkInterface[0]); } return nics; } /** * Returns the instance of <code>InetAddress</code> that represents - * the i-th IP address assigned to the network interface having the + * the i-th IPv4 address assigned to the network interface having the * given <code>name</code> (where i is specified by the value of the * <code>index</code> parameter). * <p> @@ -268,7 +261,7 @@ * <code>localHost</code> is <code>false</code>. * * @throws IllegalArgumentException if the value input for - * <code>index</code> is negtive. + * <code>index</code> is negative. * * @throws IndexOutOfBoundsException if the value input for * <code>index</code> is out of range; that is if the value @@ -282,10 +275,14 @@ { // Validate input parameters if( (name == null) && (host == null) && (localHost == false) ) { - throw new NullPointerException("name cannot be null"); + throw new NullPointerException( + "Name and host cannot be null with localHost set to false."); } - if(index < 0) throw new IllegalArgumentException - ("index cannot be negative"); + + if(index < 0) { + throw new IllegalArgumentException("index cannot be negative"); + } + // Primary retrieval attempt NetworkInterface nic = null; try { @@ -376,6 +373,7 @@ public static String getMacAddress(String name) throws SocketException { String macAddr = null; NetworkInterface nic = NicUtil.getNetworkInterface(name); + if (nic==null) return null; byte[] hwAddr = nic.getHardwareAddress(); if( (hwAddr != null) && (hwAddr.length > 0) ) { StringBuffer strBuf = new StringBuffer(); @@ -390,6 +388,9 @@ macAddr = strBuf.toString(); } return macAddr; + //TODO - verify MAC address string representation + // (e.g. MSB->LSB ordering) + //TODO - verify that HW address is a MAC address } // /** @@ -530,7 +531,7 @@ * above. * <p> * Note that in all cases, if <code>true</code> is input - * for the <code>loopOk</code> parameter, then upon failing + * for the <code>loopbackOk</code> parameter, then upon failing * to find a valid ip address using the specified search * mechanism, this method will return the <i>loop back</i> * address; otherwise, <code>null</code> is returned. @@ -663,6 +664,8 @@ //get all nics on the current node Enumeration<NetworkInterface> nics = NetworkInterface.getNetworkInterfaces(); + if (nics == null) return null; + while( nics.hasMoreElements() ) { NetworkInterface curNic = nics.nextElement(); List<InterfaceAddress> interfaceAddrs = Modified: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigDeployUtilTest.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigDeployUtilTest.java 2010-10-11 17:26:00 UTC (rev 3767) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigDeployUtilTest.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -115,6 +115,22 @@ String expected = value; assertEquals(actual, expected); } + + /** + * Test method for {@link com.bigdata.util.config.ConfigDeployUtil#getString(java.lang.String)}. + * @throws ConfigurationException + */ + @Test + public void testGetString_valid_name_default() throws ConfigurationException { + String key = "testGetString_valid_name"; + String value = key + "_value"; + Properties p = ConfigDeployUtil.getDeploymentProperties(); + p.setProperty(key+".default", value); + p.setProperty(key+".stringvals", value + ",other"); + String actual = ConfigDeployUtil.getString(key); + String expected = value; + assertEquals(actual, expected); + } /** * Test method for {@link com.bigdata.util.config.ConfigDeployUtil#getString(java.lang.String)}. @@ -135,6 +151,25 @@ //ignore -- exception } } + /** + * Test method for {@link com.bigdata.util.config.ConfigDeployUtil#getString(java.lang.String)}. + * @throws ConfigurationException + */ + @Test + public void testGetString_invalid_name_default() throws ConfigurationException { + String key = "testGetString_invalid_name"; + String value = key + "_value"; + Properties p = ConfigDeployUtil.getDeploymentProperties(); + p.setProperty(key+".default", value); + p.setProperty(key+".stringvals", "other1,other2"); + try { + ConfigDeployUtil.getString(key); + fail("Successfully called getString with bogus parameter string: [" + + key + ":" + value + "]"); + } catch (ConfigurationException e) { + //ignore -- exception + } + } /** * Test method for {@link com.bigdata.util.config.ConfigDeployUtil#getStringArray(java.lang.String)}. Added: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigurationUtilTest.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigurationUtilTest.java (rev 0) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/ConfigurationUtilTest.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -0,0 +1,549 @@ +package com.bigdata.util.config; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ConfigurationUtilTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testConcat_null() throws SecurityException, NoSuchMethodException { + String[][][] args = { + {null, null}, + {new String[0], null}, + {null, new String[0]} + }; + for (String[][] arg: args) { + try { + ConfigurationUtil.concat(arg[0], arg[1]); + fail("Successfully called concat() with null arg."); + } catch (NullPointerException e) { + // ignore -- expected. + } + } + } + + @Test + public void testConcat_empty() throws SecurityException, NoSuchMethodException { + String [] actual = ConfigurationUtil.concat(new String[0], new String[0]); + assertTrue(actual.length==0); + } + + @Test + public void testConcat_non_empty_singleton() throws SecurityException, NoSuchMethodException { + String[] a = { "abc" }; + String[] b = { "xyz" }; + String [] actual = ConfigurationUtil.concat(a, b); + assertTrue(actual.length==(a.length + b.length)); + assertArrayEquals("Arrays don't match", actual, combine(a, b)); + } + + @Test + public void testConcat_non_empty_multiple() throws SecurityException, NoSuchMethodException { + String[] a = { "abc", "def", "ghi", "jklmnop" }; + String[] b = { "qrs", "tuv", "wxyz" }; + String [] actual = ConfigurationUtil.concat(a, b); + assertTrue(actual.length==(a.length + b.length)); + List<String> l = new ArrayList<String>(); + l.addAll(Arrays.asList(a)); + l.addAll(Arrays.asList(b)); + assertArrayEquals("Arrays don't match", actual, l.toArray(new String[0])); + } + + @Test + public void testConcat_non_empty_multiple_embedded_null() + throws SecurityException, NoSuchMethodException + { + String[] a = { "abc", "def", null, "jklmnop" }; + String[] b = { "qrs", null, "wxyz" }; + String [] actual = ConfigurationUtil.concat(a, b); + assertTrue(actual.length==(a.length + b.length)); + List<String> l = new ArrayList<String>(); + l.addAll(Arrays.asList(a)); + l.addAll(Arrays.asList(b)); + assertArrayEquals("Arrays don't match", actual, l.toArray(new String[0])); + } + + @Test + public void testCreateArgList_null() { + Object[][] args = { + {null, null}, + {new String[0], null}, + }; + for (Object[] arg: args) { + try { + ConfigurationUtil.createArgList((String[])arg[0], (String)arg[1]); + fail("Successfully called createArgList() with null arg: " + + Arrays.asList(arg)); + } catch (NullPointerException e) { + // ignore -- expected. + } + } + } + + @Test + public void testCreateArgList_null2() { + //if second arg is empty or whitespace, return first arg + assertNull(ConfigurationUtil.createArgList(null, "")); + } + + @Test + public void testCreateArgList_null3() { + //if second arg is empty or whitespace, return first arg + assertNull(ConfigurationUtil.createArgList(null, " \t \t\t ")); + } + + @Test + public void testCreateArgList_empty() { + String[] orig = {"abc"}; + String extra = ""; + String[] sum = ConfigurationUtil.createArgList(orig, extra); + assertTrue(orig==sum);//identity obviates equals test + } + + @Test + public void testCreateArgList_single() { + String[] orig = {"abc"}; + String[] extraArray = new String[] {"xyz"}; + List<String> l = Arrays.asList(extraArray); + String extra = join(l); + String[] sum = ConfigurationUtil.createArgList(orig, extra); + assertArrayEquals(sum, combine(orig, extraArray)); + } + + @Test + public void testCreateArgList_multiple() { + String[] orig = {"abc"}; + String[] extraArray = new String[] {"rst", "uvw", "xyz"}; + List<String> l = Arrays.asList(extraArray); + String extra = join(l); + String[] sum = ConfigurationUtil.createArgList(orig, extra); + assertArrayEquals(sum, combine(orig, extraArray)); + } + + @Test + public void testComputeCodebase_5_arg_null() throws IOException { + Integer port = new Integer(5); + String jarf = "bogus_jar_file.jar"; + String name = "bogus_name"; + Object[][] args = { + {null, null, port, null, "sha"}, + {null, jarf, port, null, "sha"}, + {name, null, port, null, "sha"}, + {name, jarf, port, null, "sha"}, + }; + //Test int port method + for (Object[] arg: args) { + try { + ConfigurationUtil.computeCodebase( + (String)arg[0], (String)arg[1], + ((Integer)arg[2]).intValue(), + (String)arg[3], (String)arg[4]); + fail("Successfully called computeCodebase() with null arg: " + + Arrays.asList(arg)); + } catch (NullPointerException e) { + // ignore -- expected. + } + } + //Test String port method + for (Object[] arg: args) { + try { + ConfigurationUtil.computeCodebase( + (String)arg[0], (String)arg[1], + ((Integer)arg[2]).toString(), + (String)arg[3], (String)arg[4]); + fail("Successfully called computeCodebase() with null arg: " + + Arrays.asList(arg)); + } catch (NullPointerException e) { + // ignore -- expected. + } + } + + } + + @Test + public void testComputeCodebase_5_arg_invalid_port() throws IOException { + Integer port = new Integer(-1); + //Try int port + try { + ConfigurationUtil.computeCodebase( + "bogus_name", "bogus_jar_file.jar", + port.intValue(), "bogus_src", "bogus_md"); + fail("Successfully called computeCodebase() with negative port: " + + port); + } catch (IllegalArgumentException e) { + // ignore -- expected. + } + //Try String port + try { + ConfigurationUtil.computeCodebase( + "bogus_name", "bogus_jar_file.jar", + port.toString(), "bogus_src", "bogus_md"); + fail("Successfully called computeCodebase() with negative port: " + + port); + } catch (IllegalArgumentException e) { + // ignore -- expected. + } + } + + @Test + public void testComputeCodebase_5_arg_valid_hostname_http() throws IOException { + Integer port = new Integer(1234); + String hostname = InetAddress.getLocalHost().getHostAddress(); + String jarFile = "file.jar"; + Object[][] args = { + {hostname, jarFile, port, "src", null}, + {hostname, jarFile, port, "src", ""}, + {hostname, jarFile, port, "src", "off"}, + {hostname, jarFile, port, "src", "none"}, + }; + + //Test int port + for (Object[] arg: args) { + String urlString = ConfigurationUtil.computeCodebase( + (String)arg[0], (String)arg[1], + ((Integer)arg[2]).intValue(), + (String)arg[3], (String)arg[4]); //gen http url + testComputeCodebase_5_arg_valid_hostname_http_assert( + urlString, hostname, port, jarFile); + } + + //Test String port + for (Object[] arg: args) { + String urlString = ConfigurationUtil.computeCodebase( + (String)arg[0], (String)arg[1], + (String)arg[2].toString(), + (String)arg[3], (String)arg[4]); //gen http url + testComputeCodebase_5_arg_valid_hostname_http_assert( + urlString, hostname, port, jarFile); + } + } + + private void testComputeCodebase_5_arg_valid_hostname_http_assert( + String urlString, String hostname, Integer port, String jarFile) + throws MalformedURLException + { + URL url = new URL(urlString); + assertEquals(url.getProtocol(), "http"); + assertEquals(url.getHost(), hostname); //use IP to avoid DNS conversion + assertEquals(url.getPort(), port.intValue()); + assertEquals(url.getFile(), "/" + jarFile); + } + + @Test + public void testComputeCodebase_5_arg_invalid_hostname_http() throws IOException { + Integer port = new Integer(1234); + String hostname = UUID.randomUUID().toString(); + String jarFile = "file.jar"; + + //Test int port + try { + ConfigurationUtil.computeCodebase( + hostname, jarFile, port.intValue(), "", ""); + fail("Successfully called computeCodebase() with invalid hostname: " + + hostname); + + } catch (UnknownHostException e) { + //ignore -- expected + } + + //Test String port + try { + ConfigurationUtil.computeCodebase( + hostname, jarFile, port.toString(), "", ""); + fail("Successfully called computeCodebase() with invalid hostname: " + + hostname); + + } catch (UnknownHostException e) { + //ignore -- expected + } + } + + @Test + /** + * Note: this test assumes a well-known jar file under Sun's JDK and as + * such is JDK specific. + */ + public void testComputeCodebase_5_arg_valid_hostname_httpmd_md5() + throws IOException, URISyntaxException + { + Integer port = new Integer(1234); + String hostname = InetAddress.getLocalHost().getHostAddress(); + String jarFile = "rt.jar"; + String jarSrcPath = System.getProperty("java.home") + + File.separator + "lib"; + //Test int port + String urlString = ConfigurationUtil.computeCodebase( + hostname, jarFile, port.intValue(), jarSrcPath, "md5"); //gen httpmd url + testComputeCodebase_5_arg_valid_hostname_httpmd_md5_assert( + urlString, hostname, port, jarFile); + //Test String port + urlString = ConfigurationUtil.computeCodebase( + hostname, jarFile, port.toString(), jarSrcPath, "md5"); //gen httpmd url + testComputeCodebase_5_arg_valid_hostname_httpmd_md5_assert( + urlString, hostname, port, jarFile); + } + + private void testComputeCodebase_5_arg_valid_hostname_httpmd_md5_assert( + String urlString, String hostname, Integer port, String jarFile) + throws MalformedURLException, URISyntaxException + { + URI uri = new URI(urlString); + assertEquals(uri.getScheme(), "httpmd"); + assertEquals(uri.getHost(), hostname); //use IP to avoid DNS conversion + assertEquals(uri.getPort(), port.intValue()); + assertTrue(uri.getPath().startsWith("/" + jarFile + ";md5=")); + } + + @Test + /** + * Note: this test assumes a well-known jar file under Sun's JDK and as + * such is JDK specific. + */ + public void testComputeCodebase_5_arg_valid_hostname_httpmd_sha() + throws IOException, URISyntaxException + { + Integer port = new Integer(1234); + String hostname = InetAddress.getLocalHost().getHostAddress(); + String jarFile = "rt.jar"; + String jarSrcPath = System.getProperty("java.home") + + File.separator + "lib"; + //Test int port + String urlString = ConfigurationUtil.computeCodebase( + hostname, jarFile, port.intValue(), jarSrcPath, "sha"); //gen httpmd url + testComputeCodebase_5_arg_valid_hostname_httpmd_sha_assert( + urlString, hostname, port, jarFile); + //Test String port + urlString = ConfigurationUtil.computeCodebase( + hostname, jarFile, port.toString(), jarSrcPath, "sha"); //gen httpmd url + testComputeCodebase_5_arg_valid_hostname_httpmd_sha_assert( + urlString, hostname, port, jarFile); + } + + private void testComputeCodebase_5_arg_valid_hostname_httpmd_sha_assert( + String urlString, String hostname, Integer port, String jarFile) + throws MalformedURLException, URISyntaxException + { + URI uri = new URI(urlString); + assertEquals(uri.getScheme(), "httpmd"); + assertEquals(uri.getHost(), hostname); //use IP to avoid DNS conversion + assertEquals(uri.getPort(), port.intValue()); + assertTrue(uri.getPath().startsWith("/" + jarFile + ";sha=")); + } + + @Test + public void testComputeCodebase_3_arg_null() throws IOException { + Integer port = new Integer(5); + String jarf = "bogus_jar_file.jar"; + String name = "bogus_name"; + Object[][] args = { + {null, null, port}, + {null, jarf, port}, + {name, null, port}, + }; + //Test int port method + for (Object[] arg: args) { + try { + ConfigurationUtil.computeCodebase( + (String)arg[0], (String)arg[1], + ((Integer)arg[2]).intValue()); + fail("Successfully called computeCodebase() with null arg: " + + Arrays.asList(arg)); + } catch (NullPointerException e) { + // ignore -- expected. + } + } + //Test String port method + for (Object[] arg: args) { + try { + ConfigurationUtil.computeCodebase( + (String)arg[0], (String)arg[1], + ((Integer)arg[2]).toString()); + fail("Successfully called computeCodebase() with null arg: " + + Arrays.asList(arg)); + } catch (NullPointerException e) { + // ignore -- expected. + } + } + } + + @Test + public void testComputeCodebase_3_arg_invalid_port() throws IOException { + Integer port = new Integer(-1); + //Try int port + try { + ConfigurationUtil.computeCodebase( + "bogus_name", "bogus_jar_file.jar", port.intValue()); + fail("Successfully called computeCodebase() with negative port: " + + port); + } catch (IllegalArgumentException e) { + // ignore -- expected. + } + //Try String port + try { + ConfigurationUtil.computeCodebase( + "bogus_name", "bogus_jar_file.jar", port.toString()); + fail("Successfully called computeCodebase() with negative port: " + + port); + } catch (IllegalArgumentException e) { + // ignore -- expected. + } + } + + @Test + public void testComputeCodebase_3_arg_valid_hostname_http() throws IOException { + Integer port = new Integer(1234); + String hostname = InetAddress.getLocalHost().getHostAddress(); + String jarFile = "file.jar"; + + String urlString = ConfigurationUtil.computeCodebase( + hostname, jarFile, port.intValue()); //gen http url + testComputeCodebase_3_arg_valid_hostname_http_assert( + urlString, hostname, port, jarFile); + + + urlString = ConfigurationUtil.computeCodebase( + hostname, jarFile, port.toString()); //gen http url + testComputeCodebase_3_arg_valid_hostname_http_assert( + urlString, hostname, port, jarFile); + } + + private void testComputeCodebase_3_arg_valid_hostname_http_assert( + String urlString, String hostname, Integer port, String jarFile) + throws MalformedURLException + { + URL url = new URL(urlString); + assertEquals(url.getProtocol(), "http"); + assertEquals(url.getHost(), hostname); //use IP to avoid DNS conversion + assertEquals(url.getPort(), port.intValue()); + assertEquals(url.getFile(), "/" + jarFile); + } + + @Test + public void testComputeCodebase_3_arg_invalid_hostname_http() throws IOException { + Integer port = new Integer(1234); + String hostname = UUID.randomUUID().toString(); + String jarFile = "file.jar"; + + //Test int port + try { + ConfigurationUtil.computeCodebase( + hostname, jarFile, port.intValue()); + fail("Successfully called computeCodebase() with invalid hostname: " + + hostname); + } catch (UnknownHostException e) { + //ignore -- expected + } + + //Test String port + try { + ConfigurationUtil.computeCodebase( + hostname, jarFile, port.toString()); + fail("Successfully called computeCodebase() with invalid hostname: " + + hostname); + } catch (UnknownHostException e) { + //ignore -- expected + } + } + + @Test + /** + * Note: this test assumes a well-known jar file under Sun's JDK and as + * such is JDK specific. + */ + public void testComputeCodebase_port_not_number() + throws IOException, URISyntaxException + { + String port = "cafebabe"; + String hostname = InetAddress.getLocalHost().getHostAddress(); + String jarFile = "rt.jar"; + String jarSrcPath = System.getProperty("java.home") + + File.separator + "lib"; + // 5 arg method + try { + ConfigurationUtil.computeCodebase( + hostname, jarFile, port, jarSrcPath, "md5"); //gen httpmd url + fail("Successfully called computeCodebase() with invalid port: " + + port); + } catch (NumberFormatException e) { + //ignore -- expected + } + // 3 arg method + try { + ConfigurationUtil.computeCodebase( + hostname, jarFile, port); //gen httpmd url + fail("Successfully called computeCodebase() with invalid port: " + + port); + } catch (NumberFormatException e) { + //ignore -- expected + } + } + + /** + * Helper method that joins the given collection of <code>String</code> using the + * ASCII record separator (RS) character(\036). + * @param s the collection of strings to join + * @return String containing the collection's elements separated by + * the RS delimiter. + */ + private static String join(Collection<String> s) { + if (s == null || s.isEmpty()) return ""; + Iterator<String> iter = s.iterator(); + StringBuilder builder = new StringBuilder(iter.next()); + while( iter.hasNext() ) + { + builder.append('\036').append(iter.next()); + } + return builder.toString(); + } + + /** + * Helper method that combines two <code>String</code> arrays in order in which + * they are provided. + * @param s the collection of strings to join + * @return String containing the collection's elements separated by + * the RS delimiter. + */ + private static String[] combine(String[]a, String[]b) { + List<String> l = new ArrayList<String>(); + l.addAll(Arrays.asList(a)); + l.addAll(Arrays.asList(b)); + return l.toArray(new String[0]); + } +} Added: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/Log4jLoggingHandlerTest.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/Log4jLoggingHandlerTest.java (rev 0) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/Log4jLoggingHandlerTest.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -0,0 +1,52 @@ +package com.bigdata.util.config; + +import static org.junit.Assert.*; + +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class Log4jLoggingHandlerTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testFlush() { + Log4jLoggingHandler l = new Log4jLoggingHandler(); + l.flush(); + } + + @Test + public void testClose() { + Log4jLoggingHandler l = new Log4jLoggingHandler(); + l.close(); + } + + @Test + public void testPublishLogRecord() { + Log4jLoggingHandler l = new Log4jLoggingHandler(); + LogRecord rec = new LogRecord(Level.SEVERE, "Log message"); + l.publish(rec); + } + + //TODO - test static logic +} Added: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/LogUtilTest.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/LogUtilTest.java (rev 0) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/LogUtilTest.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -0,0 +1,52 @@ +package com.bigdata.util.config; + +import static org.junit.Assert.*; + +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LogUtilTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testGetLog4jLoggerString() { + Logger l = LogUtil.getLog4jLogger(LogUtilTest.class); + assertNotNull(l); + assertEquals(LogUtilTest.class.getName(), l.getName()); + } + + @Test + public void testGetLog4jLoggerClass() { + Logger l = LogUtil.getLog4jLogger(LogUtilTest.class.getName()); + assertNotNull(l); + assertEquals(LogUtilTest.class.getName(), l.getName()); + } + + @Test + public void testGetLog4jRootLogger() { + Logger l = LogUtil.getLog4jRootLogger(); + assertNotNull(l); + assertEquals("root", l.getName()); + } + + //TODO - Still need to test static property logic +} Added: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/NicUtilTest.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/NicUtilTest.java (rev 0) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/config/NicUtilTest.java 2010-10-11 18:33:47 UTC (rev 3768) @@ -0,0 +1,1293 @@ +package com.bigdata.util.config; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.InetAddress; +import java.net.InterfaceAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class NicUtilTest { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testGetNetworkInterface_null() throws SocketException { + try { + NicUtil.getNetworkInterface(null); + fail("Successfully called getNetworkInterface() with null."); + } catch (NullPointerException e) { + //ignore -- expected + } + } + + @Test + public void testGetNetworkInterface_by_if_name() throws SocketException { + Enumeration<NetworkInterface> ifs = + NetworkInterface.getNetworkInterfaces(); + while(ifs.hasMoreElements()) { + NetworkInterface i = ifs.nextElement(); + assertEquals(NicUtil.getNetworkInterface(i.getName()), i); + } + } + + @Test + public void testGetNetworkInterface_by_host_name() throws SocketException { + boolean found = false; + for (Enumeration<NetworkInterface> ifs = + NetworkInterface.getNetworkInterfaces(); + ifs.hasMoreElements();) + { + NetworkInterface i = ifs.nextElement(); + for (Enumeration<InetAddress> ips = i.getInetAddresses(); + ips.hasMoreElements();) + { + InetAddress ip = ips.nextElement(); + NetworkInterface x = NicUtil.getNetworkInterface(ip.getHostAddress()); + assertNotNull(x); + assertEquals(i, x); + found = true; + } + } + + if (!found) fail("Could not find any IP address for testing."); + } + + @Test + public void testGetNetworkInterface_by_unknown_host_name() throws SocketException { + String unknownHost = getBogusHostname(); + NetworkInterface x = NicUtil.getNetworkInterface(unknownHost); + assertNull(x); + } + + @Test + public void testGetNetworkInterface_by_empty_host_name() throws SocketException { + NetworkInterface x = NicUtil.getNetworkInterface(""); + assertNotNull(x); + assertTrue(x.isLoopback()); + } + + @Test + public void testGetInetAddressMap() throws SocketException { + Map<InetAddress, String> expected = new HashMap<InetAddress, String>(); + for (Enumeration<NetworkInterface> ifs = + NetworkInterface.getNetworkInterfaces(); + ifs.hasMoreElements();) + { + NetworkInterface i = ifs.nextElement(); + for (Enumeration<InetAddress> ips = i.getInetAddresses(); + ips.hasMoreElements();) + { + InetAddress ip = ips.nextElement(); + expected.put(ip, i.getName()); + } + } + Map<InetAddress, String> actual = NicUtil.getInetAddressMap(); + assertEquals(actual, expected); + } + + @Test + public void testGetNetworkInterfaceArray_null() throws SocketException { + try { + NicUtil.getNetworkInterfaceArray(null); + fail("Successfully called getNetworkInterfaceArray with null."); + } catch (NullPointerException e) { + // ignore -- expected + } + } + + @Test + public void testGetNetworkInterfaceArray_unknown_host_name() throws SocketException { + String unknownHost = getBogusHostname(); + NetworkInterface[] x = NicUtil.getNetworkInterfaceArray(unknownHost); + assertEquals(x.length, 0); + } + + @Test + public void testGetNetworkInterfaceArray_known_host_name() throws SocketException { + Enumeration<NetworkInterface> ns = NetworkInterface.getNetworkInterfaces(); + List<NetworkInterface> ls = Collections.list(ns); + assert(ls.size()>0); + //Pick random entry from list. + Random r = new Random(); + //Note: size() must > 0, here. + String ifName = ls.get(r.nextInt(ls.size())).getName(); + NetworkInterface[] nifs = NicUtil.getNetworkInterfaceArray(ifName); + assertTrue(nifs.length > 0); + assertEquals(nifs[0].getName(), ifName); + } + + @Test + public void testGetNetworkInterfaceArray_known_ip_address() throws SocketException { + Result<String, NetworkInterface> res = + getNonLoopbackNetworkInterfaceWithInet4Address(); + NetworkInterface[] nifs = NicUtil.getNetworkInterfaceArray(res.getValue1()); + assertEquals(nifs.length, 1); + assertEquals(nifs[0], res.getValue2()); + } + + @Test + public void testGetNetworkInterfaceArray_all() throws SocketException { + List<NetworkInterface> expected = + Collections.list(NetworkInterface.getNetworkInterfaces()); + assertTrue(expected.size() > 0); + List<NetworkInterface> actual = + Arrays.asList(NicUtil.getNetworkInterfaceArray("all")); + assertEquals(actual.size(), expected.size()); + //Remove actual from expected -- should leave an empty list + expected.removeAll(actual); + assertTrue(expected.isEmpty()); + } + + @Test + public void testGetInetAddress_npe_args() { + String name = null; + int index = 0; + String host = null; + boolean localHost = false; + try{ + NicUtil.getInetAddress(name, index, host, localHost); + fail("Successfully called getInetAddress with invalid arguments."); + } catch (NullPointerException e) { + //ignore -- expected + } + } + + @Test + public void testGetInetAddress_negative_index() { + String name = null; + int index = -1; + String host = null; + boolean localHost = true; + try{ + NicUtil.getInetAddress(name, index, host, localHost); + fail("Successfully called getInetAddress with negative index."); + } catch (IllegalArgumentException e) { + //ignore -- expected + } + } + + @Test + public void testGetInetAddress_get_localhost_only() throws UnknownHostException { + String name = null; + int index = 0; + String host = null; + boolean localHost = true; + InetAddress expected = InetAddress.getLocalHost(); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertEquals(expected, actual); + } + + @Test + public void testGetInetAddress_get_unknown_host_no_localhost() + throws UnknownHostException + { + String name = null; + int index = 0; + String host = getBogusHostname(); + boolean localHost = false; + InetAddress expected = null; + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertEquals(expected, actual); + } + + @Test + public void testGetInetAddress_get_unknown_host_with_localhost() + throws UnknownHostException + { + String name = null; + int index = 0; + String host = getBogusHostname(); + boolean localHost = true; + InetAddress expected = InetAddress.getLocalHost(); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertEquals(expected, actual); + } + + @Test + public void testGetInetAddress_get_known_host_no_localhost() + throws UnknownHostException + { + String name = null; + int index = 0; + String host = InetAddress.getLocalHost().getHostName(); + boolean localHost = false; + InetAddress expected = InetAddress.getLocalHost(); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertEquals(expected, actual); + } + + @Test + public void testGetInetAddress_get_known_host_with_localhost() + throws UnknownHostException + { + String name = null; + int index = 0; + String host = InetAddress.getLocalHost().getHostName(); + boolean localHost = true; + InetAddress expected = InetAddress.getLocalHost(); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertEquals(expected, actual); + } + + @Test + public void testGetInetAddress_get_known_name_no_host_or_localhost() + throws UnknownHostException, SocketException + { + Result<String, NetworkInterface> res = + getNonLoopbackNetworkInterfaceWithInet4Address(); + String name = res.getValue2().getName(); + int index = 0; + String host = null; + boolean localHost = false; + List<InetAddress> expectedAddresses = + Collections.list(res.getValue2().getInetAddresses()); + assertFalse(expectedAddresses.isEmpty()); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertNotNull(actual); + assertTrue(expectedAddresses.contains(actual)); + } + + @Test + public void testGetInetAddress_get_known_name_with_host_no_localhost() + throws UnknownHostException, SocketException + { + Result<String, NetworkInterface> res = + getNonLoopbackNetworkInterfaceWithInet4Address(); + String name = res.getValue2().getName(); + int index = 0; + String host = InetAddress.getLocalHost().getHostName(); + boolean localHost = false; + List<InetAddress> expectedAddresses = + Collections.list(res.getValue2().getInetAddresses()); + assertFalse(expectedAddresses.isEmpty()); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertNotNull(actual); + assertTrue(expectedAddresses.contains(actual)); + } + + //TODO - Another test which ensures host != result.NetworkInterface + + @Test + public void testGetInetAddress_get_known_name_with_host_with_localhost() + throws UnknownHostException, SocketException + { + Result<String, NetworkInterface> res = + getNonLoopbackNetworkInterfaceWithInet4Address(); + String name = res.getValue1(); + int index = 0; + String host = InetAddress.getLocalHost().getHostName(); + boolean localHost = true; + List<InetAddress> expectedAddresses = + Collections.list(res.getValue2().getInetAddresses()); + assertFalse(expectedAddresses.isEmpty()); + InetAddress actual = NicUtil.getInetAddress(name, index, host, localHost); + assertNotNull(actual); + assertTrue(expectedAddresses.contains(actual)); + } + + //TODO - Another test which ensures host != result.NetworkInterface + + //TODO - test for multi-IP interface (e.g. index arg > 0) + + @Test + public void testGetMacAddress_null() throws SocketException { + try { + NicUtil.getMacAddress(null); + fail("Successfully called getMacAddress with null."); + } catch (NullPointerException e) { + //ignore -- exception + } + } + + @Test + public void testGetMacAddress_unknown() throws SocketException { + String mac = NicUtil.getMacAddress(UUID.randomUUID().toString()); + assertNull(mac); + } + + @Test + public void testGetMacAddress_known() throws SocketException, ParseException { + Result<byte[], NetworkInterface> res = + getFirstNetworkInterfaceWithMacAddress(); + String mac = NicUtil.getMacAddress(res.getValue2().getName()); + assertNotNull(mac); + //Note: not verifying MAC address via string representation + } + + + @Test + public void testGetIpAddressString_npe() { + try { + NicUtil.getIpAddress(null); + fail("Successfully called getIpAddress with null."); + } catch (NullPointerException e) { + //ignore -- expected + } + } + + @Test + public void testGetIpAddressString_unknown_host() { + assertNull(NicUtil.getIpAddress(getBogusHostname())); + } + + @Test + public void testGetIpAddressString_known_host() throws SocketException { + Result<String, NetworkInterface> res = + getNonLoopbackNetworkInterfaceWithInet4Address(); + String actual = + NicUtil.getIpAddress(res.getValue2().getName()); + assertEquals(res.getValue1(), actual); + } + + @Test + public void testGetIpAddressStringInt_npe() { + try { + NicUtil.getIpAddress(null, 0); + fail("Successfully called getIpAddress with null."); + } catch (NullPointerException e) { + //ignore -- expected + } + } + + @Test + public void testGetIpAddressStringInt_negative_index() { + try { + NicUtil.getIpAddress("", -1); + fail("Successfully called getIpAddress with" + + "a negative index."); + } catch (IllegalArgumentException e) { + //ignore -- expected + } + } + + @Test + public void testGetIpAddressStringInt_unknown_host() { + assertNull(NicUtil.getIpAddress(getBogusHostname(), 0)); + } + + @Test + public void testGetIpAddressStringInt_known_host() throws SocketException { + Result<String, NetworkInterface> res = + getNonLoopbackNetworkInterfaceWithInet4Address(); + String actual = + NicUtil.getIpAddress(res.getValue2().getName(), 0); + assertEquals(res.getValue1(), actual); + } + + @Test + public void testGetIpAddressStringString_npe() { + try { + NicUtil.getIpAddress((String)null, (String)null); + fail("Succcessfully called getIpAddress with null."); + }catch (NullPointerException e... [truncated message content] |
From: <tho...@us...> - 2010-10-11 17:26:09
|
Revision: 3767 http://bigdata.svn.sourceforge.net/bigdata/?rev=3767&view=rev Author: thompsonbry Date: 2010-10-11 17:26:00 +0000 (Mon, 11 Oct 2010) Log Message: ----------- Integrated the Sesame TCK with the embedded federation for use in CI validation of the scale-out quads mode query. This commit also fixes a few places in the query plan generation, including: - SliceOp wraps the query plan so the top-level operator always runs on the query controller. - Access path flags set the READONLY bit. - Cost estimates needed to use the glocal index view (remoteAccessPath=true). As of now, all but 4 of the tests in the SPARQL test suite are passing against the embedded federation in quads mode. The four tests which are still failing are: - "Union is not optional". - "graph-07" - "graph-11" - "SELECT DISTINCT *" Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/mutation/InsertOp.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/SimpleDataServiceProcedureTask.java branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedFederation.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/AbstractBigdataSailTestCase.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/QuadsTestCase.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestAll.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestOptionals.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java Added Paths: ----------- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailEmbeddedFederationWithQuads.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataEmbeddedFederationSparqlTest.java Removed Paths: ------------- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/TestBaseball.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest2.java Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -37,7 +37,9 @@ import com.bigdata.bop.join.PipelineJoin; import com.bigdata.btree.IRangeQuery; import com.bigdata.btree.ITuple; +import com.bigdata.btree.ITupleCursor; import com.bigdata.btree.ITupleIterator; +import com.bigdata.btree.filter.Advancer; import com.bigdata.btree.filter.TupleFilter; import com.bigdata.mdi.PartitionLocator; import com.bigdata.relation.IRelation; @@ -256,15 +258,23 @@ int DEFAULT_FULLY_BUFFERED_READ_THRESHOLD = 100; /** - * Flags for the iterator ({@link IRangeQuery#KEYS}, - * {@link IRangeQuery#VALS}, {@link IRangeQuery#PARALLEL}). + * Specify the {@link IRangeQuery} flags for the {@link IAccessPath} ( + * default is {@link IRangeQuery#KEYS}, {@link IRangeQuery#VALS}). * <p> - * Note: The {@link IRangeQuery#PARALLEL} flag here is an indication - * that the iterator may run in parallel across the index partitions. - * This only effects scale-out and only for simple triple patterns since - * the pipeline join does something different (it runs inside the index - * partition using the local index, not the client's view of a - * distributed index). + * Note: Most access paths are read-only so it is nearly always a good + * idea to set the {@link IRangeQuery#READONLY} flag. + * <p> + * Note: Access paths used to support high-level query can nearly always + * use {@link IRangeQuery#PARALLEL} iterator semantics, which permits + * the iterator to run in parallel across index partitions in scale-out. + * This flag only effects operations which use a global index view in + * scale-out ( pipeline joins do something different). + * <p> + * Note: Some expanders may require the {@link IRangeQuery#CURSOR} flag. + * For example, {@link Advancer} patterns use an {@link ITupleCursor} + * rather than an {@link ITupleIterator}. However, since the cursors are + * <i>slightly</i> slower, they should only be specified when their + * semantics are necessary. * * @see #DEFAULT_FLAGS */ @@ -272,13 +282,11 @@ /** * The default flags will visit the keys and values of the non-deleted - * tuples and allows parallelism in the iterator (when supported). - * - * @todo consider making parallelism something that the query planner - * must specify explicitly. + * tuples. */ final int DEFAULT_FLAGS = IRangeQuery.KEYS | IRangeQuery.VALS - | IRangeQuery.PARALLEL; +// | IRangeQuery.PARALLEL + ; } Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -34,6 +34,12 @@ */ public ScanCostReport(final long rangeCount, final double cost) { + if (rangeCount < 0) + throw new IllegalArgumentException(); + + if (cost < 0) + throw new IllegalArgumentException(); + this.rangeCount = rangeCount; this.shardCount = 1; Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -31,7 +31,7 @@ * An estimated cost (latency in milliseconds) based on the samples and * adjusted for the #of graphs. */ - public final double subqueryCost; + public final double cost; /** * @@ -44,18 +44,38 @@ * @param rangeCount * An estimated range count based on the samples and adjusted * for the #of graphs. - * @param subqueryCost + * @param cost * An estimated cost (latency in milliseconds) based on the * samples and adjusted for the #of graphs. */ public SubqueryCostReport(final int ngraphs, final int limit, - final int nsamples, final long rangeCount, - final double subqueryCost) { + final int nsamples, final long rangeCount, final double cost) { + + if (ngraphs < 0) + throw new IllegalArgumentException(); + + if (limit < 1) + throw new IllegalArgumentException(); + + if (nsamples < 0) + throw new IllegalArgumentException(); + + if (rangeCount < 0) + throw new IllegalArgumentException(); + + if (cost < 0) + throw new IllegalArgumentException(); + this.ngraphs = ngraphs; + this.limit = limit; + this.nsamples = nsamples; + this.rangeCount = rangeCount; - this.subqueryCost = subqueryCost; + + this.cost = cost; + } /** @@ -67,7 +87,7 @@ ",limit=" + limit + // ",nsamples=" + nsamples + // ",rangeCount=" + rangeCount + // - ",subqueryCost=" + subqueryCost + // + ",cost=" + cost + // "}"; } Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-10-11 17:26:00 UTC (rev 3767) @@ -1,571 +0,0 @@ -RunningQuery: - - - TestJiniFederatedQueryEngine. - - - Reconcile IElementFilter, FilterConstructor, and stackable - striterators for access paths and raw iterators. We should be - able to use any of the striterator patterns! - - There are a lot of partially duplicated classes which need to be - cleared up. This can be tested on the individual striterator / - filter and also on the local and remote access paths. - - Note: Some filters can be sent with a remote iterator request (for - example, the advancer), but most are applied after we convert from - tuples into elements. - - IFilterConstructor only works with ITupleIterator. - - IElementFilter only filters. It can not resolve, transform, chunk, - etc. It is currently sent with the iterator to the B+Tree. See - AccessPath#iterator(). - - - Break the DistinctSPOIterator into a Distinct(Element)Filter and a - StripContextFilter. - - - AbstractTuple#getObject() should cache the materialized object - since some patterns used by an AccessPath may request the object - multiple times. Of course, caching implies that we must clear the - reference when advancing the iterator. Check TupleFilter and - friends since there might need to be more than once class which - caches the object. Clear the reference when the iterator is - exhausted. - - - Adapt the code to use the cost models and decision trees and write - Unit tests of the default and named graph access path patterns. - - - Implement subquery support and subquery operators {Union, Steps, - Star}. (We can't test Star until we have the mutation API in - place.) - - - PipelineType {Vectored,OneShot}. - - A vectored operator processes its inputs in chunks, producing - output chunks each time it runs. - - An one shot operator runs exactly once for a given query and - must wait for all of its inputs to become available before it - can begin. For example, SORT is a one shot operator. - - - Mutation {Insert, Delete}. Concurrency control. Scale-out - mutation operators must go through the ConcurrencyManager in order - to respect the isolation levels imposed within AbstractTask. For - standalone, we can using either UnisolatedReadWriteIndex or the - ConcurrencyManager as appropriate (but how can we tell which is - appropriate!?!). - - Note: AccessPath currently does not allow inserts (just - deletes). Hopefully it can be extended to allow INSERT and UPDATE - so we can use the same abstraction for mutation operators. - - - Mutation {Create, Destroy}. This gets into resource management, - so defer for the moment but tackle in the context of RDFS closure - using STAR. - - - MemoryType {Chunked,Blocked}. - - Blocked operators need to inherit some interface which - declares annotations for the #of blocks allocated to an - operator. How those blocks are used is very much operator - specific. For example, an external merge sort can make - different use of its buffers than some other kind of - algorithm. - - OneShot operators may be allocated some #of blocks which they - can use to buffer their inputs. When those blocks are - exhausted, then they will have to start dumping inputs to the - disk. However, that is operator specific and not yet defined. - - /** - * Chunked means that the operator handles variable sized chunks of - * intermediate results where the results are managed on the Java heap. - * Chunked operators are useful for low-latency queries where the latency to - * the first result should be low. Chunked operators generally have low - * latency to the first query result because the data flows the operators in - * smaller chunks with high overall parallelism in the query. - * <p> - * Chunk sizes of ~ 100 appear to work well for low latency queries and - * strike a balance several factors, including: latency, heap churn, RMI - * overhead (in scale-out), and IO vectoring. Chunks may be automatically - * combined by the pipeline. In scale-out, chunks may be migrated onto the C - * heap for transfer across network boundaries and transparently inflated - * into Java objects by {@link #Chunked} operators. - * <p> - * Java GC can handle high object creation rates very nicely as long as the - * objects have short life cycles. To avoid causing Java GC problems, the - * chunk size should be kept moderate such that chunked operators do not - * create large collections of long lived objects. - */ - Chunked, - - /** - * Blocked means that the operator natively manages some number of fixed - * capacity {@link ByteBuffer}s. Such buffers are allocated on the C heap - * using {@link ByteBuffer#allocateDirect(int)}. A population of such - * buffers are managed by the {@link DirectBufferPool}. Direct buffers - * provide fast transfer between disk and network subsystems and may be used - * to share data with other devices, including GPUs. - */ - Blocked, - -Note: Many of the maxParallel annotations related to thread -consumption will go away with Java7 and async file IO. Other -annotations, such as the #of 1M buffers to allocate to an operator, -need to be introduced to handle high volume queries. - -Note: UNION, STEPS, and STAR(transitive closure) are all evaluated on -the query controller. Do these together. However, I need to handle -Insert/Delete before I can do STAR since it is predicated on the -mutation count reporting. Also, make sure that we CAN do unisolated -access path reads, just in case. - ----- - -SPARQL named graph query patterns. - -Standalone: - - Named graph queries use an expander pattern. See - NamedGraphSolutionExpander. - - The following special cases exist: - - - The named graph data set is empty (no graphs were identified which - are known to the database), in which case an empty access path is - used. - - - The named graph data set includes a single graph which is known to - the database. C is bound and we use the normal access path (this - is done by special case logic in NamedGraphSolutionExpander and - should be done by the query rewrite instead). - - - The named graph data set includes all graphs. C is left unbound - and the unmodified access path is used. - - - The named graph data set includes more than a threshold number of - graphs. The context position is left unbound and an IN filter is - applied to retrict the access path to the desired graphs. See - NamedGraphsFilteredAccessPath. - - FIXME The threshold for this case is 200, which is WAY too - low. - - For standalone the decision should be based on whether more - leaves would be read (the tuple range count may be used as a - proxy for this) by issuing individual subqueries for the - specific as bound predicates or by reading with C unbound. - - For scale-out, the decision is different since we can use - multi-block iterators on the index segments and do fewer - disk seeks. - - - The named graph data set includes more than one graph but less - than some threshold #of graphs. Parallel subtasks are evaluated - for each graph in the data set and write on a shared - BlockingBuffer. See NamedGraphsParallelEvaluationAccessPath. - - @todo This is equivelant to an in-memory join, but does not - scale-out. Replace this with a DataSetJoin rather than the - expander pattern since that will work for both standalone and - scale-out. - -Scale-out: - - As per above, except: - - - When the #of named graphs is moderate, the data set graphs are - joined with the source binding sets (using DataSetJoin) to produce - a cross product in which each source binding set is replicated for - each distinct data set graph. Those binding sets are then fed - into a second join which reads on the access path for the - appropriate statement index. - - - When the #of named graphs is large we need to do something special - to avoid sending huge graph sets around with the query. - ----- - -SPARQL default graph query patterns. - -Note: Default graph queries require us to apply a distinct {s,p,o} -filter to each default graph access path. The default graph queries -uses an expander pattern. See DefaultGraphSolutionExpander and its -inner classes. - -The following special cases exist: - - - The default graph data set is empty (no graphs were identified - which are known to the database), in which case an empty access - path is used. - - - The default graph data set includes a single graph which is known - to the database. C is bound and we impose a filter which strips - off the context position. Because C takes on only one value, a - distinct filter is not required. This means that scale-out can - use normal pipeline joins. - - See StripContextAccessPath. - - - ___C index: We know that C is strictly ascending in index order - within each triple. Use an advancer pattern or ignore quads until - the data changes to a new triple. Apply a filter to strip off the - context position. - - The same optimization works in scale-out using shard-wise pipeline - joins if the ___C index was created with the constraint that the - all quads for a given triple are on the same shard. - - - SCALEOUT and VERY HIGH VOLUME: Use a distributed external merge - sort to impose distinct and do operator at a time processing. - - - SCAN and FILTER: The default graph data set includes all graphs OR - the cost of scanning with C unbound is less than the cost of - subqueries with C bound (for scale-out, subquery cost must be - estimated for a remote access path). C is left unbound and we - impose a distinct SPO filter which strips off the context - position. Unless all graphs are being merged, we also apply an IN - filter. - - SCALEOUT: The join evaluation context is ANY, uses a remote access - path, and the access path should be configured to move a - lot of data efficiently over the remote range iterator. - - - It is possible to partition the IN filter based on the - shard on which it will be applied (split the ordered - list of contexts based on the contexts found in a - given shard). - - See MergeAllGraphsAccessPath. - - - SUBQUERY: Parallel subtasks are evaluated for each graph in the - data set and write on a shared BlockingBuffer. The BlockingBuffer - is wrapped with an SPORelation.distinctSPOIterator(). - - SCALEOUT: Mark the join evaluation context as ANY and mark the - access path as remote. - - Tune the capacity for the remote access path iterator. - When the remote access path will be selective, the - capacity should be small and we will pay a high price if - there are a lot of nested subqueries. When the remote - access path is less selective the capacity should be - larger to reduce the #of RMI requests made per access - path. - - Note: The way the code is written, the access path will - do RMI for the range count before issuing the RMI - iterator request. Look at ways to optimize this. - - See DefaultGraphParallelEvaluationAccessPath. - -- @todo Lazily create the hash map for the distinctSPOIterator when we - observe the 2nd distinct SPO value. - ---- -UNION(ops)[maxParallel(default all)] - -Executes each of the operands in the union as subqueries. Each -subquery is run as a separate RunningQuery but is linked to the parent -query in which the UNION is being evaluated. The subqueries do not -receive bindings from the parent and may be executed independently. - -Note: In order to avoid materializing all of the intemediate results -on the query controller, the target for the subqueries SHOULD be -overriden to be whatever operator is the parent of the UNION. - ---- -STEPS(ops)[maxParallel(default 1)] - -The operands are executed as independent subqueries. - -@todo It should be possible to write the results for each step onto a -named query local resource so they can be reused in subsequent steps. - -@todo This operator is really no different from UNION. UNION defaults -to running all in parallel while STEPS defaults to running them -sequentially. In addition, there is an assumption for UNION that the -operands return binding sets and an assumption for STEPS that they are -mutation operators. However, what makes the UNION work is that the -operands target the UNION's parent. - ---- - -STAR(op) [maxItr(default all)] - -Evaluate the operand until its mutation count remains unchanged from -one round to the next. The operand must write on a resource. The -fixed point is determined by examining BOPStats.mutationCount. - -Do with INSERT/REMOVE since all involve mutation. - ---- -INSERT(op,pred) : insert elements into an index. -DELETE(op,pred) : remove elements from an index. - -The access path mutation operators construct elements from the source -binding sets and the asBBound predicates. For each element so -constructed, they insert/ remove the corresponding element into/from -the access path. These operators update a mutation counter IFF the -access path was modified for the constructed element. STAR relies on -the mutation operator to detect a fixed point. - -The mutation access paths need to use the appropriate concurrency -control to ensure the constraint on the mutable B+Tree is respected. -This is either the UnisolatedReadWriteIndex or the LockManager / -ConcurrencyManager. - -The basic mutation operators write on an access path and may be -combined using STEPS in order to update all of the indices associated -with a relation. - - - For incremental TM, we also need to construct an element for the just index - from the rule and assert it onto that index. - - - For the lexicon, we also need to write on the full text index. - - - For SIDs mode, we also need to capture the logic to ground the statements by - binding the SIDs. - - - triggers could be integrated here. perhaps events backed by a queue which - could be either restart safe or query local? - ----- -Parallel distributed closure : TBD. Review notes posted on trak. - ----- -Lexicon joins - - -==== -Features: - - - operator-at-once evaluation. The operator is triggered once its possible - triggers are done. This is just an application of the same utility method - which we use to decide when a query is done. - - - ISimpleSplitHandler to enforce constraint on the SPOC index such - that an SPO prefix is never split. This has to be IV aware since - we are now using variable length keys in the statement indices. - - - query and connection local resources: creating, destroying and - using resources. references to query local resources permit reuse - of intermediate results across different. - - CREATE FOO AS TEMP GRAPH ON LOCAL TEMP STORE SPO ONLY SHARED LEXICON - - - subquery evaluation (linked parent to child). a subquery may be cancelled - by a slice without cancelling the parent. cancelling the parent terminates - all subqueries. whenever a query or subquery is terminated, we need to go - through its operator and query life cycle tear down methods (unit tests). - - - "thick" resources which can be sent along with the query or access either by - RMI or copied to the node where the query is running on demand. (This could - be just alternative access path instantiations which are selected by the query - optimizer or defaulted based on the amount of data to be moved to/from the - node if not specified.) - - - The Predicate could have fromRevision/toRevision annotations which would be - used for fast computation of the delta between two historical commit points. - - - * FIXME Unit tests for non-distinct {@link IElementFilter}s on an - * {@link IPredicate}, unit tests for distinct element filter on an - * {@link IPredicate} which is capable of distributed operations. Do not use - * distinct where not required (SPOC, only one graph, etc). - * <p> - * It seems like the right way to approach this is by unifying the stackable CTC - * striterator pattern with the chunked iterator pattern and passing the query - * engine (or the bop context) into the iterator construction process (or simply - * requesting that the query engine construct the iterator stack). - * <p> - * In terms of harmonization, it is difficult to say which way would work - * better. In the short term we could simply allow both and mask the differences - * in how we construct the filters, but the conversion to/from striterators and - * chunked iterators seems to waste a bit of effort. - * <p> - * The trickiest part of all of this is to allow a distributed filter pattern - * where the filter gets created on a set of nodes identified by the operator - * and the elements move among those nodes using the query engine's buffers. - * <p> - * To actually implement the distributed distinct filter we need to stack the - * following: - * - * <pre> - * - ITupleIterator - * - Resolve ITuple to Element (e.g., SPOC). - * - Layer on optional IElementFilter associated with the IPredicate. - * - Layer on SameVariableConstraint iff required (done by AccessPath) - * - Resolve SPO to SPO, stripping off the context position. - * - Chunk SPOs (SPO[], IKeyOrder), where the key order is from the access path. - * - Filter SPO[] using DHT constructed on specified nodes of the cluster. - * The SPO[] chunks should be packaged into NIO buffers and shipped to those - * nodes. The results should be shipped back as a bit vectors packaged into - * a NIO buffers. - * - Dechunk SPO[] to SPO since that is the current expectation for the filter - * stack. - * - The result then gets wrapped as a {@link IChunkedOrderedIterator} by - * the AccessPath using a {@link ChunkedArrayIterator}. - * </pre> - * - * This stack is a bit complex(!). But it is certainly easy enough to generate - * the necessary bits programmatically. - * - * FIXME Handling the {@link Union} of binding sets. Consider whether the chunk - * combiner logic from the {@link DistributedJoinTask} could be reused. - * - * FIXME INSERT and DELETE which will construct elements using - * {@link IRelation#newElement(java.util.List, IBindingSet)} from a binding set - * and then use {@link IMutableRelation#insert(IChunkedOrderedIterator)} and - * {@link IMutableRelation#delete(IChunkedOrderedIterator)}. For s/o, we first - * need to move the bits into the right places so it makes sense to unpack the - * processing of the loop over the elements and move the data around, writing on - * each index as necessary. There could be eventually consistent approaches to - * this as well. For justifications we need to update some additional indices, - * in which case we are stuck going through {@link IRelation} rather than - * routing data directly or using the {@link IAsynchronousWriteBufferFactory}. - * For example, we could handle routing and writing in s/o as follows: - * - * <pre> - * INSERT(relation,bindingSets) - * - * expands to - * - * SEQUENCE( - * SELECT(s,p,o), // drop bindings that we do not need - * PARALLEL( - * INSERT_INDEX(spo), // construct (s,p,o) elements and insert - * INSERT_INDEX(pos), // construct (p,o,s) elements and insert - * INSERT_INDEX(osp), // construct (o,s,p) elements and insert - * )) - * - * </pre> - * - * The output of the SELECT operator would be automatically mapped against the - * shards on which the next operators need to write. Since there is a nested - * PARALLEL operator, the mapping will be against the shards of each of the - * given indices. (A simpler operator would invoke - * {@link SPORelation#insert(IChunkedOrderedIterator)}. Handling justifications - * requires that we also formulate the justification chain from the pattern of - * variable bindings in the rule). - * - * FIXME Handle {@link Program}s. There are three flavors, which should probably - * be broken into three operators: sequence(ops), set(ops), and closure(op). The - * 'set' version would be parallelized, or at least have an annotation for - * parallel evaluation. These things belong in the same broad category as the - * join graph since they are operators which control the evaluation of other - * operators (the current pipeline join also has that characteristic which it - * uses to do the nested index subqueries). - * - * FIXME SPARQL to BOP translation - * <p> - * The initial pass should translate from {@link IRule} to {@link BOp}s so we - * can immediately begin running SPARQL queries against the {@link QueryEngine}. - * A second pass should explore a rules base translation from the openrdf SPARQL - * operator tree into {@link BOp}s, perhaps using an embedded {@link Prolog} - * engine. What follows is a partial list of special considerations for that - * translation: - * <ul> - * <li>Distinct can be trivially enforced for default graph queries against the - * SPOC index.</li> - * <li>Local distinct should wait until there is more than one tuple from the - * index since a single tuple does not need to be made distinct using a hash - * map.</li> - * <li>Low volume distributed queries should use solution modifiers which - * evaluate on the query controller node rather than using distributed sort, - * distinct, slice, or aggregation operators.</li> - * <li></li> - * <li></li> - * <li></li> - * <li>High volume queries should use special operators (different - * implementations of joins, use an external merge sort, etc).</li> - * </ul> - * - * FIXME SPARQL Coverage: Add native support for all SPARQL operators. A lot of - * this can be picked up from Sesame. Some things, such as isIRI() can be done - * natively against the {@link IV}. Likewise, there is already a set of - * comparison methods for {@link IV}s which are inlined values. Add support for - * <ul> - * <li></li> - * <li></li> - * <li></li> - * <li></li> - * <li></li> - * <li></li> - * </ul> - ------------------------------------------------------------- - -Some problems: - -- Should Appender simply pass the other iterator as m_state to the FilterBase? - -- Mapper uses non-Serializable "Method". We should provide defered reflection for the method. - -- Merger uses non-Serializable "Iterator". - -- Sorter and Resolver should use an annotation pattern so we can create instances from Prolog of the operator. I do not think we can do this when the operator is abstract. Maybe we can have two "Resolver" classes, two "Sorter" classes, etc. The simple one can be used for inline programming. The other one will extend BOpBase and implement IFilter and will be used for query. - -- CONTRACTOR should be able to break an iterator into many chunks, not just one. Maybe the API should return an Iterator from an Iterator in which the chunkiness is changed (from element to element[])? - -Here is what I have not done yet: - -- Striterator unit tests. - -- Bop-erator tests. - -- MGC: All of the Filterators need to use deferred prefetch. Prefetch - during the constructor causes problems when we are stacking filters - using FilterBase. (This is also true for the ITuple filters). - -- BT: Write unit tests at the IPredicate/AccessPath level to use - stackable filters (for the LOCAL and REMOTE access paths). Write - unit tests for correct range counts with and without local/remote - filters. Write unit tests for caching of those range counts. - -- Write BOp versions for TupleResolver, ... - -- done. IPredicate#getConstraint() must go (issues remain with - BackchainAccessPath). - -- The (SPO|Magic)Predicate contructors must be touched up. - - - They assume an IElementFilter (IFilterTest). However, the filters - are now specified with IFilter. - - - Those constructor will DROP the local and remote filters unless - the logic is modified. - - - When used remotely, IElementFilter must do ITuple.getObject() and - MUST be wrapped using ElementFilter to be "Tuple" aware (problem - with Tuple reference reuse inside the ITupleIterators). - - - Reconcile IElementFilter and implementations. IElementFilter was - transparently wrapped by AccessPath to resolve the ITuple to the - element before applying the filter. If IElementFilter is used in - contexts other than the index then all implementations must be - modified to conditionally resolve the element while filtering. - - - IPredicates created from a Relation MUST use the timestamp of that - relation NOT READ_COMMITTED. Remove the default timestamp - annotation. Make this required where it is used so we can track - all of this down. - -MikeP: - - - BackchainAccessPath. Are the index local or access path filters - ever used here? If they are then this code needs to be modified. - - - The timestamp on which the backchain access path will read MUST be - passed through to the IPredicate constructors. - - - Leave C as an anonymous variable when it will not be used, not - null. - - - Replace NamedGraph and DefaultGraph access paths per the decision - tree and cost model. - - - Set the TIMESTAMP on the predicate. - - - Additional SPOPredicate and Predicate constructor cleanup. - -Reconcile the com.bigdata.striterators package. Can we get rid of it? -Incrementally? Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -284,7 +284,8 @@ * by the client. */ throw new RuntimeException( - "The top-level of a query must be evaluated on the query controller."); + "The top-level of a query must be evaluated on the query controller: query=" + + getQuery()); } Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/mutation/InsertOp.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/mutation/InsertOp.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/mutation/InsertOp.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -294,6 +294,12 @@ * * FIXME Allow remote writes as well if a remote access path is * marked on the {@link IPredicate}. + * + * FIXME There is currently a problem obtaining the UNISOLATED + * index in scale-out using the DelegateIndexManager. The issue + * is down in the guts of how AbstractTask exposes its views of + * the indices and manifests as a problem an assert in Name2Addr + * concerning the dirtyListener implementation. */ public <T> ILocalBTreeView getMutableLocalIndexView( final IRelation<T> relation, final IKeyOrder<T> keyOrder, Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -288,53 +288,22 @@ * The order in which the elements would be visited for this * access path. */ -// * @param ndx -// * The index on which the access path is reading. -// * @param flags -// * The default {@link IRangeQuery} flags. -// * @param chunkOfChunksCapacity -// * The #of chunks that can be held by an {@link IBuffer} that is -// * the target or one or more producers. This is generally a small -// * number on the order of the #of parallel producers that might -// * be writing on the {@link IBuffer} since the capacity of the -// * {@link UnsynchronizedArrayBuffer}s is already quite large (10k -// * or better elements, defining a single "chunk" from a single -// * producer). -// * @param chunkCapacity -// * The maximum size for a single chunk (generally 10k or better). -// * @param fullyBufferedReadThreshold -// * If the estimated remaining rangeCount for an -// * {@link #iterator(long, long, int)} is LTE this threshold then -// * we will do a fully buffered (synchronous) read. Otherwise we -// * will do an asynchronous read. public AccessPath(// final IRelation<R> relation,// final IIndexManager localIndexManager, // -// final long timestamp,// final IPredicate<R> predicate,// final IKeyOrder<R> keyOrder // -// final IIndex ndx,// -// final int flags, // -// final int chunkOfChunksCapacity, -// final int chunkCapacity, -// final int fullyBufferedReadThreshold ) { if (relation == null) throw new IllegalArgumentException(); -// if (indexManager == null) -// throw new IllegalArgumentException(); - if (predicate == null) throw new IllegalArgumentException(); if (keyOrder == null) throw new IllegalArgumentException(); -// if (ndx == null) -// throw new IllegalArgumentException(); - this.relation = relation; final boolean remoteAccessPath = predicate.getProperty( @@ -358,7 +327,6 @@ } this.indexManager = localIndexManager; } -// this.indexManager = indexManager; this.timestamp = relation.getTimestamp(); @@ -389,8 +357,12 @@ final String name = DataService.getIndexPartitionName(namespace + "." + keyOrder.getIndexName(), partitionId); - // MUST be a local index view. - ndx = (ILocalBTreeView) indexManager.getIndex(name, timestamp); + try { + // MUST be a local index view. + ndx = (ILocalBTreeView) indexManager.getIndex(name, timestamp); + } catch (Throwable t) { + throw new RuntimeException(predicate.toString(), t); + } if (ndx == null) { @@ -458,9 +430,7 @@ IPredicate.Annotations.FULLY_BUFFERED_READ_THRESHOLD, IPredicate.Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD); - this.flags = flags - | (TimestampUtility.isReadOnly(timestamp) ? IRangeQuery.READONLY - : 0); + this.flags = flags; this.chunkOfChunksCapacity = chunkOfChunksCapacity; @@ -603,25 +573,10 @@ } - /** - * Resolved lazily if not specified to the ctor. - */ -// @SuppressWarnings("unchecked") public IRelation<R> getRelation() { + return relation; -// -// IRelation<R> tmp = relation.get(); -// -// if (tmp == null) { -// -// tmp = (IRelation<R>) indexManager.getResourceLocator().locate( -// predicate.getOnlyRelationName(), timestamp); -// -// relation.compareAndSet(null/*expect*/, tmp/*update*/); -// -// } -// -// return relation.get(); + } public IIndexManager getIndexManager() { @@ -638,8 +593,6 @@ public IPredicate<R> getPredicate() { -// assertInitialized(); - return predicate; } @@ -1425,19 +1378,13 @@ * The predicate. * * @return The estimated cost of a scan on that predicate. - * - * @todo This tunnels through to the {@link AbstractBTree} class and is thus - * specific to standalone and also may run into trouble once we - * support unisolated access paths for reads or mutation since it may - * encounter an {@link UnisolatedReadWriteIndex} instead of an - * {@link AbstractBTree}. */ public ScanCostReport estimateCost() { if(ndx instanceof UnisolatedReadWriteIndex) { return ((UnisolatedReadWriteIndex) ndx).estimateCost(diskCostModel, - rangeCount); + rangeCount(false/* exact */)); } @@ -1589,7 +1536,7 @@ final AbstractClient<?> client = ndx.getFederation().getClient(); - // maximum parallelization by the client. + // maximum parallelization by the client : @todo not used yet. final int maxParallel = client.getMaxParallelTasksPerRequest(); // the metadata index for that scale-out index. @@ -1604,13 +1551,13 @@ final long partitionCount = mdi.rangeCount(fromKey, toKey); if (partitionCount == 0) { - + /* * SWAG in case zero partition count is reported (I am not sure that * this code path is possible). */ - return new ScanCostReport(rangeCount, partitionCount, 100/* millis */); - + return new ScanCostReport(0L/* rangeCount */, partitionCount, 100/* millis */); + } // fast range count (may be cached by the access path). @@ -1622,7 +1569,8 @@ * Delegate the operation to the remote shard. */ - return (ScanCostReport) ndx.submit(fromKey, + return (ScanCostReport) ndx.submit( + fromKey == null ? BytesUtil.EMPTY : fromKey, new EstimateShardScanCost(rangeCount, fromKey, toKey)); } @@ -1652,6 +1600,7 @@ final double costPerShard = costPerJournal + 2 * costPerSegment; + // @todo ignores potential parallelism. final double cost = costPerShard * partitionCount; return new ScanCostReport(rangeCount, partitionCount, cost); Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -1873,9 +1873,17 @@ * iterator can keep looking for another element but the * source is no longer writing on the buffer and nothing * will show up. + * + * Whether or not this is an error depends on whether or + * not you are intending to chain together producers and + * consumers using blocking buffers or if the output of + * the producer will be collected and then passed onto + * another process (perhaps on another node) once the + * producer is done. */ - log.error("Future not set on buffer."); + if (log.isInfoEnabled()) + log.info("Future not set on buffer."); } else { Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/SimpleDataServiceProcedureTask.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/SimpleDataServiceProcedureTask.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/SimpleDataServiceProcedureTask.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -43,7 +43,7 @@ super(ndx, ts, split, proc, resultHandler); if (key == null) - throw new IllegalArgumentException(); + throw new IllegalArgumentException("name="+ndx.getName()+", proc="+proc); this.key = key; Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedFederation.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedFederation.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedFederation.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -932,14 +932,18 @@ if (metadataService != null) { - // the file flagging this as the MDS rather than a DS. - final File tmp = new File(metadataService.getResourceManager() - .getDataDir(), EmbeddedFederation.MDS); + if (!isTransient()) { - if(!tmp.delete()) { + // the file flagging this as the MDS rather than a DS. + final File tmp = new File(metadataService.getResourceManager() + .getDataDir(), EmbeddedFederation.MDS); - log.warn(ERR_COULD_NOT_DELETE + tmp); + if (!tmp.delete()) { + log.warn(ERR_COULD_NOT_DELETE + tmp); + + } + } metadataService.destroy(); Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -54,6 +54,7 @@ import com.bigdata.BigdataStatics; import com.bigdata.bop.BOp; +import com.bigdata.bop.BOpEvaluationContext; import com.bigdata.bop.Constant; import com.bigdata.bop.HashBindingSet; import com.bigdata.bop.IBindingSet; @@ -74,6 +75,8 @@ import com.bigdata.bop.engine.QueryEngine; import com.bigdata.bop.engine.RunningQuery; import com.bigdata.bop.solutions.ISortOrder; +import com.bigdata.bop.solutions.SliceOp; +import com.bigdata.btree.IRangeQuery; import com.bigdata.btree.keys.IKeyBuilderFactory; import com.bigdata.rdf.internal.DummyIV; import com.bigdata.rdf.internal.IV; @@ -103,10 +106,10 @@ import com.bigdata.relation.accesspath.IBuffer; import com.bigdata.relation.accesspath.IElementFilter; import com.bigdata.relation.accesspath.ThickAsynchronousIterator; +import com.bigdata.relation.rule.IAccessPathExpander; import com.bigdata.relation.rule.IProgram; import com.bigdata.relation.rule.IQueryOptions; import com.bigdata.relation.rule.IRule; -import com.bigdata.relation.rule.IAccessPathExpander; import com.bigdata.relation.rule.IStep; import com.bigdata.relation.rule.Program; import com.bigdata.relation.rule.QueryOptions; @@ -1523,6 +1526,21 @@ // timestamp anns.add(new NV(IPredicate.Annotations.TIMESTAMP, database .getSPORelation().getTimestamp())); + + /* + * Explicitly set the access path / iterator flags. + * + * Note: High level query generally permits iterator level parallelism. + * We set the PARALLEL flag here so it can be used if a global index + * view is chosen for the access path. + * + * Note: High level query for SPARQL always uses read-only access paths. + * If you are working with a SPARQL extension with UPDATE or INSERT INTO + * semantics then you will need to remote the READONLY flag for the + * mutable access paths. + */ + anns.add(new NV(IPredicate.Annotations.FLAGS, IRangeQuery.DEFAULT + | IRangeQuery.PARALLEL | IRangeQuery.READONLY)); return new SPOPredicate(vars, anns.toArray(new NV[anns.size()])); // return new SPOPredicate( @@ -1682,19 +1700,44 @@ final QueryEngine queryEngine = tripleSource.getSail().getQueryEngine(); - /* - * Note: The ids are assigned using incrementAndGet() so ONE (1) is the - * first id that will be assigned when we pass in ZERO (0) as the - * initial state of the AtomicInteger. - */ final int startId = 1; - final PipelineOp query = Rule2BOpUtility.convert(step, - new AtomicInteger(0), database, queryEngine); + final PipelineOp query; + { - if (log.isInfoEnabled()) { - log.info(query); + /* + * Note: The ids are assigned using incrementAndGet() so ONE (1) is + * the first id that will be assigned when we pass in ZERO (0) as + * the initial state of the AtomicInteger. + */ + final AtomicInteger idFactory = new AtomicInteger(0); + + /* + * Convert the step to a bigdata operator tree. + */ + PipelineOp tmp = Rule2BOpUtility.convert(step, idFactory, database, + queryEngine); + + if (!tmp.getEvaluationContext().equals( + BOpEvaluationContext.CONTROLLER)) { + /* + * Wrap with an operator which will be evaluated on the query + * controller. + */ + tmp = new SliceOp(new BOp[] { tmp }, NV.asMap(// + new NV(BOp.Annotations.BOP_ID, idFactory + .incrementAndGet()), // + new NV(BOp.Annotations.EVALUATION_CONTEXT, + BOpEvaluationContext.CONTROLLER))); + + } + + query = tmp; + + if (log.isInfoEnabled()) + log.info(query); + } - + final UUID queryId = UUID.randomUUID(); final RunningQuery runningQuery = queryEngine.eval(queryId, query, new LocalChunkMessage<IBindingSet>(queryEngine, queryId, Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -613,22 +613,35 @@ /* * Estimate cost of SCAN with C unbound. * + * Note: We need to use global index view in order to estimate the cost + * of the scan even though the scan will be shard-wise when we actually + * run the query. + * * @todo must pass estimateCost() to the underlying access path plus * layer on any cost for the optional expander. */ final IRelation r = context.getRelation(pred); final ScanCostReport scanCostReport = ((AccessPath) context - .getAccessPath(r, pred)).estimateCost(); + .getAccessPath(r, (Predicate<?>) pred.setProperty( + IPredicate.Annotations.REMOTE_ACCESS_PATH, true))) + .estimateCost(); anns.add(new NV(Annotations.COST_SCAN, scanCostReport)); - // Estimate cost of SUBQUERY with C bound (sampling). + /* + * Estimate cost of SUBQUERY with C bound (sampling). + * + * Note: Again, we need to use a remote index view in order to estimate + * the cost of the subqueries even though we will use sharded joins when + * actually running the query. + */ final SubqueryCostReport subqueryCostReport = summary - .estimateSubqueryCost(context, SAMPLE_LIMIT, pred); + .estimateSubqueryCost(context, SAMPLE_LIMIT, (Predicate<?>) pred.setProperty( + IPredicate.Annotations.REMOTE_ACCESS_PATH, true)); anns.add(new NV(Annotations.COST_SUBQUERY, subqueryCostReport)); - if (scanCostReport.cost < subqueryCostReport.subqueryCost) { + if (scanCostReport.cost < subqueryCostReport.cost) { /* * Scan and filter. C is left unbound. We do a range scan on the @@ -790,7 +803,7 @@ // pred = (Predicate<?>) pred.setProperty(IPredicate.Annotations.FLAGS, // pred.getProperty(IPredicate.Annotations.FLAGS, // IPredicate.Annotations.DEFAULT_FLAGS) -// | IRangeQuery.CURSOR); +// | IRangeQuery.CURSOR); // @todo also READONLY // // // Set Advancer (runs at the index). // pred = pred.addIndexLocalFilter(new ContextAdvancer()); @@ -838,22 +851,35 @@ // // } - // Estimate cost of SCAN with C unbound. + /* + * Estimate cost of SCAN with C unbound. + * + * Note: We need to use the global index view in order to estimate the + * cost of the scan regardless of whether the query runs with + * partitioned or global index views when it is evaluated. + */ final IRelation r = context.getRelation(pred); final ScanCostReport scanCostReport = ((AccessPath) context - .getAccessPath(r, pred)).estimateCost(); + .getAccessPath(r, (Predicate<?>) pred.setProperty( + IPredicate.Annotations.REMOTE_ACCESS_PATH, true))) + .estimateCost(); anns.add(new NV(Annotations.COST_SCAN, scanCostReport)); /* * Estimate cost of SUBQUERY with C bound (sampling). + * + * Note: We need to use the global index view in order to estimate the + * cost of the scan regardless of whether the query runs with + * partitioned or global index views when it is evaluated. */ final SubqueryCostReport subqueryCostReport = dataset == null ? null - : summary.estimateSubqueryCost(context, SAMPLE_LIMIT, pred); + : summary.estimateSubqueryCost(context, SAMPLE_LIMIT, (Predicate<?>) pred.setProperty( + IPredicate.Annotations.REMOTE_ACCESS_PATH, true)); anns.add(new NV(Annotations.COST_SUBQUERY, subqueryCostReport)); if (subqueryCostReport == null - || scanCostReport.cost < subqueryCostReport.subqueryCost) { + || scanCostReport.cost < subqueryCostReport.cost) { /* * SCAN AND FILTER. C is not bound. Unless all graphs are used, @@ -888,7 +914,7 @@ anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); pred = (Predicate) pred.setProperty( - Predicate.Annotations.REMOTE_ACCESS_PATH, false); + Predicate.Annotations.REMOTE_ACCESS_PATH, true); } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); @@ -936,7 +962,7 @@ anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); pred = (Predicate) pred.setProperty( - Predicate.Annotations.REMOTE_ACCESS_PATH, false); + Predicate.Annotations.REMOTE_ACCESS_PATH, true); } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); Deleted: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/TestBaseball.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/TestBaseball.java 2010-10-11 16:06:05 UTC (rev 3766) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/TestBaseball.java 2010-10-11 17:26:00 UTC (rev 3767) @@ -1,137 +0,0 @@ -package com.bigdata.samples; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Properties; - -import org.openrdf.repository.Repository; -import org.openrdf.rio.RDFFormat; - -import com.bigdata.btree.IndexMetadata; -import com.bigdata.rdf.sail.BigdataSail; -import com.bigdata.rdf.sail.BigdataSailRepository; -import com.bigdata.rdf.sail.BigdataSailRepositoryConnection; - -public class TestBaseball extends SampleCode { - - public static final void main(String[] args) { - try { - SampleCode sampleCode = new TestBaseball(); - - // use one of our pre-configured option-sets or "modes" - Properties properties = - sampleCode.loadProperties("fullfeature.properties"); - - // this option can be faster and make better use of disk if you have - // enough ram and are doing large writes. - properties.setProperty( - IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY, - "8000"); - - // when loading a large data file, it's sometimes better to do - // database-at-once closure rather than incremental closure. this - // is how you do it. - properties.setProperty(BigdataSail.Options.TRUTH_MAINTENANCE, "false"); - - // we won't be doing any retraction, so no justifications either - properties.setProperty(BigdataSail.Options.JUSTIFY, "false"); - - // no free text search - properties.setProperty(BigdataSail.Options.TEXT_INDEX, "false"); - - // no statement identifiers - properties.setProperty(BigdataSail.Options.STATEMENT_IDENTIFIERS, - "false"); - - // triples only. - properties.setProperty( - com.bigdata.rdf.store.AbstractTripleStore.Options.QUADS, - "false"); - - if (properties.getProperty(com.bigdata.journal.Options.FILE) == null) { - // create backing tmp file iff none was specified by properties. -// File journal = File.createTempFile("baseball", ".jnl"); - File journal = new File("d:/baseball.jnl"); - System.out.println("journalFile="+journal.getAbsolutePath()); - // ... [truncated message content] |
From: <dm...@us...> - 2010-10-11 16:06:12
|
Revision: 3766 http://bigdata.svn.sourceforge.net/bigdata/?rev=3766&view=rev Author: dmacgbr Date: 2010-10-11 16:06:05 +0000 (Mon, 11 Oct 2010) Log Message: ----------- Add class to run SPARQL test suite against a running federation. Supply the config file via the system property '-Dbigdata.configuration' Added Paths: ----------- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/log4j.properties Added: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java (rev 0) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-11 16:06:05 UTC (rev 3766) @@ -0,0 +1,146 @@ +/* + +Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. + +Contact: + SYSTAP, LLC + 4501 Tower Road + Greensboro, NC 27410 + lic...@bi... + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +*/ +package com.bigdata.rdf.sail.tck; + +import java.util.Properties; +import java.util.UUID; + +import junit.framework.Test; + +import org.apache.log4j.Logger; +import org.openrdf.query.Dataset; +import org.openrdf.query.parser.sparql.ManifestTest; +import org.openrdf.query.parser.sparql.SPARQLQueryTest; +import org.openrdf.repository.Repository; +import org.openrdf.repository.dataset.DatasetRepository; + +import com.bigdata.journal.ITx; +import com.bigdata.rdf.sail.BigdataSail; +import com.bigdata.rdf.sail.BigdataSailRepository; +import com.bigdata.rdf.store.ScaleOutTripleStore; +import com.bigdata.service.jini.JiniClient; +import com.bigdata.service.jini.JiniFederation; + +/** + * @author <a href="mailto:dm...@us...">David MacMillan</a> + * @version $Id$ + */ +public class BigdataFederationSparqlTest extends SPARQLQueryTest +{ + public static Test suite () + throws Exception + { + return ManifestTest.suite + ( + new Factory () + { + public SPARQLQueryTest createSPARQLQueryTest ( String URI, String name, String query, String results, Dataset dataSet, boolean laxCardinality) + { + return new BigdataFederationSparqlTest ( URI, name, query, results, dataSet, laxCardinality ) ; + } + } + ) ; + } + + public BigdataFederationSparqlTest ( String URI, String name, String query, String results, Dataset dataSet, boolean laxCardinality ) + { + super ( URI, name, query, results, dataSet, laxCardinality ) ; + } + + @Override public void tearDown () + throws Exception + { + super.tearDown () ; + _ts.destroy () ; + _ts = null ; + } + + @Override protected Repository newRepository () + throws Exception + { + return new DatasetRepository ( new BigdataSailRepository ( new BigdataSail ( newTripleStore () ) ) ) ; + } + + private ScaleOutTripleStore newTripleStore () + throws Exception + { + _ts = new ScaleOutTripleStore ( getFederation (), newNamespace (), ITx.UNISOLATED, getProperties () ) ; + _ts.create () ; + return _ts ; + } + + private JiniFederation<Object> getFederation () + throws Exception + { + if ( null == _fed ) + { + JiniClient<Object> jc = new JiniClient<Object> ( new String [] { getConfiguration () } ) ; + _fed = jc.connect () ; + } + return _fed ; + } + + private String getConfiguration () + throws Exception + { + String c = System.getProperty ( CONFIG_PROPERTY ) ; + if ( null == c ) + throw new Exception ( String.format ( "Configuration property not set. Specify as: -D%s=<filename or URL>", CONFIG_PROPERTY ) ) ; + return c ; + } + + private String newNamespace () + { + return "SPARQLTest_" + UUID.randomUUID ().toString () ; + } + + private Properties getProperties () + { + if ( null == _properties ) + { + // + // TODO What do we really need here? Don't some of these entail others? + // + _properties = new Properties () ; + _properties.put ( BigdataSail.Options.QUADS_MODE, "true" ) ; + _properties.put ( BigdataSail.Options.TRUTH_MAINTENANCE, "false" ) ; + _properties.put ( BigdataSail.Options.NATIVE_JOINS, "true" ) ; + _properties.put ( BigdataSail.Options.QUERY_TIME_EXPANDER, "true" ) ; + _properties.put ( BigdataSail.Options.ALLOW_AUTO_COMMIT, "true" ) ; + _properties.put ( BigdataSail.Options.ISOLATABLE_INDICES, "false" ) ; + _properties.put ( BigdataSail.Options.STAR_JOINS, "false" ) ; + _properties.put ( BigdataSail.Options.TEXT_INDEX, "false" ) ; + } + return _properties ; + } + + public static final String CONFIG_PROPERTY = "bigdata.configuration" ; + + private static final Logger _logger = Logger.getLogger ( BigdataFederationSparqlTest.class ) ; + + private JiniFederation<Object> _fed = null ; + private ScaleOutTripleStore _ts = null ; + private Properties _properties = null ; +} \ No newline at end of file Added: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/log4j.properties =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/log4j.properties (rev 0) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/log4j.properties 2010-10-11 16:06:05 UTC (rev 3766) @@ -0,0 +1,101 @@ +# Default log4j configuration. See the individual classes for the +# specific loggers, but generally they are named for the class in +# which they are defined. + +# Default log4j configuration for testing purposes. + +log4j.rootCategory=WARN, dest1 +#log4j.rootCategory=DEBUG, dest1 + +# Loggers. +#log4j.logger.com.bigdata=ERROR +#log4j.logger.com.bigdata.rdf.sail.TestNamedGraphs=INFO +#log4j.logger.com.bigdata.rdf.sail.BigdataEvaluationStrategyImpl=INFO +log4j.logger.com.bigdata.rdf.sail.tck.DavidsBigdataSparqlTest=INFO +#log4j.logger.com.bigdata.btree.compression=INFO +#log4j.logger.com.bigdata.journal.IJournal=INFO +#log4j.logger.com.bigdata.resources=INFO +#log4j.logger.com.bigdata.journal.AbstractTask=INFO +#log4j.logger.com.bigdata.journal.WriteExecutorService=INFO +#log4j.logger.com.bigdata.journal.QueueLengthTask=INFO +#log4j.logger.com.bigdata.journal.AbstractBufferStrategy=INFO, A2 +#log4j.logger.com.bigdata.resources.ResourceManager=INFO, dest1, A2 +#log4j.logger.com.bigdata.resources.StoreManager=INFO +#log4j.logger.com.bigdata.resources.OverflowManager=INFO +#log4j.logger.com.bigdata.resources.PostProcessOldJournalTask=INFO +#log4j.logger.com.bigdata.resources.AbstractResourceManagerTask=INFO +#log4j.logger.com.bigdata.resources.IndexManager=INFO +#log4j.logger.com.bigdata.resources.ResourceFileFilter=INFO +#log4j.logger.com.bigdata.btree.UnisolatedReadWriteIndex=DEBUG +# +#log4j.logger.com.bigdata.service=DEBUG +#log4j.logger.com.bigdata.service.jini.AbstractServer=INFO +#log4j.logger.com.bigdata.service.IBigdataFederation=INFO +#log4j.logger.com.bigdata.service.ClientIndexView=INFO +#log4j.logger.com.bigdata.service.DataServiceIndex=INFO +#log4j.logger.com.bigdata.service.LoadBalancerService=INFO +#log4j.logger.com.bigdata.service.LoadBalancerService$UpdateTask=INFO +#log4j.logger.com.bigdata.service.DataService$StartPerformanceCounterCollectionTask=INFO +#log4j.logger.com.bigdata.service.DataService$ReportTask=INFO +#log4j.logger.com.bigdata.service.AbstractFederation$ReportTask=INFO +# +#log4j.logger.com.bigdata.service.ResultSet=INFO +#log4j.logger.com.bigdata.search=INFO +#log4j.logger.com.bigdata.relation=INFO +#log4j.logger.com.bigdata.relation.locator=DEBUG +#log4j.logger.com.bigdata.rdf.rules=INFO +#log4j.logger.com.bigdata.relation.rule.eval=INFO +#log4j.logger.com.bigdata.relation.rule.eval.RuleState=DEBUG +#log4j.logger.com.bigdata.relation.rule.eval.NestedSubqueryEvaluator=DEBUG +#log4j.logger.com.bigdata.relation.accesspath.AbstractArrayBuffer=DEBUG +#log4j.logger.com.bigdata.relation.accesspath.BlockingBuffer=INFO +#log4j.logger.com.bigdata.rdf.spo.DistinctTermScanner$DistinctTermScanTask=INFO +#log4j.logger.com.bigdata.service.IBigdataFederation=DEBUG +#log4j.logger.com.bigdata.rdf=INFO +#log4j.logger.com.bigdata.rdf.store.ITripleStore=DEBUG +#log4j.logger.com.bigdata.rdf.rio=DEBUG +#log4j.logger.com.bigdata.rdf.rio.StatementBuffer=INFO +#log4j.logger.com.bigdata.rdf.rio.AbstractStatementBuffer=INFO +#log4j.logger.com.bigdata.rdf.lexicon.LexiconRelation=INFO +#log4j.logger.com.bigdata.rdf.lexicon.TermIdEncoder=DEBUG +#log4j.logger.com.bigdata.rdf.store.BigdataStatementIteratorImpl=DEBUG +#log4j.logger.com.bigdata.rdf.lexicon.Term2IdWriteProc=DEBUG +#log4j.logger.com.bigdata.rdf.store.IndexWriteProc=WARN +#log4j.logger.com.bigdata.rdf.store.AbstractTripleStore=INFO +#log4j.logger.com.bigdata.rdf.store.ConcurrentDataLoader=INFO +#log4j.logger.com.bigdata.rdf.store.ConcurrentDataLoader$WorkflowTask=INFO +#log4j.logger.com.bigdata.rdf.store.ConcurrentDataLoader$VerifyStatementBuffer=INFO +#log4j.logger.com.bigdata.rdf.store.DataLoader=INFO +# Note: turns on ground truth testing (not scalable). +#log4j.logger.com.bigdata.rdf.store.Term2IdWriteProc=DEBUG +#log4j.logger.com.bigdata.rdf.spo.ISPOBuffer=DEBUG +#log4j.logger.com.bigdata.rdf.spo.SPOIterator=WARN +#log4j.logger.com.bigdata.rdf.inf.TruthMaintenance=DEBUG +#log4j.logger.com.bigdata.rdf.inf.InferenceEngine=INFO +#log4j.logger.com.bigdata.rdf.inf.Justification=DEBUG +#log4j.logger.com.bigdata.rdf.inf.BackchainOwlSameAs2=DEBUG + +# Test case logger (logging of statements does lots of RMI when using a federation). +#log4j.logger.junit.framework.Test=INFO + +# WARNING Generating caller location information is extremely slow. +# It's use should be avoided unless execution speed is not an issue. +# This applies to %L %l %M %m %F %C etc. Pretty much anything useful :-( + +log4j.appender.dest1=org.apache.log4j.ConsoleAppender +log4j.appender.dest1.layout=org.apache.log4j.PatternLayout +#log4j.appender.dest1.layout.ConversionPattern=%-5p: %m%n +#log4j.appender.dest1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n +#log4j.appender.dest1.layout.ConversionPattern=%-4r(%d) [%t] %-5p %c(%l:%M) %x - %m%n +#log4j.appender.dest1.layout.ConversionPattern=%-5p: %r %X{trial} %X{depth} [%t] %l: %m%n +log4j.appender.dest1.layout.ConversionPattern=%d %t %-5p: %c: %m%n + +# A2 is set to be a FileAppender. +log4j.appender.A2=org.apache.log4j.FileAppender +log4j.appender.A2.Threshold=DEBUG +log4j.appender.A2.File=ResourceManager.log +log4j.appender.A2.Append=true + +# A2 uses PatternLayout. +log4j.appender.A2.layout=org.apache.log4j.PatternLayout +log4j.appender.A2.layout.ConversionPattern=%5p [%t] %l %d - %m%n \ No newline at end of file This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ble...@us...> - 2010-10-11 12:44:30
|
Revision: 3765 http://bigdata.svn.sourceforge.net/bigdata/?rev=3765&view=rev Author: blevine218 Date: 2010-10-11 12:44:24 +0000 (Mon, 11 Oct 2010) Log Message: ----------- Comment out asserts in discardAllCaches() method which cause sporadic failures in certain unit/stress tests. discardAllCaches() does not appear to be called outside of unit/stress tests and given the comments regarding consistency, these asserts might not be appropriate anyway. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRU.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecyclerExplicitDeleteRequired.java Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRU.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-10-10 21:37:24 UTC (rev 3764) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRU.java 2010-10-11 12:44:24 UTC (rev 3765) @@ -297,10 +297,15 @@ } - assert size == 0; - assert first == null; - assert last == null; + // blevine (10/11/2010) + // Stress tests against this class fail sporadically due to the asserts below. Comments + // in the code indicate that this method may not be consistent. And given that it doesn't + // appear to be called except by test code, I'm commenting out these asserts for now. + //assert size == 0; + //assert first == null; + //assert last == null; + // size = 0; // // first = last = null; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-10 21:37:24 UTC (rev 3764) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-11 12:44:24 UTC (rev 3765) @@ -315,10 +315,15 @@ } - assert size == 0; - assert first == null; - assert last == null; + // blevine (10/11/2010) + // Stress tests against this class fail sporadically due to the asserts below. Comments + // in the code indicate that this method may not be consistent. And given that it doesn't + // appear to be called except by test code, I'm commenting out these asserts for now. + //assert size == 0; + //assert first == null; + //assert last == null; + // size = 0; // // first = last = null; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecyclerExplicitDeleteRequired.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecyclerExplicitDeleteRequired.java 2010-10-10 21:37:24 UTC (rev 3764) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecyclerExplicitDeleteRequired.java 2010-10-11 12:44:24 UTC (rev 3765) @@ -340,10 +340,15 @@ } - assert size == 0; - assert first == null; - assert last == null; + // blevine (10/11/2010) + // Stress tests against this class fail sporadically due to the asserts below. Comments + // in the code indicate that this method may not be consistent. And given that it doesn't + // appear to be called except by test code, I'm commenting out these asserts for now. + //assert size == 0; + //assert first == null; + //assert last == null; + // size = 0; // // first = last = null; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fko...@us...> - 2010-10-10 21:37:30
|
Revision: 3764 http://bigdata.svn.sourceforge.net/bigdata/?rev=3764&view=rev Author: fkoliver Date: 2010-10-10 21:37:24 +0000 (Sun, 10 Oct 2010) Log Message: ----------- Remove unintended change in pom (experiment which had no affect). Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-10-09 09:11:41 UTC (rev 3763) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-10-10 21:37:24 UTC (rev 3764) @@ -170,9 +170,6 @@ <slf4j.jar>${deploy.lib}/slf4j.jar</slf4j.jar> <nxparser.jar>${deploy.lib}/nxparser.jar</nxparser.jar> <zookeeper.jar>${deploy.lib}/zookeeper.jar</zookeeper.jar> - - <basedir>${deploy.dir}/testing</basedir> - </systemPropertyVariables> </configuration> </execution> This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mar...@us...> - 2010-10-09 09:11:47
|
Revision: 3763 http://bigdata.svn.sourceforge.net/bigdata/?rev=3763&view=rev Author: martyncutcher Date: 2010-10-09 09:11:41 +0000 (Sat, 09 Oct 2010) Log Message: ----------- Call RootBlockCommitter directly rather than via registration and immediately free meta-rallocation blocks to support "null" commit. Modified Paths: -------------- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RootBlockCommitter.java branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-10-09 01:36:31 UTC (rev 3762) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-10-09 09:11:41 UTC (rev 3763) @@ -633,6 +633,8 @@ private final long initialExtent; private final long minimumExtension; + private RootBlockCommitter m_rootBlockCommitter; + /** * The maximum extent before a {@link #commit()} will {@link #overflow()}. * In practice, overflow tries to trigger before this point in order to @@ -2300,8 +2302,13 @@ return 0L; } - + /* + * Explicitly call the RootBlockCommitter + */ + rootAddrs[PREV_ROOTBLOCK] = this.m_rootBlockCommitter.handleCommit(commitTime); + + /* * Write the commit record onto the store. * * @todo Modify to log the current root block and set the address of @@ -2767,9 +2774,11 @@ setupName2AddrBTree(getRootAddr(ROOT_NAME2ADDR)); /** - * Register committer to write previous root block + * Do not register committer to write previous root block, but + * instead just create it and call explicitly when required. This + * is a workaround to allow "void" transactions. */ - setCommitter(PREV_ROOTBLOCK, new RootBlockCommitter(this)); + m_rootBlockCommitter = new RootBlockCommitter(this); /** * If the strategy is a RWStrategy, then register the delete Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RootBlockCommitter.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RootBlockCommitter.java 2010-10-09 01:36:31 UTC (rev 3762) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/journal/RootBlockCommitter.java 2010-10-09 09:11:41 UTC (rev 3763) @@ -47,7 +47,9 @@ * to be stored in the CommitRecord. */ public long handleCommit(final long commitTime) { - final ByteBuffer rbv = journal.getRootBlockView().asReadOnlyBuffer(); + IRootBlockView view = journal.getRootBlockView(); + + final ByteBuffer rbv = view.asReadOnlyBuffer(); /* * FIXME There is an API issue with the RWStore which does not allow * us to pass in a read-only buffer. Write unit tests for this on Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-10-09 01:36:31 UTC (rev 3762) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/rwstore/RWStore.java 2010-10-09 09:11:41 UTC (rev 3763) @@ -1584,7 +1584,9 @@ throw new IllegalStateException("Returned MetaBits Address not valid!"); } - free(oldMetaBits, oldMetaBitsSize); + // Call immediateFree - no need to defer freeof metaBits, this + // has to stop somewhere! + immediateFree((int) oldMetaBits, oldMetaBitsSize); // save allocation headers Iterator iter = m_commitList.iterator(); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <ble...@us...> - 2010-10-09 01:36:38
|
Revision: 3762 http://bigdata.svn.sourceforge.net/bigdata/?rev=3762&view=rev Author: blevine218 Date: 2010-10-09 01:36:31 +0000 (Sat, 09 Oct 2010) Log Message: ----------- fixed typo in setting java.net.preferIPv4Stack system property for unit tests Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 22:21:56 UTC (rev 3761) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-10-09 01:36:31 UTC (rev 3762) @@ -143,7 +143,7 @@ <!-- These system properties are required by the unit tests. --> <systemPropertyVariables> <java.security.policy>${java.security.policy}</java.security.policy> - <java.net.preferIPv4Stack>{java.net.preferIPv4Stack}</java.net.preferIPv4Stack> + <java.net.preferIPv4Stack>${java.net.preferIPv4Stack}</java.net.preferIPv4Stack> <log4j.configuration>${log4j.configuration}</log4j.configuration> <app.home>${app.home}</app.home> <!-- This is the deployment directory, easily accessed by the DataFinder class. --> @@ -170,13 +170,9 @@ <slf4j.jar>${deploy.lib}/slf4j.jar</slf4j.jar> <nxparser.jar>${deploy.lib}/nxparser.jar</nxparser.jar> <zookeeper.jar>${deploy.lib}/zookeeper.jar</zookeeper.jar> - - - - <basedir>${deploy.dir}/testing</basedir> - - - + + <basedir>${deploy.dir}/testing</basedir> + </systemPropertyVariables> </configuration> </execution> This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fko...@us...> - 2010-10-08 22:22:05
|
Revision: 3761 http://bigdata.svn.sourceforge.net/bigdata/?rev=3761&view=rev Author: fkoliver Date: 2010-10-08 22:21:56 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Change some subclasses of BTree into uses of BTree: EventBTree, CommitTimeIndex, CommitRecordIndex, JournalIndex, IndexSegmentIndex Users of these specialized trees should see only a specialized set of methods rather than the full inheritance from BTree. These classes should hide key/value conversion. Users should never need to see ITuple or ITupleSerializer. Removed unused CounterSetBTree. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/httpd/DummyEventReportingService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/AbstractJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/CommitRecordIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/journal/DumpJournal.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/loadbalancer/EmbeddedLoadBalancer.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/IndexSegmentIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/JournalIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/ResourceEvents.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/resources/StoreManager.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/AbstractTransactionService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/CommitTimeIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/DistributedTransactionService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/EventReceiver.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/LoadBalancerService.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/MetadataIndexCache.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/transaction/EmbeddedTransactionService.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/journal/TestCommitRecordIndex.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestDistributedTransactionServiceRestart.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestEventReceiver.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/service/TestSnapshotHelper.java Added Paths: ----------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/service/EventBTree.java Removed Paths: ------------- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/query/CounterSetBTreeSelector.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/query/CounterSetLoader.java branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/counters/store/CounterSetBTree.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/store/TestAll.java branches/maven_scaleout/bigdata-core/src/test/java/com/bigdata/counters/store/TestCounterSetBTree.java Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 22:21:56 UTC (rev 3761) @@ -60,6 +60,8 @@ java files were getting recompiled and put into the bigdata jar. This setting forces javac to only look for source in the current maven source directory. --> <sourcepath>${project.build.sourceDirectory}</sourcepath> + <!-- <Xlint></Xlint> --> + <!-- <Xlint:unchecked></Xlint:unchecked> --> </compilerArguments> </configuration> </plugin> @@ -141,7 +143,7 @@ <!-- These system properties are required by the unit tests. --> <systemPropertyVariables> <java.security.policy>${java.security.policy}</java.security.policy> - <java.net.preferIPv4Stack>{java.net.preferIPv4Stack}"</java.net.preferIPv4Stack> + <java.net.preferIPv4Stack>{java.net.preferIPv4Stack}</java.net.preferIPv4Stack> <log4j.configuration>${log4j.configuration}</log4j.configuration> <app.home>${app.home}</app.home> <!-- This is the deployment directory, easily accessed by the DataFinder class. --> @@ -160,7 +162,7 @@ <fastutil.jar>${deploy.lib}/fastutil.jar</fastutil.jar> <icu4j.jar>${deploy.lib}/icu4j.jar</icu4j.jar> <jsk-lib.jar>${deploy.lib}/jsk-lib.jar</jsk-lib.jar> - <jsk-platform.jar>${deploy.lib}jsk-platform.jar</jsk-platform.jar> + <jsk-platform.jar>${deploy.lib}/jsk-platform.jar</jsk-platform.jar> <log4j.jar>${deploy.lib}/log4j.jar</log4j.jar> <iris.jar>${deploy.lib}/iris.jar</iris.jar> <jgrapht.jar>${deploy.lib}/jgrapht.jar</jgrapht.jar> @@ -168,6 +170,13 @@ <slf4j.jar>${deploy.lib}/slf4j.jar</slf4j.jar> <nxparser.jar>${deploy.lib}/nxparser.jar</nxparser.jar> <zookeeper.jar>${deploy.lib}/zookeeper.jar</zookeeper.jar> + + + + <basedir>${deploy.dir}/testing</basedir> + + + </systemPropertyVariables> </configuration> </execution> Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/AbstractBTree.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -128,7 +128,6 @@ * </p> * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * @see KeyBuilder */ @@ -164,7 +163,7 @@ /** * Log for btree opeations. */ - protected static final Logger log = Logger.getLogger(AbstractBTree.class); + private static final Logger log = Logger.getLogger(AbstractBTree.class); /** * True iff the {@link #log} level is INFO or less. @@ -338,7 +337,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ static class ChildMemoizer extends Memoizer<LoadChildRequest/* request */, AbstractNode<?>/* child */> { @@ -1362,7 +1360,6 @@ * Static class since must be {@link Serializable}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ static final class TransientResourceMetadata implements IResourceMetadata { @@ -2588,6 +2585,7 @@ } + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator() { return rangeIterator(null, null); @@ -2602,6 +2600,7 @@ * @param toKey * @return */ + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(Object fromKey, Object toKey) { fromKey = fromKey == null ? null : metadata.getTupleSerializer() @@ -2614,6 +2613,7 @@ } + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { return rangeIterator(fromKey, toKey, 0/* capacity */, @@ -2665,6 +2665,7 @@ * @param toKey * @return */ + //fko ===== needs generic type ===== final public ITupleIterator rangeIterator(Object fromKey, Object toKey, final int capacity,// final int flags,// @@ -2705,208 +2706,13 @@ * @todo add support to the iterator construct for filtering by a tuple * revision timestamp range. */ - public ITupleIterator rangeIterator(// - final byte[] fromKey,// - final byte[] toKey,// - final int capacityIsIgnored,// - final int flags,// - final IFilterConstructor filter// - ) { + //fko ===== needs generic type ===== + abstract public ITupleIterator rangeIterator(final byte[] fromKey, + final byte[] toKey, + final int capacityIsIgnored, + final int flags, + final IFilterConstructor filter); -// btreeCounters.nrangeIterator.incrementAndGet(); - - /* - * Does the iterator declare that it will not write back on the index? - */ - final boolean readOnly = ((flags & IRangeQuery.READONLY) != 0); - - if (readOnly && ((flags & IRangeQuery.REMOVEALL) != 0)) { - - throw new IllegalArgumentException(); - - } - - /* - * Note: this does not work out since it is not so easy to determine when - * the iterator is a point test as toKey is the exclusive upper bound. - */ -// * Note: this method will automatically apply the optional bloom filter to -// * reject range iterator requests that correspond to a point test. However -// * this can only be done when the fromKey and toKey are both non-null and -// * equals and further when the iterator was not requested with any options -// * that would permit concurrent modification of the index. -// if (isBloomFilter() -// && fromKey != null -// && toKey != null -// && (readOnly || (((flags & REMOVEALL) == 0) && ((flags & CURSOR) == 0))) -// && BytesUtil.bytesEqual(fromKey, toKey)) { -// -// /* -// * Do a fast rejection test using the bloom filter. -// */ -// if(!getBloomFilter().contains(fromKey)) { -// -// /* -// * The key is known to not be in the index so return an empty -// * iterator. -// */ -// return EmptyTupleIterator.INSTANCE; -// -// } -// -// /* -// * Since the bloom filter accepts the key we fall through into the -// * normal iterator logic. Using this code path is still possible -// * that the filter gave us a false positive and that the key is not -// * (in fact) in the index. Either way, the logic below will sort -// * things out. -// */ -// -// } - - /* - * Figure out what base iterator implementation to use. We will layer - * on the optional filter(s) below. - */ - ITupleIterator src; - - if ((this instanceof BTree) && ((flags & REVERSE) == 0) - && ((flags & REMOVEALL) == 0) && ((flags & CURSOR) == 0)) { - - /* - * Use the recursion-based striterator since it is faster for a - * BTree (but not for an IndexSegment). - * - * Note: The recursion-based striterator does not support remove()! - * - * @todo we could pass in the Tuple here to make the APIs a bit more - * consistent across the recursion-based and the cursor based - * iterators. - * - * @todo when the capacity is one and REVERSE is specified then we - * can optimize this using a reverse traversal striterator - this - * will have lower overhead than the cursor for the BTree (but not - * for an IndexSegment). - */ - -// src = fastForwardIterator(fromKey, toKey, capacity, flags); - - src = getRoot().rangeIterator(fromKey, toKey, flags); - - } else { - - final Tuple tuple = new Tuple(this, flags); - - if (this instanceof IndexSegment) { - - final IndexSegment seg = (IndexSegment) this; - - /* - * @todo we could scan the list of pools and chose the best fit - * pool and then allocate a buffer from that pool. Best fit - * would mean either the byte range fits without "too much" slop - * or the #of reads will have to perform is not too large. We - * might also want to limit the maximum size of the reads. - */ - -// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M; - final DirectBufferPool pool = DirectBufferPool.INSTANCE; - - if (true - && ((flags & REVERSE) == 0) - && ((flags & CURSOR) == 0) - && (seg.getStore().getCheckpoint().maxNodeOrLeafLength <= pool - .getBufferCapacity()) - && ((rangeCount(fromKey, toKey) / branchingFactor) > 2)) { - - src = new IndexSegmentMultiBlockIterator(seg, pool, - fromKey, toKey, flags); - - } else { - - src = new IndexSegmentTupleCursor(seg, tuple, fromKey, - toKey); - - } - - } else if (this instanceof BTree) { - - if (isReadOnly()) { - - // Note: this iterator does not allow removal. - src = new ReadOnlyBTreeTupleCursor(((BTree) this), tuple, - fromKey, toKey); - - } else { - - // Note: this iterator supports traversal with concurrent - // modification. - src = new MutableBTreeTupleCursor(((BTree) this), - new Tuple(this, flags), fromKey, toKey); - - } - - } else { - - throw new UnsupportedOperationException( - "Unknown B+Tree implementation: " - + this.getClass().getName()); - - } - - if ((flags & REVERSE) != 0) { - - /* - * Reverse scan iterator. - * - * Note: The reverse scan MUST be layered directly over the - * ITupleCursor. Most critically, REMOVEALL combined with a - * REVERSE scan needs to process the tuples in reverse index - * order and then delete them as it goes. - */ - - src = new Reverserator((ITupleCursor) src); - - } - - } - - if (filter != null) { - - /* - * Apply the optional filter. - * - * Note: This needs to be after the reverse scan and before - * REMOVEALL (those are the assumptions for the flags). - */ - - src = filter.newInstance(src); - - } - - if ((flags & REMOVEALL) != 0) { - - assertNotReadOnly(); - - /* - * Note: This iterator removes each tuple that it visits from the - * source iterator. - */ - - src = new TupleRemover() { - @Override - protected boolean remove(ITuple e) { - // remove all visited tuples. - return true; - } - }.filter(src); - - } - - return src; - - } - /** * Copy all data, including deleted index entry markers and timestamps iff * supported by the source and target. The goal is an exact copy of the data @@ -3893,7 +3699,6 @@ * {@link Reference} (a runtime security manager exception will result). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * @param <T> */ Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/BTree.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -32,9 +32,13 @@ import com.bigdata.BigdataStatics; import com.bigdata.btree.AbstractBTreeTupleCursor.MutableBTreeTupleCursor; +import com.bigdata.btree.AbstractBTreeTupleCursor.ReadOnlyBTreeTupleCursor; import com.bigdata.btree.Leaf.ILeafListener; import com.bigdata.btree.data.ILeafData; import com.bigdata.btree.data.INodeData; +import com.bigdata.btree.filter.IFilterConstructor; +import com.bigdata.btree.filter.Reverserator; +import com.bigdata.btree.filter.TupleRemover; import com.bigdata.journal.AbstractJournal; import com.bigdata.journal.ICommitter; import com.bigdata.journal.IIndexManager; @@ -42,6 +46,7 @@ import com.bigdata.journal.Name2Addr.Entry; import com.bigdata.mdi.IResourceMetadata; import com.bigdata.rawstore.IRawStore; +import org.apache.log4j.Logger; /** * <p> @@ -151,10 +156,14 @@ * several published papers. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class BTree extends AbstractBTree implements ICommitter, ILocalBTreeView { + /** + * Log for btree operations. + */ + private static final Logger log = Logger.getLogger(BTree.class); + final public int getHeight() { return height; @@ -188,6 +197,7 @@ } + @Override public final IResourceMetadata[] getResourceMetadata() { //override to make final so sub-classes cannot modify behavior. return super.getResourceMetadata(); @@ -204,23 +214,23 @@ */ public ICounter getCounter() { - ICounter counter = new Counter(this); + ICounter tmpCounter = new Counter(this); final LocalPartitionMetadata pmd = metadata.getPartitionMetadata(); if (pmd != null) { - counter = new PartitionedCounter(pmd.getPartitionId(), counter); + tmpCounter = new PartitionedCounter(pmd.getPartitionId(), tmpCounter); } if (isReadOnly()) { - return new ReadOnlyCounter(counter); + return new ReadOnlyCounter(tmpCounter); } - return counter; + return tmpCounter; } @@ -393,7 +403,7 @@ * <p> * Note: The {@link #getCounter()} is NOT changed by this method. */ - final private void newRootLeaf() { + private void newRootLeaf() { height = 0; @@ -1339,14 +1349,8 @@ // view of this BTree. newResources[1] = new JournalMetadata((AbstractJournal) getStore(), priorCommitTime); + System.arraycopy(oldResources, 1, newResources, 2, oldResources.length - 1); - // any other stores in the view are copied. - for (int i = 1; i < oldResources.length; i++) { - - newResources[i + 1] = oldResources[i]; - - } - final LocalPartitionMetadata newPmd = new LocalPartitionMetadata( oldPmd.getPartitionId(), // partitionId -1, // sourcePartitionId @@ -1567,7 +1571,6 @@ * @throws IllegalArgumentException * if store is <code>null</code>. */ - @SuppressWarnings("unchecked") public static BTree load(final IRawStore store, final long addrCheckpoint, final boolean readOnly) { @@ -1653,6 +1656,147 @@ } /** + * Core implementation. + * <p> + * Note: If {@link IRangeQuery#CURSOR} is specified the returned iterator + * supports traversal with concurrent modification by a single-threaded + * process (the {@link BTree} is NOT thread-safe for writers). Write are + * permitted iff {@link AbstractBTree} allows writes. + * <p> + * Note: {@link IRangeQuery#REVERSE} is handled here by wrapping the + * underlying {@link ITupleCursor}. + * <p> + * Note: {@link IRangeQuery#REMOVEALL} is handled here by wrapping the + * iterator. + * <p> + * Note: + * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)} + * is also responsible for constructing an {@link ITupleIterator} in a + * manner similar to this method. If you are updating the logic here, then + * check the logic in that method as well! + * + * @todo add support to the iterator construct for filtering by a tuple + * revision timestamp range. + */ + public ITupleIterator rangeIterator(// + final byte[] fromKey,// + final byte[] toKey,// + final int capacityIsIgnored,// + final int flags,// + final IFilterConstructor filter// + ) { + + /* + * Does the iterator declare that it will not write back on the index? + */ + final boolean ro = ((flags & IRangeQuery.READONLY) != 0); + + if (ro && ((flags & IRangeQuery.REMOVEALL) != 0)) { + + throw new IllegalArgumentException(); + + } + + /* + * Figure out what base iterator implementation to use. We will layer + * on the optional filter(s) below. + */ + ITupleIterator src; + + if (((flags & REVERSE) == 0) && + ((flags & REMOVEALL) == 0) && + ((flags & CURSOR) == 0)) { + + /* + * Use the recursion-based striterator since it is faster for a + * BTree (but not for an IndexSegment). + * + * Note: The recursion-based striterator does not support remove()! + * + * @todo we could pass in the Tuple here to make the APIs a bit more + * consistent across the recursion-based and the cursor based + * iterators. + * + * @todo when the capacity is one and REVERSE is specified then we + * can optimize this using a reverse traversal striterator - this + * will have lower overhead than the cursor for the BTree (but not + * for an IndexSegment). + */ + src = getRoot().rangeIterator(fromKey, toKey, flags); + + } else { + + final Tuple tuple = new Tuple(this, flags); + + if (isReadOnly()) { + + // Note: this iterator does not allow removal. + src = new ReadOnlyBTreeTupleCursor(((BTree) this), tuple, + fromKey, toKey); + + } else { + + // Note: this iterator supports traversal with concurrent + // modification. + src = new MutableBTreeTupleCursor(((BTree) this), + new Tuple(this, flags), fromKey, toKey); + + } + + if ((flags & REVERSE) != 0) { + + /* + * Reverse scan iterator. + * + * Note: The reverse scan MUST be layered directly over the + * ITupleCursor. Most critically, REMOVEALL combined with a + * REVERSE scan needs to process the tuples in reverse index + * order and then delete them as it goes. + */ + + src = new Reverserator((ITupleCursor) src); + + } + + } + + if (filter != null) { + + /* + * Apply the optional filter. + * + * Note: This needs to be after the reverse scan and before + * REMOVEALL (those are the assumptions for the flags). + */ + + src = filter.newInstance(src); + + } + + if ((flags & REMOVEALL) != 0) { + + assertNotReadOnly(); + + /* + * Note: This iterator removes each tuple that it visits from the + * source iterator. + */ + + src = new TupleRemover() { + @Override + protected boolean remove(ITuple e) { + // remove all visited tuples. + return true; + } + }.filter(src); + + } + + return src; + + } + + /** * Factory for mutable nodes and leaves used by the {@link NodeSerializer}. */ protected static class NodeFactory implements INodeFactory { @@ -1682,7 +1826,6 @@ * Mutable counter. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class Counter implements ICounter { @@ -1742,7 +1885,6 @@ * int32 word. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class PartitionedCounter implements ICounter { @@ -1851,7 +1993,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ protected static class Stack { @@ -2027,7 +2168,6 @@ * Note: The {@link MutableBTreeTupleCursor} does register such listeners. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class LeafCursor implements ILeafCursor<Leaf> { @@ -2125,6 +2265,7 @@ } + @Override public LeafCursor clone() { return new LeafCursor(this); @@ -2177,7 +2318,7 @@ } - public Leaf first() { + final public Leaf first() { stack.clear(); @@ -2198,7 +2339,7 @@ } - public Leaf last() { + final public Leaf last() { stack.clear(); @@ -2224,7 +2365,7 @@ * the leaf may not actually contain the key, in which case it is the * leaf that contains the insertion point for the key. */ - public Leaf seek(final byte[] key) { + final public Leaf seek(final byte[] key) { stack.clear(); Deleted: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/DelegateIndex.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -1,169 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -*/ -/* - * Created on Feb 20, 2008 - */ - -package com.bigdata.btree; - -import com.bigdata.btree.filter.IFilterConstructor; -import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor; -import com.bigdata.btree.proc.IKeyRangeIndexProcedure; -import com.bigdata.btree.proc.IResultHandler; -import com.bigdata.btree.proc.ISimpleIndexProcedure; -import com.bigdata.counters.ICounterSet; -import com.bigdata.mdi.IResourceMetadata; - -/** - * An object that delegates its {@link IIndex} interface. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class DelegateIndex implements IIndex { - - private final IIndex delegate; - - /** - * @param delegate - * The delegate. - */ - public DelegateIndex(IIndex delegate) { - - if (delegate == null) { - - throw new IllegalArgumentException(); - - } - - this.delegate = delegate; - - } - - public String toString() { - - final StringBuilder sb = new StringBuilder(); - - sb.append(getClass().getSimpleName()); - - sb.append("{ "); - - sb.append(delegate.toString()); - - sb.append("}"); - - return sb.toString(); - - } - - public boolean contains(byte[] key) { - return delegate.contains(key); - } - - public ICounter getCounter() { - return delegate.getCounter(); - } - - public IndexMetadata getIndexMetadata() { - return delegate.getIndexMetadata(); - } - - public IResourceMetadata[] getResourceMetadata() { - return delegate.getResourceMetadata(); - } - - public ICounterSet getCounters() { - return delegate.getCounters(); - } - - public byte[] insert(byte[] key, byte[] value) { - return delegate.insert(key, value); - } - - public byte[] lookup(byte[] key) { - return delegate.lookup(key); - } - - public long rangeCount() { - return delegate.rangeCount(); - } - - public long rangeCount(byte[] fromKey, byte[] toKey) { - return delegate.rangeCount(fromKey, toKey); - } - - public long rangeCountExact(byte[] fromKey, byte[] toKey) { - return delegate.rangeCountExact(fromKey, toKey); - } - - public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { - return delegate.rangeCountExactWithDeleted(fromKey, toKey); - } - - public ITupleIterator rangeIterator() { - return rangeIterator(null,null); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey, int capacity, int flags, IFilterConstructor filter) { - return delegate.rangeIterator(fromKey, toKey, capacity, flags, filter); - } - - public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { - return delegate.rangeIterator(fromKey, toKey); - } - - public byte[] remove(byte[] key) { - return delegate.remove(key); - } - - public Object submit(byte[] key, ISimpleIndexProcedure proc) { - return delegate.submit(key, proc); - } - - public void submit(byte[] fromKey, byte[] toKey, IKeyRangeIndexProcedure proc, IResultHandler handler) { - delegate.submit(fromKey, toKey, proc, handler); - } - - public void submit(int fromIndex, int toIndex, byte[][] keys, byte[][] vals, AbstractKeyArrayIndexProcedureConstructor ctor, IResultHandler handler) { - delegate.submit(fromIndex, toIndex, keys, vals, ctor, handler); - } - - public boolean contains(Object key) { - return delegate.contains(key); - } - - public Object insert(Object key, Object value) { - return delegate.insert(key, value); - } - - public Object lookup(Object key) { - return delegate.lookup(key); - } - - public Object remove(Object key) { - return delegate.remove(key); - } - -} Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/IndexSegment.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -29,15 +29,20 @@ import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf; import com.bigdata.btree.data.ILeafData; import com.bigdata.btree.data.INodeData; +import com.bigdata.btree.filter.IFilterConstructor; +import com.bigdata.btree.filter.Reverserator; +import com.bigdata.btree.filter.TupleRemover; import com.bigdata.btree.raba.IRaba; import com.bigdata.btree.raba.ReadOnlyKeysRaba; import com.bigdata.btree.raba.ReadOnlyValuesRaba; import com.bigdata.io.AbstractFixedByteArrayBuffer; +import com.bigdata.io.DirectBufferPool; import com.bigdata.io.FixedByteArrayBuffer; import com.bigdata.mdi.IResourceMetadata; import com.bigdata.service.Event; import com.bigdata.service.EventResource; import com.bigdata.service.EventType; +import org.apache.log4j.Logger; /** * An index segment is read-only btree corresponding to some key range of a @@ -51,11 +56,15 @@ * leaves will all refuse mutation operations). * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class IndexSegment extends AbstractBTree { /** + * Log for btree opeations. + */ + private static final Logger log = Logger.getLogger(IndexSegment.class); + + /** * Type safe reference to the backing store. */ private final IndexSegmentStore fileStore; @@ -160,6 +169,7 @@ * {@link #getEntryCount()} uses the {@link IndexSegmentCheckpoint} and that * is only available while the {@link IndexSegmentStore} is open. */ + @Override public String toString() { // make sure the fileStore will remain open. @@ -303,7 +313,7 @@ } @Override - protected void _reopen() { + final protected void _reopen() { // prevent concurrent close. fileStore.lock.lock(); @@ -684,6 +694,138 @@ }; } + /** + * Core implementation. + * <p> + * Note: If {@link IRangeQuery#CURSOR} is specified the returned iterator + * supports traversal with concurrent modification by a single-threaded + * process (the {@link BTree} is NOT thread-safe for writers). Write are + * permitted iff {@link AbstractBTree} allows writes. + * <p> + * Note: {@link IRangeQuery#REVERSE} is handled here by wrapping the + * underlying {@link ITupleCursor}. + * <p> + * Note: {@link IRangeQuery#REMOVEALL} is handled here by wrapping the + * iterator. + * <p> + * Note: + * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)} + * is also responsible for constructing an {@link ITupleIterator} in a + * manner similar to this method. If you are updating the logic here, then + * check the logic in that method as well! + * + * @todo add support to the iterator construct for filtering by a tuple + * revision timestamp range. + */ + public ITupleIterator rangeIterator(// + final byte[] fromKey,// + final byte[] toKey,// + final int capacityIsIgnored,// + final int flags,// + final IFilterConstructor filter// + ) { + + /* + * Does the iterator declare that it will not write back on the index? + */ + final boolean readOnly = ((flags & IRangeQuery.READONLY) != 0); + + if (readOnly && ((flags & IRangeQuery.REMOVEALL) != 0)) { + + throw new IllegalArgumentException(); + + } + + /* + * Figure out what base iterator implementation to use. We will layer + * on the optional filter(s) below. + */ + ITupleIterator src; + + final Tuple tuple = new Tuple(this, flags); + + final IndexSegment seg = (IndexSegment) this; + + /* + * @todo we could scan the list of pools and chose the best fit + * pool and then allocate a buffer from that pool. Best fit + * would mean either the byte range fits without "too much" slop + * or the #of reads will have to perform is not too large. We + * might also want to limit the maximum size of the reads. + */ + +// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M; + final DirectBufferPool pool = DirectBufferPool.INSTANCE; + + if (true + && ((flags & REVERSE) == 0) + && ((flags & CURSOR) == 0) + && (seg.getStore().getCheckpoint().maxNodeOrLeafLength <= pool + .getBufferCapacity()) + && ((rangeCount(fromKey, toKey) / branchingFactor) > 2)) { + + src = new IndexSegmentMultiBlockIterator(seg, pool, + fromKey, toKey, flags); + + } else { + + src = new IndexSegmentTupleCursor(seg, tuple, fromKey, toKey); + + } + + + if ((flags & REVERSE) != 0) { + + /* + * Reverse scan iterator. + * + * Note: The reverse scan MUST be layered directly over the + * ITupleCursor. Most critically, REMOVEALL combined with a + * REVERSE scan needs to process the tuples in reverse index + * order and then delete them as it goes. + */ + + src = new Reverserator((ITupleCursor) src); + + } + + + if (filter != null) { + + /* + * Apply the optional filter. + * + * Note: This needs to be after the reverse scan and before + * REMOVEALL (those are the assumptions for the flags). + */ + + src = filter.newInstance(src); + + } + + if ((flags & REMOVEALL) != 0) { + + assertNotReadOnly(); + + /* + * Note: This iterator removes each tuple that it visits from the + * source iterator. + */ + + src = new TupleRemover() { + @Override + protected boolean remove(ITuple e) { + // remove all visited tuples. + return true; + } + }.filter(src); + + } + + return src; + + } + /* * INodeFactory */ @@ -719,7 +861,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ public static class ImmutableNode extends Node { @@ -768,8 +909,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id: IndexSegment.java 2265 2009-10-26 12:51:06Z thompsonbry - * $ */ private static class EmptyReadOnlyLeafData implements ILeafData { @@ -869,7 +1008,6 @@ * * @author <a href="mailto:tho...@us...">Bryan * Thompson</a> - * @version $Id$ */ public static class ImmutableLeaf extends Leaf { @@ -1007,7 +1145,6 @@ // * // * @author <a href="mailto:tho...@us...">Bryan // * Thompson</a> -// * @version $Id$ // */ // static public class ImmutableEmptyLastLeaf extends ImmutableLeaf { // @@ -1046,7 +1183,6 @@ * A position for the {@link IndexSegmentTupleCursor}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * @param <E> * The generic type for objects de-serialized from the values in * the index. @@ -1054,6 +1190,7 @@ static private class CursorPosition<E> extends AbstractCursorPosition<ImmutableLeaf,E> { @SuppressWarnings("unchecked") + @Override public IndexSegmentTupleCursor<E> getCursor() { return (IndexSegmentTupleCursor)cursor; @@ -1102,7 +1239,6 @@ * listeners for concurrent modifications. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * @param <E> * The generic type for the objects de-serialized from the index. */ @@ -1265,7 +1401,6 @@ * Cursor using the double-linked leaves for efficient scans. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class ImmutableLeafCursor implements ILeafCursor<ImmutableLeaf> { @@ -1283,6 +1418,7 @@ } + @Override public ImmutableLeafCursor clone() { return new ImmutableLeafCursor(this); @@ -1333,7 +1469,7 @@ } - public ImmutableLeaf seek(final byte[] key) { + final public ImmutableLeaf seek(final byte[] key) { leaf = findLeaf(key); @@ -1363,7 +1499,7 @@ } - public ImmutableLeaf first() { + final public ImmutableLeaf first() { final long addr = getStore().getCheckpoint().addrFirstLeaf; @@ -1373,7 +1509,7 @@ } - public ImmutableLeaf last() { + final public ImmutableLeaf last() { final long addr = getStore().getCheckpoint().addrLastLeaf; Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndex.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -29,13 +29,13 @@ import java.io.ObjectOutput; import java.util.UUID; -import com.bigdata.mdi.PartitionLocator; import org.CognitiveWeb.extser.LongPacker; import com.bigdata.btree.keys.IKeyBuilderFactory; import com.bigdata.btree.view.FusedView; import com.bigdata.journal.ICommitter; import com.bigdata.journal.IResourceManager; +import com.bigdata.mdi.PartitionLocator; import com.bigdata.rawstore.IRawStore; import com.bigdata.service.MetadataService; @@ -44,7 +44,7 @@ * metadata index for each distributed index. The keys of the metadata index are * the first key that would be directed into the corresponding index segment, * e.g., a <em>separator key</em> (this is just the standard btree semantics). - * The values are serialized {@link com.bigdata.mdi.PartitionLocator} objects. + * The values are serialized {@link PartitionLocator} objects. * <p> * Note: At this time the recommended scale-out approach for the metadata index * is to place the metadata indices on a {@link MetadataService} (the same @@ -63,7 +63,6 @@ * taking the database offline. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * @todo The {@link MetadataIndex} does NOT support either overflow (it may NOT * be a {@link FusedView}) NOR key-range splits. There are several issues @@ -97,6 +96,7 @@ */ private transient final MetadataIndexView view; + @Override public MetadataIndexMetadata getIndexMetadata() { return (MetadataIndexMetadata) super.getIndexMetadata(); @@ -210,6 +210,7 @@ * Extended to require a checkpoint if {@link #incrementAndGetNextPartitionId()} has been * invoked. */ + @Override public boolean needsCheckpoint() { if(nextPartitionId != ((MetadataIndexCheckpoint)getCheckpoint()).getNextPartitionId()) { @@ -227,7 +228,6 @@ * identifier to be assigned by the metadata index. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class MetadataIndexCheckpoint extends Checkpoint { @@ -329,7 +329,6 @@ * for the managed scale-out index. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class MetadataIndexMetadata extends IndexMetadata implements Externalizable { @@ -381,6 +380,7 @@ private static final transient int VERSION0 = 0x0; + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { @@ -398,6 +398,7 @@ } + @Override public void writeExternal(ObjectOutput out) throws IOException { super.writeExternal(out); @@ -430,7 +431,6 @@ * {@link MetadataIndex}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public static class PartitionLocatorTupleSerializer extends DefaultTupleSerializer<byte[]/*key*/, PartitionLocator/*val*/> { @@ -473,6 +473,7 @@ */ private final static transient byte VERSION = VERSION0; + @Override public void readExternal(final ObjectInput in) throws IOException, ClassNotFoundException { @@ -490,6 +491,7 @@ } + @Override public void writeExternal(final ObjectOutput out) throws IOException { super.writeExternal(out); Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/MetadataIndexView.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -28,27 +28,27 @@ package com.bigdata.btree; -import com.bigdata.mdi.PartitionLocator; +import com.bigdata.btree.MetadataIndex.MetadataIndexMetadata; +import com.bigdata.btree.filter.IFilterConstructor; import org.apache.log4j.Logger; import com.bigdata.cache.LRUCache; -import com.bigdata.btree.MetadataIndex.MetadataIndexMetadata; +import com.bigdata.mdi.PartitionLocator; /** * The extension semantics for the {@link IMetadataIndex} are implemented by * this class. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ -public class MetadataIndexView extends DelegateIndex implements IMetadataIndex { +public class MetadataIndexView implements IMetadataIndex { protected static final Logger log = Logger.getLogger(MetadataIndexView.class); // protected static final boolean INFO = log.isInfoEnabled(); // protected static final boolean DEBUG = log.isDebugEnabled(); - private final AbstractBTree delegate; + private final MetadataIndex metadataIndex; /** * <code>true</code> iff this is a read-only view. this is used to @@ -58,19 +58,14 @@ */ final private boolean readOnly; - public MetadataIndexView(AbstractBTree delegate) { - - super(delegate); - - this.delegate = delegate; - - this.readOnly = delegate.isReadOnly(); - + public MetadataIndexView(MetadataIndex metadataIndex) { + this.metadataIndex = metadataIndex; + this.readOnly = metadataIndex.isReadOnly(); } public MetadataIndexMetadata getIndexMetadata() { - return (MetadataIndexMetadata) super.getIndexMetadata(); + return (MetadataIndexMetadata) metadataIndex.getIndexMetadata(); } @@ -87,7 +82,7 @@ * de-serialization using the ITupleSerializer. */ - return (PartitionLocator) delegate.lookup((Object) key); + return (PartitionLocator) metadataIndex.lookup((Object) key); } @@ -112,7 +107,7 @@ */ private PartitionLocator find_with_iterator(byte[] key) { - final ITupleIterator<PartitionLocator> itr = delegate.rangeIterator( + final ITupleIterator<PartitionLocator> itr = metadataIndex.rangeIterator( null/* fromKey */, key/* toKey */, 1/* capacity */, IRangeQuery.VALS | IRangeQuery.REVERSE, null/* filter */); @@ -147,7 +142,7 @@ if (key == null) { // use the index of the last partition. - index = delegate.getEntryCount() - 1; + index = metadataIndex.getEntryCount() - 1; } else { @@ -212,7 +207,7 @@ /** * Remove the locator from the {@link #locatorCache}. It will be re-read on - * demand from the {@link #delegate}. + * demand from the {@link #metadataIndex}. */ public void staleLocator(PartitionLocator locator) { @@ -230,8 +225,8 @@ */ private PartitionLocator getLocatorAtIndex(int index) { - final ITuple<PartitionLocator> tuple = delegate.valueAt(index, - delegate.getLookupTuple()); + final ITuple<PartitionLocator> tuple = metadataIndex.valueAt(index, + metadataIndex.getLookupTuple()); return tuple.getObject(); @@ -252,7 +247,7 @@ */ private int findIndexOf(byte[] key) { - int pos = delegate.indexOf(key); + int pos = metadataIndex.indexOf(key); if (pos < 0) { @@ -266,7 +261,7 @@ if(pos == 0) { - if(delegate.getEntryCount() != 0) { + if(metadataIndex.getEntryCount() != 0) { throw new IllegalStateException( "Partition not defined for empty key."); @@ -294,4 +289,34 @@ } + public long rangeCount() { + return metadataIndex.rangeCount(); + } + + public long rangeCount(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeCount(fromKey, toKey); + } + + public long rangeCountExact(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeCountExact(fromKey, toKey); + } + + public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeCountExactWithDeleted(fromKey, toKey); + } + + public ITupleIterator rangeIterator() { + return metadataIndex.rangeIterator(); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { + return metadataIndex.rangeIterator(fromKey, toKey); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey, + int capacity, int flags, + IFilterConstructor filterCtor) { + return metadataIndex.rangeIterator(fromKey, toKey, capacity, flags, filterCtor); + } + } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/Node.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -56,6 +56,7 @@ import cutthecrap.utils.striterators.Expander; import cutthecrap.utils.striterators.SingleValueIterator; import cutthecrap.utils.striterators.Striterator; +import org.apache.log4j.Logger; /** * <p> @@ -82,11 +83,15 @@ * we can prune the search before we materialize the child. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ public class Node extends AbstractNode<Node> implements INodeData { /** + * Log for btree opeations. + */ + private static final Logger log = Logger.getLogger(Node.class); + + /** * The data record. {@link MutableNodeData} is used for all mutation * operations. {@link ReadOnlyNodeData} is used when the {@link Node} is * made persistent. A read-only data record is automatically converted into @@ -623,7 +628,7 @@ btree.getBtreeCounters().rootsSplit++; - if (BTree.log.isInfoEnabled() || BigdataStatics.debug) { + if (log.isInfoEnabled() || BigdataStatics.debug) { // Note: nnodes and nleaves might not reflect rightSibling yet. @@ -632,8 +637,8 @@ + ", m=" + btree.getBranchingFactor() + ", nentries=" + btree.nentries; - if (BTree.log.isInfoEnabled()) - BTree.log.info(msg); + if (log.isInfoEnabled()) + log.info(msg); if (BigdataStatics.debug) System.err.println(msg); @@ -2400,8 +2405,8 @@ // one less node in the tree. btree.nnodes--; - if (BTree.INFO) { - BTree.log.info("reduced tree height: height=" + if (INFO) { + log.info("reduced tree height: height=" + btree.height + ", newRoot=" + btree.root); } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/btree/ReadOnlyIndex.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -27,6 +27,9 @@ package com.bigdata.btree; +import com.bigdata.btree.proc.IKeyRangeIndexProcedure; +import com.bigdata.btree.proc.ISimpleIndexProcedure; +import com.bigdata.counters.ICounterSet; import java.util.Iterator; import com.bigdata.btree.filter.IFilterConstructor; @@ -47,20 +50,19 @@ * @see {@link IResourceManager#getIndex(String, long)} * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ */ -public class ReadOnlyIndex extends DelegateIndex { +public class ReadOnlyIndex implements IIndex { + + private IIndex src; public ReadOnlyIndex(IIndex src) { - - super(src); - + this.src = src; } /** {@link IndexMetadata} is cloned to disallow modification. */ final public IndexMetadata getIndexMetadata() { - return super.getIndexMetadata().clone(); + return src.getIndexMetadata().clone(); } @@ -71,7 +73,7 @@ */ final public IResourceMetadata[] getResourceMetadata() { - return super.getResourceMetadata().clone(); + return src.getResourceMetadata().clone(); } @@ -80,7 +82,7 @@ */ final public ICounter getCounter() { - return new ReadOnlyCounter(super.getCounter()); + return new ReadOnlyCounter(src.getCounter()); } @@ -121,7 +123,7 @@ /* * Must explicitly disable Iterator#remove(). */ - return new ReadOnlyEntryIterator(super.rangeIterator(fromKey, toKey, + return new ReadOnlyEntryIterator(src.rangeIterator(fromKey, toKey, capacity, flags, filter)); } @@ -173,4 +175,65 @@ } + public ICounterSet getCounters() { + return src.getCounters(); + } + + public Object submit(byte[] key, ISimpleIndexProcedure proc) { + return src.submit(key, proc); + } + + public void submit(byte[] fromKey, byte[] toKey, + IKeyRangeIndexProcedure proc, IResultHandler handler) { + src.submit(fromKey, toKey, proc, handler); + } + + public byte[] lookup(byte[] key) { + return src.lookup(key); + } + + public boolean contains(byte[] key) { + return src.contains(key); + } + + public Object insert(Object key, Object value) { + return src.insert(key, value); + } + + public Object lookup(Object key) { + return src.lookup(key); + } + + public boolean contains(Object key) { + return src.contains(key); + } + + public Object remove(Object key) { + return src.remove(key); + } + + public long rangeCount() { + return src.rangeCount(); + } + + public long rangeCount(byte[] fromKey, byte[] toKey) { + return src.rangeCount(fromKey, toKey); + } + + public long rangeCountExact(byte[] fromKey, byte[] toKey) { + return src.rangeCountExact(fromKey, toKey); + } + + public long rangeCountExactWithDeleted(byte[] fromKey, byte[] toKey) { + return src.rangeCountExactWithDeleted(fromKey, toKey); + } + + public ITupleIterator rangeIterator() { + return src.rangeIterator(); + } + + public ITupleIterator rangeIterator(byte[] fromKey, byte[] toKey) { + return src.rangeIterator(fromKey, toKey); + } + } Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/HardReferenceGlobalLRURecycler.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -613,6 +613,7 @@ e.prior = last; last = e; } + //System.err.println("++size"); size++; counters.bytesInMemory.addAndGet(e.bytesInMemory); counters.bytesOnDisk.addAndGet(e.bytesOnDisk); @@ -654,6 +655,7 @@ e.cache = null; // clear reference to the cache. e.k = null; // clear the key. e.v = null; // clear the value reference. + //System.err.println("--size"); size--; counters.bytesInMemory.addAndGet(-e.bytesInMemory); counters.bytesOnDisk.addAndGet(-e.bytesOnDisk); Modified: branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java =================================================================== --- branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java 2010-10-08 19:57:43 UTC (rev 3760) +++ branches/maven_scaleout/bigdata-core/src/main/java/com/bigdata/cache/LRUNexus.java 2010-10-08 22:21:56 UTC (rev 3761) @@ -47,7 +47,6 @@ * system properties defined by {@link Options}. * * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ * * FIXME LR... [truncated message content] |
From: <ble...@us...> - 2010-10-08 19:57:49
|
Revision: 3760 http://bigdata.svn.sourceforge.net/bigdata/?rev=3760&view=rev Author: blevine218 Date: 2010-10-08 19:57:43 +0000 (Fri, 08 Oct 2010) Log Message: ----------- use <includes> instead of <test> to designate top-level suite to run. This allows the -Dtest=... override to work. Modified Paths: -------------- branches/maven_scaleout/bigdata-core/pom.xml Modified: branches/maven_scaleout/bigdata-core/pom.xml =================================================================== --- branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 17:23:07 UTC (rev 3759) +++ branches/maven_scaleout/bigdata-core/pom.xml 2010-10-08 19:57:43 UTC (rev 3760) @@ -132,7 +132,11 @@ <testFailureIgnore>true</testFailureIgnore> <skip>false</skip> <!-- Run only the top-level suite which in turn runs the other suites and tests --> - <test>com.bigdata.TestAll</test> + <includes> + <include>com/bigdata/TestAll.java</include> + </includes> + + <forkMode>pertest</forkMode> <!-- These system properties are required by the unit tests. --> <systemPropertyVariables> This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 17:23:14
|
Revision: 3759 http://bigdata.svn.sourceforge.net/bigdata/?rev=3759&view=rev Author: thompsonbry Date: 2010-10-08 17:23:07 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Modifications to the decision tree (simplified for named graphs). Modifications to the annotations for predicates and joins for named and default graph code paths in scale-out. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/query-cost-model.xls branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/query-cost-model.xls =================================================================== (Binary files differ) Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-08 16:42:34 UTC (rev 3758) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-08 17:23:07 UTC (rev 3759) @@ -78,6 +78,7 @@ import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.InGraphHashSetFilter; import com.bigdata.rdf.spo.NamedGraphSolutionExpander; +import com.bigdata.rdf.spo.SPORelation; import com.bigdata.rdf.store.AbstractTripleStore; import com.bigdata.relation.IRelation; import com.bigdata.relation.accesspath.AccessPath; @@ -485,12 +486,17 @@ * @return The join operator. */ private static PipelineOp triplesModeJoin(final QueryEngine queryEngine, - final PipelineOp left, final List<NV> anns, final Predicate<?> pred) { + final PipelineOp left, final List<NV> anns, Predicate<?> pred) { final boolean scaleOut = queryEngine.isScaleOut(); if (scaleOut) { + /* + * All triples queries can run shard-wise. + */ anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.SHARDED)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); @@ -527,8 +533,13 @@ final boolean scaleOut = queryEngine.isScaleOut(); if (scaleOut) { + /* + * All named graph patterns in scale-out are partitioned (sharded). + */ anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.SHARDED)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); @@ -667,17 +678,15 @@ summary.getGraphs()) // })); - if (scaleOut) { - anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, - BOpEvaluationContext.SHARDED)); - pred = (Predicate) pred.setProperty( - Predicate.Annotations.REMOTE_ACCESS_PATH, false); - } else { - anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, - BOpEvaluationContext.ANY)); - pred = (Predicate) pred.setProperty( - Predicate.Annotations.REMOTE_ACCESS_PATH, false); - } +// if (scaleOut) { +// anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, +// BOpEvaluationContext.SHARDED)); +// pred = (Predicate) pred.setProperty( +// Predicate.Annotations.REMOTE_ACCESS_PATH, false); +// } else { +// anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, +// BOpEvaluationContext.ANY)); +// } return new PipelineJoin(new BOp[] { dataSetJoin, pred }, anns .toArray(new NV[anns.size()])); @@ -744,6 +753,14 @@ pred = pred.asBound((IVariable<?>) pred.get(3), new Constant<IV<?, ?>>(summary.firstContext)); + if (scaleOut) { + // use a partitioned join. + anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, + BOpEvaluationContext.SHARDED)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); + } + // Strip of the context position. pred = pred.addAccessPathFilter(StripContextFilter.newInstance()); @@ -786,19 +803,16 @@ // /* // * When true, an ISimpleSplitHandler guarantees that no triple // * on that index spans more than one shard. -// * -// * @todo Implement the split handler and detect when it is being -// * used. The implementation can use ContextAdvancer to skip to -// * the end of the "triple" identified by the default split code. // */ -// final boolean shardTripleConstraint = false; +// final SPORelation r = (SPORelation)context.getRelation(pred); +// final boolean shardTripleConstraint = r.getContainer().isConstrainXXXCShards(); // // if (shardTripleConstraint) { // // // JOIN is SHARDED. -// pred = (Predicate<?>) pred.setProperty( +// anns.add(new NV( // BOp.Annotations.EVALUATION_CONTEXT, -// BOpEvaluationContext.SHARDED); +// BOpEvaluationContext.SHARDED)); // // // AP is LOCAL. // pred = (Predicate<?>) pred.setProperty( @@ -807,9 +821,9 @@ // } else { // // // JOIN is ANY. -// pred = (Predicate<?>) pred.setProperty( +// anns.add(new NV( // BOp.Annotations.EVALUATION_CONTEXT, -// BOpEvaluationContext.ANY); +// BOpEvaluationContext.ANY)); // // // AP is REMOTE. // pred = (Predicate<?>) pred.setProperty( @@ -866,6 +880,20 @@ // Filter for distinct SPOs. pred = pred.addAccessPathFilter(DistinctFilter.newInstance()); + if (scaleOut) { + /* + * Use the global index view so we can impose the distinct + * filter. + */ + anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, + BOpEvaluationContext.ANY)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); + } else { + anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, + BOpEvaluationContext.ANY)); + } + return new PipelineJoin(new BOp[] { left, pred }, anns .toArray(new NV[anns.size()])); @@ -901,6 +929,10 @@ pred = pred.addAccessPathFilter(DistinctFilter.newInstance()); if (scaleOut) { + /* + * Use the global index view so we can impose the distinct + * filter. + */ anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); pred = (Predicate) pred.setProperty( @@ -908,8 +940,6 @@ } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); - pred = (Predicate) pred.setProperty( - Predicate.Annotations.REMOTE_ACCESS_PATH, false); } return new PipelineJoin(new BOp[] { left, pred }, anns This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 16:42:40
|
Revision: 3758 http://bigdata.svn.sourceforge.net/bigdata/?rev=3758&view=rev Author: thompsonbry Date: 2010-10-08 16:42:34 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Modified to set the REMOTE_ACCESS_PATH annotation on the Predicate rather than the Join Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-08 16:22:13 UTC (rev 3757) +++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java 2010-10-08 16:42:34 UTC (rev 3758) @@ -670,15 +670,13 @@ if (scaleOut) { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.SHARDED)); - anns - .add(new NV(Predicate.Annotations.REMOTE_ACCESS_PATH, - false)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); - anns - .add(new NV(Predicate.Annotations.REMOTE_ACCESS_PATH, - false)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); } return new PipelineJoin(new BOp[] { dataSetJoin, pred }, anns @@ -905,15 +903,13 @@ if (scaleOut) { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); - anns - .add(new NV(Predicate.Annotations.REMOTE_ACCESS_PATH, - true)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); } else { anns.add(new NV(Predicate.Annotations.EVALUATION_CONTEXT, BOpEvaluationContext.ANY)); - anns - .add(new NV(Predicate.Annotations.REMOTE_ACCESS_PATH, - false)); + pred = (Predicate) pred.setProperty( + Predicate.Annotations.REMOTE_ACCESS_PATH, false); } return new PipelineJoin(new BOp[] { left, pred }, anns This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 16:22:22
|
Revision: 3757 http://bigdata.svn.sourceforge.net/bigdata/?rev=3757&view=rev Author: thompsonbry Date: 2010-10-08 16:22:13 +0000 (Fri, 08 Oct 2010) Log Message: ----------- added more information in an exception. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-10-08 14:57:36 UTC (rev 3756) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-10-08 16:22:13 UTC (rev 3757) @@ -354,7 +354,7 @@ } else { if (localIndexManager == null) { throw new RuntimeException("Local index manager not given but" - + "access path specifies local index."); + + " access path specifies local index: pred="+predicate); } this.indexManager = localIndexManager; } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 14:57:42
|
Revision: 3756 http://bigdata.svn.sourceforge.net/bigdata/?rev=3756&view=rev Author: thompsonbry Date: 2010-10-08 14:57:36 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Modified the test suite to explicitly specify local access paths. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java 2010-10-08 14:56:30 UTC (rev 3755) +++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java 2010-10-08 14:57:36 UTC (rev 3756) @@ -467,8 +467,8 @@ .asMap(new NV[] {// new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// -// new NV(Predicate.Annotations.KEY_ORDER, -// R.primaryKeyOrder),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.BOP_ID, predId),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })), @@ -643,6 +643,8 @@ .asMap(new NV[] {// new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.BOP_ID, predId1),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })); @@ -652,6 +654,8 @@ .asMap(new NV[] {// new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.BOP_ID, predId2),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 14:56:37
|
Revision: 3755 http://bigdata.svn.sourceforge.net/bigdata/?rev=3755&view=rev Author: thompsonbry Date: 2010-10-08 14:56:30 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Modified the default for REMOTE_ACCESS_PATH to be "true" and updated the Javadoc on IPredicate.Options.REMOTE_ACCESS_PATH. Updated the TestFederatedQueryEngine test suite to explicitly specify local (partitioned) access paths. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-10-08 14:40:48 UTC (rev 3754) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-10-08 14:56:30 UTC (rev 3755) @@ -48,6 +48,7 @@ import com.bigdata.relation.rule.IAccessPathExpander; import com.bigdata.relation.rule.eval.IEvaluationPlan; import com.bigdata.relation.rule.eval.pipeline.JoinMasterTask; +import com.bigdata.service.ndx.IClientIndex; import com.bigdata.striterator.IKeyOrder; import cutthecrap.utils.striterators.FilterBase; @@ -194,36 +195,46 @@ int DEFAULT_PARTITION_ID = -1; /** - * Boolean option determines whether the predicate will use a local - * access path or a remote access path (default + * Boolean option determines whether the predicate will use a data + * service local access path (partitioned index view) or a remote access + * path (global index view) (default * {@value #DEFAULT_REMOTE_ACCESS_PATH}). * <p> - * <em>Local access paths</em> are much more efficient and should be - * used for most purposes. However, it is not possible to impose certain - * kinds of filters on a sharded or hash partitioned operations across - * local access paths. In particular, a DISTINCT filter can not be - * imposed using sharded or hash partitioned. + * Note: "Remote" has the semantics that the access path has a total + * view of the index. In scale-out this is achieved using RMI and an + * {@link IClientIndex}. "Local" has the semantics that the access path + * has a partitioned view of the index. In scale-out, this corresponds + * to a shard. In standalone, there is no difference between "local" and + * "remote" index views since the indices are not partitioned. * <p> - * When the access path is local, the parent operator must be annotated - * to use a {@link BOpEvaluationContext#SHARDED shard wise} or - * {@link BOpEvaluationContext#HASHED node-wise} mapping of the binding - * sets. + * Local access paths (in scale-out) are much more efficient and should + * be used for most purposes. However, it is not possible to impose + * certain kinds of filters on a partitioned index. For example, a + * DISTINCT filter requires a global index view. * <p> - * <em>Remote access paths</em> use a scale-out index view. This view - * makes the scale-out index appear as if it were monolithic rather than - * sharded or hash partitioned. The monolithic view of a scale-out index - * can be used to impose a DISTINCT filter since all tuples will flow - * back to the caller. + * When the access path is local (aka partitioned), the parent operator + * must be annotated to use a {@link BOpEvaluationContext#SHARDED shard + * wise} or {@link BOpEvaluationContext#HASHED node-wise} mapping of the + * binding sets. * <p> - * When the access path is remote, the parent operator should use + * Remote access paths (in scale-out) use a scale-out index view. This + * view makes the scale-out index appear as if it were monolithic rather + * than partitioned. The monolithic view of a scale-out index can be + * used to impose a DISTINCT filter since all tuples will flow back to + * the caller. + * <p> + * When the access path is remote the parent operator should use * {@link BOpEvaluationContext#ANY} to prevent the binding sets from - * being moved around when the access path is remote. + * being moved around when the access path is remote. Note that the + * {@link BOpEvaluationContext} is basically ignored for standalone + * since there is only one place for the operator to run - on the query + * controller. * * @see BOpEvaluationContext */ String REMOTE_ACCESS_PATH = "remoteAccessPath"; - boolean DEFAULT_REMOTE_ACCESS_PATH = false; + boolean DEFAULT_REMOTE_ACCESS_PATH = true; /** * If the estimated rangeCount for an {@link AccessPath#iterator()} is Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java =================================================================== --- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java 2010-10-08 14:40:48 UTC (rev 3754) +++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java 2010-10-08 14:56:30 UTC (rev 3755) @@ -540,8 +540,8 @@ new NV(Predicate.Annotations.BOP_ID, predId),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// -// new NV(Predicate.Annotations.KEY_ORDER, -// R.primaryKeyOrder),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), })); final PipelineJoin<E> joinOp = new PipelineJoin<E>(startOp/* left */, @@ -708,8 +708,8 @@ new NV(Predicate.Annotations.BOP_ID, predId),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// -// new NV(Predicate.Annotations.KEY_ORDER, -// R.primaryKeyOrder),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), })); final PipelineJoin<E> joinOp = new PipelineJoin<E>(startOp/* left */, @@ -858,8 +858,8 @@ .asMap(new NV[] {// new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// -// new NV(Predicate.Annotations.KEY_ORDER, -// R.primaryKeyOrder),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.BOP_ID, predId),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// @@ -1013,6 +1013,8 @@ new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// new NV(Predicate.Annotations.BOP_ID, predId1),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })); @@ -1022,6 +1024,8 @@ new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// new NV(Predicate.Annotations.BOP_ID, predId2),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })); @@ -1207,6 +1211,8 @@ .asMap(new NV[] {// new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.BOP_ID, predId1),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })); @@ -1216,6 +1222,8 @@ .asMap(new NV[] {// new NV(Predicate.Annotations.RELATION_NAME, new String[] { namespace }),// + // Note: local access path! + new NV( Predicate.Annotations.REMOTE_ACCESS_PATH,false), new NV(Predicate.Annotations.BOP_ID, predId2),// new NV(Predicate.Annotations.TIMESTAMP, ITx.READ_COMMITTED),// })); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mrp...@us...> - 2010-10-08 14:40:54
|
Revision: 3754 http://bigdata.svn.sourceforge.net/bigdata/?rev=3754&view=rev Author: mrpersonick Date: 2010-10-08 14:40:48 +0000 (Fri, 08 Oct 2010) Log Message: ----------- finishing up the change sets implementation Added Paths: ----------- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java Added: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java (rev 0) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java 2010-10-08 14:40:48 UTC (rev 3754) @@ -0,0 +1,163 @@ +package com.bigdata.rdf.changesets; + +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import org.apache.log4j.Logger; +import com.bigdata.rdf.model.BigdataStatement; +import com.bigdata.rdf.spo.ISPO; +import com.bigdata.rdf.store.AbstractTripleStore; +import com.bigdata.rdf.store.BigdataStatementIterator; +import com.bigdata.striterator.ChunkedArrayIterator; + +/** + * This is a very simple implementation of a change log. NOTE: This is not + * a particularly great implementation. First of all it ends up storing + * two copies of the change set. Secondly it needs to be smarter about + * concurrency, or maybe we can be smart about it when we do the + * implementation on the other side (the SAIL connection can just write + * change events to a buffer and then the buffer can be drained by + * another thread that doesn't block the actual read/write operations, + * although then we need to be careful not to issue the committed() + * notification before the buffer is drained). + * + * @author mike + * + */ +public class InMemChangeLog implements IChangeLog { + + protected static final Logger log = Logger.getLogger(InMemChangeLog.class); + + /** + * Running tally of new changes since the last commit notification. + */ + private final Map<ISPO,IChangeRecord> changeSet = + new HashMap<ISPO, IChangeRecord>(); + + /** + * Keep a record of the change set as of the last commit. + */ + private final Map<ISPO,IChangeRecord> committed = + new HashMap<ISPO, IChangeRecord>(); + + /** + * See {@link IChangeLog#changeEvent(IChangeRecord)}. + */ + public synchronized void changeEvent(final IChangeRecord record) { + + if (log.isInfoEnabled()) + log.info(record); + + changeSet.put(record.getStatement(), record); + + } + + /** + * See {@link IChangeLog#transactionCommited()}. + */ + public synchronized void transactionCommited() { + + if (log.isInfoEnabled()) + log.info("transaction committed"); + + committed.clear(); + + committed.putAll(changeSet); + + changeSet.clear(); + + } + + /** + * See {@link IChangeLog#transactionAborted()}. + */ + public synchronized void transactionAborted() { + + if (log.isInfoEnabled()) + log.info("transaction aborted"); + + changeSet.clear(); + + } + + /** + * Return the change set as of the last commmit point. + * + * @return + * a collection of {@link IChangeRecord}s as of the last commit + * point + */ + public Collection<IChangeRecord> getLastCommit() { + + return committed.values(); + + } + + /** + * Return the change set as of the last commmit point, using the supplied + * database to resolve ISPOs to BigdataStatements. + * + * @return + * a collection of {@link IChangeRecord}s as of the last commit + * point + */ + public Collection<IChangeRecord> getLastCommit(final AbstractTripleStore db) { + + return resolve(db, committed.values()); + + } + + /** + * Use the supplied database to turn a set of ISPO change records into + * BigdataStatement change records. BigdataStatements also implement + * ISPO, the difference being that BigdataStatements also contain + * materialized RDF terms for the 3 (or 4) positions, in addition to just + * the internal identifiers (IVs) for those terms. + * + * @param db + * the database containing the lexicon needed to materialize + * the BigdataStatement objects + * @param unresolved + * the ISPO change records that came from IChangeLog notification + * events + * @return + * the fully resolves BigdataStatement change records + */ + private Collection<IChangeRecord> resolve(final AbstractTripleStore db, + final Collection<IChangeRecord> unresolved) { + + final Collection<IChangeRecord> resolved = + new LinkedList<IChangeRecord>(); + + // collect up the ISPOs out of the unresolved change records + final ISPO[] spos = new ISPO[unresolved.size()]; + int i = 0; + for (IChangeRecord rec : unresolved) { + spos[i++] = rec.getStatement(); + } + + // use the database to resolve them into BigdataStatements + final BigdataStatementIterator it = + db.asStatementIterator( + new ChunkedArrayIterator<ISPO>(i, spos, null/* keyOrder */)); + + /* + * the BigdataStatementIterator will produce BigdataStatement objects + * in the same order as the original ISPO array + */ + for (IChangeRecord rec : unresolved) { + + final BigdataStatement stmt = it.next(); + + resolved.add(new ChangeRecord(stmt, rec.getChangeAction())); + + } + + return resolved; + + } + + + +} This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 14:37:05
|
Revision: 3753 http://bigdata.svn.sourceforge.net/bigdata/?rev=3753&view=rev Author: thompsonbry Date: 2010-10-08 14:36:59 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Modified the WriteCacheService to set open:=false at the top of close() and to trap a thrown IllegalStateException in read() and return null so that the caller will read through to the backing store if the write cache service is concurrently closed (this is per its API). Modified the TestConcurrentJournal to check for isCancelled() since the test_concurrentReaders() was running with a timeout and failing if any tasks were cancelled. Modified Paths: -------------- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java 2010-10-08 12:50:48 UTC (rev 3752) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java 2010-10-08 14:36:59 UTC (rev 3753) @@ -226,6 +226,8 @@ * write lock prevents {@link #acquireForWriter()} when we must either reset * the {@link #current} cache buffer or change the {@link #current} * reference. E.g., {@link #flush(boolean, long, TimeUnit)}. + * <p> + * Note: {@link #read(long)} is non-blocking. It does NOT use this lock!!! */ final private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @@ -957,6 +959,9 @@ return; } + // Closed. + open = false; + // Interrupt the write task. localWriteFuture.cancel(true/* mayInterruptIfRunning */); final Future<?> rwf = remoteWriteFuture; @@ -1031,9 +1036,6 @@ // if (haServer != null) // haServer.interrupt(); - // Closed. - open = false; - } finally { writeLock.unlock(); } @@ -1910,8 +1912,6 @@ * Not open. Return [null] rather than throwing an exception per the * contract for this implementation. */ - log.warn("Reading from closed writeCacheService"); - return null; } @@ -1934,9 +1934,17 @@ * Ask the cache buffer if it has the record still. It will not if the * cache buffer has been concurrently reset. */ + try { + return cache.read(off.longValue()); + } catch (IllegalStateException ex) { + /* + * The write cache was closed. Per the API for this method, return + * [null] so that the caller will read through to the backing store. + */ + assert !open; + return null; + } - return cache.read(off.longValue()); - } /** Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-10-08 12:50:48 UTC (rev 3752) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/journal/TestConcurrentJournal.java 2010-10-08 14:36:59 UTC (rev 3753) @@ -1784,7 +1784,7 @@ for(Future f : futures) { - if(f.isDone()) { + if(f.isDone()&&!f.isCancelled()) { // all tasks that complete should have done so without error. f.get(); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <tho...@us...> - 2010-10-08 12:50:55
|
Revision: 3752 http://bigdata.svn.sourceforge.net/bigdata/?rev=3752&view=rev Author: thompsonbry Date: 2010-10-08 12:50:48 +0000 (Fri, 08 Oct 2010) Log Message: ----------- Removed NESTED_SUBQUERY option. Removed MAX_PARALLEL option. Modified Paths: -------------- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config branches/QUADS_QUERY_BRANCH/src/resources/config/standalone/bigdataStandalone.config Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config =================================================================== --- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config 2010-10-08 11:51:47 UTC (rev 3751) +++ branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config 2010-10-08 12:50:48 UTC (rev 3752) @@ -1441,30 +1441,6 @@ */ new NV(BigdataSail.Options.JUSTIFY, "false"), - /* - * Choice of the join algorithm. - * - * false is pipeline, which scales-out. - * - * true is nested, which is also the default right now but - * does not scale-out. - */ - new NV(BigdataSail.Options.NESTED_SUBQUERY, "false"), - - /* - * Maximum #of subqueries to evaluate concurrently for the 1st - * join dimension for native rules. Zero disables the use of - * an executor service. One forces a single thread, but runs - * the subquery on the executor service. N>1 is concurrent - * subquery evaluation. - * - * Note: parallel subquery does not work for pipeline joins at - * this time so this option is only safe for nested subquery - * join, and nested subquery joins do not scale-out. - */ - // new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "5"), - new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "0"), - new NV(com.bigdata.config.Configuration.getOverrideProperty ( namespace, IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config =================================================================== --- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config 2010-10-08 11:51:47 UTC (rev 3751) +++ branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config 2010-10-08 12:50:48 UTC (rev 3752) @@ -1539,30 +1539,6 @@ */ new NV(BigdataSail.Options.JUSTIFY, "false"), - /* - * Choice of the join algorithm. - * - * false is pipeline, which scales-out. - * - * true is nested, which is also the default right now but - * does not scale-out. - */ - new NV(BigdataSail.Options.NESTED_SUBQUERY, "false"), - - /* - * Maximum #of subqueries to evaluate concurrently for the 1st - * join dimension for native rules. Zero disables the use of - * an executor service. One forces a single thread, but runs - * the subquery on the executor service. N>1 is concurrent - * subquery evaluation. - * - * Note: parallel subquery does not work for pipeline joins at - * this time so this option is only safe for nested subquery - * join, and nested subquery joins do not scale-out. - */ - // new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "5"), - new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "0"), - new NV(com.bigdata.config.Configuration.getOverrideProperty ( namespace, IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config =================================================================== --- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-10-08 11:51:47 UTC (rev 3751) +++ branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-10-08 12:50:48 UTC (rev 3752) @@ -1466,30 +1466,6 @@ */ new NV(BigdataSail.Options.JUSTIFY, "false"), - /* - * Choice of the join algorithm. - * - * false is pipeline, which scales-out. - * - * true is nested, which is also the default right now but - * does not scale-out. - */ - new NV(BigdataSail.Options.NESTED_SUBQUERY, "false"), - - /* - * Maximum #of subqueries to evaluate concurrently for the 1st - * join dimension for native rules. Zero disables the use of - * an executor service. One forces a single thread, but runs - * the subquery on the executor service. N>1 is concurrent - * subquery evaluation. - * - * Note: parallel subquery does not work for pipeline joins at - * this time so this option is only safe for nested subquery - * join, and nested subquery joins do not scale-out. - */ - // new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "5"), - new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "0"), - new NV(com.bigdata.config.Configuration.getOverrideProperty ( namespace, IndexMetadata.Options.WRITE_RETENTION_QUEUE_CAPACITY Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/standalone/bigdataStandalone.config =================================================================== --- branches/QUADS_QUERY_BRANCH/src/resources/config/standalone/bigdataStandalone.config 2010-10-08 11:51:47 UTC (rev 3751) +++ branches/QUADS_QUERY_BRANCH/src/resources/config/standalone/bigdataStandalone.config 2010-10-08 12:50:48 UTC (rev 3752) @@ -1278,30 +1278,6 @@ */ new NV(BigdataSail.Options.JUSTIFY, "false"), - /* - * Choice of the join algorithm. - * - * false is pipeline, which scales-out. - * - * true is nested, which is also the default right now but - * does not scale-out. - */ - new NV(BigdataSail.Options.NESTED_SUBQUERY, "false"), - - /* - * Maximum #of subqueries to evaluate concurrently for the 1st - * join dimension for native rules. Zero disables the use of - * an executor service. One forces a single thread, but runs - * the subquery on the executor service. N>1 is concurrent - * subquery evaluation. - * - * Note: parallel subquery does not work for pipeline joins at - * this time so this option is only safe for nested subquery - * join, and nested subquery joins do not scale-out. - */ - // new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "5"), - new NV(BigdataSail.Options.MAX_PARALLEL_SUBQUERIES, "0"), - // This has a large effect on the B+Tree throughput, but raises RAM demands. new NV(com.bigdata.config.Configuration.getOverrideProperty ( namespace, This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mar...@us...> - 2010-10-08 11:51:53
|
Revision: 3751 http://bigdata.svn.sourceforge.net/bigdata/?rev=3751&view=rev Author: martyncutcher Date: 2010-10-08 11:51:47 +0000 (Fri, 08 Oct 2010) Log Message: ----------- add debug output to isolate test failures Modified Paths: -------------- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/io/TestFileChannelUtility.java branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java 2010-10-08 01:45:59 UTC (rev 3750) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/java/com/bigdata/io/writecache/WriteCacheService.java 2010-10-08 11:51:47 UTC (rev 3751) @@ -1411,6 +1411,9 @@ log.trace("offset: " + offset + ", length: " + data.limit() + ", chk=" + chk + ", useChecksum=" + useChecksum); } + + if (!open) + throw new IllegalStateException("WriteCacheService has been closed"); if (offset < 0) throw new IllegalArgumentException(); @@ -1907,6 +1910,8 @@ * Not open. Return [null] rather than throwing an exception per the * contract for this implementation. */ + log.warn("Reading from closed writeCacheService"); + return null; } Modified: branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/io/TestFileChannelUtility.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/io/TestFileChannelUtility.java 2010-10-08 01:45:59 UTC (rev 3750) +++ branches/JOURNAL_HA_BRANCH/bigdata/src/test/com/bigdata/io/TestFileChannelUtility.java 2010-10-08 11:51:47 UTC (rev 3751) @@ -358,7 +358,7 @@ final RandomAccessFile source = new RandomAccessFile(sourceFile, "rw"); - final RandomAccessFile target = new RandomAccessFile(sourceFile, "rw"); + final RandomAccessFile target = new RandomAccessFile(targetFile, "rw"); try { @@ -377,6 +377,7 @@ // write ground truth onto the file. FileChannelUtility.writeAll(source.getChannel(), ByteBuffer .wrap(expected), 0L/* pos */); + target.setLength(FILE_SIZE); // do a bunch of trials of random transfers. for(int trial=0; trial<1000; trial++) { Modified: branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java =================================================================== --- branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java 2010-10-08 01:45:59 UTC (rev 3750) +++ branches/JOURNAL_HA_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java 2010-10-08 11:51:47 UTC (rev 3751) @@ -619,12 +619,15 @@ String baseURI; try { + + System.out.println("looking for " + resource); is = new FileInputStream(new File(resource)); baseURI = new File(resource).toURI().toString(); } catch (FileNotFoundException ex) { + System.out.println("no file, retrieving from resource"); is = getClass().getResourceAsStream(resource); baseURI = getClass().getResource(resource).toURI() .toString(); This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mrp...@us...> - 2010-10-08 01:46:10
|
Revision: 3750 http://bigdata.svn.sourceforge.net/bigdata/?rev=3750&view=rev Author: mrpersonick Date: 2010-10-08 01:45:59 +0000 (Fri, 08 Oct 2010) Log Message: ----------- finishing up the change sets implementation Modified Paths: -------------- branches/CHANGE_SET_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPORetractionBuffer.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexRemover.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriter.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/StatementWriter.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java branches/CHANGE_SET_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestChangeSets.java Added Paths: ----------- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java Removed Paths: ------------- branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/ChangeRecord.java branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/IChangeLog.java branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/IChangeRecord.java branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/StatementWriter.java Modified: branches/CHANGE_SET_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -39,6 +39,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.io.OutputStream; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -795,18 +796,34 @@ IResultHandler<ResultBitBuffer, ResultBitBuffer> { private final boolean[] results; + + /** + * I added this so I could encode information about tuple modification + * that takes more than one boolean to encode. For example, SPOs can + * be: INSERTED, REMOVED, UPDATED, NO_OP (2 bits). + */ + private final int multiplier; + private final AtomicInteger onCount = new AtomicInteger(); public ResultBitBufferHandler(final int nkeys) { + + this(nkeys, 1); + + } + + public ResultBitBufferHandler(final int nkeys, final int multiplier) { - results = new boolean[nkeys]; + results = new boolean[nkeys*multiplier]; + this.multiplier = multiplier; } public void aggregate(final ResultBitBuffer result, final Split split) { - System.arraycopy(result.getResult(), 0, results, split.fromIndex, - split.ntuples); + System.arraycopy(result.getResult(), 0, results, + split.fromIndex*multiplier, + split.ntuples*multiplier); onCount.addAndGet(result.getOnCount()); Copied: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java (from rev 3729, branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/ChangeRecord.java) =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java (rev 0) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -0,0 +1,98 @@ +package com.bigdata.rdf.changesets; + +import java.util.Comparator; +import com.bigdata.rdf.spo.ISPO; +import com.bigdata.rdf.spo.SPOComparator; + +public class ChangeRecord implements IChangeRecord { + + private final ISPO stmt; + + private final ChangeAction action; + +// private final StatementEnum oldType; + + public ChangeRecord(final ISPO stmt, final ChangeAction action) { + +// this(stmt, action, null); +// +// } +// +// public ChangeRecord(final BigdataStatement stmt, final ChangeAction action, +// final StatementEnum oldType) { +// + this.stmt = stmt; + this.action = action; +// this.oldType = oldType; + + } + + public ChangeAction getChangeAction() { + + return action; + + } + +// public StatementEnum getOldStatementType() { +// +// return oldType; +// +// } + + public ISPO getStatement() { + + return stmt; + + } + + @Override + public boolean equals(Object o) { + + if (o == this) + return true; + + if (o == null || o instanceof IChangeRecord == false) + return false; + + final IChangeRecord rec = (IChangeRecord) o; + + final ISPO stmt2 = rec.getStatement(); + + // statements are equal + if (stmt == stmt2 || + (stmt != null && stmt2 != null && stmt.equals(stmt2))) { + + // actions are equal + return action == rec.getChangeAction(); + + } + + return false; + + } + + public String toString() { + + StringBuilder sb = new StringBuilder(); + + sb.append(action).append(": ").append(stmt); + + return sb.toString(); + + } + + public static final Comparator<IChangeRecord> COMPARATOR = + new Comparator<IChangeRecord>() { + + public int compare(final IChangeRecord r1, final IChangeRecord r2) { + + final ISPO spo1 = r1.getStatement(); + final ISPO spo2 = r2.getStatement(); + + return SPOComparator.INSTANCE.compare(spo1, spo2); + + } + + }; + +} Copied: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java (from rev 3668, branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/IChangeLog.java) =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java (rev 0) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -0,0 +1,38 @@ +package com.bigdata.rdf.changesets; + +/** + * Provides detailed information on changes made to statements in the database. + * Change records are generated for any statements that are used in + * addStatement() or removeStatements() operations on the SAIL connection, as + * well as any inferred statements that are added or removed as a result of + * truth maintenance when the database has inference enabled. Change records + * will be sent to an instance of this class via the + * {@link #changeEvent(IChangeRecord)} method. These events will + * occur on an ongoing basis as statements are added to or removed from the + * indices. It is the change log's responsibility to collect change records. + * When the transaction is actually committed (or aborted), the change log will + * receive notification via {@link #transactionCommited()} or + * {@link #transactionAborted()}. + */ +public interface IChangeLog { + + /** + * Occurs when a statement add or remove is flushed to the indices (but + * not yet committed). + * + * @param record + * the {@link IChangeRecord} + */ + void changeEvent(final IChangeRecord record); + + /** + * Occurs when the current SAIL transaction is committed. + */ + void transactionCommited(); + + /** + * Occurs if the current SAIL transaction is aborted. + */ + void transactionAborted(); + +} Copied: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java (from rev 3729, branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/IChangeRecord.java) =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java (rev 0) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -0,0 +1,120 @@ +package com.bigdata.rdf.changesets; + +import com.bigdata.rdf.model.BigdataStatement; +import com.bigdata.rdf.model.StatementEnum; +import com.bigdata.rdf.spo.ISPO; + +/** + * Provides detailed information on changes made to statements in the database. + * Change records are generated for any statements that are used in + * addStatement() or removeStatements() operations on the SAIL connection, as + * well as any inferred statements that are added or removed as a result of + * truth maintenance when the database has inference enabled. + * <p> + * See {@link IChangeLog}. + */ +public interface IChangeRecord { + + /** + * Attempting to add or remove statements can have a number of different + * effects. This enum captures the different actions that can take place as + * a result of trying to add or remove a statement from the database. + */ + public enum ChangeAction { + + /** + * The focus statement was not in the database before and will be + * in the database after the commit. This can be the result of either + * explicit addStatement() operations on the SAIL connection, or from + * new inferences being generated via truth maintenance when the + * database has inference enabled. If the focus statement has a + * statement type of explicit then it was added via an addStatement() + * operation. If the focus statement has a statement type of inferred + * then it was added via truth maintenance. + */ + INSERTED, + + /** + * The focus statement was in the database before and will not + * be in the database after the commit. When the database has inference + * and truth maintenance enabled, the statement that is the focus of + * this change record was either an explicit statement that was the + * subject of a removeStatements() operation on the connection, or it + * was an inferred statement that was removed as a result of truth + * maintenance. Either way, the statement is no longer provable as an + * inference using other statements still in the database after the + * commit. If it were still provable, the explicit statement would have + * had its type changed to inferred, and the inferred statement would + * have remained untouched by truth maintenance. If an inferred + * statement was the subject of a removeStatement() operation on the + * connection it would have resulted in a no-op, since inferences can + * only be removed via truth maintenance. + */ + REMOVED, + + /** + * This change action can only occur when inference and truth + * maintenance are enabled on the database. Sometimes an attempt at + * statement addition or removal via an addStatement() or + * removeStatements() operation on the connection will result in a type + * change rather than an actual assertion or deletion. When in + * inference mode, statements can have one of three statement types: + * explicit, inferred, or axiom (see {@link StatementEnum}). There are + * several reasons why a statement will change type rather than be + * asserted or deleted: + * <p> + * <ul> + * <li> A statement is asserted, but already exists in the database as + * an inference or an axiom. The existing statement will have its type + * changed from inference or axiom to explicit. </li> + * <li> An explicit statement is retracted, but is still provable by + * other means. It will have its type changed from explicit to + * inference. </li> + * <li> An explicit statement is retracted, but is one of the axioms + * needed for inference. It will have its type changed from explicit to + * axiom. </li> + * </ul> + */ + UPDATED, + +// /** +// * This change action can occur for one of two reasons: +// * <p> +// * <ul> +// * <li> A statement is asserted, but already exists in the database as +// * an explicit statement. </li> +// * <li> An inferred statement or an axiom is retracted. Only explicit +// * statements can be retracted via removeStatements() operations. </li> +// * </ul> +// */ +// NO_OP + + } + + /** + * Return the ISPO that is the focus of this change record. + * + * @return + * the {@link ISPO} + */ + ISPO getStatement(); + + /** + * Return the change action for this change record. + * + * @return + * the {@link ChangeAction} + */ + ChangeAction getChangeAction(); + +// /** +// * If the change action is {@link ChangeAction#TYPE_CHANGE}, this method +// * will return the old statement type of the focus statement. The +// * new statement type is available on the focus statement itself. +// * +// * @return +// * the old statement type of the focus statement +// */ +// StatementEnum getOldStatementType(); + +} Copied: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java (from rev 3736, branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/StatementWriter.java) =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java (rev 0) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -0,0 +1,208 @@ +package com.bigdata.rdf.changesets; + +import java.util.Iterator; +import java.util.Map; +import org.apache.log4j.Logger; +import com.bigdata.rdf.changesets.IChangeRecord.ChangeAction; +import com.bigdata.rdf.internal.IV; +import com.bigdata.rdf.model.BigdataBNode; +import com.bigdata.rdf.spo.ISPO; +import com.bigdata.rdf.spo.SPO; +import com.bigdata.rdf.spo.ISPO.ModifiedEnum; +import com.bigdata.rdf.store.AbstractTripleStore; +import com.bigdata.relation.accesspath.IElementFilter; +import com.bigdata.striterator.ChunkedArrayIterator; +import com.bigdata.striterator.IChunkedOrderedIterator; + +public class StatementWriter { + + protected static final Logger log = Logger.getLogger(StatementWriter.class); + + public static long addStatements(final AbstractTripleStore database, + final AbstractTripleStore statementStore, + final boolean copyOnly, + final IElementFilter<ISPO> filter, + final IChunkedOrderedIterator<ISPO> itr, + final IChangeLog changeLog) { + + long n = 0; + + if (itr.hasNext()) { + +// final BigdataStatementIteratorImpl itr2 = +// new BigdataStatementIteratorImpl(database, bnodes, itr) +// .start(database.getExecutorService()); +// +// final BigdataStatement[] stmts = +// new BigdataStatement[database.getChunkCapacity()]; + final SPO[] stmts = new SPO[database.getChunkCapacity()]; + + int i = 0; + while ((i = nextChunk(itr, stmts)) > 0) { + n += addStatements(database, statementStore, copyOnly, filter, + stmts, i, changeLog); + } + + } + + return n; + + } + + private static long addStatements(final AbstractTripleStore database, + final AbstractTripleStore statementStore, + final boolean copyOnly, + final IElementFilter<ISPO> filter, + final ISPO[] stmts, + final int numStmts, + final IChangeLog changeLog) { + +// final SPO[] tmp = allocateSPOs(stmts, numStmts); + + final long n = database.addStatements(statementStore, copyOnly, + new ChunkedArrayIterator<ISPO>(numStmts, stmts, + null/* keyOrder */), filter); + + // Copy the state of the isModified() flag and notify changeLog + for (int i = 0; i < numStmts; i++) { + + if (stmts[i].isModified()) { + +// stmts[i].setModified(true); + + if (changeLog != null) { + + switch(stmts[i].getModified()) { + case INSERTED: + changeLog.changeEvent(new ChangeRecord(stmts[i], ChangeAction.INSERTED)); + break; + case UPDATED: + changeLog.changeEvent(new ChangeRecord(stmts[i], ChangeAction.UPDATED)); + break; + case REMOVED: + throw new AssertionError(); + default: + break; + } + + } + + } + + } + + return n; + + } + + public static long removeStatements(final AbstractTripleStore database, + final IChunkedOrderedIterator<ISPO> itr, + final boolean computeClosureForStatementIdentifiers, + final IChangeLog changeLog) { + + long n = 0; + + if (itr.hasNext()) { + +// final BigdataStatementIteratorImpl itr2 = +// new BigdataStatementIteratorImpl(database, bnodes, itr) +// .start(database.getExecutorService()); +// +// final BigdataStatement[] stmts = +// new BigdataStatement[database.getChunkCapacity()]; + final SPO[] stmts = new SPO[database.getChunkCapacity()]; + + int i = 0; + while ((i = nextChunk(itr, stmts)) > 0) { + n += removeStatements(database, stmts, i, + computeClosureForStatementIdentifiers, changeLog); + } + + } + + return n; + + } + + private static long removeStatements(final AbstractTripleStore database, + final ISPO[] stmts, + final int numStmts, + final boolean computeClosureForStatementIdentifiers, + final IChangeLog changeLog) { + + final long n = database.removeStatements( + new ChunkedArrayIterator<ISPO>(numStmts, stmts, + null/* keyOrder */), + computeClosureForStatementIdentifiers); + + // Copy the state of the isModified() flag and notify changeLog + for (int i = 0; i < numStmts; i++) { + + if (stmts[i].isModified()) { + + // just to be safe + stmts[i].setModified(ModifiedEnum.REMOVED); + + changeLog.changeEvent( + new ChangeRecord(stmts[i], ChangeAction.REMOVED)); + + } + + } + + return n; + + } + + private static int nextChunk(final Iterator<ISPO> itr, + final ISPO[] stmts) { + + assert stmts != null && stmts.length > 0; + + int i = 0; + while (itr.hasNext()) { + stmts[i++] = itr.next(); + if (i == stmts.length) { + // stmts[] is full + return i; + } + } + + /* + * stmts[] is empty (i = 0) or partially + * full (i > 0 && i < stmts.length) + */ + return i; + + } + +// private static SPO[] allocateSPOs(final BigdataStatement[] stmts, +// final int numStmts) { +// +// final SPO[] tmp = new SPO[numStmts]; +// +// for (int i = 0; i < tmp.length; i++) { +// +// final BigdataStatement stmt = stmts[i]; +// +// final SPO spo = new SPO(stmt); +// +// if (log.isDebugEnabled()) +// log.debug("writing: " + stmt.toString() + " (" + spo + ")"); +// +// if(!spo.isFullyBound()) { +// +// throw new AssertionError("Not fully bound? : " + spo); +// +// } +// +// tmp[i] = spo; +// +// } +// +// return tmp; +// +// +// } + +} Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -35,9 +35,9 @@ import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; +import com.bigdata.rdf.changesets.IChangeLog; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.model.BigdataBNode; -import com.bigdata.rdf.sail.changesets.IChangeLog; import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.ISPOAssertionBuffer; import com.bigdata.rdf.spo.JustificationWriter; @@ -106,10 +106,11 @@ */ protected final boolean justify; + /** + * Used for change set notification (optional). + */ protected final IChangeLog changeLog; - protected final Map<IV, BigdataBNode> bnodes; - /** * Create a buffer. * @@ -135,14 +136,35 @@ boolean justified) { this(focusStore, db, filter, capacity, justified, - null/* changeLog */, null/* bnodes */); + null/* changeLog */); } + /** + * Create a buffer. + * + * @param focusStore + * The focusStore on which the entailments computed by closure + * will be written (required). This is either the database or a + * temporary focusStore used during incremental TM. + * @param db + * The database in which the terms are defined (required). + * @param filter + * Option filter. When present statements matched by the filter + * are NOT retained by the {@link SPOAssertionBuffer} and will + * NOT be added to the <i>focusStore</i>. + * @param capacity + * The maximum {@link SPO}s that the buffer can hold before it + * is {@link #flush()}ed. + * @param justified + * true iff the Truth Maintenance strategy requires that we + * focusStore {@link Justification}s for entailments. + * @param changeLog + * optional change log for change notification + */ public SPOAssertionBuffer(AbstractTripleStore focusStore, AbstractTripleStore db, IElementFilter<ISPO> filter, int capacity, - boolean justified, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes + boolean justified, final IChangeLog changeLog ) { super(db, filter, capacity); @@ -163,8 +185,6 @@ this.changeLog = changeLog; - this.bnodes = bnodes; - } /** @@ -214,15 +234,13 @@ } else { - n = com.bigdata.rdf.sail.changesets. - StatementWriter.addStatements( + n = com.bigdata.rdf.changesets.StatementWriter.addStatements( db, focusStore, true/* copyOnly */, null/* filter */, new ChunkedArrayIterator<ISPO>(numStmts, stmts, null/*keyOrder*/), - changeLog, - bnodes); + changeLog); } @@ -249,7 +267,7 @@ tasks.add(new StatementWriter(getTermDatabase(), focusStore, false/* copyOnly */, new ChunkedArrayIterator<ISPO>( numStmts, stmts, null/*keyOrder*/), nwritten, - changeLog, bnodes)); + changeLog)); // task will write justifications on the justifications index. final AtomicLong nwrittenj = new AtomicLong(); Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPORetractionBuffer.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPORetractionBuffer.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPORetractionBuffer.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -28,10 +28,10 @@ package com.bigdata.rdf.inf; import java.util.Map; +import com.bigdata.rdf.changesets.IChangeLog; +import com.bigdata.rdf.changesets.StatementWriter; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.model.BigdataBNode; -import com.bigdata.rdf.sail.changesets.IChangeLog; -import com.bigdata.rdf.sail.changesets.StatementWriter; import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.SPO; import com.bigdata.rdf.store.AbstractTripleStore; @@ -54,11 +54,12 @@ private final AbstractTripleStore store; private final boolean computeClosureForStatementIdentifiers; - + + /** + * Optional change log for change notification. + */ protected final IChangeLog changeLog; - protected final Map<IV, BigdataBNode> bnodes; - /** * @param store * The database from which the statement will be removed when the @@ -73,13 +74,25 @@ boolean computeClosureForStatementIdentifiers) { this(store, capacity, computeClosureForStatementIdentifiers, - null/* changeLog */, null/* bnodes */); + null/* changeLog */); } + /** + * @param store + * The database from which the statement will be removed when the + * buffer is {@link #flush()}ed. + * @param capacity + * The capacity of the retraction buffer. + * @param computeClosureForStatementIdentifiers + * See + * {@link AbstractTripleStore#removeStatements(com.bigdata.rdf.spo.ISPOIterator, boolean)} + * @param changeLog + * optional change log for change notification + */ public SPORetractionBuffer(AbstractTripleStore store, int capacity, boolean computeClosureForStatementIdentifiers, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes) { + final IChangeLog changeLog) { super(store, null/*filter*/, capacity); @@ -92,8 +105,6 @@ this.changeLog = changeLog; - this.bnodes = bnodes; - } public int flush() { @@ -114,8 +125,7 @@ new ChunkedArrayIterator<ISPO>( numStmts,stmts,null/*keyOrder*/), computeClosureForStatementIdentifiers, - changeLog, - bnodes); + changeLog); } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -54,13 +54,13 @@ import org.apache.log4j.MDC; import com.bigdata.journal.TemporaryStore; +import com.bigdata.rdf.changesets.IChangeLog; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.model.BigdataBNode; import com.bigdata.rdf.model.BigdataStatement; import com.bigdata.rdf.model.StatementEnum; import com.bigdata.rdf.rio.IStatementBuffer; import com.bigdata.rdf.rules.InferenceEngine; -import com.bigdata.rdf.sail.changesets.IChangeLog; import com.bigdata.rdf.spo.ExplicitSPOFilter; import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.SPO; @@ -244,15 +244,39 @@ ) { return applyExistingStatements(focusStore, database, filter, - null/* changeLog */, null/* bnodes */); + null/* changeLog */); } + + /** + * Any statements in the <i>fousStore</i> that are already in the database + * are converted to explicit statements (iff they are not already explicit) + * and <strong>removed</strong> from the <i>focusStore</i> as a + * side-effect. This prevents the application of the rules to data that is + * already known to the database. + * + * @param focusStore + * The store whose closure is being computed. + * @param database + * The database. + * @param filter + * An optional filter. Statements matching the filter are NOT + * written on the database, but they are still removed from the + * focusStore. + * @param changeLog + * optional change log for change notification + * + * @return The #of statements that were removed from the focusStore. + * + * @todo this uses some techniques that are not scaleable if the focusStore + * is extremely large. + */ static public int applyExistingStatements( final AbstractTripleStore focusStore, final AbstractTripleStore database, final IElementFilter<ISPO> filter, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes + final IChangeLog changeLog ) { if(INFO) @@ -286,7 +310,7 @@ final SPOAssertionBuffer assertionBuffer = new SPOAssertionBuffer( database, database, filter, capacity, false/* justified */, - changeLog, bnodes); + changeLog); /* * This buffer will retract statements from the tempStore that are @@ -385,12 +409,26 @@ */ public ClosureStats assertAll(final TempTripleStore tempStore) { - return assertAll(tempStore, null/* changeLog */, null/* bnodes */); + return assertAll(tempStore, null/* changeLog */); } + /** + * Perform truth maintenance for statement assertion. + * <p> + * This method computes the closure of the temporary store against the + * database, writing entailments into the temporary store. Once all + * entailments have been computed, it then copies the all statements in the + * temporary store into the database and deletes the temporary store. + * + * @param tempStore + * A temporary store containing statements to be asserted. The + * tempStore will be closed as a post-condition. + * @param changeLog + * optional change log for change notification + */ public ClosureStats assertAll(final TempTripleStore tempStore, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes) { + final IChangeLog changeLog) { if (tempStore == null) { @@ -436,7 +474,8 @@ * consistent if we change our mind about that practice. */ - applyExistingStatements(tempStore, database, inferenceEngine.doNotAddFilter, changeLog, bnodes); + applyExistingStatements(tempStore, database, + inferenceEngine.doNotAddFilter, changeLog); final ClosureStats stats = inferenceEngine.computeClosure(tempStore); @@ -457,7 +496,7 @@ final long ncopied = tempStore.copyStatements(database, null/* filter */, true /* copyJustifications */, - changeLog, bnodes); + changeLog); // database.dumpStore(database,true,true,false,true); @@ -506,12 +545,35 @@ */ public ClosureStats retractAll(final TempTripleStore tempStore) { - return retractAll(tempStore, null/* changeLog */, null/* bnodes */); + return retractAll(tempStore, null/* changeLog */); } + /** + * Perform truth maintenance for statement retraction. + * <p> + * When the closure is computed, each statement to be retracted is examined + * to determine whether or not it is still entailed by the database without + * the support of the statements that were explicitly retracted. Statements + * that were explicit in the database that are still provable are converted + * to inferences. Statements which can no longer be proven (i.e., that are + * not supported by a grounded {@link Justification} chain) are retracted + * from the database and added into another temporary store and their + * justifications are deleted from the database. This process repeats with + * the new temporary store until no fixed point (no more ungrounded + * statements are identified). + * + * @param tempStore + * A temporary store containing explicit statements to be + * retracted from the database. The tempStore will be closed and + * as a post-condition. + * @param changeLog + * optional change log for change notification + * + * @return statistics about the closure operation. + */ public ClosureStats retractAll(final TempTripleStore tempStore, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes) { + final IChangeLog changeLog) { final long begin = System.currentTimeMillis(); @@ -547,7 +609,7 @@ } // do truth maintenance. - retractAll(stats, tempStore, 0, changeLog, bnodes); + retractAll(stats, tempStore, 0, changeLog); MDC.remove("depth"); @@ -624,10 +686,12 @@ * Recursive depth - this is ZERO(0) the first time the method is * called. At depth ZERO(0) the tempStore MUST contain only the * explicit statements to be retracted. + * @param changeLog + * optional change log for change notification */ private void retractAll(final ClosureStats stats, final TempTripleStore tempStore, final int depth, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes) { + final IChangeLog changeLog) { MDC.put("depth", "depth=" + depth); @@ -651,7 +715,8 @@ // final TempTripleStore focusStore = new TempTripleStore(database.getProperties(), database); // consider each statement in the tempStore. - final IChunkedOrderedIterator<ISPO> itr = tempStore.getAccessPath(SPOKeyOrder.SPO).iterator(); + final IChunkedOrderedIterator<ISPO> itr = + tempStore.getAccessPath(SPOKeyOrder.SPO).iterator(); final long nretracted; final long ndowngraded; @@ -677,8 +742,7 @@ null, //filter @todo was inferenceEngine.doNotAddFilter, capacity,// false,// justify - changeLog, - bnodes + changeLog ); /* @@ -696,7 +760,7 @@ */ final SPORetractionBuffer retractionBuffer = new SPORetractionBuffer( database, capacity, false/* computeClosureForStatementIdentifiers */, - changeLog, bnodes); + changeLog); /* * Note: when we enter this method recursively statements in the @@ -1003,7 +1067,7 @@ * Recursive processing. */ - retractAll(stats, focusStore, depth + 1, changeLog, bnodes); + retractAll(stats, focusStore, depth + 1, changeLog); } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -66,7 +66,8 @@ private StatementEnum type; private boolean userFlag; private transient boolean override = false; - private transient boolean modified = false; +// private transient boolean modified = false; + private transient ModifiedEnum modified = ModifiedEnum.NONE; /** * Used by {@link BigdataValueFactory} @@ -227,7 +228,7 @@ return "<" + s + ", " + p + ", " + o + (c == null ? "" : ", " + c) + ">" + (type == null ? "" : " : " + type) - + (modified ? " : modified" : ""); + + (isModified() ? " : modified("+modified+")" : ""); } @@ -353,14 +354,20 @@ public boolean isModified() { - return modified; + return modified != ModifiedEnum.NONE; } - public void setModified(final boolean modified) { + public void setModified(final ModifiedEnum modified) { this.modified = modified; } + + public ModifiedEnum getModified() { + + return modified; + + } } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -40,6 +40,9 @@ import org.openrdf.model.URI; import org.openrdf.model.Value; +import com.bigdata.rdf.changesets.ChangeRecord; +import com.bigdata.rdf.changesets.IChangeLog; +import com.bigdata.rdf.changesets.IChangeRecord.ChangeAction; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.model.BigdataBNode; import com.bigdata.rdf.model.BigdataResource; @@ -48,9 +51,6 @@ import com.bigdata.rdf.model.BigdataValue; import com.bigdata.rdf.model.BigdataValueFactory; import com.bigdata.rdf.model.StatementEnum; -import com.bigdata.rdf.sail.changesets.ChangeRecord; -import com.bigdata.rdf.sail.changesets.IChangeLog; -import com.bigdata.rdf.sail.changesets.IChangeRecord.ChangeAction; import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.SPO; import com.bigdata.rdf.store.AbstractTripleStore; @@ -879,28 +879,6 @@ // final long nwritten = writeSPOs(sids ? tmp.clone() : tmp, numStmts); final long nwritten = writeSPOs(tmp.clone(), numStmts); - // Copy the state of the isModified() flag - { - - for (int i = 0; i < numStmts; i++) { - - if (tmp[i].isModified()) { - - stmts[i].setModified(true); - - if (changeLog != null) { - - changeLog.changeEvent( - new ChangeRecord(stmts[i], ChangeAction.ADDED)); - - } - - } - - } - - } - if (sids) { /* @@ -972,6 +950,34 @@ } + // Copy the state of the isModified() flag + for (int i = 0; i < numStmts; i++) { + + if (tmp[i].isModified()) { + + stmts[i].setModified(tmp[i].getModified()); + + if (changeLog != null) { + + switch(stmts[i].getModified()) { + case INSERTED: + changeLog.changeEvent(new ChangeRecord(stmts[i], ChangeAction.INSERTED)); + break; + case UPDATED: + changeLog.changeEvent(new ChangeRecord(stmts[i], ChangeAction.UPDATED)); + break; + case REMOVED: + throw new AssertionError(); + default: + break; + } + + } + + } + + } + return nwritten; } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -240,7 +240,7 @@ * Modification can indicate that the statement was inserted, retracted, or * had its associated {@link StatementEnum} in the database updated. */ - public void setModified(boolean modified); + public void setModified(ModifiedEnum modified); /** * Return the state of the transient <i>modified</i> flag. This flag @@ -265,6 +265,8 @@ * Because this information is set at a low-level it can not currently * be used in combination with truth maintenance mechanisms. */ + public ModifiedEnum getModified(); + public boolean isModified(); /** @@ -294,5 +296,60 @@ * The database whose lexicon will be used. */ public String toString(IRawTripleStore db); + + public enum ModifiedEnum { + + INSERTED, REMOVED, UPDATED, NONE; + + public static boolean[] toBooleans(final ModifiedEnum[] modified, final int n) { + + final boolean[] b = new boolean[n*2]; + for (int i = 0; i < n; i++) { + switch(modified[i]) { + case INSERTED: + b[i*2] = true; + b[i*2+1] = false; + break; + case REMOVED: + b[i*2] = false; + b[i*2+1] = true; + break; + case UPDATED: + b[i*2] = true; + b[i*2+1] = true; + break; + case NONE: + default: + b[i*2] = false; + b[i*2+1] = false; + break; + } + } + + return b; + + } + + public static ModifiedEnum[] fromBooleans(final boolean[] b, final int n) { + + assert n < b.length && n % 2 == 0; + + final ModifiedEnum[] m = new ModifiedEnum[n/2]; + for (int i = 0; i < n; i+=2) { + if (b[i] && !b[i+1]) + m[i/2] = INSERTED; + else if (!b[i] && b[i+1]) + m[i/2] = REMOVED; + else if (b[i] && b[i+1]) + m[i/2] = UPDATED; + else + m[i/2] = NONE; + } + + return m; + + } + + } } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -88,7 +88,8 @@ */ private transient boolean override = false; - private transient boolean modified = false; +// private transient boolean modified = false; + private transient ModifiedEnum modified = ModifiedEnum.NONE; final public IV get(final int index) { switch(index) { @@ -621,7 +622,7 @@ + (c == null ? "" : ", " + toString(c)) + (type == null ? "" : " : " + type + (override ? ", override" : "")) - + (modified ? ", modified" : "") + " >"; + + (isModified() ? ", modified ("+modified+")" : "") + " >"; } @@ -670,7 +671,7 @@ t = "Unknown "; } - return t + (modified ? "(*)" : "") + " : " + return t + (isModified() ? "(*)" : "") + " : " + store.toString(s, p, o, c); } else { @@ -713,14 +714,20 @@ public boolean isModified() { - return modified; + return modified != ModifiedEnum.NONE; } - public void setModified(final boolean modified) { + public void setModified(final ModifiedEnum modified) { this.modified = modified; } + + public ModifiedEnum getModified() { + + return modified; + + } } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexRemover.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexRemover.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexRemover.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -12,6 +12,7 @@ import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedure.ResultBitBufferHandler; import com.bigdata.btree.proc.BatchRemove.BatchRemoveConstructor; import com.bigdata.rdf.inf.Justification; +import com.bigdata.rdf.spo.ISPO.ModifiedEnum; /** * Class writes on a statement index, removing the specified statements (batch @@ -183,7 +184,7 @@ final boolean[] bits = modified.getResult(); writeCount = modified.getOnCount(); - + for (int i = 0; i < numStmts; i++) { if (bits[i]) { @@ -198,8 +199,8 @@ * explicitly cleared it in between those writes). */ - a[i].setModified(bits[i]); - + a[i].setModified(ModifiedEnum.REMOVED); + } } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.util.Arrays; import org.apache.log4j.Logger; import com.bigdata.btree.BytesUtil; import com.bigdata.btree.IIndex; @@ -43,6 +44,7 @@ import com.bigdata.rdf.internal.TermId; import com.bigdata.rdf.internal.VTE; import com.bigdata.rdf.model.StatementEnum; +import com.bigdata.rdf.spo.ISPO.ModifiedEnum; import com.bigdata.rdf.store.IRawTripleStore; import com.bigdata.relation.IMutableRelationIndexWriteProcedure; @@ -190,7 +192,12 @@ // .endsWith(SPOKeyOrder.SPO.getIndexName()) : false; // Array used to report by which statements were modified by this operation. - final boolean[] modified = reportMutation ? new boolean[n] : null; + final ModifiedEnum[] modified = reportMutation ? new ModifiedEnum[n] : null; + if (reportMutation) { + for (int i = 0; i < n; i++) { + modified[i] = ModifiedEnum.NONE; + } + } for (int i = 0; i < n; i++) { @@ -255,7 +262,7 @@ writeCount++; if (reportMutation) - modified[i] = true; + modified[i] = ModifiedEnum.INSERTED; } else { @@ -291,7 +298,7 @@ writeCount++; if (reportMutation) - modified[i] = true; + modified[i] = ModifiedEnum.UPDATED; } @@ -337,7 +344,7 @@ writeCount++; if (reportMutation) - modified[i] = true; + modified[i] = ModifiedEnum.UPDATED; } @@ -351,9 +358,26 @@ log.info("Wrote " + writeCount + " SPOs on ndx=" + ndx.getIndexMetadata().getName()); - return reportMutation ? new ResultBitBuffer(n, modified, writeCount) - : Long.valueOf(writeCount); - + if (reportMutation) { + + final boolean[] b = ModifiedEnum.toBooleans(modified, n); + + int onCount = 0; + for (int i = 0; i < b.length; i++) { + if (b[i]) + onCount++; + } + + ResultBitBuffer rbb = new ResultBitBuffer(b.length, b, onCount); + + return rbb; + + } else { + + return Long.valueOf(writeCount); + + } + } /** Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriter.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriter.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriter.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -57,6 +57,7 @@ import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedure.ResultBitBuffer; import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedure.ResultBitBufferHandler; import com.bigdata.io.ByteArrayBuffer; +import com.bigdata.rdf.spo.ISPO.ModifiedEnum; import com.bigdata.rdf.spo.SPOIndexWriteProc.IndexWriteProcConstructor; import com.bigdata.relation.accesspath.IElementFilter; @@ -283,7 +284,7 @@ */ final long _begin = System.currentTimeMillis(); - final long writeCount; + long writeCount = 0; if (reportMutation) { /* @@ -294,7 +295,7 @@ */ final ResultBitBufferHandler aggregator = new ResultBitBufferHandler( - numToAdd); + numToAdd,2); ndx.submit(0/* fromIndex */, numToAdd/* toIndex */, keys, vals, IndexWriteProcConstructor.REPORT_MUTATION, aggregator); @@ -303,11 +304,11 @@ final boolean[] bits = modified.getResult(); - writeCount = modified.getOnCount(); - + final ModifiedEnum[] m = ModifiedEnum.fromBooleans(bits, bits.length); + for (int i = 0; i < numToAdd; i++) { - if (bits[i]) { + if (m[i] != ModifiedEnum.NONE) { /* * Note: This only turns on the modified flag. It will not @@ -319,7 +320,9 @@ * explicitly cleared it in between those writes). */ - denseStmts[i].setModified(bits[i]); + denseStmts[i].setModified(m[i]); + + writeCount++; } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/StatementWriter.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/StatementWriter.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/StatementWriter.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -4,12 +4,12 @@ import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicLong; import org.apache.log4j.Logger; +import com.bigdata.rdf.changesets.ChangeRecord; +import com.bigdata.rdf.changesets.IChangeLog; +import com.bigdata.rdf.changesets.IChangeRecord.ChangeAction; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.model.BigdataBNode; import com.bigdata.rdf.model.BigdataStatement; -import com.bigdata.rdf.sail.changesets.ChangeRecord; -import com.bigdata.rdf.sail.changesets.IChangeLog; -import com.bigdata.rdf.sail.changesets.IChangeRecord.ChangeAction; import com.bigdata.rdf.store.AbstractTripleStore; import com.bigdata.rdf.store.BigdataStatementIteratorImpl; import com.bigdata.relation.accesspath.IElementFilter; @@ -40,8 +40,6 @@ private final IChangeLog changeLog; - private final Map<IV, BigdataBNode> bnodes; - /** * @param database * The database. If statement identifiers are being generated @@ -67,14 +65,14 @@ IChunkedOrderedIterator<ISPO> itr, AtomicLong nwritten) { this(database, statementStore, copyOnly, itr, nwritten, - null/* changeLog */, null/* bnodes */); + null/* changeLog */); } public StatementWriter(final AbstractTripleStore database, final AbstractTripleStore statementStore, final boolean copyOnly, final IChunkedOrderedIterator<ISPO> itr, final AtomicLong nwritten, - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes) { + final IChangeLog changeLog) { if (database == null) throw new IllegalArgumentException(); @@ -100,8 +98,6 @@ this.changeLog = changeLog; - this.bnodes = bnodes; - } /** @@ -122,14 +118,13 @@ } else { - n = com.bigdata.rdf.sail.changesets.StatementWriter.addStatements( + n = com.bigdata.rdf.changesets.StatementWriter.addStatements( database, statementStore, copyOnly, null/* filter */, itr, - changeLog, - bnodes); + changeLog); } Modified: branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java =================================================================== --- branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-10-07 20:23:05 UTC (rev 3749) +++ branches/CHANGE_SET_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-10-08 01:45:59 UTC (rev 3750) @@ -75,6 +75,7 @@ import com.bigdata.rdf.axioms.BaseAxioms; import com.bigdata.rdf.axioms.NoAxioms; import com.bigdata.rdf.axioms.OwlAxioms; +import com.bigdata.rdf.changesets.IChangeLog; import com.bigdata.rdf.inf.IJustificationIterator; import com.bigdata.rdf.inf.Justification; import com.bigdata.rdf.inf.JustificationIterator; @@ -103,7 +104,6 @@ import com.bigdata.rdf.rules.MatchRule; import com.bigdata.rdf.rules.RDFJoinNexusFactory; import com.bigdata.rdf.rules.RuleContextEnum; -import com.bigdata.rdf.sail.changesets.IChangeLog; import com.bigdata.rdf.spo.BulkCompleteConverter; import com.bigdata.rdf.spo.BulkFilterConverter; import com.bigdata.rdf.spo.ExplicitSPOFilter; @@ -2203,6 +2203,13 @@ } + final public BigdataStatement getStatement(final Statement s) { + + return getStatement(s.getSubject(), s.getPredicate(), + s.getObject(), s.getContext()); + + } + final public BigdataStatement getStatement(final Resource s, final URI p, final Value o) { @@ -2987,7 +2994,7 @@ ) { return copyStatements(dst, filter, copyJustifications, - null/* changeLog */, null /* bnodes */); + null/* changeLog */); } @@ -2995,7 +3002,7 @@ final AbstractTripleStore dst,// final IElementFilter<ISPO> filter,// final boolean copyJustifications,// - final IChangeLog changeLog, final Map<IV, BigdataBNode> bnodes + final IChangeLog changeLog ) { if (dst == this) @@ -3017,15 +3024,13 @@ } else { - return com.bigdata.rdf.sail.changesets. - StatementWriter.addStatements( + return com.bigdata.rdf.changesets.StatementWriter.addStatements( dst, dst, true/* copyOnly */, null/* filter */, itr, - changeLog, - bnodes); + changeLog); } @@ -3051,7 +3056,7 @@ // task will write SPOs on the statement indices. tasks.add(new StatementWriter(dst, dst, true/* copyOnly */, - itr, nwritten, changeLog, bnodes)); + itr, nwritten, changeLog)); // task will write justifications on the justifications index. final AtomicLong nwrittenj = new AtomicLong(); Modified: branches/CHANGE_SET_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java ===================================... [truncated message content] |