Menu
▾
▴
bigdata-commit
|
From: <tho...@us...> - 2010-08-04 18:53:45
|
Revision: 3408
http://bigdata.svn.sourceforge.net/bigdata/?rev=3408&view=rev
Author: thompsonbry
Date: 2010-08-04 18:53:35 +0000 (Wed, 04 Aug 2010)
Log Message:
-----------
Merged from trunk [r3368:r3407].
Rolled back changes by MikeP to IPredicate. He will roll in new versions of those changes based on updated work on handling options.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/ResourceEvents.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DefaultServiceFederationDelegate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/GlobalRowStoreHelper.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/KeyDecoder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/Schema.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/TPS.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedConvertingIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractBTreeTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractTupleCursorTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestBTreeLeafCursors.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestBigdataMap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestCopyOnWrite.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestDirtyIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIncrementalWrite.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderCacheInteraction.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithCompactingMerge.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithIncrementalBuild.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentBuilderWithSmallTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentWithBloomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestInsertLookupRemoveKeysInRootLeaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestLinearListMethods.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestMutableBTreeCursors.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestReopen.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestSplitJoinRootLeaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestSplitJoinThreeLevels.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestSplitRootLeaf.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestTouch.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestTransientBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestTupleFilters.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/AbstractUnicodeKeyBuilderTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestKeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/keys/TestSuccessorUtil.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/raba/codec/AbstractRabaCoderTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/raba/codec/RandomURIGenerator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerBootstrapTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestBuildTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestBuildTask2.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestMergeTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestOverflow.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestResourceManagerBootstrap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestMove.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestRestartSafe.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestScatterSplit.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestSplitJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTaskWithSplits.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/service/jini/TestBigdataClient.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9.txt
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteProc.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/Term2IdWriteTask.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/BlobOverflowHandler.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/small.rdf
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOTupleSerializer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOValueCoders.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/AbstractTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/QUADS_QUERY_BRANCH/src/resources/analysis/queries/benchmark.txt
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config
branches/QUADS_QUERY_BRANCH/src/resources/config/log4j.properties
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DHTFilterFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/bsbm/src/resources/bsbm-data/queries/query9-modified.txt
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/CVSROOT/
branches/QUADS_QUERY_BRANCH/bigdata-master-pom/
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/config/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/btc/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/lib/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/answers (U1)/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/resources/scripts/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/resources/
branches/QUADS_QUERY_BRANCH/bigdata-perf/uniprot/src/resources/logging/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/dsi-utils/LEGAL/
branches/QUADS_QUERY_BRANCH/dsi-utils/lib/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/compression/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/io/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/java/it/unimi/dsi/util/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/io/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/test/it/unimi/dsi/util/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/osgi/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -1525,45 +1525,63 @@
}
- /**
- * Load an instance of a {@link BTree} or derived class from the store. The
- * {@link BTree} or derived class MUST declare a constructor with the
- * following signature: <code>
+ /**
+ * Load an instance of a {@link BTree} or derived class from the store. The
+ * {@link BTree} or derived class MUST declare a constructor with the
+ * following signature: <code>
*
* <i>className</i>(IRawStore store, Checkpoint checkpoint, BTreeMetadata metadata, boolean readOnly)
*
* </code>
- *
- * @param store
- * The store.
- * @param addrCheckpoint
- * The address of a {@link Checkpoint} record for the index.
- * @param readOnly
- * When <code>true</code> the {@link BTree} will be marked as
- * read-only. Marking has some advantages relating to the locking
- * scheme used by {@link Node#getChild(int)} since the root node
- * is known to be read-only at the time that it is allocated as
- * per-child locking is therefore in place for all nodes in the
- * read-only {@link BTree}. It also results in much higher
- * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
- *
- * @return The {@link BTree} or derived class loaded from that
- * {@link Checkpoint} record.
- */
+ *
+ * @param store
+ * The store.
+ * @param addrCheckpoint
+ * The address of a {@link Checkpoint} record for the index.
+ * @param readOnly
+ * When <code>true</code> the {@link BTree} will be marked as
+ * read-only. Marking has some advantages relating to the locking
+ * scheme used by {@link Node#getChild(int)} since the root node
+ * is known to be read-only at the time that it is allocated as
+ * per-child locking is therefore in place for all nodes in the
+ * read-only {@link BTree}. It also results in much higher
+ * concurrency for {@link AbstractBTree#touch(AbstractNode)}.
+ *
+ * @return The {@link BTree} or derived class loaded from that
+ * {@link Checkpoint} record.
+ *
+ * @throws IllegalArgumentException
+ * if store is <code>null</code>.
+ */
@SuppressWarnings("unchecked")
public static BTree load(final IRawStore store, final long addrCheckpoint,
final boolean readOnly) {
+ if (store == null)
+ throw new IllegalArgumentException();
+
/*
* Read checkpoint record from store.
*/
- final Checkpoint checkpoint = Checkpoint.load(store, addrCheckpoint);
+ final Checkpoint checkpoint;
+ try {
+ checkpoint = Checkpoint.load(store, addrCheckpoint);
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not load Checkpoint: store="
+ + store + ", addrCheckpoint="
+ + store.toString(addrCheckpoint), t);
+ }
- /*
- * Read metadata record from store.
- */
- final IndexMetadata metadata = IndexMetadata.read(store, checkpoint
- .getMetadataAddr());
+ /*
+ * Read metadata record from store.
+ */
+ final IndexMetadata metadata;
+ try {
+ metadata = IndexMetadata.read(store, checkpoint.getMetadataAddr());
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not read IndexMetadata: store="
+ + store + ", checkpoint=" + checkpoint, t);
+ }
if (log.isInfoEnabled()) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -58,21 +58,21 @@
* Note: The total order of the {@link BigdataMap} is completely determined by
* {@link ITupleSerializer#serializeKey(Object)}. There is NO concept of a
* {@link Comparator}. The {@link ITupleSerializer} is responsible for coercing
- * application keys into variable length <strong>unsigned</strong> byte[]s
- * which are the keys for the underlying B+Tree. The order for the B+Tree is the
+ * application keys into variable length <strong>unsigned</strong> byte[]s which
+ * are the keys for the underlying B+Tree. The order for the B+Tree is the
* natural order for the <strong>unsigned byte[]</strong>s. {@link KeyBuilder}
* supports the generation of unsigned byte[]s from various kinds of Java
* primitives and Unicode {@link String}s and is typically used to write the
* {@link ITupleSerializer#serializeKey(Object)} method.
* <p>
- * Note: The coercion of the application keys into unsigned byte[]s is not
- * typesafe unless you either consistently use a strongly typed instance of this
+ * Note: The coercion of the application keys into unsigned byte[]s is not type
+ * safe unless you either consistently use a strongly typed instance of this
* class or specify an {@link ITupleSerializer} for the backing B+Tree that only
* allows application keys that are instances of acceptable classes. This issue
* is more critical for keys than for values since the keys define the total
* index order and the default coercion rules for keys are provided by
- * {@link KeyBuilder#asSortKey(Object)} which does not attenpt to partition the
- * key space by the application key type (keys are not safely polymorphic by
+ * {@link KeyBuilder#append(Object)} which does not attempt to partition the key
+ * space by the application key type (keys are not safely polymorphic by
* default).
* <p>
* Note: When storing Java objects in the tuple value, the value MUST be treated
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexMetadata.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -2049,10 +2049,14 @@
// Note: default assumes NOT an index partition.
this.pmd = null;
+ /* Intern'd to reduce duplication on the heap. Will be com.bigdata.btree.BTree or
+ * com.bigdata.btree.IndexSegment and occasionally a class derived from BTree.
+ */
this.btreeClassName = getProperty(indexManager, properties, namespace,
- Options.BTREE_CLASS_NAME, BTree.class.getName().toString());
+ Options.BTREE_CLASS_NAME, BTree.class.getName()).intern();
- this.checkpointClassName = Checkpoint.class.getName();
+ // Intern'd to reduce duplication on the heap.
+ this.checkpointClassName = Checkpoint.class.getName().intern();
// this.addrSer = AddressSerializer.INSTANCE;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/DefaultKeyBuilderFactory.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -207,7 +207,7 @@
if (properties != null) {
- val = properties.getProperty(key, def);
+ val = properties.getProperty(key);//, def);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/ICUSortKeyGenerator.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -108,7 +108,7 @@
}
- ICUSortKeyGenerator(Locale locale, Object strength, DecompositionEnum mode) {
+ ICUSortKeyGenerator(final Locale locale, final Object strength, final DecompositionEnum mode) {
if (locale == null)
throw new IllegalArgumentException();
@@ -132,7 +132,7 @@
} else {
- StrengthEnum str = (StrengthEnum) strength;
+ final StrengthEnum str = (StrengthEnum) strength;
if (log.isInfoEnabled())
log.info("strength=" + str);
@@ -200,9 +200,9 @@
* Buffer is reused for each {@link String} from which a sort key is
* derived.
*/
- private RawCollationKey raw = new RawCollationKey(128);
+ final private RawCollationKey raw = new RawCollationKey(128);
- public void appendSortKey(KeyBuilder keyBuilder, String s) {
+ public void appendSortKey(final KeyBuilder keyBuilder, final String s) {
// RawCollationKey raw = collator.getRawCollationKey(s, null);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/IKeyBuilder.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -33,6 +33,7 @@
import java.util.Locale;
import java.util.Properties;
import java.util.UUID;
+
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder.Options;
@@ -88,7 +89,6 @@
* {@link #appendText(String, boolean, boolean)}.
* </p>
*
- * @see KeyBuilder#asSortKey(Object)
* @see KeyBuilder#newInstance()
* @see KeyBuilder#newUnicodeInstance()
* @see KeyBuilder#newUnicodeInstance(Properties)
@@ -97,7 +97,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface IKeyBuilder extends ISortKeyBuilder {
+public interface IKeyBuilder extends ISortKeyBuilder<Object> {
/**
* The #of bytes of data in the key.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/keys/KeyBuilder.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -1065,78 +1065,8 @@
}
- /*
- * static helper methods.
- */
-
- /**
- * Used to unbox an application key (convert it to an unsigned byte[]).
- */
- static private final IKeyBuilder _keyBuilder = newUnicodeInstance();
-
- /**
- * Utility method converts an application key to a sort key (an unsigned
- * byte[] that imposes the same sort order).
- * <p>
- * Note: This method is thread-safe.
- * <p>
- * Note: Strings are Unicode safe for the default locale. See
- * {@link Locale#getDefault()}. If you require a specific local or different
- * locals at different times or for different indices then you MUST
- * provision and apply your own {@link KeyBuilder}.
- *
- * @param val
- * An application key.
- *
- * @return The unsigned byte[] equivalent of that key. This will be
- * <code>null</code> iff the <i>key</i> is <code>null</code>. If the
- * <i>key</i> is a byte[], then the byte[] itself will be returned.
- *
- * @deprecated This method circumvents explicit configuration of the
- * {@link KeyBuilder} and is used nearly exclusively by unit
- * tests. While explicit configuration is not required for keys
- * which do not include Unicode sort key components, this method
- * also relies on a single global {@link KeyBuilder} instance
- * protected by a lock. That lock is therefore a bottleneck. The
- * correct practice is to use thread-local or per task
- * {@link IKeyBuilder}s to avoid lock contention.
- */
- @SuppressWarnings("unchecked")
- public static final byte[] asSortKey(Object val) {
+ public byte[] getSortKey(final Object val) {
- if (val == null) {
-
- return null;
-
- }
-
- if (val instanceof byte[]) {
-
- return (byte[]) val;
-
- }
-
- /*
- * Synchronize on the keyBuilder to avoid concurrent modification of its
- * state.
- */
-
- synchronized (_keyBuilder) {
-
- return _keyBuilder.getSortKey(val);
-
-// _keyBuilder.reset();
-//
-// _keyBuilder.append( key );
-//
-// return _keyBuilder.getKey();
-
- }
-
- }
-
- public byte[] getSortKey(Object val) {
-
reset();
append( val );
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -277,19 +277,19 @@
AbstractStatisticsCollector
.addGarbageCollectorMXBeanCounters(serviceRoot
.makePath(ICounterHierarchy.Memory_GarbageCollectors));
-
- /*
- * Add counters reporting on the various DirectBufferPools.
- */
- {
- // general purpose pool.
- serviceRoot.makePath(
- IProcessCounters.Memory + ICounterSet.pathSeparator
- + "DirectBufferPool").attach(
- DirectBufferPool.getCounters());
-
- }
+ // Moved since counters must be dynamically reattached to reflect pool hierarchy.
+// /*
+// * Add counters reporting on the various DirectBufferPools.
+// */
+// {
+//
+// serviceRoot.makePath(
+// IProcessCounters.Memory + ICounterSet.pathSeparator
+// + "DirectBufferPool").attach(
+// DirectBufferPool.getCounters());
+//
+// }
if (LRUNexus.INSTANCE != null) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/WriteCache.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -51,7 +51,7 @@
import com.bigdata.counters.Instrument;
import com.bigdata.journal.AbstractBufferStrategy;
import com.bigdata.journal.DiskOnlyStrategy;
-import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
+//import com.bigdata.journal.DiskOnlyStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.rwstore.RWStore;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractJournal.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -1027,33 +1027,33 @@
}
- case Disk: {
+// case Disk: {
+//
+// /*
+// * Setup the buffer strategy.
+// */
+//
+// fileMetadata = new FileMetadata(file, BufferMode.Disk,
+// useDirectBuffers, initialExtent, maximumExtent, create,
+// isEmptyFile, deleteOnExit, readOnly, forceWrites,
+// offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
+// //readOnly ? null : writeCache,
+// writeCacheEnabled,
+// validateChecksum,
+// createTime, checker, alternateRootBlock);
+//
+// _bufferStrategy = new DiskOnlyStrategy(
+// 0L/* soft limit for maximumExtent */,
+//// minimumExtension,
+// fileMetadata);
+//
+// this._rootBlock = fileMetadata.rootBlock;
+//
+// break;
+//
+// }
- /*
- * Setup the buffer strategy.
- */
-
- fileMetadata = new FileMetadata(file, BufferMode.Disk,
- useDirectBuffers, initialExtent, maximumExtent, create,
- isEmptyFile, deleteOnExit, readOnly, forceWrites,
- offsetBits, //readCacheCapacity, readCacheMaxRecordSize,
- //readOnly ? null : writeCache,
- writeCacheEnabled,
- validateChecksum,
- createTime, checker, alternateRootBlock);
-
- _bufferStrategy = new DiskOnlyStrategy(
- 0L/* soft limit for maximumExtent */,
-// minimumExtension,
- fileMetadata);
-
- this._rootBlock = fileMetadata.rootBlock;
-
- break;
-
- }
-
-// case Disk:
+ case Disk:
case DiskWORM: {
/*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-08-04 17:08:49 UTC (rev 3407)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/DiskOnlyStrategy.java 2010-08-04 18:53:35 UTC (rev 3408)
@@ -46,6 +46,7 @@
import com.bigdata.io.DirectBufferPool;
import com.bigdata.io.FileChannelUtility;
import com.bigdata.io.IReopenChannel;
+import com.bigdata.journal.WORMStrategy.StoreCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.rawstore.IRawStore;
import com.bigdata.resources.StoreManager.ManagedJournal;
@@ -501,7 +502,7 @@
writeCache.flush();
- storeCounters.ncacheFlush++;
+// storeCounters.ncacheFlush++;
}
@@ -544,551 +545,551 @@
}
- /**
- * Counters for {@link IRawStore} access, including operations that read or
- * write through to the underlying media.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- *
- * @todo report elapsed time and average latency for force, reopen, and
- * writeRootBlock.
- *
- * @todo counters need to be atomic if we want to avoid the possibility of
- * concurrent <code>x++</code> operations failing to correctly
- * increment <code>x</code> for each request.
- */
- public static class StoreCounters {
-
- /**
- * #of read requests.
- */
- public long nreads;
-
- /**
- * #of read requests that are satisfied by our write cache (vs the
- * OS or disk level write cache).
- */
- public long ncacheRead;
-
- /**
- * #of read requests that read through to the backing file.
- */
- public long ndiskRead;
-
- /**
- * #of bytes read.
- */
- public long bytesRead;
-
- /**
- * #of bytes that have been read from the disk.
- */
- public long bytesReadFromDisk;
-
- /**
- * The size of the largest record read.
- */
- public long maxReadSize;
-
- /**
- * Total elapsed time for reads.
- */
- public long elapsedReadNanos;
-
- /**
- * Total elapsed time checking the disk write cache for records to be
- * read.
- */
- public long elapsedCacheReadNanos;
-
- /**
- * Total elapsed time for reading on the disk.
- */
- public long elapsedDiskReadNanos;
-
- /**
- * #of write requests.
- */
- public long nwrites;
-
- /**
- * #of write requests that are absorbed by our write cache (vs the OS or
- * disk level write cache).
- */
- public long ncacheWrite;
-
- /**
- * #of times the write cache was flushed to disk.
- */
- public long ncacheFlush;
-
- /**
- * #of write requests that write through to the backing file.
- */
- public long ndiskWrite;
-
- /**
- * The size of the largest record written.
- */
- public long maxWriteSize;
-
- /**
- * #of bytes written.
- */
- public long bytesWritten;
-
- /**
- * #of bytes that have been written on the disk.
- */
- public long bytesWrittenOnDisk;
-
- /**
- * Total elapsed time for writes.
- */
- public long elapsedWriteNanos;
-
- /**
- * Total elapsed time writing records into the cache (does not count
- * time to flush the cache when it is full or to write records that do
- * not fit in the cache directly to the disk).
- */
- public long elapsedCacheWriteNanos;
-
- /**
- * Total elapsed time for writing on the disk.
- */
- public long elapsedDiskWriteNanos;
-
- /**
- * #of times the data were forced to the disk.
- */
- public long nforce;
-
- /**
- * #of times the length of the file was changed (typically, extended).
- */
- public long ntruncate;
-
- /**
- * #of times the file has been reopened after it was closed by an
- * interrupt.
- */
- public long nreopen;
-
- /**
- * #of times one of the root blocks has been written.
- */
- public long nwriteRootBlock;
-
- /**
- * Initialize a new set of counters.
- */
- public StoreCounters() {
-
- }
-
- /**
- * Copy ctor.
- * @param o
- */
- public StoreCounters(final StoreCounters o) {
-
- add( o );
-
- }
-
- /**
- * Adds counters to the current counters.
- *
- * @param o
- */
- public void add(final StoreCounters o) {
-
- nreads += o.nreads;
- ncacheRead += o.ncacheRead;
- ndiskRead += o.ndiskRead;
- bytesRead += o.bytesRead;
- bytesReadFromDisk += o.bytesReadFromDisk;
- maxReadSize += o.maxReadSize;
- elapsedReadNanos += o.elapsedReadNanos;
- elapsedCacheReadNanos += o.elapsedCacheReadNanos;
- elapsedDiskReadNanos += o.elapsedDiskReadNanos;
-
- nwrites += o.nwrites;
- ncacheWrite += o.ncacheWrite;
- ncacheFlush += o.ncacheFlush;
- ndiskWrite += o.ndiskWrite;
- maxWriteSize += o.maxWriteSize;
- bytesWritten += o.bytesWritten;
- bytesWrittenOnDisk += o.bytesWrittenOnDisk;
- elapsedWriteNanos += o.elapsedWriteNanos;
- elapsedCacheWriteNanos += o.elapsedCacheWriteNanos;
- elapsedDiskWriteNanos += o.elapsedDiskWriteNanos;
-
- nforce += o.nforce;
- ntruncate += o.ntruncate;
- nreopen += o.nreopen;
- nwriteRootBlock += o.nwriteRootBlock;
-
- }
-
- /**
- * Returns a new {@link StoreCounters} containing the current counter values
- * minus the given counter values.
- *
- * @param o
- *
- * @return
- */
- public StoreCounters subtract(final StoreCounters o) {
-
- // make a copy of the current counters.
- final StoreCounters t = new StoreCounters(this);
-
- // subtract out the given counters.
- t.nreads -= o.nreads;
- t.ncacheRead -= o.ncacheRead;
- t.ndiskRead -= o.ndiskRead;
- t.bytesRead -= o.bytesRead;
- t.bytesReadFromDisk -= o.bytesReadFromDisk;
- t.maxReadSize -= o.maxReadSize;
- t.elapsedReadNanos -= o.elapsedReadNanos;
- t.elapsedCacheReadNanos -= o.elapsedCacheReadNanos;
- t.elapsedDiskReadNanos -= o.elapsedDiskReadNanos;
-
- t.nwrites -= o.nwrites;
- t.ncacheWrite -= o.ncacheWrite;
- t.ncacheFlush -= o.ncacheFlush;
- t.ndiskWrite -= o.ndiskWrite;
- t.maxWriteSize -= o.maxWriteSize;
- t.bytesWritten -= o.bytesWritten;
- t.bytesWrittenOnDisk -= o.bytesWrittenOnDisk;
- t.elapsedWriteNanos -= o.elapsedWriteNanos;
- t.elapsedCacheWriteNanos -= o.elapsedCacheWriteNanos;
- t.elapsedDiskWriteNanos -= o.elapsedDiskWriteNanos;
-
- t.nforce -= o.nforce;
- t.ntruncate -= o.ntruncate;
- t.nreopen -= o.nreopen;
- t.nwriteRootBlock -= o.nwriteRootBlock;
-
- return t;
-
- }
-
- synchronized public CounterSet getCounters() {
-
- if (root == null) {
-
- root = new CounterSet();
-
- // IRawStore API
- {
-
- /*
- * reads
- */
-
- root.addCounter("nreads", new Instrument<Long>() {
- public void sample() {
- setValue(nreads);
- }
- });
-
- root.addCounter("bytesRead", new Instrument<Long>() {
- public void sample() {
- setValue(bytesRead);
- }
- });
-
- root.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- final double elapsedReadSecs = (elapsedReadNanos / 1000000000.);
- setValue(elapsedReadSecs);
- }
- });
-
- root.addCounter("bytesReadPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double readSecs = (elapsedReadNanos / 1000000000.);
- final double bytesReadPerSec = (readSecs == 0L ? 0d
- : (bytesRead / readSecs));
- setValue(bytesReadPerSec);
- }
- });
-
- root.addCounter("maxReadSize", new Instrument<Long>() {
- public void sample() {
- setValue(maxReadSize);
- }
- });
-
- /*
- * writes
- */
-
- root.addCounter("nwrites", new Instrument<Long>() {
- public void sample() {
- setValue(nwrites);
- }
- });
-
- root.addCounter("bytesWritten", new Instrument<Long>() {
- public void sample() {
- setValue(bytesWritten);
- }
- });
-
- root.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- final double writeSecs = (elapsedWriteNanos / 1000000000.);
- setValue(writeSecs);
- }
- });
-
- root.addCounter("bytesWrittenPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double writeSecs = (elapsedWriteNanos / 1000000000.);
- final double bytesWrittenPerSec = (writeSecs == 0L ? 0d
- : (bytesWritten / writeSecs));
- setValue(bytesWrittenPerSec);
- }
- });
-
- root.addCounter("maxWriteSize", new Instrument<Long>() {
- public void sample() {
- setValue(maxWriteSize);
- }
- });
-
- }
-
- /*
- * write cache statistics
- */
- {
-
- final CounterSet writeCache = root.makePath("writeCache");
-
- /*
- * read
- */
- writeCache.addCounter("nread", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheRead);
- }
- });
-
- writeCache.addCounter("readHitRate", new Instrument<Double>() {
- public void sample() {
- setValue(nreads == 0L ? 0d : (double) ncacheRead
- / nreads);
- }
- });
-
- writeCache.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- setValue(elapsedCacheReadNanos / 1000000000.);
- }
- });
-
- /*
- * write
- */
-
- // #of writes on the write cache.
- writeCache.addCounter("nwrite", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheWrite);
- }
- });
-
- /*
- * % of writes that are buffered vs writing through to the
- * disk.
- *
- * Note: This will be 1.0 unless you are writing large
- * records. Large records are written directly to the disk
- * rather than first into the write cache. When this happens
- * the writeHitRate on the cache can be less than one.
- */
- writeCache.addCounter("writeHitRate", new Instrument<Double>() {
- public void sample() {
- setValue(nwrites == 0L ? 0d : (double) ncacheWrite
- / nwrites);
- }
- });
-
- writeCache.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- setValue(elapsedCacheWriteNanos / 1000000000.);
- }
- });
-
- // #of times the write cache was flushed to the disk.
- writeCache.addCounter("nflush", new Instrument<Long>() {
- public void sample() {
- setValue(ncacheFlush);
- }
- });
-
- }
-
- // disk statistics
- {
- final CounterSet disk = root.makePath("disk");
-
- /*
- * read
- */
-
- disk.addCounter("nreads", new Instrument<Long>() {
- public void sample() {
- setValue(ndiskRead);
- }
- });
-
- disk.addCounter("bytesRead", new Instrument<Long>() {
- public void sample() {
- setValue(bytesReadFromDisk);
- }
- });
-
- disk.addCounter("bytesPerRead", new Instrument<Double>() {
- public void sample() {
- final double bytesPerDiskRead = (ndiskRead == 0 ? 0d
- : (bytesReadFromDisk / (double)ndiskRead));
- setValue(bytesPerDiskRead);
- }
- });
-
- disk.addCounter("readSecs", new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- setValue(diskReadSecs);
- }
- });
-
- disk.addCounter("bytesReadPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- final double bytesReadPerSec = (diskReadSecs == 0L ? 0d
- : bytesReadFromDisk / diskReadSecs);
- setValue(bytesReadPerSec);
- }
- });
-
- disk.addCounter("secsPerRead", new Instrument<Double>() {
- public void sample() {
- final double diskReadSecs = (elapsedDiskReadNanos / 1000000000.);
- final double readLatency = (diskReadSecs == 0 ? 0d
- : diskReadSecs / ndiskRead);
- setValue(readLatency);
- }
- });
-
- /*
- * write
- */
-
- disk.addCounter("nwrites", new Instrument<Long>() {
- public void sample() {
- setValue(ndiskWrite);
- }
- });
-
- disk.addCounter("bytesWritten", new Instrument<Long>() {
- public void sample() {
- setValue(bytesWrittenOnDisk);
- }
- });
-
- disk.addCounter("bytesPerWrite", new Instrument<Double>() {
- public void sample() {
- final double bytesPerDiskWrite = (ndiskWrite == 0 ? 0d
- : (bytesWrittenOnDisk / (double)ndiskWrite));
- setValue(bytesPerDiskWrite);
- }
- });
-
- disk.addCounter("writeSecs", new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- setValue(diskWriteSecs);
- }
- });
-
- disk.addCounter("bytesWrittenPerSec",
- new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- final double bytesWrittenPerSec = (diskWriteSecs == 0L ? 0d
- : bytesWrittenOnDisk
- / diskWriteSecs);
- setValue(bytesWrittenPerSec);
- }
- });
-
- disk.addCounter("secsPerWrite", new Instrument<Double>() {
- public void sample() {
- final double diskWriteSecs = (elapsedDiskWriteNanos / 1000000000.);
- final double writeLatency = (diskWriteSecs == 0 ? 0d
- : diskWriteSecs / ndiskWrite);
- setValue(writeLatency);
- }
- });
-
- /*
- * other
- */
-
- disk.addCounter("nforce", new Instrument<Long>() {
- public void sample() {
- setValue(nforce);
- }
- });
-
- disk.addCounter("nextend", new Instrument<Long>() {
- public void sample() {
- setValue(ntruncate);
- }
- });
-
- disk.addCounter("nreopen", new Instrument<Long>() {
- public void sample() {
- setValue(nreopen);
- }
- });
-
- disk.addCounter("rootBlockWrites", new Instrument<Long>() {
- public void sample() {
- setValue(nwriteRootBlock);
- }
- });
-
- }
-
- }
-
- return root;
-
- }
- private CounterSet root;
-
- /**
- * Human readable representation of the counters.
- */
- public String toString() {
-
- return getCounters().toString();
-
- }
-
- }
+// /**
+// * Counters for {@link IRawStore} access, including operations that read or
+// * write through to the underlying media.
+// *
+// * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+// * @version $Id$
+// *
+// * @todo report elapsed time and average latency for force, reopen, and
+// * writeRootBlock.
+// *
+// * @todo counters need to be atomic if we want to avoid the possibility of
+// * concurrent <code>x++</code> operations failing to correctly
+// * increment <code>x</code> for each request.
+// */
+// public static class StoreCounters {
+//
+// /**
+// * #of read requests.
+// */
+// public long nreads;
+//
+// /**
+// * #of read requests that are satisfied by our write cache (vs the
+// * OS or disk level write cache).
+// */
+// public long ncacheRead;
+//
+// /**
+// * #of read requests that read through to the backing file.
+// */
+// public long ndiskRead;
+//
+// /**
+// * #of bytes read.
+// */
+// public long bytesRead;
+//
+// /**
+// * #of bytes that have been read from the disk.
+// */
+// public long bytesReadFromDisk;
+//
+// /**
+// * The size of the largest record read.
+// */
+// public long maxReadSize;
+//
+// /**
+// * Total elapsed time for reads.
+// */
+// public long elapsedReadNanos;
+//
+// /**
+// * Total elapsed time checking the disk write cache for records to be
+// * read.
+// */
+// public long elapsedCacheReadNanos;
+//
+// /**
+// * Total elapsed time for reading on the disk.
+// */
+// public long elapsedDiskReadNanos;
+//
+// /**
+// * #of write requests.
+// */
+// public long nwrites;
+//
+// /**
+// * #of write requests that are absorbed by our write cache (vs the OS or
+// * disk level write cache).
+// */
+// public long ncacheWrite;
+//
+// /**
+// * #of times the write cache was flushed to disk.
+// */
+// public long ncacheFlush;
+//
+// /**
+// * #of write requests that write through to the backing file.
+// */
+// public long ndiskWrite;
+//
+// /**
+// * The size of the largest record written.
+// */
+// public long maxWriteSize;
+//
+// /**
+// * #of bytes written.
+// */
+// public long bytesWritten;
+//
+// /**
+// * #of bytes that have been written on the disk.
+// */
+// public long bytesWrittenOnDisk;
+//
+// /**
+// * Total elapsed time for writes.
+// */
+// public long elapsedWriteNanos;
+//
+// /**
+// * Total elapsed time writing records into the cache (does not count
+// * time to flush the cache when it is full or to write records that do
+// * not fit in the cache directly to the disk).
+// */
+// public long elapsedCacheWriteNanos;
+//
+// /**
+// * Total elapsed time for writing on the disk.
+// */
+// public long elapsedDiskWriteNanos;
+//
+// /**
+// * #of times the data were forced to the disk.
+// */
+// public long nforce;
+//
+// /**
+// * #of times the length of the file was changed (typically, extended).
+// */
+// public long ntruncate;
+//
+// /**
+// * #of times the file has been reopened after it was closed by an
+// * interrupt.
+// */
+// public long nreopen;
+//
+// /**
+// * #of times one of the root blocks has been written.
+// */
+// public long nwriteRootBlock;
+//
+// /**
+// * Initialize a new set of counters.
+// */
+// public StoreCounters() {
+//
+// }
+//
+// /**
+// * Copy ctor.
+// * @param o
+// */
+// public StoreCounters(final StoreCounters o) {
+//
+// add( o );
+//
+// }
+//
+// /**
+// * Adds counters to the current counters.
+// *
+// * @param o
+// */
+// public void add(final StoreCounters o) {
+//
+// nreads += o.nreads;
+// ncacheRead += o.ncacheRead;
+// ndiskRead += o.ndiskRead;
+// bytesRead += o.bytesRead;
+// bytesReadFromDisk += o.bytesReadFromDisk;
+// maxReadSize += o.maxReadSize;
+// elapsedReadNanos += o.elapsedReadNanos;
+// elapsedCacheReadNanos += o.elapsedCacheReadNanos;
+// elapsedDiskReadNanos += o.elapsedDiskReadNanos;
+//
+// nwrites += o.nwrites;
+// ncacheWrite += o.ncacheWrite;
+// ncacheFlush += o.ncacheFlush;
+// ndiskWrite += o.ndiskWrite;
+// maxWriteSize += o.maxWriteSize;
+// bytesWritten += o.bytesWritten;
+// bytesWrittenOnDisk += o.bytesWrittenOnDisk;
+// elapsedWriteNanos += o.elapsedWriteNanos;
+// elapsedCacheWriteNanos += o.elapsedCacheWriteNanos;
+// elapsedDiskWriteNanos += o.elapsedDiskWriteNanos;
+//
+// nforce += o.nforce;
+// ntruncate += o.ntruncate;
+// nreopen += o.nreopen;
+// nwriteRootBlock += o.nwriteRootBlock;
+//
+// }
+//
+// /**
+// * Returns a new {@link StoreCounters} containing the current counter values
+// * minus the given counter values.
+// *
+// * @param o
+// *
+// * @return
+// */
+// public StoreCounters subtract(final StoreCounters o) {
+//
+// // make a copy of the current counters.
+// final StoreCounters t = new StoreCounters(this);
+//
+// // subtract out the given counters.
+// t.nreads -= o.nreads;
+// t.ncacheRead -= o.ncacheRead;
+// t.ndiskRead -= o.ndiskRead;
+// t.bytesRead -= o.bytesRead;
+// t.bytesReadFromDisk -= o.bytesReadFromDisk;
+// t.maxReadSize -= o.maxReadSize;
+// t.elapsedReadNanos -= o.elapsedReadNanos;
+// t.elapsedCacheReadNanos -= o.elapsedCacheReadNanos;
+// t.elapsedDiskReadNanos -= o.elapsedDiskReadNanos;
+//
+// t.nwrites -= o.nwrites;
+// t.ncacheWrite -= o.ncacheWrite;
+// t.ncacheFlush -= o.ncacheFlush;
+// t.ndiskWrite -= o.ndiskWrite;
+// t.maxWriteSize -= o.maxWriteSize;
+// t.bytesWritten -= o.bytesWritten;
+// t.bytesWrittenOnDisk -= o.bytesWrittenOnDisk;
+// t.elapsedWriteNanos -= o.elapsedWriteNanos;
+// t.elapsedCacheWriteNanos -= o.elapsedCacheWriteNanos;
+// t.elapsedDiskWriteNanos -= o.elapsedDiskWriteNanos;
+//
+// t.nforce -= o.nforce;
+// t.ntruncate -= o.ntruncate;
+// t.nreopen -= o.nreopen;
+// t.nwriteRootBlock -= o.nwriteRootBlock;
+//
+// return t;
+//
+// }
+//
+// synchronized public CounterSet getCounters() {
+//
+// if (root == null) {
+//
+// root = new CounterSet();
+//
+// // IRawStore API
+// {
+//
+// /*
+// * reads
+// */
+//
+// root.addCounter("nreads", new Instrument<Long>() {
+// public void sample() {
+// setValue(nreads);
+// }
+// });
+//
+// root.addCounter("bytesRead", new Instrument<Long>() {
+// public void sample() {
+// setValue(bytesRead);
+// }
+// });
+//
+// root.addCounter("readSecs", new Instrument<Double>() {
+// public void sample() {
+// final double elapsedReadSecs = (elapsedReadNanos / 1000000000.);
+// setValue(elapsedReadSecs);
+// }
+// });
+//
+// root.addCounter("bytesReadPerSec",
+// new Instrument<Double>() {
+// public void sample() {
+// final double readSecs = (elapsedReadNanos / 1000000000.);
+// final double bytesReadPerSec = (readSecs == 0L ? 0d
+// : (bytesRead / readSecs));
+// setValue(bytesReadPerSec);
+// }
+// });
+//
+// root.addCounter("maxReadSize", new Instrument<Long>() {
+// public void sample() {
+// setValue(maxReadSize);
+...
[truncated message content] |
|
From: <tho...@us...> - 2010-08-06 15:59:49
|
Revision: 3424
http://bigdata.svn.sourceforge.net/bigdata/?rev=3424&view=rev
Author: thompsonbry
Date: 2010-08-06 15:59:39 +0000 (Fri, 06 Aug 2010)
Log Message:
-----------
Merged from trunk [r3407:r3423].
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DistributedTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/AbstractIndexSegmentTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIndexSegmentMultiBlockIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestReleaseResources.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/StressTestConcurrent.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestDistributedTransactionServiceRestart.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/TransactionServer.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/master/TaskMaster.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/util/DumpFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DefaultExtensionFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtension.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IExtensionFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/ILexiconConfiguration.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/LexiconConfiguration.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/LocalTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.config
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/ColorsEnumExtension.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/EpochExtension.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/SampleExtensionFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/TestEncodeDecodeKeys.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithEmbeddedFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/TestScaleOutTripleStoreWithJiniFederation.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/NanoSparqlServer.java
branches/QUADS_QUERY_BRANCH/build.xml
branches/QUADS_QUERY_BRANCH/src/resources/config/README
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
branches/QUADS_QUERY_BRANCH/src/resources/scripts/dumpFed.sh
branches/QUADS_QUERY_BRANCH/src/resources/scripts/nanoSparqlServer.sh
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/dsi-utils/LEGAL/
branches/QUADS_QUERY_BRANCH/dsi-utils/lib/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/osgi/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -420,7 +420,7 @@
}
}
-
+
/**
* Note: A commit is required in order for a read-committed view to have
* access to the registered indices. When running against an
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -2840,7 +2840,8 @@
* might also want to limit the maximum size of the reads.
*/
- final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+// final DirectBufferPool pool = DirectBufferPool.INSTANCE_10M;
+ final DirectBufferPool pool = DirectBufferPool.INSTANCE;
if (true
&& ((flags & REVERSE) == 0)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -644,7 +644,18 @@
this.lastCommitTime = lastCommitTime;
}
- private long lastCommitTime = 0L;// Until the first commit.
+
+ /**
+ * The lastCommitTime of the {@link Checkpoint} record from which the
+ * {@link BTree} was loaded.
+ * <p>
+ * Note: Made volatile on 8/2/2010 since it is not otherwise obvious what
+ * would guarantee visibility of this field, through I do seem to remember
+ * that visibility might be guaranteed by how the BTree class is discovered
+ * and returned to the class. Still, it does no harm to make this a volatile
+ * read.
+ */
+ volatile private long lastCommitTime = 0L;// Until the first commit.
/**
* Return the {@link IDirtyListener}.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DumpIndexSegment.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -36,6 +36,7 @@
import org.apache.log4j.Logger;
import com.bigdata.btree.IndexSegment.ImmutableNodeFactory.ImmutableLeaf;
+import com.bigdata.io.DirectBufferPool;
import com.bigdata.journal.DumpJournal;
import com.bigdata.rawstore.IRawStore;
@@ -154,6 +155,16 @@
}
+ // multi-block scan of the index segment.
+ boolean multiBlockScan = false; // @todo command line option.
+ if (multiBlockScan) {
+
+ writeBanner("dump leaves using multi-block forward scan");
+
+ dumpLeavesMultiBlockForwardScan(store);
+
+ }
+
// dump the leaves using a fast reverse scan.
boolean fastReverseScan = true;// @todo command line option
if (fastReverseScan) {
@@ -524,6 +535,36 @@
}
+ /**
+ * Dump leaves using the {@link IndexSegmentMultiBlockIterator}.
+ *
+ * @param store
+ */
+ static void dumpLeavesMultiBlockForwardScan(final IndexSegmentStore store) {
+
+ final long begin = System.currentTimeMillis();
+
+ final IndexSegment seg = store.loadIndexSegment();
+
+ final ITupleIterator<?> itr = new IndexSegmentMultiBlockIterator(seg, DirectBufferPool.INSTANCE,
+ null/* fromKey */, null/* toKey */, IRangeQuery.DEFAULT/* flags */);
+
+ int nscanned = 0;
+
+ while(itr.hasNext()) {
+
+ itr.next();
+
+ nscanned++;
+
+ }
+
+ final long elapsed = System.currentTimeMillis() - begin;
+
+ System.out.println("Visited "+nscanned+" tuples using multi-block forward scan in "+elapsed+" ms");
+
+ }
+
static void writeBanner(String s) {
System.out.println(bar);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/io/DirectBufferPool.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -218,12 +218,12 @@
*/
public final static DirectBufferPool INSTANCE;
- /**
- * A JVM-wide pool of direct {@link ByteBuffer}s with a default
- * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
- * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
- */
- public final static DirectBufferPool INSTANCE_10M;
+// /**
+// * A JVM-wide pool of direct {@link ByteBuffer}s with a default
+// * {@link Options#BUFFER_CAPACITY} of <code>10 MB</code>. The main use case
+// * for the 10M buffers are multi-block IOs for the {@link IndexSegment}s.
+// */
+// public final static DirectBufferPool INSTANCE_10M;
/**
* An unbounded list of all {@link DirectBufferPool} instances.
@@ -251,11 +251,11 @@
bufferCapacity//
);
- INSTANCE_10M = new DirectBufferPool(//
- "10M",//
- Integer.MAX_VALUE, // poolCapacity
- 10 * Bytes.megabyte32 // bufferCapacity
- );
+// INSTANCE_10M = new DirectBufferPool(//
+// "10M",//
+// Integer.MAX_VALUE, // poolCapacity
+// 10 * Bytes.megabyte32 // bufferCapacity
+// );
/*
* This configuration will block if there is a concurrent demand for
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/AbstractLocalTransactionManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -7,6 +7,7 @@
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
+import com.bigdata.resources.StoreManager;
import com.bigdata.service.IBigdataFederation;
import com.bigdata.service.IDataService;
@@ -171,16 +172,18 @@
* Delay between attempts reach the remote service (ms).
*/
final long delay = 10L;
-
- /**
- * #of attempts to reach the remote service.
- *
- * Note: delay*maxtries == 1000ms of trying before we give up.
- *
- * If this is not enough, then consider adding an optional parameter giving
- * the time the caller will wait and letting the StoreManager wait longer
- * during startup to discover the timestamp service.
- */
+
+ /**
+ * #of attempts to reach the remote service.
+ * <p>
+ * Note: delay*maxtries == 1000ms of trying before we give up, plus however
+ * long we are willing to wait for service discovery if the problem is
+ * locating the {@link ITransactionService}.
+ * <p>
+ * If this is not enough, then consider adding an optional parameter giving
+ * the time the caller will wait and letting the {@link StoreManager} wait
+ * longer during startup to discover the timestamp service.
+ */
final int maxtries = 100;
/**
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/WriteExecutorService.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1752,11 +1752,11 @@
//
// }
- /**
- * Flag may be set to force overflow processing during the next group
- * commit. The flag is cleared once an overflow has occurred.
- */
- public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
+// /**
+// * Flag may be set to force overflow processing during the next group
+// * commit. The flag is cleared once an overflow has occurred.
+// */
+// public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
/**
* Return <code>true</code> if the pre-conditions for overflow processing
@@ -1765,7 +1765,8 @@
private boolean isShouldOverflow() {
return resourceManager.isOverflowEnabled()
- && (forceOverflow.get() || resourceManager.shouldOverflow());
+// && (forceOverflow.get() || resourceManager.shouldOverflow());
+ && resourceManager.shouldOverflow();
}
@@ -1815,10 +1816,10 @@
log.error("Overflow error: "+serviceName+" : "+t, t);
- } finally {
-
- // clear force flag.
- forceOverflow.set(false);
+// } finally {
+//
+// // clear force flag.
+// forceOverflow.set(false);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -582,9 +582,21 @@
}
/**
+ * The default implementation only logs the event.
+ */
+ public AbstractResource<E> init() {
+
+ if (log.isInfoEnabled())
+ log.info(toString());
+
+ return this;
+
+ }
+
+ /**
*
* @todo Lock service supporting shared locks, leases and lease renewal,
- * excalation of shared locks to exclusive locks, deadlock detection,
+ * escalation of shared locks to exclusive locks, deadlock detection,
* and possibly a resource hierarchy. Leases should be Callable
* objects that are submitted by the client to its executor service so
* that they will renew automatically until cancelled (and will cancel
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IMutableResource.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -38,7 +38,10 @@
public interface IMutableResource<T> extends ILocatableResource<T> {
/**
- * Create any logically contained resources (relations, indices).
+ * Create any logically contained resources (relations, indices). There is
+ * no presumption that {@link #init()} is suitable for invocation from
+ * {@link #create()}. Instead, you are responsible for invoking {@link #init()}
+ * from this method IFF it is appropriate to reuse its initialization logic.
*/
void create();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -21,8 +21,8 @@
*/
public class RelationFusedView<E> implements IRelation<E> {
- private IRelation<E> relation1;
- private IRelation<E> relation2;
+ final private IRelation<E> relation1;
+ final private IRelation<E> relation2;
public IRelation<E> getRelation1() {
@@ -36,6 +36,13 @@
}
+ // NOP
+ public RelationFusedView<E> init() {
+
+ return this;
+
+ }
+
/**
*
* @param relation1
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/DefaultResourceLocator.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -586,6 +586,8 @@
properties //
});
+ r.init();
+
if(INFO) {
log.info("new instance: "+r);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/locator/ILocatableResource.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -45,6 +45,13 @@
*/
public interface ILocatableResource<T> {
+ /**
+ * Deferred initialization method is automatically invoked when the resource
+ * is materialized by the {@link IResourceLocator}. The implementation is
+ * encouraged to strengthen the return type.
+ */
+ public ILocatableResource<T> init();
+
/**
* The identifying namespace.
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1074,16 +1074,20 @@
final UUID sinkUUID = locator.getDataServiceUUID();
+ final IDataService dataService;
if (sinkUUID.equals(fed.getServiceUUID())) {
- /*
- * @todo As an optimization, special case when the downstream
- * data service is _this_ data service.
- */
+ /*
+ * As an optimization, special case when the downstream
+ * data service is _this_ data service.
+ */
+ dataService = (IDataService)fed.getService();
+ } else {
+
+ dataService = fed.getDataService(sinkUUID);
+
}
-
- final IDataService dataService = fed.getDataService(sinkUUID);
sink = new JoinTaskSink(fed, locator, this);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -310,6 +310,7 @@
private final OverflowActionEnum action;
private final ViewMetadata vmd;
+ private final boolean forceCompactingMerge;
private final AbstractTask<T> task;
/**
@@ -319,11 +320,17 @@
* @param vmd
* The {@link ViewMetadata} for the index partition for which
* that action will be taken.
+ * @param forceCompactingMerge
+ * if a compacting merge should be taken even if the view was
+ * simply copied to the new journal.
* @param task
* The task which implements that action.
*/
- public AtomicCallable(final OverflowActionEnum action,
- final ViewMetadata vmd, final AbstractTask<T> task) {
+ public AtomicCallable(final OverflowActionEnum action,//
+ final ViewMetadata vmd,//
+ final boolean forceCompactingMerge, //
+ final AbstractTask<T> task//
+ ) {
if (action == null)
throw new IllegalArgumentException();
@@ -337,6 +344,8 @@
this.action = action;
this.vmd = vmd;
+
+ this.forceCompactingMerge = forceCompactingMerge;
this.task = task;
@@ -407,110 +416,112 @@
}
- /**
- * Schedule a build for each shard and a merge for each shard with a
- * non-zero merge priority. Whether a build or a merge is performed for a
- * shard will depend on which action is initiated first. When an build or
- * merge action is initiated, that choice is atomically registered on the
- * {@link ViewMetadata} and any subsequent attempt (within this method
- * invocation) to start a build or merge for the same shard will be dropped.
- * Processing ends once all tasks scheduled on a "build" service are
- * complete.
- * <p>
- * After actions are considered for each shard for which a compacting merge
- * is executed. These after actions can cause a shard split, join, or move.
- * Deferring such actions until we have a compact view (comprised of one
- * journal and one index segment) greatly improves our ability to decide
- * whether a shard should be split or joined and simplifies the logic and
- * effort required to split, join or move a shard.
- * <p>
- * The following is a brief summary of some after actions on compact shards.
- * <dl>
- * <dt>split</dt>
- * <dd>A shard is split when its size on the disk exceeds the (adjusted)
- * nominal size of a shard (overflow). By waiting until the shard view is
- * compact we have exact information about the size of the shard (it is
- * contained in a single {@link IndexSegment}) and we are able to easily
- * select the separator key to split the shard.</dd>
- * <dt>tailSplit</dt>
- * <dd>A tail split may be selected for a shard which has a mostly append
- * access pattern. For such access patterns, a normal split would leave the
- * left sibling 50% full and the right sibling would quickly fill up with
- * continued writes on the tail of the key range. To compensate for this
- * access pattern, a tail split chooses a separator key near the end of the
- * key range of a shard. This results in a left sibling which is mostly full
- * and a right sibling which is mostly empty. If the pattern of heavy tail
- * append continues, then the left sibling will remain mostly full and the
- * new writes will flow mostly into the right sibling.</dd>
- * <dt>scatterSplit</dt>
- * <dd>A scatter split breaks the first shard for a new scale-out index into
- * N shards and scatters those shards across the data services in a
- * federation in order to improve the data distribution and potential
- * concurrency of the index. By waiting until the shard view is compact we
- * are able to quickly select appropriate separator keys for the shard
- * splits.</dd>
- * <dt>move</dt>
- * <dd>A move transfer a shard from this data service to another data
- * service in order to reduce the load on this data service. By waiting
- * until the shard view is compact we are able to rapidly transfer the bulk
- * of the data in the form of a single {@link IndexSegment}.</dd>
- * <dt>join</dt>
- * <dd>A join combines a shard which is under 50% of its (adjusted) nominal
- * maximum size on the disk (underflow) with its right sibling. Joins are
- * driven by deletes of tuples from a key range. Since deletes are handled
- * as writes where a delete marker is set on the tuple, neither the shard
- * size on the disk nor the range count of the shard will decrease until a
- * compacting merge. A join is indicated if the size on disk for the shard
- * has shrunk considerably since the last time a compacting merge was
- * performed for the view (this covers both the case of deletes, which
- * reduce the range count, and updates which replace the values in the
- * tuples with more compact data). <br>
- * There are actually three cases for a join.
- * <ol>
- * <li>If the right sibling is local, then the shard will be joined with its
- * right sibling.</li>
- * <li>If the right sibling is remote, then the shard will be moved to the
- * data service on which the right sibling is found.</li>
- * <li>If the right sibling does not exist, then nothing is done (the last
- * shard in a scale-out index does not have a right sibling). The right most
- * sibling will remain undercapacity until and unless its left sibling also
- * underflows, at which point the left sibling will cause itself to be
- * joined with the right sibling (this is done to simplify the logic which
- * searches for a sibling with which to join an undercapacity shard).</li>
- * </ol>
- * </dl>
- *
- * @param forceCompactingMerges
- * When <code>true</code> a compacting merge will be forced for
- * each non-compact view.
- *
- * @throws InterruptedException
- *
- * @todo The size of the merge queue (or its sum of priorities) may be an
- * indication of the load of the node which could be used to decide
- * that index partitions should be shed/moved.
- *
- * @todo For HA, this needs to be a shared priority queue using zk or the
- * like since any node in the failover set could do the merge (or
- * build). [Alternatively, nodes do the build/merge for the shards for
- * which they have the highest affinity out of the failover set.]
- *
- * FIXME tailSplits currently operate on the mutable BTree rather than
- * a compact view). This task does not require a compact view (at
- * least, not yet) and generating one for it might be a waste of time.
- * Instead it examines where the inserts are occurring in the index
- * and splits of the tail if the index is heavy for write append. It
- * probably could defer that choice until a compact view was some
- * percentage of a split (maybe .6?) So, probably an after action for
- * the mergeQ.
- *
- * FIXME joins must track metadata about the previous size on disk of
- * the compact view in order to decide when underflow has resulted. In
- * order to handle the change in the value of the acceleration factor,
- * this data should be stored as the percentage of an adjusted split
- * of the last compact view. We can update that metadata each time we
- * do a compacting merge.
- */
+ /**
+ * Schedule a build for each shard and a merge for each shard with a
+ * non-zero merge priority. Whether a build or a merge is performed for a
+ * shard will depend on which action is initiated first. When an build or
+ * merge action is initiated, that choice is atomically registered on the
+ * {@link ViewMetadata} and any subsequent attempt (within this method
+ * invocation) to start a build or merge for the same shard will be dropped.
+ * Processing ends once all tasks scheduled on a "build" service are
+ * complete.
+ * <p>
+ * After actions are considered for each shard for which a compacting merge
+ * is executed. These after actions can cause a shard split, join, or move.
+ * Deferring such actions until we have a compact view (comprised of one
+ * journal and one index segment) greatly improves our ability to decide
+ * whether a shard should be split or joined and simplifies the logic and
+ * effort required to split, join or move a shard.
+ * <p>
+ * The following is a brief summary of some after actions on compact shards.
+ * <dl>
+ * <dt>split</dt>
+ * <dd>A shard is split when its size on the disk exceeds the (adjusted)
+ * nominal size of a shard (overflow). By waiting until the shard view is
+ * compact we have exact information about the size of the shard (it is
+ * contained in a single {@link IndexSegment}) and we are able to easily
+ * select the separator key to split the shard.</dd>
+ * <dt>tailSplit</dt>
+ * <dd>A tail split may be selected for a shard which has a mostly append
+ * access pattern. For such access patterns, a normal split would leave the
+ * left sibling 50% full and the right sibling would quickly fill up with
+ * continued writes on the tail of the key range. To compensate for this
+ * access pattern, a tail split chooses a separator key near the end of the
+ * key range of a shard. This results in a left sibling which is mostly full
+ * and a right sibling which is mostly empty. If the pattern of heavy tail
+ * append continues, then the left sibling will remain mostly full and the
+ * new writes will flow mostly into the right sibling.</dd>
+ * <dt>scatterSplit</dt>
+ * <dd>A scatter split breaks the first shard for a new scale-out index into
+ * N shards and scatters those shards across the data services in a
+ * federation in order to improve the data distribution and potential
+ * concurrency of the index. By waiting until the shard view is compact we
+ * are able to quickly select appropriate separator keys for the shard
+ * splits.</dd>
+ * <dt>move</dt>
+ * <dd>A move transfer a shard from this data service to another data
+ * service in order to reduce the load on this data service. By waiting
+ * until the shard view is compact we are able to rapidly transfer the bulk
+ * of the data in the form of a single {@link IndexSegment}.</dd>
+ * <dt>join</dt>
+ * <dd>A join combines a shard which is under 50% of its (adjusted) nominal
+ * maximum size on the disk (underflow) with its right sibling. Joins are
+ * driven by deletes of tuples from a key range. Since deletes are handled
+ * as writes where a delete marker is set on the tuple, neither the shard
+ * size on the disk nor the range count of the shard will decrease until a
+ * compacting merge. A join is indicated if the size on disk for the shard
+ * has shrunk considerably since the last time a compacting merge was
+ * performed for the view (this covers both the case of deletes, which
+ * reduce the range count, and updates which replace the values in the
+ * tuples with more compact data). <br>
+ * There are actually three cases for a join.
+ * <ol>
+ * <li>If the right sibling is local, then the shard will be joined with its
+ * right sibling.</li>
+ * <li>If the right sibling is remote, then the shard will be moved to the
+ * data service on which the right sibling is found.</li>
+ * <li>If the right sibling does not exist, then nothing is done (the last
+ * shard in a scale-out index does not have a right sibling). The right most
+ * sibling will remain undercapacity until and unless its left sibling also
+ * underflows, at which point the left sibling will cause itself to be
+ * joined with the right sibling (this is done to simplify the logic which
+ * searches for a sibling with which to join an undercapacity shard).</li>
+ * </ol>
+ * </dl>
+ *
+ * @param forceCompactingMerges
+ * When <code>true</code> a compacting merge will be forced for
+ * each non-compact view. Compacting merges will be taken in
+ * priority order and will continue until finished or until the
+ * journal is nearing its nominal maximum extent.
+ *
+ * @throws InterruptedException
+ *
+ * @todo The size of the merge queue (or its sum of priorities) may be an
+ * indication of the load of the node which could be used to decide
+ * that index partitions should be shed/moved.
+ *
+ * @todo For HA, this needs to be a shared priority queue using zk or the
+ * like since any node in the failover set could do the merge (or
+ * build). [Alternatively, nodes do the build/merge for the shards for
+ * which they have the highest affinity out of the failover set.]
+ *
+ * FIXME tailSplits currently operate on the mutable BTree rather than
+ * a compact view). This task does not require a compact view (at
+ * least, not yet) and generating one for it might be a waste of time.
+ * Instead it examines where the inserts are occurring in the index
+ * and splits of the tail if the index is heavy for write append. It
+ * probably could defer that choice until a compact view was some
+ * percentage of a split (maybe .6?) So, probably an after action for
+ * the mergeQ.
+ *
+ * FIXME joins must track metadata about the previous size on disk of
+ * the compact view in order to decide when underflow has resulted. In
+ * order to handle the change in the value of the acceleration factor,
+ * this data should be stored as the percentage of an adjusted split
+ * of the last compact view. We can update that metadata each time we
+ * do a compacting merge.
+ */
private List<Future<?>> scheduleAndAwaitTasks(
final boolean forceCompactingMerges) throws InterruptedException {
@@ -554,21 +565,30 @@
if (log.isInfoEnabled())
log.info("was copied : " + vmd);
- continue;
+ } else {
+ buildList.add(new Priority<ViewMetadata>(vmd.buildPriority, vmd));
+
}
- buildList.add(new Priority<ViewMetadata>(vmd.buildPriority, vmd));
+ if (vmd.mergePriority > 0d || forceCompactingMerges) {
- if (vmd.mergePriority > 0d) {
+ /*
+ * Schedule a merge if the priority is non-zero or if compacting
+ * merges are being forced.
+ */
- mergeList
- .add(new Priority<ViewMetadata>(vmd.mergePriority, vmd));
+ mergeList
+ .add(new Priority<ViewMetadata>(vmd.mergePriority, vmd));
}
} // itr.hasNext()
+ if(log.isInfoEnabled()) {
+ log.info("Scheduling tasks: buildList="+buildList.size()+", mergeList="+mergeList.size());
+ }
+
/*
* Schedule build and merge tasks and await their futures. The tasks are
* submitted from a PriorityQueue, so the order in which the tasks are
@@ -606,18 +626,23 @@
resourceManager.mergeServiceCorePoolSize);
// Schedule merge tasks.
- if (!forceCompactingMerges) {
-
for (Priority<ViewMetadata> p : mergeList) {
final ViewMetadata vmd = p.v;
- if (vmd.mergePriority > 0) {
+ if (vmd.mergePriority > 0 || forceCompactingMerges) {
+ if(forceCompactingMerges && vmd.getAction().equals(OverflowActionEnum.Copy)) {
+
+ vmd.clearCopyAction();
+
+ }
+
// Schedule a compacting merge.
final FutureTask<?> ft = new FutureTask(
new AtomicCallable(OverflowActionEnum.Merge,
- vmd, new CompactingMergeTask(vmd)));
+ vmd, forceCompactingMerges,
+ new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
@@ -625,8 +650,6 @@
}
- }
-
// Schedule build tasks.
for (Priority<ViewMetadata> p : buildList) {
@@ -636,7 +659,8 @@
// Force a compacting merge.
final FutureTask<?> ft = new FutureTask(new AtomicCallable(
- OverflowActionEnum.Merge, vmd,
+ OverflowActionEnum.Merge, vmd,
+ forceCompactingMerges,
new CompactingMergeTask(vmd)));
mergeFutures.add(ft);
mergeService.execute(ft);
@@ -646,6 +670,7 @@
// Schedule a build.
final FutureTask<?> ft = new FutureTask(new AtomicCallable(
OverflowActionEnum.Build, vmd,
+ forceCompactingMerges,
new IncrementalBuildTask(vmd)));
buildFutures.add(ft);
buildService.execute(ft);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/BTreeMetadata.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -280,6 +280,25 @@
actionRef.set(action);
}
+
+ /**
+ * Used to force clear a {@link OverflowActionEnum#Copy} action
+ * when we will force a compacting merge. This allows us to do
+ * compacting merges on shard views which would otherwise simply
+ * be copied onto the new journal.
+ */
+ void clearCopyAction() {
+
+ lock.lock();
+ try {
+ if(actionRef.get().equals(OverflowActionEnum.Copy)) {
+ actionRef.set(null/*clear*/);
+ }
+ } finally {
+ lock.unlock();
+ }
+
+ }
/**
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/IndexManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1684,16 +1684,28 @@
final StringBuilder sb = new StringBuilder();
final AbstractJournal journal = getJournal(timestamp);
+
+ if (journal == null) {
+ /*
+ * This condition can occur if there are no shard views on the
+ * previous journal and the releaseAge is zero since the previous
+ * journal can be purged (deleted) before this method is invoked.
+ * This situation arises in a few of the unit tests which begin with
+ * an empty journal and copy everything onto the new journal such
+ * that the old journal can be immediately released.
+ */
+ return "No journal: timestamp=" + timestamp;
+ }
sb.append("timestamp="+timestamp+"\njournal="+journal.getResourceMetadata());
// historical view of Name2Addr as of that timestamp.
- final ITupleIterator itr = journal.getName2Addr(timestamp)
+ final ITupleIterator<?> itr = journal.getName2Addr(timestamp)
.rangeIterator();
while (itr.hasNext()) {
- final ITuple tuple = itr.next();
+ final ITuple<?> tuple = itr.next();
final Entry entry = EntrySerializer.INSTANCE
.deserialize(new DataInputBuffer(tuple.getValue()));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/OverflowManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -287,6 +287,14 @@
*/
protected final AtomicBoolean asyncOverflowEnabled = new AtomicBoolean(true);
+ /**
+ * Flag may be set to force overflow processing during the next group
+ * commit. The flag is cleared by {@link #overflow()}.
+ *
+ * @see DataService#forceOverflow(boolean, boolean)
+ */
+ public final AtomicBoolean forceOverflow = new AtomicBoolean(false);
+
/**
* A flag that may be set to force the next asynchronous overflow to perform
* a compacting merge for all indices that are not simply copied over to the
@@ -295,6 +303,8 @@
* made compact and SHOULD NOT be used for deployed federations</strong>).
* The state of the flag is cleared each time asynchronous overflow
* processing begins.
+ *
+ * @see DataService#forceOverflow(boolean, boolean)
*/
public final AtomicBoolean compactingMerge = new AtomicBoolean(false);
@@ -1849,6 +1859,19 @@
*/
public boolean shouldOverflow() {
+ if(forceOverflow.get()) {
+
+ /*
+ * Note: forceOverflow trumps everything else.
+ */
+
+ if (log.isInfoEnabled())
+ log.info("Forcing overflow.");
+
+ return true;
+
+ }
+
if (isTransient()) {
/*
@@ -1886,7 +1909,7 @@
return false;
}
-
+
/*
* Look for overflow condition on the "live" journal.
*/
@@ -1959,8 +1982,18 @@
*/
public Future<Object> overflow() {
- assert overflowAllowed.get();
+// assert overflowAllowed.get();
+ /*
+ * Atomically test and clear the flag. The local boolean is inspected
+ * below. When true, asynchronous overflow processing will occur unless
+ * an error occurs during synchronous overflow processing. This ensures
+ * that we can force a compacting merge on the shards of a data service
+ * even if that data service has not buffer sufficient writes to warrant
+ * a build on any of the index segments.
+ */
+ final boolean forceOverflow = this.forceOverflow.getAndSet(false/* newValue */);
+
final Event e = new Event(getFederation(), new EventResource(),
EventType.SynchronousOverflow).addDetail(
"synchronousOverflowCounter",
@@ -1982,7 +2015,12 @@
if (asyncOverflowEnabled.get()) {
- if (overflowMetadata.postProcess) {
+ /*
+ * Do overflow processing if overflow is being forced OR if we
+ * need to do a build for at least one index partition.
+ */
+
+ if (forceOverflow || overflowMetadata.postProcess) {
/*
* Post-processing SHOULD be performed.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/StoreManager.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -1416,22 +1416,45 @@
* Verify that the concurrency manager has been set and wait a while
* it if is not available yet.
*/
- if (log.isInfoEnabled())
- log.info("Waiting for concurrency manager");
- for (int i = 0; i < 5; i++) {
- try {
- getConcurrencyManager(); break;
- } catch (IllegalStateException ex) {
- Thread.sleep(100/* ms */);
- }
+ {
+ int nwaits = 0;
+ while (true) {
+ try {
+ getConcurrencyManager();
+ break;
+ } catch (IllegalStateException ex) {
+ Thread.sleep(100/* ms */);
+ if (++nwaits % 50 == 0)
+ log.warn("Waiting for concurrency manager");
+ }
+ }
}
- getConcurrencyManager();
- if (Thread.interrupted())
- throw new InterruptedException();
- /*
- * Look for pre-existing data files.
- */
+ try {
+ final IBigdataFederation<?> fed = getFederation();
+ if (fed == null) {
+ /*
+ * Some of the unit tests do not start the txs until after
+ * the DataService. For those unit tests getFederation()
+ * will return null during startup() of the DataService. To
+ * have a common code path, we throw the exception here
+ * which is caught below.
+ */
+ throw new UnsupportedOperationException();
+ }
+ while (true) {
+ if (fed.getTransactionService() != null) {
+ break;
+ }
+ log.warn("Waiting for transaction service discovery");
+ }
+ } catch (UnsupportedOperationException ex) {
+ log.warn("Federation not available - running in test case?");
+ }
+
+ /*
+ * Look for pre-existing data files.
+ */
if (!isTransient) {
if (log.isInfoEnabled())
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -829,7 +829,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public T getService() {
@@ -840,7 +840,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public String getServiceName() {
@@ -851,7 +851,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public Class getServiceIface() {
@@ -862,7 +862,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public UUID getServiceUUID() {
@@ -873,7 +873,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public boolean isServiceReady() {
@@ -894,7 +894,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public void reattachDynamicCounters() {
@@ -905,7 +905,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public void didStart() {
@@ -916,7 +916,7 @@
}
/**
- * Delegated.
+ * Delegated. {@inheritDoc}
*/
public AbstractHTTPD newHttpd(final int httpdPort,
final CounterSet counterSet) throws IOException {
@@ -927,7 +927,10 @@
}
- public void serviceJoin(IService service, UUID serviceUUID) {
+ /**
+ * Delegated. {@inheritDoc}
+ */
+ public void serviceJoin(final IService service, final UUID serviceUUID) {
if (!isOpen()) return;
@@ -941,7 +944,10 @@
}
- public void serviceLeave(UUID serviceUUID) {
+ /**
+ * Delegated. {@inheritDoc}
+ */
+ public void serviceLeave(final UUID serviceUUID) {
if(!isOpen()) return;
@@ -1129,9 +1135,9 @@
// notify delegates that deferred startup has occurred.
AbstractFederation.this.didStart();
+
}
-
/**
* Setup sampling on the client's thread pool. This collects interesting
* statistics about the thread pool for reporting to the load balancer
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -47,6 +47,7 @@
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.IndexSegment;
import com.bigdata.journal.ITransactionService;
import com.bigdata.journal.ITx;
import com.bigdata.mdi.IMetadataIndex;
@@ -492,40 +493,56 @@
}
- /**
- * Force overflow of each data service in the scale-out federation (only
- * scale-out federations support overflow processing). This method is
- * synchronous. It will not return until all {@link DataService}s have
- * initiated and completed overflow processing. Any unused resources (as
- * determined by the {@link StoreManager}) will have been purged.
- *
- * @param truncateJournal
- * When <code>true</code>, the live journal will be truncated
- * to its minimum extent (all writes will be preserved but there
- * will be no free space left in the journal). This may be used
- * to force the {@link DataService} to its minimum possible
- * footprint.
- *
- * @todo when overflow processing is enabled for the {@link MetadataService}
- * we will have to modify this to also trigger overflow for those
- * services.
- */
- public void forceOverflow(final boolean truncateJournal) {
+ /**
+ * Force overflow of each data service in the scale-out federation (only
+ * scale-out federations support overflow processing). This method is
+ * synchronous. It will not return until all {@link DataService}s have
+ * initiated and completed overflow processing. Any unused resources (as
+ * determined by the {@link StoreManager}) will have been purged.
+ * <p>
+ * This is a relatively fast operation when
+ * <code>compactingMerge := false</code>. By specifying both
+ * <code>compactingMerge := false</code> and
+ * <code>truncateJournal := false</code> you can cause the data services to
+ * close out their current journals against further writes. While this is
+ * not a global synchronous operation, it can provide a basis to obtain a
+ * "near synchronous" snapshot from the federation consisting of all writes
+ * up to the point where overflow was triggered on each data service.
+ *
+ * @param compactingMerge
+ * When <code>true</code>, each shard on each
+ * {@link IDataService} will undergo a compacting merge.
+ * Synchronous parallel compacting merge of all shards is an
+ * expensive operation. This parameter shoudl normally be
+ * <code>false</code> unless you are requesting a compacting
+ * merge for specific purposes, such as benchmarking when all
+ * data is known to exist in one {@link IndexSegment} per shard.
+ * @param truncateJournal
+ * When <code>true</code>, the live journal will be truncated to
+ * its minimum extent (all writes will be preserved but there
+ * will be no free space left in the journal). This may be used
+ * to force the {@link DataService} to its minimum possible
+ * footprint.
+ *
+ * @todo when overflow processing is enabled for the {@link MetadataService}
+ * we will have to modify this to also trigger overflow for those
+ * services.
+ */
+ public void forceOverflow(final boolean compactingMerge, final boolean truncateJournal) {
// find UUID for each data service.
final UUID[] dataServiceUUIDs = getDataServiceUUIDs(0/* maxCount */);
final int ndataServices = dataServiceUUIDs.length;
- if(log.isInfoEnabled())
- log.info("#dataServices=" + ndataServices + ", now=" + new Date());
+ log.warn("Forcing overflow: #dataServices=" + ndataServices + ", now=" + new Date());
final List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(ndataServices);
for (UUID serviceUUID : dataServiceUUIDs) {
tasks.add(new ForceOverflowTask(getDataService(serviceUUID),
- truncateJournal));
+ compactingMerge, truncateJournal));
}
@@ -570,8 +587,7 @@
}
- if(log.isInfoEnabled())
- log.info("Did overflow: #ok=" + nok + ", #dataServices="
+ log.warn("Did overflow: #ok=" + nok + ", #dataServices="
+ ndataServices + ", now=" + new Date());
if (nok != tasks.size()) {
@@ -643,16 +659,19 @@
.getLogger(ForceOverflowTask.class);
private final IDataService dataService;
+ private final boolean compactingMerge;
private final boolean truncateJournal;
- public ForceOverflowTask(final IDataService dataService,
- final boolean truncateJournal) {
+ public ForceOverflowTask(final IDataService dataService,
+ final boolean compactingMerge, final boolean truncateJournal) {
if (dataService == null)
throw new IllegalArgumentException();
this.dataService = dataService;
+ this.compactingMerge = compactingMerge;
+
this.truncateJournal = truncateJournal;
}
@@ -663,8 +682,7 @@
log.info("dataService: " + dataService.getServiceName());
// returns once synchronous overflow is complete.
- dataService
- .forceOverflow(true/* immediate */, true/* compactingMerge */);
+ dataService.forceOverflow(true/* immediate */, compactingMerge);
if (log.isInfoEnabled())
log.info("Synchronous overflow is done: "
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2010-08-06 15:46:07 UTC (rev 3423)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractTransactionService.java 2010-08-06 15:59:39 UTC (rev 3424)
@@ -43,7 +43,6 @@
import org.apache.log4j.Logger;
-import com.bigdata.concurrent.LockManager;
import com.bigdata.config.LongValidator;
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.Instrument;
@@ -80,9 +79,9 @@
*/
protected static final Logger log = Logger.getLogger(AbstractTransactionService.class);
- protected static final boolean INFO = log.isInfoEnabled();
+// protected static final boolean INFO = log.isInfoEnabled();
- protected static final boolean DEBUG = log.isDebugEnabled();
+// protected static final boolean DEBUG = log.isDebugEnabled();
/**
* Options understood by this service.
@@ -91,29 +90,39 @@
* @version $Id$
*/
public interface Options {
-
- /**
- * How long you want to hold onto the database history (in milliseconds)
- * or {@link Long#MAX_VALUE} for an (effectively) immortal database. The
- * {@link ITransactionService} tracks the timestamp corresponding to the
- * earliest running transaction (if any). When such a transaction
- * exists, the actual release time is:
- *
- * <pre>
- * releaseTime = min(earliestRunningTx, now - minimumReleaseAge) - 1
- * </pre>
- *
- * This ensures that history in use by running transactions is not
- * released even when the minimumReleaseAge is ZERO (0).
- *
- * @see #DEFAULT_MIN_RELEASE_AGE
- * @see #MIN_RELEASE_AGE_1H
- * @see #MIN_RELEASE_AGE_1D
- * @see #MIN_RELEASE_AGE_1W
- * @see #MIN_RELEASE_AGE_NEVER
- *
- * @see AbstractTransactionService#updateReleaseTime(long)
- */
+
+ /**
+ * How long you want to hold onto the database history (in milliseconds)
+ * or {@link Long#MAX_VALUE} for an (effectively) immortal database. The
+ * {@link ITransactionService} tracks the timestamp corresponding to the
+ * earliest running transaction (if any). When such a transaction
+ * exists, the actual release time is:
+ *
+ * <pre>
+ * releaseTime = min(lastCommitTime - 1, min(earliestRunningTx, now - minimumReleaseAge))
+ * </pre>
+ *
+ * This ensures that history in use by running transactions is not
+ * released even when the minimumReleaseAge is ZERO (0).
+ * <p>
+ * When no transactions exist the actual release time is:
+ *
+ * <pre>
+ * releaseTime = min(commitTime - 1, now - minimumReleaseAge)
+ * </pre>
+ *
+ * This ensures that the the release time advances when no transactions
+ * are in use, but that the mi...
[truncated message content] |
|
From: <tho...@us...> - 2010-08-18 20:56:09
|
Revision: 3448
http://bigdata.svn.sourceforge.net/bigdata/?rev=3448&view=rev
Author: thompsonbry
Date: 2010-08-18 20:55:58 +0000 (Wed, 18 Aug 2010)
Log Message:
-----------
Mapped out the "
Added tuprolog dependency (LGPL), but I have not done anything yet to integrate it. It will be used to do query plan rewrites.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/.classpath
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IRelation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPathFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/EmptyAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/DefaultSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IProgram.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IQueryOptions.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IRule.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/ISolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/ISortOrder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Rule.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/SortOrder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/AbstractStepTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultEvaluationPlan2.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRangeCountFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/IJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/IRangeCountFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/IRuleState.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/ISolution.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/NestedSubqueryWithJoinThreadsTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/NoReorderEvaluationPlan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/RuleLog.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/RuleState.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/RuleStats.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/Solution.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/SolutionFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/IJoinMaster.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTaskFactoryTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinTaskSink.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/LocalJoinMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/LocalJoinTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsyncDistributedOutputBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsyncLocalOutputBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsynchronizedOutputBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsynchronizedSolutionBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/search/FullTextIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/FederationCallable.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IFederationCallable.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/proxy/RemoteAsynchronousIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/Filter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/IKeyOrder.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/NV.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/accesspath/TestSameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/AbstractRuleTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestRule.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestDefaultEvaluationPlan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestRuleState.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/Justification.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/OwlSameAsPropertiesExpandingIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/DummyIV.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/AbstractInlineConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineEQ.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineGE.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineGT.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineLE.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineLT.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/InlineNE.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/IRISUtils.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicTuple.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/relation/rule/BindingSetSortKeyBuilder.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_11_13.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_3_5_6_7_9.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleFastClosure_5_6_7_9.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/BackchainAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/FastClosure.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/MatchRule.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexusFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RejectAnythingSameAsItself.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleContextEnum.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleFastClosure11.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleFastClosure13.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleFastClosure3.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlInverseOf1.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlSameAs1.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlSameAs1b.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlSameAs2.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlSameAs3.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlTransitiveProperty1.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleOwlTransitiveProperty2.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleRdfs05.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleRdfs07.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleRdfs09.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RuleRdfs11.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/SPOBindingSetSerializer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/TMUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NoAxiomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/BigdataSolutionResolverator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/Vocabulary.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestInlineConstraints.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestIRIS.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestMagicStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/AbstractRuleTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestJustifications.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestMatch.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleExpansion.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleFastClosure_3_5_6_7_9.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestSlice.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Bigdata2Sesame2BindingSetIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl2.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/lib/tuprolog/
branches/QUADS_QUERY_BRANCH/bigdata/lib/tuprolog/tuprolog-v2.1.1.jar
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractBOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractPipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ArrayBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpList.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Constant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConstantEval.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Distinct.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/EmptyBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IVariable.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IVariableOrConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/NV.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/QuoteOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/SampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/SampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Var.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Union.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bop-notes.txt
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/EQ.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/EQConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/IN.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/NE.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/NEConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/OR.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/constraint/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/eval/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/eval/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/eval/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/MapBindingSetsOverShards.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/ReceiveBindingSets.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/IJoinMaster.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/JoinStats.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinStats.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestVar.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/constraint/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/constraint/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/constraint/TestEQ.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/eval/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/eval/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/eval/TestJoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestMapBindingSetsOverNodes.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestMapBindingSetsOverShards.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestSendReceiveBuffers.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedDataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/EmbeddedMetadataService.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/Binding.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/IBinding.java
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/ArrayBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Binding.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Constant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/EQ.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/EQConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/EmptyBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/HashBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IBinding.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IN.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IVariable.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IVariableOrConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/NE.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/NEConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/OR.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/Var.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DHTFilterFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/JoinStats.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractEmbeddedDataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractEmbeddedLoadBalancerService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractEmbeddedTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/EmbeddedClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/EmbeddedFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/EmbeddedMetadataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestBindingSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestConstant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestVar.java
Modified: branches/QUADS_QUERY_BRANCH/.classpath
===================================================================
--- branches/QUADS_QUERY_BRANCH/.classpath 2010-08-18 20:04:48 UTC (rev 3447)
+++ branches/QUADS_QUERY_BRANCH/.classpath 2010-08-18 20:55:58 UTC (rev 3448)
@@ -21,6 +21,7 @@
<classpathentry kind="lib" path="bigdata/lib/dsi-utils-1.0.6-020610.jar"/>
<classpathentry kind="lib" path="bigdata/lib/lgpl-utils-1.0.6-020610.jar"/>
<classpathentry kind="lib" path="bigdata-rdf/lib/nxparser-6-22-2010.jar"/>
+ <classpathentry kind="lib" path="bigdata/lib/tuprolog/tuprolog-v2.1.1.jar"/>
<classpathentry kind="src" path="lgpl-utils/src/java"/>
<classpathentry kind="src" path="lgpl-utils/src/test"/>
<classpathentry exported="true" kind="lib" path="bigdata/lib/icu/icu4j-3_6.jar"/>
Added: branches/QUADS_QUERY_BRANCH/bigdata/lib/tuprolog/tuprolog-v2.1.1.jar
===================================================================
(Binary files differ)
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/lib/tuprolog/tuprolog-v2.1.1.jar
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractBOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractBOp.java 2010-08-18 20:55:58 UTC (rev 3448)
@@ -0,0 +1,212 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 16, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.bigdata.bop.constraint.EQ;
+
+/**
+ * Abstract base class for {@link BOp}s.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractBOp implements BOp {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The argument values.
+ * <p>
+ * Note: This field is reported out as a {@link List} so we can make it
+ * thread safe and, if desired, immutable. However, it is internally a
+ * simple array and exposed to subclasses so they can benefit from fast
+ * positional access to the arguments in operations which would otherwise
+ * become hot, such as {@link EQ#accept(IBindingSet)}.
+ * <p>
+ * If we allow mutation of the arguments then caching of the arguments (or
+ * annotations) by classes such as {@link EQ} will cause {@link #clone()} to
+ * fail because (a) it will do a field-by-field copy on the concrete
+ * implementation class; and (b) it will not consistently update the cached
+ * references. In order to "fix" this problem, any classes which cache
+ * arguments or annotations would have to explicitly overrides
+ * {@link #clone()} in order to set those fields based on the arguments on
+ * the cloned {@link AbstractBOp} class.
+ */
+ protected final BOp[] args;
+
+ /**
+ * The operator annotations.
+ */
+ protected final Map<String,Object> annotations;
+
+ /**
+ * Check the operator argument.
+ *
+ * @param args
+ * The arguments.
+ *
+ * @throws IllegalArgumentException
+ * if the arguments are not valid for the operator.
+ */
+ protected void checkArgs(final Object[] args) {
+
+ }
+
+ /**
+ * Deep copy clone semantics for {@link #args} and {@link #annotations}.
+ * <p>
+ * {@inheritDoc}
+ *
+ * @todo This will deep copy {@link BOp} structures but does not do a deep
+ * copy of other kinds of embedded structures.
+ */
+ public AbstractBOp clone() {
+ try {
+ final AbstractBOp tmp = (AbstractBOp) super.clone();
+ // deep copy the arguments.
+ {
+ final int arity = arity();
+ for (int i = 0; i < arity; i++) {
+ tmp.args[i] = (BOp) (args[i].clone());
+ }
+ }
+ // deep copy the annotations.
+ {
+ final Iterator<Map.Entry<String, Object>> itr = annotations
+ .entrySet().iterator();
+ while (itr.hasNext()) {
+ final Map.Entry<String, Object> e = itr.next();
+ if (e.getValue() instanceof BOp) {
+ tmp.annotations.put(e.getKey(), ((BOp) e.getValue())
+ .clone());
+ }
+ }
+ }
+ return tmp;
+ } catch (CloneNotSupportedException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ /**
+ * @param args
+ * The arguments to the operator.
+ */
+ protected AbstractBOp(final BOp[] args) {
+
+ this(args, null/* annotations */);
+
+ }
+
+ /**
+ * @param args
+ * The arguments to the operator.
+ * @param annotations
+ * The annotations for the operator (optional).
+ */
+ protected AbstractBOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ if (args == null)
+ throw new IllegalArgumentException();
+
+ checkArgs(args);
+
+ final ArrayList<BOp> tmp = new ArrayList<BOp>(args.length);
+
+ for (int i = 0; i < args.length; i++) {
+
+ tmp.add(args[i]);
+
+ }
+
+ this.args = args;
+
+ this.annotations = (annotations == null ? new LinkedHashMap<String, Object>()
+ : annotations);
+
+ }
+
+ final public Map<String, Object> annotations() {
+
+ return Collections.unmodifiableMap(annotations);
+
+ }
+
+ public BOp get(final int index) {
+
+ return args[index];
+
+ }
+
+ public int arity() {
+
+ return args.length;
+
+ }
+
+ final public List<BOp> args() {
+
+ return Collections.unmodifiableList(Arrays.asList(args));
+
+ }
+
+ /**
+ * Return the value of the named annotation.
+ *
+ * @param name
+ * The name of the annotation.
+ * @param defaultValue
+ * The default value.
+ * @return The annotation value -or- the <i>defaultValue</i> if the
+ * annotation was not bound.
+ * @param <T>
+ * The generic type of the annotation value.
+ */
+ @SuppressWarnings("unchecked")
+ public <T> T getProperty(final String name, final T defaultValue) {
+
+ if (!annotations.containsKey(name))
+ return defaultValue;
+
+ return (T) annotations.get(name);
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractBOp.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java 2010-08-18 20:55:58 UTC (rev 3448)
@@ -0,0 +1,97 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 18, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractChunkedOrderedIteratorOp<E> extends AbstractBOp
+ implements ChunkedOrderedIteratorOp<E> {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends ChunkedOrderedIteratorOp.Annotations {
+
+ }
+
+ /**
+ * @param args
+ */
+ protected AbstractChunkedOrderedIteratorOp(BOp[] args) {
+ super(args);
+ }
+
+ /**
+ * @param args
+ * @param annotations
+ */
+ protected AbstractChunkedOrderedIteratorOp(BOp[] args, Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ protected int getChunkCapacity() {
+
+ return getProperty(Annotations.CHUNK_CAPACITY,
+ Annotations.DEFAULT_CHUNK_CAPACITY);
+
+ }
+
+ protected int getChunkOfChunksCapacity() {
+
+ return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
+ Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+
+ }
+
+ protected int getFullyBufferedReadThreshold() {
+
+ return getProperty(Annotations.FULLY_BUFFERED_READ_THRESHOLD,
+ Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
+
+ }
+
+ protected long getChunkTimeout() {
+
+ return getProperty(Annotations.CHUNK_TIMEOUT,
+ Annotations.DEFAULT_CHUNK_TIMEOUT);
+
+ }
+
+ /**
+ * The {@link TimeUnit}s in which the {@link #chunkTimeout} is measured.
+ */
+ protected static transient final TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractPipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractPipelineOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractPipelineOp.java 2010-08-18 20:55:58 UTC (rev 3448)
@@ -0,0 +1,96 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 18, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.bigdata.relation.accesspath.BlockingBuffer;
+import com.bigdata.relation.accesspath.IBlockingBuffer;
+
+/**
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractPipelineOp<E> extends AbstractBOp implements
+ PipelineOp<E> {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BindingSetPipelineOp.Annotations {
+
+ }
+
+ /**
+ * @param args
+ * @param annotations
+ */
+ protected AbstractPipelineOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ super(args, annotations);
+
+ }
+
+ protected int getChunkCapacity() {
+
+ return getProperty(Annotations.CHUNK_CAPACITY,
+ Annotations.DEFAULT_CHUNK_CAPACITY);
+
+ }
+
+ protected int getChunkOfChunksCapacity() {
+
+ return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
+ Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+
+ }
+
+ protected long getChunkTimeout() {
+
+ return getProperty(Annotations.CHUNK_TIMEOUT,
+ Annotations.DEFAULT_CHUNK_TIMEOUT);
+
+ }
+
+ /**
+ * The {@link TimeUnit}s in which the {@link #chunkTimeout} is measured.
+ */
+ protected static transient final TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
+
+ public IBlockingBuffer<E[]> newBuffer() {
+
+ return new BlockingBuffer<E[]>(getChunkOfChunksCapacity(),
+ getChunkCapacity(), getChunkTimeout(), chunkTimeoutUnit);
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractPipelineOp.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractSampleIndex.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractSampleIndex.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractSampleIndex.java 2010-08-18 20:55:58 UTC (rev 3448)
@@ -0,0 +1,101 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 16, 2010
+ */
+
+package com.bigdata.bop;
+
+
+import com.bigdata.btree.IIndex;
+import com.bigdata.relation.accesspath.IAccessPath;
+
+/**
+ * Abstract base class for sampling operator for an {@link IIndex}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ * @param <E>
+ * The generic type of the elements materialized from that index.
+ *
+ * @todo Implement sample operator. E.g., sampleRange(fromKey,toKey,limit). This
+ * could be on {@link IIndex} or on {@link IAccessPath}. For a shard view,
+ * it must proportionally select from among the ordered components of the
+ * view. For a hash table it would be sample(limit) since range based
+ * operations are not efficient.
+ * <p>
+ * This should accept an index, not a predicate (for RDF we determine the
+ * index an analysis of the bound and unbound arguments on the predicate
+ * and always have a good index, but this is not true in the general
+ * case). When the index is remote, it should be executed at the remote
+ * index.
+ *
+ * @todo This needs to operation on element chunks, not {@link IBindingSet}
+ * chunks. It also may not require pipelining.
+ */
+abstract public class AbstractSampleIndex<E> extends AbstractPipelineOp<E> {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Known annotations.
+ */
+ public interface Annotations extends BOp.Annotations {
+ /**
+ * The sample limit.
+ */
+ String LIMIT = "limit";
+ }
+
+ protected AbstractSampleIndex(final IPredicate<E> pred, final int limit) {
+
+ super(new BOp[] { pred }, NV.asMap(new NV[] {//
+ new NV(Annotations.LIMIT, Integer.valueOf(limit)) //
+ }));
+
+ if (pred == null)
+ throw new IllegalArgumentException();
+
+ if (limit <= 0)
+ throw new IllegalArgumentException();
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public IPredicate<E> pred() {
+
+ return (IPredicate<E>) args[0];
+
+ }
+
+ public int limit() {
+
+ return (Integer) annotations.get(Annotations.LIMIT);
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractSampleIndex.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ArrayBindingSet.java (from rev 3423, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/ArrayBindingSet.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ArrayBindingSet.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ArrayBindingSet.java 2010-08-18 20:55:58 UTC (rev 3448)
@@ -0,0 +1,443 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Jun 20, 2008
+ */
+
+package com.bigdata.bop;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.log4j.Logger;
+
+/**
+ * An {@link IBindingSet} backed by an dense array (no gaps). This
+ * implementation is more efficient for fixed or small N (N LTE ~20). It simples
+ * scans the array looking for the variable using references tests for equality.
+ * Since the #of variables is generally known in advance this can be faster and
+ * lighter than {@link HashBindingSet} for most applications.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class ArrayBindingSet implements IBindingSet {
+
+ private static final long serialVersionUID = -6468905602211956490L;
+
+ protected static final Logger log = Logger.getLogger(ArrayBindingSet.class);
+
+ /**
+ * True iff the {@link #log} level is INFO or less.
+ */
+ protected static final boolean INFO = log.isInfoEnabled();
+
+ /**
+ * True iff the {@link #log} level is DEBUG or less.
+ */
+ protected static final boolean DEBUG = log.isDebugEnabled();
+
+ /**
+ * A dense array of the bound variables.
+ */
+ private final IVariable[] vars;
+ /**
+ * A dense array of the values bound to the variables (correlated with
+ * {@link #vars}).
+ */
+ private final IConstant[] vals;
+
+ private int nbound = 0;
+
+ /**
+ * Copy constructor.
+ */
+ protected ArrayBindingSet(ArrayBindingSet bindingSet) {
+
+ if (bindingSet == null)
+ throw new IllegalArgumentException();
+
+ nbound = bindingSet.nbound;
+
+ vars = bindingSet.vars.clone();
+
+ vals = bindingSet.vals.clone();
+
+ }
+
+ /**
+ * Initialized with the given bindings (assumes for efficiency that all
+ * elements of bound arrays are non-<code>null</code> and that no
+ * variables are duplicated).
+ *
+ * @param vars
+ * The variables.
+ * @param vals
+ * Their bound values.
+ */
+ public ArrayBindingSet(IVariable[] vars, IConstant[] vals) {
+
+// if (vars == null)
+// throw new IllegalArgumentException();
+//
+// if (vals == null)
+// throw new IllegalArgumentException();
+
+ assert vars != null;
+ assert vals != null;
+ assert vars.length == vals.length;
+
+ // for (int i = 0; i < vars.length; i++) {
+ //
+ // if (vars[i] == null)
+ // throw new IllegalArgumentException();
+ //
+ // if (vals[i] == null)
+ // throw new IllegalArgumentException();
+ //
+ // }
+
+ this.vars = vars;
+
+ this.vals = vals;
+
+ this.nbound = vars.length;
+
+ }
+
+ /**
+ * Initialized with the given capacity.
+ *
+ * @param capacity
+ * The capacity.
+ *
+ * @throws IllegalArgumentException
+ * if the <i>capacity</i> is negative.
+ */
+ public ArrayBindingSet(int capacity) {
+
+ if (capacity < 0)
+ throw new IllegalArgumentException();
+
+ vars = new IVariable[capacity];
+
+ vals = new IConstant[capacity];
+
+ }
+
+ public Iterator<IVariable> vars() {
+
+ return Collections.unmodifiableList(Arrays.asList(vars)).iterator();
+
+ }
+
+ /**
+ * Iterator does not support either removal or concurrent modification of
+ * the binding set.
+ */
+ public Iterator<Map.Entry<IVariable,IConstant>> iterator() {
+
+ return new BindingSetIterator();
+
+ }
+
+ private class BindingSetIterator implements Iterator<Map.Entry<IVariable,IConstant>> {
+
+ private int i = 0;
+
+ public boolean hasNext() {
+
+ return i < nbound;
+
+ }
+
+ public Entry<IVariable, IConstant> next() {
+
+ // the index whose bindings are being returned.
+ final int index = i++;
+
+ return new Map.Entry<IVariable, IConstant>() {
+
+ public IVariable getKey() {
+
+ return vars[index];
+
+ }
+
+ public IConstant getValue() {
+
+ return vals[index];
+
+ }
+
+ public IConstant setValue(IConstant value) {
+
+ if (value == null)
+ throw new IllegalArgumentException();
+
+ final IConstant t = vals[index];
+
+ vals[index] = value;
+
+ return t;
+
+ }
+
+ };
+
+ }
+
+ public void remove() {
+
+ throw new UnsupportedOperationException();
+
+ }
+
+ }
+
+ public int size() {
+
+ return nbound;
+
+ }
+
+ public void clearAll() {
+
+ for (int i = nbound - 1; nbound > 0; i--, nbound--) {
+
+ vars[i] = null;
+
+ vals[i] = null;
+
+ }
+
+ assert nbound == 0;
+
+ }
+
+ /**
+ * Since the array is dense (no gaps), {@link #clear(IVariable)} requires
+ * that we copy down any remaining elements in the array by one position.
+ */
+ public void clear(IVariable var) {
+
+ if (var == null)
+ throw new IllegalArgumentException();
+
+ for (int i = 0; i < nbound; i++) {
+
+ if (vars[i] == var) {
+
+ final int nremaining = nbound-(i+1);
+
+ if (nremaining >= 0) {
+
+ // Copy down to close up the gap!
+ System.arraycopy(vars, i+1, vars, i, nremaining);
+
+ System.arraycopy(vals, i+1, vals, i, nremaining);
+
+ } else {
+
+ // Just clear the reference.
+
+ vars[i] = null;
+
+ vals[i] = null;
+
+ }
+
+ nbound--;
+
+ break;
+
+ }
+
+ }
+
+ }
+
+ public IConstant get(IVariable var) {
+
+ if (var == null)
+ throw new IllegalArgumentException();
+
+ for (int i = 0; i < nbound; i++) {
+
+ if (vars[i] == var) {
+
+ return vals[i];
+
+ }
+
+ }
+
+ return null;
+
+ }
+
+ public boolean isBound(IVariable var) {
+
+ return get(var) != null;
+
+ }
+
+ public void set(final IVariable var, final IConstant val) {
+
+ if (var == null)
+ throw new IllegalArgumentException();
+
+ if (val == null)
+ throw new IllegalArgumentException();
+
+ if(DEBUG) {
+
+ log.debug("var=" + var + ", val=" + val + ", nbound=" + nbound+", capacity="+vars.length);
+
+ }
+
+ for (int i = 0; i < nbound; i++) {
+
+ if (vars[i] == var) {
+
+ vals[i] = val;
+
+ return;
+
+ }
+
+ }
+
+ vars[nbound] = var;
+
+ vals[nbound] = val;
+
+ nbound++;
+
+ }
+
+ public String toString() {
+
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("{");
+
+ for(int i=0; i<nbound; i++) {
+
+ if(i>0) sb.append(", ");
+
+ sb.append(vars[i]);
+
+ sb.append("=");
+
+ sb.append(vals[i]);
+
+ }
+
+ sb.append("}");
+
+ return sb.toString();
+
+ }
+
+ public ArrayBindingSet clone() {
+
+ return new ArrayBindingSet(this);
+
+ }
+
+ /**
+ * Return a shallow copy of the binding set, eliminating unecessary
+ * variables.
+ */
+ public ArrayBindingSet copy(final IVariable[] variablesToKeep) {
+
+ // bitflag for the old binding set
+ final boolean[] keep = new boolean[nbound];
+
+ // for each var in the old binding set, see if we need to keep it
+ for (int i = 0; i < nbound; i++) {
+
+ final IVariable v = vars[i];
+
+ keep[i] = false;
+ for (IVariable k : variablesToKeep) {
+ if (v == k) {
+ keep[i] = true;
+ break;
+ }
+ }
+
+ }
+
+ // allocate the new vars
+ final IVariable[] newVars = new IVariable[vars.length];
+
+ // allocate the new vals
+ final IConstant[] newVals = new IConstant[vals.length];
+
+ // fill in the new binding set based on the keep bitflag
+ int newbound = 0;
+ for (int i = 0; i < nbound; i++) {
+ if (keep[i]) {
+ newVars[newbound] = vars[i];
+ newVals[newbound] = vals[i];
+ newbound++;
+ }
+ }
+
+ ArrayBindingSet bs = new ArrayBindingSet(newVars, newVals);
+ bs.nbound = newbound;
+
+ return bs;
+
+ }
+
+ public boolean equals(IBindingSet o) {
+
+ if (o == this)
+ return true;
+
+ if (nbound != o.size())
+ return false;
+
+ for(int i=0; i<nbound; i++) {
+
+// if (!o.isBound(vars[i]))
+// return false;
+
+ if (!vals[i].equals(o.get(vars[i])))
+ return false;
+
+ }
+
+ return true;
+
+ }
+
+}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2010-08-18 20:55:58 UTC (rev 3448)
@@ -0,0 +1,218 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 12, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * An operator, such as a constant, variable, join, sort, filter, etc. Operators
+ * are organized in a tree of operators. The arity of an operator is the number
+ * of children declared by that operator class. The children of an operator are
+ * themselves operators and traversal is supported between a parent and its
+ * children. In addition to their arguments, operators may have a variety of
+ * annotations, including those specific to an operator (such as the maximum
+ * number of iterators for a closure operator), those shared by many operators
+ * (such as set of variables which are selected by a join or distributed hash
+ * table), or those shared by all operators (such as a cost model).
+ * <p>
+ * Operators are mutable, thread-safe, {@link Serializable} to facilitate
+ * distributed computing, and {@link Cloneable} to facilitate non-destructive
+ * tree rewrites.
+ * <p>
+ * What follows is a summary of some of the more important kinds of operations.
+ * For each type of operation, there may be several implementations. One common
+ * way in which implementations of the same operator may differ is whether they
+ * are designed for low-volume selective queries or high volume unselective
+ * queries.
+ * <dl>
+ * <dt>JOINs</dt>
+ * <dd></dd>
+ * <dt>Mapping binding sets across shards (key-range partitions) or nodes (hash
+ * partitioned)</dt>
+ * <dd></dd>
+ * <dt>Predicates and access paths</dt>
+ * <dd></dd>
+ * <dt>SORT</dt>
+ * <dd></dd>
+ * <dt>DISTINCT</dt>
+ * <dd></dd>
+ * <dt>Element filters</dt>
+ * <dd></dd>
+ * <dt>Rule constraints</dt>
+ * <dd></dd>
+ * <dt>Binding set filters (removing binding sets which are not required outside
+ * of some context)</dt>
+ * <dd></dd>
+ * <dt>Identifiers for sinks to which binding sets can be written and
+ * conditional routing of binding sets, for example based on variable value or
+ * type or join success or failure</dt>
+ * <dd></dd>
+ * <dt>Sequential or iterative programs.</dt>
+ * <dd></dd>
+ * <dt>Creating or destroying transient or persistent resources (graphs, tables,
+ * DHTs, etc). Such life cycle operators dominate the subtree within which the
+ * resource will be utilized.</dt>
+ * <dd></dd>
+ * <dt>Export of proxy objects, especially for query or mutation buffers.</dt>
+ * <dd></dd>
+ * </dl>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ *
+ * @todo javadoc review and reconcile with notes.
+ */
+public interface BOp extends Cloneable, Serializable {
+
+ /**
+ * The #of arguments to the operation.
+ */
+ int arity();
+
+ /**
+ * Return an argument to the operation.
+ *
+ * @param index
+ * The argument index in [0:{@link #arity()}-1].
+ *
+ * @return The argument.
+ */
+ BOp get(int index);
+
+// /**
+// * Bind an argument of the operation.
+// *
+// * @param index
+// * The argument index in [0:{@link #arity()}-1].
+// * @param newValue
+// * The bound value.
+// *
+// * @return A copy of the operation having the bound value for the argument.
+// */
+// BOp<?> setArg(int index, BOp<?> newValue);
+
+// /**
+// * Return the value of the named operator annotation.
+// *
+// * @param name
+// * The annotation name.
+// *
+// * @return The value of the named operator annotation.
+// */
+// Object getProperty(String name);
+
+// /**
+// * Set the value of the named operator annotation.
+// *
+// * @param name
+// * The annotation name.
+// * @param newValue
+// * The new value for the named annotation,
+// *
+// * @return The old value of the named operator annotation.
+// */
+// Object setProperty(String name,Object newValue);
+
+// /**
+// * Return the type constraint on the specified argument.
+// *
+// * @param index
+// * The argument index in [0:{@link #arity()}-1].
+// *
+// * @return The type constraint on that argument.
+// */
+// Class<?> getArgType(int index);
+
+// /**
+// * The type of the values produced by the operation (Constant or variable,
+// * primitive?, relation, triple store, index, file, bat, ...).
+// */
+// Class<T> getResultType();
+
+// /**
+// * @TODO There needs to be some simple evaluation path for things such as
+// * native SPARQL operations. This is currently
+// * {@link IConstraint#accept(IBindingSet)}, which returns a truth
+// * value. This seems quite adequate.
+// */
+// boolean accept(IBindingSet bset);
+
+// /**
+// * The #of arguments to this operation which are variables. This method does
+// * not report on...
[truncated message content] |
|
From: <mar...@us...> - 2010-09-27 16:28:50
|
Revision: 3637
http://bigdata.svn.sourceforge.net/bigdata/?rev=3637&view=rev
Author: martyncutcher
Date: 2010-09-27 16:28:41 +0000 (Mon, 27 Sep 2010)
Log Message:
-----------
Checkin of CTC Striterators with revised license
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/ctc-striterators/
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appender.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appenderator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ArrayIterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractor.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractorator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EmptyIterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EnumIterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expanderator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IFilter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IStriterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXProperty.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXPropertyIterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapper.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapperator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Merger.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mergerator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolverator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/SingleValueIterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Sorter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Sorterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Visitor.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Visitorator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/XProperty.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/XPropertyIterator.java
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appender.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appender.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appender.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,52 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * Appender
+ *
+ * Used with Appenderator by Striterator to filter returned objects.
+ */
+
+public class Appender implements IFilter {
+ protected Iterator m_xtra = null;
+
+ public Appender() {}
+
+ public Appender(Iterator xtra) {
+ m_xtra = xtra;
+ }
+
+ //-------------------------------------------------------------
+
+ final public Iterator filter(Iterator src) {
+ return new Appenderator(src, (Iterator) m_xtra);
+ }
+
+ //-------------------------------------------------------------
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appender.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appenderator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appenderator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appenderator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,75 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/**
+ * Appenderator
+ **/
+
+public class Appenderator implements Iterator {
+
+ Iterator m_src;
+ Iterator m_xtra;
+
+ Iterator m_current;
+
+ public Appenderator(Iterator src, Iterator xtra) {
+ m_src = src;
+ m_xtra = xtra;
+
+ m_current = m_src;
+ }
+
+ //-------------------------------------------------------------
+
+ public boolean hasNext() {
+ if (m_current.hasNext()) {
+ return true;
+ }
+
+ m_current = m_xtra;
+
+ return m_current.hasNext();
+ }
+
+ //-------------------------------------------------------------
+ // must call hasNext() to ensure m_current is correct
+ public Object next() {
+ if (hasNext()) {
+ return m_current.next();
+ }
+
+ throw new NoSuchElementException("FilterIterator");
+ }
+
+ //-------------------------------------------------------------
+
+ public void remove() {
+ m_current.remove();
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Appenderator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ArrayIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ArrayIterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ArrayIterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,56 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * Supports standard iteration over an object Array, allowing this to
+ * be used as a source for a <code>Striterator</code>.
+ */
+public class ArrayIterator implements Iterator {
+ Object[] m_src = null;
+ int m_index = 0;
+
+ /** Constructor takes source object array **/
+ public ArrayIterator(Object[] src) {
+ m_src = src;
+ }
+
+ /** checks with current index and array size **/
+ public boolean hasNext() {
+ return m_src != null && m_src.length > m_index;
+ }
+
+ /** @return current index from array **/
+ public Object next() {
+ return m_src[m_index++];
+ }
+
+ /** void .. does nothing **/
+ public void remove() {
+ }
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ArrayIterator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractor.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractor.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractor.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,55 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * Used with a Contractorator to contract an Expanded iterator
+ *
+ * The Contractorator will call contract on this object
+ * @author Martyn Cutcher
+ */
+public abstract class Contractor implements IFilter {
+
+ protected Object m_state = null;
+
+ public Contractor() { }
+
+ public Contractor(Object state) {
+ m_state = state;
+ }
+
+ //-------------------------------------------------------------
+
+ public Iterator filter(Iterator src) {
+ return new Contractorator(src, this);
+ }
+
+ //-------------------------------------------------------------
+
+ protected abstract Object contract(Iterator src);
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractor.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractorator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractorator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractorator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,76 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * A Contractorator is the reverse of an Expanderator. A Contractorator takes a
+ * source Iterator, and consumes elements from it in batches by passing to the
+ * Contractor which returns an object when ready. It is upto the Contractor to
+ * handle termination conditions. For example, it might batch consume the
+ * objects ten at a time with some summary info, but handle that there may be
+ * less than ten elements in the final summary.
+ *
+ * A Contractor could also be used to chunk up a number of objects for
+ * serialization, perhaps for network transmission. Such a pattern might be used
+ * with an Expander on the other side to deserialize into an iterator.
+ *
+ * @author Martyn Cutcher
+ */
+public class Contractorator implements Iterator {
+ Iterator m_src;
+ Contractor m_contractor;
+ Object m_next;
+
+ public Contractorator(Iterator src, Contractor contractor) {
+ m_src = src;
+ m_contractor = contractor;
+
+ m_next = m_contractor.contract(m_src);
+ }
+
+ public boolean hasNext() {
+ return m_next != null;
+ }
+
+ public Object next() {
+ if (m_next == null) {
+ throw new NoSuchElementException();
+ }
+
+ Object ret = m_next;
+ m_next = m_contractor.contract(m_src);
+
+ return ret;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Contractorator.java
___________________________________________________________________
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EmptyIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EmptyIterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EmptyIterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,50 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/***************************************************************************
+ * <p>Need to return an iterator to indicate that there's nothing there? Here's
+ * one ready made.</p>
+ *
+ * <p>It allows calls to be made without needing to check for a null iterator.</p>
+ */
+
+public final class EmptyIterator implements Iterator {
+ public boolean hasNext() {
+ return false;
+ }
+
+ public Object next() {
+ return null;
+ }
+
+ public void remove() {}
+
+ final static public EmptyIterator DEFAULT = new EmptyIterator();
+}
+
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EmptyIterator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EnumIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EnumIterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EnumIterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,54 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+import java.util.Enumeration;
+
+/**
+ * Wrapper for Enumeration objects to produce an Iterator.
+ */
+
+public class EnumIterator implements Iterator {
+
+ protected Enumeration m_enum = null;
+
+ public EnumIterator(Enumeration p_enum) {
+ m_enum = p_enum;
+ }
+
+ public boolean hasNext() {
+ return m_enum.hasMoreElements();
+ }
+
+ public Object next() {
+ return m_enum.nextElement();
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException("Remove Not Supported by underlying Enumeration");
+ }
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/EnumIterator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,42 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+/******************************************************************************
+ * Derived from Filter, and excludes a single object from the iteration.
+ */
+public class ExclusionFilter extends Filter {
+ public ExclusionFilter(Object state) {
+ super(state);
+ }
+
+ /***********************************************************************
+ * Just make sure that the current object is not the one to be excluded.
+ **/
+ protected boolean isValid(Object obj) {
+ return obj != m_state;
+ }
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,56 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * Expander
+ *
+ * Used with Expanderator by Striterator to expand returned objects.
+ *
+ * Derivations must implement expand method.
+ */
+
+public abstract class Expander implements IFilter {
+ protected Object m_state = null;
+
+ public Expander() { }
+
+ public Expander(Object state) {
+ m_state = state;
+ }
+
+ //-------------------------------------------------------------
+
+ final public Iterator filter(Iterator src) {
+ return new Expanderator(src, this);
+ }
+
+ //-------------------------------------------------------------
+
+ protected abstract Iterator expand(Object obj);
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expanderator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expanderator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expanderator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,81 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/**
+ * Expanderator
+ *
+ * Flattens out a two-level iteration. By combining Expanderators recursively a general tree
+ * iteration is provided.
+ *
+ * Provides resolution for both the child object and also the nested iterator.
+ * The actual expansion is via an Expander object that is passed in at construction.
+ */
+
+public class Expanderator implements Iterator {
+
+ Iterator m_src;
+ Iterator m_child = null;
+
+ Expander m_expander;
+
+ public Expanderator(Iterator src, Expander expander) {
+ m_src = src;
+ m_expander = expander;
+ }
+
+ //-------------------------------------------------------------
+
+ public boolean hasNext() {
+ if (m_child != null && m_child.hasNext()) {
+ return true;
+ } else if (m_src.hasNext()) {
+ m_child = m_expander.expand(m_src.next());
+
+ return hasNext();
+ } else {
+ return false;
+ }
+ }
+
+ //-------------------------------------------------------------
+ // must call hasNext() to ensure m_child is setup
+ public Object next() {
+ if (hasNext()) {
+ return m_child.next();
+ }
+
+ throw new NoSuchElementException("Expanderator");
+ }
+
+ //-------------------------------------------------------------
+
+ public void remove() {
+ m_child.remove();
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expanderator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,52 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * Used with Filterator by Striterator to filter returned objects.
+ */
+public abstract class Filter implements IFilter {
+
+ protected Object m_state = null;
+
+ public Filter() {}
+
+ public Filter(Object state) {
+ m_state = state;
+ }
+
+ //-------------------------------------------------------------
+
+ final public Iterator filter(Iterator src) {
+ return new Filterator(src, this);
+ }
+
+ //-------------------------------------------------------------
+
+ protected abstract boolean isValid(Object obj);
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,95 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/**
+ * <p>Example usage</p>
+ * <pre>
+ * Iterator src = new Filterator(old, new Filter(state)) {
+ * boolean isValid(Object object) {
+ * return object == m_state;
+ * }
+ * } );
+ * </pre>
+ *
+ * <p>The Filterator provide the protocol support to utulise Filter objects.</p>
+ */
+
+public class Filterator implements Iterator {
+
+ Iterator m_src;
+ Object m_value = null;
+
+ protected Filter m_filter = null;
+
+ public Filterator(Iterator src, Filter filter) {
+ m_src = src;
+ m_filter = filter;
+
+ m_value = getNext();
+ }
+
+ //-------------------------------------------------------------
+
+ public boolean hasNext() {
+ return m_value != null;
+ }
+
+ //-------------------------------------------------------------
+ // must call hasNext() to ensure m_child is setup
+ public Object next() {
+ if (hasNext()) {
+ Object val = m_value;
+ m_value = getNext();
+
+ return val;
+ }
+
+ throw new NoSuchElementException("FilterIterator");
+ }
+
+ //-------------------------------------------------------------
+
+ public void remove() {
+ m_src.remove();
+ }
+
+ //-------------------------------------------------------------
+
+ protected Object getNext() {
+ while (m_src.hasNext()) {
+ Object next = m_src.next();
+
+ if (m_filter.isValid(next)) {
+ return next;
+ }
+ }
+
+ return null;
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IFilter.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,47 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+import java.io.Serializable;
+
+/**
+ * Provides the hook interface that allows use by Striterators
+ */
+public interface IFilter extends Serializable {
+ /**
+ * <p>The filter method is provided to allow the creation of the filtering iterator.</p>
+ *
+ * <p>Any implementation should follow the following pattern:</p>
+ * <pre>
+ * public Iterator filter(Iterator src) {
+ * return new Filterator(src, this);
+ * }
+ * </pre>
+ **/
+ public abstract Iterator filter(Iterator src);
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IFilter.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IStriterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IStriterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IStriterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,62 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.lang.reflect.Method;
+import java.util.Iterator;
+import java.util.Enumeration;
+
+/**
+ * IStriterator
+ *
+ * Extends the Iterator interface to expose methods to add IFilter objects and a specific Type filter
+ */
+
+public interface IStriterator extends Iterator, Enumeration {
+ /** Adds a Discriminating IFilter object **/
+ public IStriterator addFilter(IFilter filter);
+
+ /** check each object against cls.isInstance(object) **/
+ public IStriterator addTypeFilter(Class cls);
+
+ /** check each object against object.getClass() == cls **/
+ public IStriterator addInstanceOfFilter(Class cls);
+
+ /** exclude the passed object from the iteration **/
+ public IStriterator exclude(Object object);
+
+ /** append the passed iteration **/
+ public IStriterator append(Iterator iter);
+
+ /** Ensures the returned values appear only once **/
+ public IStriterator makeUnique();
+
+ /** map the clients method against the Iteration, the Method MUST take a single Object valued parameter.
+ * can be called by :
+ * iter.map(this, MyClass.aMethod);
+ **/
+ public IStriterator map(Object client, Method method);
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IStriterator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXProperty.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXProperty.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXProperty.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,37 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+/** IXProperty.java
+ */
+
+public interface IXProperty {
+ /** Returns the key of the property **/
+ public String getKey();
+
+ /** Returns the value of the property **/
+ public Object getValue();
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXProperty.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXPropertyIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXPropertyIterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXPropertyIterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,36 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+/**
+ * IXPropertyIterator
+ *
+ * A standard iterator, extended to provide a nextProperty method returning IXProperty objects
+ **/
+
+public interface IXPropertyIterator extends IStriterator {
+ public IXProperty nextProperty();
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IXPropertyIterator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapper.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapper.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapper.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,64 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+import java.lang.reflect.Method;
+
+/**
+ * Mapper
+ *
+ * Used with Mapperator by Striterator to map instance methods against member objects.
+ */
+
+public class Mapper implements IFilter {
+ protected Object m_client = null;
+ protected Method m_method = null;
+ protected Object[] m_args = {null};
+
+ public Mapper(Object client, Method method) {
+ m_client = client;
+ m_method = method;
+ }
+
+ //-------------------------------------------------------------
+
+ final public Iterator filter(Iterator src) {
+ return new Mapperator(src, this);
+ }
+
+ //-------------------------------------------------------------
+
+ protected void apply(Object obj) {
+ try {
+ m_args[0] = obj;
+
+ m_method.invoke(m_client, m_args);
+ } catch (Exception e) {
+ throw new RuntimeException("Error on mapping method", e);
+ }
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapper.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapperator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapperator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapperator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,61 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/**
+ * Mapperator
+ *
+ * Initialized with a Mapper object, wraps a standard iterator and calls apply on each object as it is iterated
+ */
+
+public class Mapperator implements Iterator {
+
+ Iterator m_iter;
+ Mapper m_mapper;
+
+ public Mapperator(Iterator iter, Mapper mapper) {
+ m_iter = iter;
+ m_mapper = mapper;
+ }
+
+ public boolean hasNext() {
+ return m_iter.hasNext();
+ }
+
+ public Object next() {
+ Object obj = m_iter.next();
+
+ m_mapper.apply(obj);
+
+ return obj;
+ }
+
+ public void remove() {
+ m_iter.remove();
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mapperator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Merger.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Merger.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Merger.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,53 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+import java.util.Comparator;
+
+/**
+ * Used with Mergerator by Striterator to merge returned objects.
+ */
+
+public class Merger implements IFilter {
+ protected Iterator m_xtra = null;
+ protected Comparator m_comparator = null;
+
+ public Merger() {}
+
+ public Merger(Iterator xtra, Comparator comparator) {
+ m_xtra = xtra;
+ m_comparator = comparator;
+ }
+
+ //-------------------------------------------------------------
+
+ final public Iterator filter(Iterator src) {
+ return new Mergerator(src, m_xtra, m_comparator);
+ }
+
+ //-------------------------------------------------------------
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Merger.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mergerator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mergerator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mergerator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,88 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/**
+ * Merges two sorted lists removing equal objects. For example, two sorted
+ * lists of Strings to create a single list with no duplicates.
+ **/
+public class Mergerator implements Iterator {
+
+ Iterator m_setA;
+ Iterator m_setB;
+ Comparator m_comparator;
+
+ Object m_valA = null;
+ Object m_valB = null;
+
+ public Mergerator(Iterator setA, Iterator setB, Comparator comparator) {
+ m_setA = setA;
+ m_setB = setB;
+
+ m_comparator = comparator;
+
+ if (m_setA.hasNext()) {
+ m_valA = m_setA.next();
+ }
+
+ if (m_setB.hasNext()) {
+ m_valB = m_setB.next();
+ }
+ }
+
+ //-------------------------------------------------------------
+
+ public boolean hasNext() {
+ return m_valA != null || m_valB != null;
+ }
+
+ //-------------------------------------------------------------
+ // must call hasNext() to ensure m_current is correct
+ public Object next() {
+ if (hasNext()) {
+ Object retVal = null;
+
+ if (m_valB == null || (m_valA != null && (m_comparator.compare(m_valA, m_valB) <= 0))) {
+ retVal = m_valA;
+ m_valA = m_setA.hasNext() ? m_setA.next() : null;
+ } else {
+ retVal = m_valB;
+ m_valB = m_setB.hasNext() ? m_setB.next() : null;
+ }
+ return retVal;
+ } else {
+ throw new NoSuchElementException("Mergerator");
+ }
+ }
+
+ //-------------------------------------------------------------
+
+ public void remove() {
+ throw new RuntimeException("Cannot remove object from merged set");
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Mergerator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,56 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * Resolver
+ *
+ * Used with Resolverator by Striterator to resolve returned objects.
+ *
+ * Cannot be instantiated directly since an implementation of the resolve method is required.
+ */
+
+public abstract class Resolver implements IFilter {
+ protected Object m_state = null;
+
+ public Resolver() {}
+
+ public Resolver(Object state) {
+ m_state = state;
+ }
+
+ //-------------------------------------------------------------
+
+ final public Iterator filter(Iterator src) {
+ return new Resolverator(src, this);
+ }
+
+ //-------------------------------------------------------------
+
+ protected abstract Object resolve(Object obj);
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolverator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolverator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolverator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,57 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/**
+ * Resolverator
+ *
+ * Initialized with a Resolver object, wraps a standard iterator and resolves each returned object
+ */
+
+public class Resolverator implements Iterator {
+
+ Iterator m_iter;
+ Resolver m_resolver;
+
+ public Resolverator(Iterator iter, Resolver resolver) {
+ m_iter = iter;
+ m_resolver = resolver;
+ }
+
+ public boolean hasNext() {
+ return m_iter.hasNext();
+ }
+
+ public Object next() {
+ return m_resolver.resolve(m_iter.next());
+ }
+
+ public void remove() {
+ m_iter.remove();
+ }
+}
\ No newline at end of file
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolverator.java
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:mime-type
+ text/plain
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/SingleValueIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/SingleValueIterator.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/SingleValueIterator.java 2010-09-27 16:28:41 UTC (rev 3637)
@@ -0,0 +1,61 @@
+/*
+Striterator - transformation and mapping patterns over java Iterators
+
+Copyright (C) SYSTAP, LLC 2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package cutthecrap.utils.striterators;
+
+import java.util.*;
+
+/***************************************************************************
+ * SingleValueIterator
+ *
+ * Only one value but need to return an iterator? This makes it easy.
+ */
+
+ public class SingleValueIterator implements Iterator {
+ Object m_value;
+ boolean m_hasNext = true;
+
+ public SingleValueIterator(Object value) {
+ m_value = value;
+ }
+ public boolean hasNext() {
+ return m_hasNext;
+ }
+ public Object next() {
+ if (!m_hasNext) {
+ m_value = null;
+ } else {
+ m_hasNext = false;
+ }
+
+ return m_value;
+ }
+ public void remove() {
+ m_hasNext = false;
+ m_value = null;
+ }
+}
+
+
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/util...
[truncated message content] |
|
From: <tho...@us...> - 2010-09-28 10:59:21
|
Revision: 3651
http://bigdata.svn.sourceforge.net/bigdata/?rev=3651&view=rev
Author: thompsonbry
Date: 2010-09-28 10:59:12 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Dropped the ctc_utils jar in favor of the newly contributed CTC source code.
Made a number of changes to reconciled the CTC striterators with the bigdata striterators and BOPs. More are too come.
See https://sourceforge.net/apps/trac/bigdata/ticket/178.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/.classpath
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadOnlyEntryIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/Advancer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/FilterConstructor.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/ITupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/LookaheadTupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/PrefixFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleRemover.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleTransformer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/view/FusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/AtomicRowFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/DistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/Striterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestRemoverator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/DoNotAddFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ExplicitSPOFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphHashSetFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InferredSPOFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NoAxiomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/test/cutthecrap/utils/striterators/TestFilterBase.java
Modified: branches/QUADS_QUERY_BRANCH/.classpath
===================================================================
--- branches/QUADS_QUERY_BRANCH/.classpath 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/.classpath 2010-09-28 10:59:12 UTC (rev 3651)
@@ -15,6 +15,8 @@
<classpathentry kind="src" path="contrib/src/problems"/>
<classpathentry kind="src" path="bigdata/src/samples"/>
<classpathentry kind="src" path="dsi-utils/src/test"/>
+ <classpathentry kind="src" path="ctc-striterators/src/java"/>
+ <classpathentry kind="src" path="ctc-striterators/src/test"/>
<classpathentry kind="lib" path="bigdata-jini/lib/apache/zookeeper-3.2.1.jar"/>
<classpathentry kind="lib" path="bigdata-sails/lib/commons-httpclient.jar"/>
<classpathentry kind="lib" path="bigdata-sails/lib/servlet-api.jar"/>
@@ -26,7 +28,6 @@
<classpathentry kind="src" path="lgpl-utils/src/test"/>
<classpathentry exported="true" kind="lib" path="bigdata/lib/icu/icu4j-3_6.jar"/>
<classpathentry exported="true" kind="lib" path="bigdata/lib/unimi/colt-1.2.0.jar"/>
- <classpathentry exported="true" kind="lib" path="bigdata/lib/ctc_utils-5-4-2005.jar"/>
<classpathentry exported="true" kind="lib" path="bigdata/lib/cweb-commons-1.1-b2-dev.jar"/>
<classpathentry exported="true" kind="lib" path="bigdata/lib/cweb-extser-0.1-b2-dev.jar"/>
<classpathentry exported="true" kind="lib" path="bigdata/lib/cweb-junit-ext-1.1-b3-dev.jar" sourcepath="/junit-ext/src"/>
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -31,6 +31,8 @@
import java.util.List;
import java.util.Map;
+import cutthecrap.utils.striterators.IPropertySet;
+
/**
* An operator, such as a constant, variable, join, sort, filter, etc.
* <p>
@@ -53,7 +55,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public interface BOp extends Cloneable, Serializable {
+public interface BOp extends Cloneable, Serializable, IPropertySet {
/**
* The #of arguments to the operation.
@@ -114,7 +116,7 @@
* @return The value of the annotation and <code>null</code> if the
* annotation is not bound.
*/
- <T> T getProperty(final String name);
+ //<T> T getProperty(final String name);
// /**
// * Return the value of the named annotation.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -305,11 +305,17 @@
}
- @SuppressWarnings("unchecked")
- public <T> T getProperty(final String name) {
+// @SuppressWarnings("unchecked")
+// public <T> T getProperty(final String name) {
+//
+// return (T) annotations.get(name);
+//
+// }
- return (T) annotations.get(name);
+ public Object getProperty(final String name) {
+ return annotations.get(name);
+
}
// public <T> T getRequiredProperty(final String name) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -30,29 +30,25 @@
import java.io.Serializable;
+import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.mdi.PartitionLocator;
-import com.bigdata.relation.IMutableRelation;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IElementFilter;
import com.bigdata.relation.rule.IRule;
import com.bigdata.relation.rule.ISolutionExpander;
-import com.bigdata.relation.rule.eval.ActionEnum;
import com.bigdata.relation.rule.eval.IEvaluationPlan;
-import com.bigdata.relation.rule.eval.ISolution;
import com.bigdata.relation.rule.eval.pipeline.JoinMasterTask;
-import com.bigdata.service.AbstractScaleOutFederation;
-import com.bigdata.service.DataService;
import com.bigdata.striterator.IKeyOrder;
/**
* An immutable constraint on the elements visited using an {@link IAccessPath}.
* The slots in the predicate corresponding to variables are named and those
- * names establish binding patterns access {@link IPredicate}s in the context
- * of a {@link IRule}. Access is provided to slots by ordinal index regardless
- * of whether or not they are named variables.
+ * names establish binding patterns access {@link IPredicate}s. Access is
+ * provided to slots by ordinal index regardless of whether or not they are
+ * named variables.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -189,18 +185,12 @@
/**
* Resource identifier (aka namespace) identifies the {@link IRelation}
* associated with this {@link IPredicate}.
- * <p>
- * This is more or less ignored when the {@link IRule} is executed as a
- * query.
- * <p>
- * When the {@link IRule} is executed as an {@link ActionEnum#Insert} or
- * {@link ActionEnum#Delete} then this identifies the target
- * {@link IMutableRelation} on which the computed {@link ISolution}s will be
- * written.
*
* @throws IllegalStateException
* if there is more than on element in the view.
*
+ * @see Annotations#RELATION_NAME
+ *
* @todo Rename as getRelationName()
*/
public String getOnlyRelationName();
@@ -227,29 +217,12 @@
/**
* The index partition identifier and <code>-1</code> if no partition
* identifier was specified.
- * <p>
- * Note: The ability to specify an index partition identifier for a
- * predicate is provided in support of scale-out JOIN strategies. The
- * {@link AccessPath} and the {@link JoinMasterTask} are both aware
- * of this property. The {@link JoinMasterTask} sets the partition
- * identifier in order to request an access path backed by the name of the
- * local index object on a {@link DataService} rather than the name of the
- * scale-out index.
- * <p>
- * The index partition can not be specified until a choice has been made
- * concerning which {@link IAccessPath} to use for a predicate without an
- * index partition constraint. The {@link IAccessPath} choice is therefore
- * made by the {@link IEvaluationPlan} using the scale-out index view and an
- * {@link AbstractScaleOutFederation#locatorScan(String, long, byte[], byte[], boolean)}
- * is used to identify the index partitions on which the {@link IAccessPath}
- * will read. The index partition is then set on a constrained
- * {@link IPredicate} for each target index partition and the JOINs are then
- * distributed to the {@link DataService}s on which those index partitions
- * reside.
*
* @return The index partition identifier -or- <code>-1</code> if the
* predicate is not locked to a specific index partition.
*
+ * @see Annotations#PARTITION_ID
+ *
* @see PartitionLocator
* @see AccessPath
* @see JoinMasterTask
@@ -268,9 +241,10 @@
* if the index partition identified is a negative integer.
* @throws IllegalStateException
* if the index partition identifier was already specified.
+ * @see Annotations#PARTITION_ID
*/
public IPredicate<E> setPartitionId(int partitionId);
-
+
/**
* <code>true</code> iff the predicate is optional when evaluated as the
* right-hand side of a join. An optional predicate will match once after
@@ -285,8 +259,10 @@
* More control over the behavior of optionals may be gained through the use
* of an {@link ISolutionExpander} pattern.
*
- * @return <code>true</code> iff this predicate is optional when
- * evaluating a JOIN.
+ * @return <code>true</code> iff this predicate is optional when evaluating
+ * a JOIN.
+ *
+ * @deprecated By {@link PipelineJoin.Annotations#OPTIONAL}
*/
public boolean isOptional();
@@ -296,6 +272,8 @@
*
* @return The {@link ISolutionExpander}.
*
+ * @see Annotations#EXPANDER
+ *
* @todo replace with {@link ISolutionExpander#getAccessPath(IAccessPath)},
* which is the only method declared by {@link ISolutionExpander}.
*/
@@ -304,6 +282,8 @@
/**
* An optional constraint on the visitable elements.
*
+ * @see Annotations#CONSTRAINT
+ *
* @todo rename as get(Element)Filter().
*/
public IElementFilter<E> getConstraint();
@@ -328,6 +308,8 @@
* The {@link IKeyOrder}.
*
* @return The new {@link IPredicate}.
+ *
+ * @see Annotations#KEY_ORDER
*/
public IPredicate<E> setKeyOrder(final IKeyOrder<E> keyOrder);
@@ -499,4 +481,5 @@
* @return The newly annotated {@link IPredicate}.
*/
public IPredicate<E> setBOpId(int bopId);
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-28 10:59:12 UTC (rev 3651)
@@ -2,16 +2,42 @@
- TestJiniFederatedQueryEngine.
- - Join=ANY, Predicate=RMI.
+ - Reconcile IElementFilter, FilterConstructor, and stackable
+ striterators for access paths and raw iterators. We should be
+ able to use any of the striterator patterns!
- - Unit tests of the default and named graph access path patterns.
+ There are a lot of partially duplicated classes which need to be
+ cleared up. This can be tested on the individual striterator /
+ filter and also on the local and remote access paths.
- - Cost model for the default graph and named graph access path
- patterns so we can choose the right one for each query.
+ Note: Some filters can be sent with a remote iterator request (for
+ example, the advancer), but most are applied after we convert from
+ tuples into elements.
- - Subqueries {Union,Steps,Star}. Implement subquery support. (We
- can't test Star until we have the mutation API in place.)
+ IFilterConstructor only works with ITupleIterator.
+ IElementFilter only filters. It can not resolve, transform, chunk,
+ etc. It is currently sent with the iterator to the B+Tree. See
+ AccessPath#iterator().
+
+ - Break the DistinctSPOIterator into a Distinct(Element)Filter and a
+ StripContextFilter.
+
+ - AbstractTuple#getObject() should cache the materialized object
+ since some patterns used by an AccessPath may request the object
+ multiple times. Of course, caching implies that we must clear the
+ reference when advancing the iterator. Check TupleFilter and
+ friends since there might need to be more than once class which
+ caches the object. Clear the reference when the iterator is
+ exhausted.
+
+ - Adapt the code to use the cost models and decision trees and write
+ Unit tests of the default and named graph access path patterns.
+
+ - Implement subquery support and subquery operators {Union, Steps,
+ Star}. (We can't test Star until we have the mutation API in
+ place.)
+
- PipelineType {Vectored,OneShot}.
A vectored operator processes its inputs in chunks, producing
@@ -28,6 +54,10 @@
ConcurrencyManager as appropriate (but how can we tell which is
appropriate!?!).
+ Note: AccessPath currently does not allow inserts (just
+ deletes). Hopefully it can be extended to allow INSERT and UPDATE
+ so we can use the same abstraction for mutation operators.
+
- Mutation {Create, Destroy}. This gets into resource management,
so defer for the moment but tackle in the context of RDFS closure
using STAR.
@@ -461,4 +491,57 @@
* <li></li>
* <li></li>
* </ul>
-
\ No newline at end of file
+
+
+Martyn,
+
+Rationalizing CTC striterators, bigdata striterators, and access paths:
+
+Dropped the ctc_utils jar and added the source files to the classpath.
+
+I've added an IPropertySet interface:
+
+Object getProperty(String name).
+
+There are currently several varieties of IFilter. Each one knows how to create its corresponding implementation class. This exactly parallels the bops, where a concrete bop such as MemorySortOp, knows how to create the corresponding runtime class, MemorySortOp.SortTask. I've added a filterOnce() method to provide the semantics of wrapping an iterator with just that IFilter. The existing filter() method now wraps the chain, starting with _this_ IFilter and then wrapping anything which was added to the chain. The chain is empty if nothing was added. I've modified the various IFilter implementations to conform with this.
+
+I've modified all of the implementation classes (Appenderator, etc.) to accept the new context argument regardless of whether they use it. I have also modified them to always have the reference to the IFilter object so the IPropertySet will be accessible.
+
+Modified Uniqueness filter to use a linked hash map.
+
+Modified Sorter uses Arrays.sort() rather than TreeSort.
+
+Some problems:
+
+- Should Appender simply pass the other iterator as m_state to the FilterBase?
+
+- Mapper uses non-Serializable "Method". We should provide defered reflection for the method.
+
+- Merger uses non-Serializable "Iterator".
+
+- Sorter and Resolver should use an annotation pattern so we can create instances from Prolog of the operator. I do not think we can do this when the operator is abstract. Maybe we can have two "Resolver" classes, two "Sorter" classes, etc. The simple one can be used for inline programming. The other one will extend BOpBase and implement IFilter and will be used for query.
+
+- CONTRACTOR should be able to break an iterator into many chunks, not just one. Maybe the API should return an Iterator from an Iterator in which the chunkiness is changed (from element to element[])?
+
+Here is what I have not done yet:
+
+Write unit tests to sanity check these changes.
+
+Write BOp versions of Append, Expand, etc. These will extend BOpBase.
+
+The following places directly invoke filterOnce(src,ctx). The first two should be modified to pass in the correct context object. That might require raising a context object into an IRangeIterator parameter.
+
+AbstractBTree#2933
+FusedView#1281
+TestRemoverator#86
+
+Replace (I)FilterConstructor with IFilter. FilterConstructor implemented clone. I am not sure that I can recall why. Presumably because a cloned would have a distinct filter chain. We can not easily override clone() on FilterBase because of all the derived versions. We could have a deep copy constructor pattern, which is what I did for the BOpBase class and then use that for clone.
+
+Replace the IPredicate "CONSTRAINT" with "REMOTE_FILTERS" (an IFilter which is evaluated at the BTree) and "LOCAL_FILTERS" (an IFilter which is evaluated against the range iterator once it is materialized on the local node). If the index is local, then the same concepts apply but we do not have to serialize the REMOTE_FILTERS. For the moment, the AccessPath automatically resolves ITuples to relation elements and adds the SameVariableConstraint if necessary. We might shift the burden of responsibility for this to the query planner in the future.
+
+Write unit tests at the IPredicate/AccessPath level to use stackable filters.
+
+Fixed the build.xml script to accomodate the class path changes.
+
+Thanks,
+Bryan
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -2930,7 +2930,7 @@
// remove all visited tuples.
return true;
}
- }.filter(src);
+ }.filterOnce(src, null/* context */);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadOnlyEntryIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadOnlyEntryIterator.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadOnlyEntryIterator.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -34,20 +34,20 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-public class ReadOnlyEntryIterator implements ITupleIterator {
+public class ReadOnlyEntryIterator<E> implements ITupleIterator<E> {
- private final ITupleIterator src;
+ private final ITupleIterator<E> src;
/**
*
*/
- public ReadOnlyEntryIterator(ITupleIterator src) {
+ public ReadOnlyEntryIterator(final ITupleIterator<E> src) {
this.src = src;
}
- public ITuple next() {
+ public ITuple<E> next() {
return src.next();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/Advancer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/Advancer.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/Advancer.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -12,6 +12,8 @@
import com.bigdata.btree.KeyOutOfRangeException;
import com.bigdata.io.ByteArrayBuffer;
+import cutthecrap.utils.striterators.FilterBase;
+
/**
* Used to write logic that advances an {@link ITupleCursor} to another key
* after it visits some element. For example, the "distinct term scan" for
@@ -26,12 +28,15 @@
* the key for the last visited tuple and then issue remove(key)
* against the underlying index.
*/
-abstract public class Advancer<E> implements ITupleFilter<E> {
+abstract public class Advancer<E> extends FilterBase implements ITupleFilter<E> {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
protected transient static final Logger log = Logger.getLogger(Advancer.class);
- protected transient static final boolean INFO = log.isInfoEnabled();
-
/**
* Set by {@link #filter(ITupleCursor)}.
*/
@@ -48,11 +53,13 @@
* @return
*/
@SuppressWarnings("unchecked")
- final public ITupleIterator<E> filter(final Iterator src) {
+ @Override
+ final public ITupleIterator<E> filterOnce(final Iterator src,
+ final Object context) {
this.src = (ITupleCursor<E>) src;
- return new Advancer.Advancerator<E>(this.src, this);
+ return new Advancer.Advancerator<E>(this.src, context, this);
}
@@ -75,6 +82,7 @@
private static class Advancerator<E> implements ITupleIterator<E> {
final private ITupleCursor<E> src;
+ final protected Object context;
final private Advancer<E> filter;
@@ -95,10 +103,13 @@
*/
final private ByteArrayBuffer kbuf = new ByteArrayBuffer();
- public Advancerator(final ITupleCursor<E> src, final Advancer<E> filter) {
+ public Advancerator(final ITupleCursor<E> src, Object context,
+ final Advancer<E> filter) {
this.src = src;
+ this.context = context;
+
this.filter = filter;
}
@@ -115,7 +126,7 @@
final ITuple<E> tuple = src.next();
- if (INFO) {
+ if (log.isInfoEnabled()) {
log.info("next: " + tuple);
@@ -156,7 +167,7 @@
final byte[] key = this.kbuf.toByteArray();
- if (INFO) {
+ if (log.isInfoEnabled()) {
log.info("key=" + BytesUtil.toString(key));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/FilterConstructor.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/FilterConstructor.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/FilterConstructor.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -90,9 +90,9 @@
*/
private static class WrappedTupleIterator<E> implements ITupleIterator<E> {
- final private Iterator src;
+ final private Iterator<E> src;
- public WrappedTupleIterator(final Iterator src) {
+ public WrappedTupleIterator(final Iterator<E> src) {
if (src == null)
throw new IllegalArgumentException();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/ITupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/ITupleFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/ITupleFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -53,7 +53,9 @@
/**
* Strengthened return type.
+ * <p>
+ * {@inheritDoc}
*/
- public ITupleIterator<E> filter(Iterator src);
+ public ITupleIterator<E> filterOnce(Iterator src, Object context);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/LookaheadTupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/LookaheadTupleFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/LookaheadTupleFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -37,6 +37,8 @@
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.ITupleSerializer;
+import cutthecrap.utils.striterators.FilterBase;
+
/**
* Lookahead filter for an {@link ITuple}. You can push back a single
* {@link ITuple} onto the filter.
@@ -46,7 +48,7 @@
*
* @todo unit tests.
*/
-public class LookaheadTupleFilter<E> implements ITupleFilter<E> {
+public class LookaheadTupleFilter<E> extends FilterBase implements ITupleFilter<E> {
private static final long serialVersionUID = -5551926378159692251L;
@@ -77,14 +79,15 @@
}
@SuppressWarnings("unchecked")
- public ILookaheadTupleIterator<E> filter(Iterator src) {
+ @Override
+ public ILookaheadTupleIterator<E> filterOnce(Iterator src, Object context) {
if(src instanceof ITupleCursor) {
- return new LookaheadTupleCursor((ITupleCursor)src);
+ return new LookaheadTupleCursor((ITupleCursor)src, context);
}
- return new LookaheadTupleIterator((ITupleIterator)src);
+ return new LookaheadTupleIterator((ITupleIterator)src, context);
}
@@ -98,12 +101,13 @@
static private class LookaheadTupleCursor<E> implements ILookaheadTupleIterator<E> {
private final ITupleCursor<E> src;
+ private final Object context;
private boolean pushbackAllowed = false;
- public LookaheadTupleCursor(ITupleCursor<E> src) {
+ public LookaheadTupleCursor(ITupleCursor<E> src, Object context) {
this.src = src;
-
+ this.context = context;
}
public void pushback() {
@@ -151,6 +155,7 @@
static private class LookaheadTupleIterator<E> implements ILookaheadTupleIterator<E> {
private final ITupleIterator<E> src;
+ private final Object context;
/**
* <code>true</code> iff pushback is allowed. when <code>true</code>,
@@ -171,9 +176,10 @@
/**
* @param src
*/
- public LookaheadTupleIterator(ITupleIterator<E> src) {
-
+ public LookaheadTupleIterator(ITupleIterator<E> src, Object context) {
+
this.src = src;
+ this.context = context;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/PrefixFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/PrefixFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/PrefixFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -13,6 +13,8 @@
import com.bigdata.btree.keys.StrengthEnum;
import com.bigdata.btree.keys.SuccessorUtil;
+import cutthecrap.utils.striterators.FilterBase;
+
/**
* <p>
* Filter visits all {@link ITuple}s whose keys begin with any of the specified
@@ -103,16 +105,12 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
- *
- * @see TestPrefixFilter
*/
-public class PrefixFilter<E> implements ITupleFilter<E> {
+public class PrefixFilter<E> extends FilterBase implements ITupleFilter<E> {
protected transient static final Logger log = Logger
.getLogger(PrefixFilter.class);
- protected transient static final boolean INFO = log.isInfoEnabled();
-
private static final long serialVersionUID = 1828228416774862469L;
/**
@@ -163,9 +161,10 @@
}
@SuppressWarnings("unchecked")
- public ITupleIterator<E> filter(Iterator src) {
+ @Override
+ public ITupleIterator<E> filterOnce(Iterator src, Object context) {
- return new PrefixFilterator<E>((ITupleCursor<E>) src, this);
+ return new PrefixFilterator<E>((ITupleCursor<E>) src, context, this);
}
@@ -177,8 +176,10 @@
* successor of the last key prefix (formed by adding one bit, not by
* appending a <code>nul</code> byte).
*/
- protected final ITupleCursor<E> src;
+ private final ITupleCursor<E> src;
+ private final Object context;
+
private final PrefixFilter<E> filter;
/**
@@ -204,7 +205,8 @@
* @param filter
* The filter to be applied.
*/
- public PrefixFilterator(ITupleCursor<E> src, PrefixFilter<E> filter) {
+ public PrefixFilterator(final ITupleCursor<E> src,
+ final Object context, final PrefixFilter<E> filter) {
if (src == null)
throw new IllegalArgumentException();
@@ -214,6 +216,8 @@
this.src = src;
+ this.context = context;
+
this.filter = filter;
this.index = 0;
@@ -238,7 +242,7 @@
if (BytesUtil.compareBytes(key, toKey) >= 0) {
- if (INFO)
+ if (log.isInfoEnabled())
log.info("Scanned beyond prefix: toKey="
+ BytesUtil.toString(toKey) + ", tuple="
+ tuple);
@@ -261,7 +265,7 @@
}
- if(INFO)
+ if(log.isInfoEnabled())
log.info("No more prefixes.");
return false;
@@ -277,7 +281,7 @@
// no more tuples (at least in this index partition).
- if(INFO)
+ if(log.isInfoEnabled())
log.info("No more tuples.");
return false;
@@ -303,7 +307,7 @@
current = src.seek(prefix);
// current = src.tuple();
- if (INFO) {
+ if (log.isInfoEnabled()) {
log.info("index=" + index + ", prefix="
+ BytesUtil.toString(prefix) + ", current=" + current);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -17,6 +17,7 @@
import com.bigdata.btree.Tuple;
import cutthecrap.utils.striterators.Filter;
+import cutthecrap.utils.striterators.FilterBase;
/**
* <p>
@@ -47,14 +48,19 @@
* The type of the elements visited by the iterator (tuples of some
* sort).
*/
-abstract public class TupleFilter<E> implements ITupleFilter<E> {
+abstract public class TupleFilter<E> extends FilterBase implements ITupleFilter<E> {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
protected static transient final Logger log = Logger.getLogger(TupleFilter.class);
- /**
- * Optional state specified by the ctor.
- */
- protected Object state;
+// /**
+// * Optional state specified by the ctor.
+// */
+// protected Object state;
public TupleFilter() {
@@ -62,16 +68,18 @@
}
- public TupleFilter(Object state) {
+ public TupleFilter(final Object state) {
- this.state = state;
+// this.state = state;
+ super(state);
}
@SuppressWarnings("unchecked")
- public ITupleIterator<E> filter(Iterator src) {
+ @Override
+ public ITupleIterator<E> filterOnce(final Iterator src,Object context) {
- return new TupleFilter.Filterator((ITupleIterator) src);
+ return new TupleFilter.Filterator((ITupleIterator) src, context, this);
}
@@ -85,7 +93,7 @@
* @version $Id$
* @param <E>
*/
- protected class Filterator implements ITupleIterator<E> {
+ static protected class Filterator<E> implements ITupleIterator<E> {
/**
* The source iterator.
@@ -93,9 +101,16 @@
protected final ITupleIterator<E> src;
/**
+ * The context.
+ */
+ protected final Object context;
+
+ protected final TupleFilter<E> filter;
+
+ /**
* The next value to be returned by {@link #next()}.
*/
- private ITuple nextValue = null;
+ private ITuple<E> nextValue = null;
/**
* The {@link ITuple} instance that will actually be returned to the
@@ -107,11 +122,14 @@
* of the data must be made in order to avoid side-effects from the
* one-step lookahead used by the filter.
*/
- final private AbstractTuple returnValue;
+ final private AbstractTuple<E> returnValue;
- public Filterator(final ITupleIterator<E> src) {
+ public Filterator(final ITupleIterator<E> src, final Object context,
+ final TupleFilter<E> filter) {
this.src = src;
+ this.context = context;
+ this.filter = filter;
/*
* One step lookahead.
@@ -161,7 +179,7 @@
}
- @SuppressWarnings("unchecked")
+// @SuppressWarnings("unchecked")
public ITuple<E> next() {
if (!hasNext())
@@ -182,7 +200,7 @@
visit(returnValue);
// return the private instance containing a copy of the data.
- return (ITuple<E>)returnValue;
+ return (ITuple<E>) returnValue;
}
@@ -193,7 +211,7 @@
* @param tuple
* The tuple that will be visited.
*/
- protected void visit(ITuple<E> tuple) {
+ protected void visit(final ITuple<E> tuple) {
// NOP
@@ -220,7 +238,7 @@
*/
public void remove() {
- if(src instanceof ITupleCursor) {
+ if(src instanceof ITupleCursor<?>) {
/*
* The ITupleCursor supports traversal with concurrent
@@ -262,13 +280,13 @@
*
* @return The next object to be visited.
*/
- protected ITuple getNext() {
+ protected ITuple<E> getNext() {
while (src.hasNext()) {
final ITuple<E> next = src.next();
- if (!isValid(next)) {
+ if (!filter.isValid(next)) {
if(log.isInfoEnabled()) {
@@ -286,7 +304,7 @@
}
- return (ITuple) next;
+ return next;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleRemover.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleRemover.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleRemover.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -8,6 +8,8 @@
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
+import cutthecrap.utils.striterators.FilterBase;
+
/**
* Visits all elements visited by the source iterator and removes those
* matching the filter.
@@ -23,13 +25,20 @@
* @version $Id$
* @param <E>
*/
-abstract public class TupleRemover<E> implements ITupleFilter<E> {
+abstract public class TupleRemover<E> extends FilterBase implements ITupleFilter<E> {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
@SuppressWarnings("unchecked")
- public ITupleIterator<E> filter(Iterator src) {
+ @Override
+ public ITupleIterator<E> filterOnce(final Iterator src, Object context) {
- return new TupleRemover.Removerator<E>((ITupleIterator<E>) src, this);
-
+ return new TupleRemover.Removerator<E>((ITupleIterator<E>) src,
+ context, this);
+
}
/**
@@ -55,6 +64,11 @@
* The source iterator.
*/
private final ITupleIterator<E> src;
+
+ /**
+ * The context object (ignored).
+ */
+ private final Object ctx;
/**
* The filter to be applied.
@@ -65,10 +79,13 @@
* @param src
* @param filter
*/
- public Removerator(ITupleIterator<E> src, TupleRemover<E> filter) {
-
+ public Removerator(final ITupleIterator<E> src, final Object context,
+ final TupleRemover<E> filter) {
+
this.src = src;
+ this.ctx = context;
+
this.filter = filter;
}
@@ -113,4 +130,4 @@
}
-}
\ No newline at end of file
+}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleTransformer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleTransformer.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleTransformer.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -34,6 +34,8 @@
import com.bigdata.btree.ITupleSerializer;
import com.bigdata.btree.filter.LookaheadTupleFilter.ILookaheadTupleIterator;
+import cutthecrap.utils.striterators.FilterBase;
+
/**
* Abstract base class for an {@link ITupleFilter} that transforms the data type
* of the keys and/or values.
@@ -52,7 +54,8 @@
* one out) and patterns with a M:1 (many in, one out) and 1:M (1 in, many
* out)?
*/
-abstract public class TupleTransformer<E,F> implements ITupleFilter<F> {
+abstract public class TupleTransformer<E, F> extends FilterBase implements
+ ITupleFilter<F> {
/** The serialization provider for the transformed tuples. */
final protected ITupleSerializer<? extends Object/*key*/,F/*val*/> tupleSer;
@@ -76,13 +79,15 @@
* The source iterator.
*/
@SuppressWarnings("unchecked")
- public ITupleIterator<F> filter(Iterator src) {
+ @Override
+ public ITupleIterator<F> filterOnce(Iterator src, final Object context) {
// layer in one-step lookahead.
- src = new LookaheadTupleFilter().filter((ITupleIterator<E>)src);
+ src = new LookaheadTupleFilter().filterOnce((ITupleIterator<E>) src,
+ context);
// the transformer.
- return newTransformer((ILookaheadTupleIterator<E>)src);
+ return newTransformer((ILookaheadTupleIterator<E>) src, context);
}
@@ -91,7 +96,8 @@
* from the lookahead source whose tuples are of the source type and visits
* the transformed tuples.
*/
- abstract protected ITupleIterator<F> newTransformer(ILookaheadTupleIterator<E> src);
+ abstract protected ITupleIterator<F> newTransformer(
+ final ILookaheadTupleIterator<E> src, final Object context);
// /**
// * Return <code>true</code> iff another tuple of the transformed type can
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -32,9 +32,10 @@
}
@SuppressWarnings("unchecked")
- public ITupleIterator<E> filter(Iterator src) {
+ @Override
+ public ITupleIterator<E> filterOnce(Iterator src, Object context) {
- return new Updaterator((ITupleCursor<E>) src);
+ return new Updaterator((ITupleCursor<E>) src, context, this);
}
@@ -52,17 +53,19 @@
*/
abstract protected void update(IIndex ndx, ITuple<E> tuple);
- protected class Updaterator extends TupleFilter.Filterator {
+ protected class Updaterator extends TupleFilter.Filterator<E> {
- public Updaterator(ITupleIterator<E> src) {
+ public Updaterator(final ITupleIterator<E> src, final Object context,
+ final TupleFilter<E> filter) {
- super(src);
+ super(src, context, filter);
}
- protected void visit(ITuple tuple) {
+ @Override
+ protected void visit(final ITuple<E> tuple) {
- final IIndex ndx = ((ITupleCursor) src).getIndex();
+ final IIndex ndx = ((ITupleCursor<E>) src).getIndex();
update(ndx, tuple);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/view/FusedView.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/view/FusedView.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/view/FusedView.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -1278,7 +1278,7 @@
// remove all visited tuples.
return true;
}
- }.filter(src);
+ }.filterOnce(src, null/* context */);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -77,8 +77,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
* @param R
- * The generic type of the [R]elation elements of the
- * {@link IRelation}.
+ * The generic type of the elements of the {@link IRelation}.
*
* @todo This needs to be more generalized so that you can use a index that is
* best without being optimal by specifying a low-level filter to be
@@ -234,50 +233,7 @@
}
-// protected void setFromKey(final byte[] fromKey) {
-//
-// assertNotInitialized();
-//
-// if (pmd != null) {
-//
-// /*
-// * The predicate is constrained to an index partition, so constrain
-// * the fromKey so that it lies within that index partition.
-// */
-//
-// this.fromKey = AbstractKeyRangeIndexProcedure.constrainFromKey(fromKey,
-// pmd);
-//
-// } else {
-//
-// this.fromKey = fromKey;
-//
-// }
-//
-// }
-//
-// protected void setToKey(final byte[] toKey) {
-//
-// assertNotInitialized();
-//
-// if (pmd != null) {
-//
-// /*
-// * The predicate is constrained to an index partition, so constrain
-// * the toKey so that it lies within that index partition.
-// */
-//
-// this.toKey = AbstractKeyRangeIndexProcedure.constrainToKey(toKey, pmd);
-//
-// } else {
-//
-// this.toKey = toKey;
-//
-// }
-//
-// }
-
- public IKeyOrder<R> getKeyOrder() {
+ final public IKeyOrder<R> getKeyOrder() {
return keyOrder;
@@ -393,7 +349,6 @@
this.chunkCapacity = chunkCapacity;
-// this.fullyBufferedReadThreshold = 100000;
this.fullyBufferedReadThreshold = fullyBufferedReadThreshold;
this.historicalRead = TimestampUtility.isReadOnly(timestamp);
@@ -416,6 +371,9 @@
} else {
+ /*
+ * Stack filters.
+ */
filter = new FilterConstructor<R>();
if (constraint != null) {
@@ -1074,10 +1032,6 @@
static private class ChunkConsumerTask<R> implements Callable<Void> {
static protected final Logger log = Logger.getLogger(ChunkConsumerTask.class);
-
-// static protected final boolean log.isInfoEnabled() = log.isInfoEnabled();
-//
-// static protected final boolean log.isDebugEnabled() = log.isDebugEnabled();
private final AccessPath<R> accessPath;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/AtomicRowFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/AtomicRowFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/AtomicRowFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -72,9 +72,9 @@
protected static transient final Logger log = Logger
.getLogger(AtomicRowFilter.class);
- protected static transient final boolean INFO = log.isInfoEnabled();
-
- protected static transient final boolean DEBUG = log.isDebugEnabled();
+// protected static transient final boolean INFO = log.isInfoEnabled();
+//
+// protected static transient final boolean DEBUG = log.isDebugEnabled();
private final Schema schema;
@@ -119,10 +119,10 @@
@Override
protected ITupleIterator<TPS> newTransformer(
- ILookaheadTupleIterator<TPV> src) {
+ ILookaheadTupleIterator<TPV> src, final Object context) {
- return new Transformerator<TPV, TPS>(src);
-
+ return new Transformerator<TPV, TPS>(src, context);
+
}
private class Transformerator<E extends TPV/* src */, F extends TPS/* out */>
@@ -147,6 +147,8 @@
private final ILookaheadTupleIterator<E> src;
+ private final Object context;
+
/**
* Builds iterator that reads the source tuples and visits the transformed
* tuples.
@@ -154,13 +156,15 @@
* @param src
* Visits the source tuples.
*/
- public Transformerator(final ILookaheadTupleIterator<E> src) {
+ public Transformerator(final ILookaheadTupleIterator<E> src, final Object context) {
- if (src == null)
- throw new IllegalArgumentException();
+ if (src == null)
+ throw new IllegalArgumentException();
- this.src = src;
+ this.src = src;
+ this.context = context;
+
}
/**
@@ -336,7 +340,7 @@
// Skip property names that have been filtered out.
- if (DEBUG) {
+ if (log.isDebugEnabled()) {
log.debug("Skipping property: name=" + col + " (filtered)");
@@ -355,7 +359,7 @@
if (columnValueTimestamp < fromTime) {
- if (DEBUG) {
+ if (log.isDebugEnabled()) {
log.debug("Ignoring earlier revision: col=" + col
+ ", fromTime=" + fromTime + ", timestamp="
@@ -369,7 +373,7 @@
if (toTime != CURRENT_ROW && columnValueTimestamp >= toTime) {
- if (DEBUG) {
+ if (log.isDebugEnabled()) {
log.debug("Ignoring later revision: col=" + col
+ ", toTime=" + toTime + ", timestamp="
@@ -398,7 +402,7 @@
tps.set(col, columnValueTimestamp, v);
- if (INFO)
+ if (log.isInfoEnabled())
log.info("Accept: name=" + col + ", timestamp="
+ columnValueTimestamp + ", value=" + v + ", key="
+ BytesUtil.toString(key));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -46,12 +46,17 @@
* will differ from {@link E} iff the filter is transforming the
* element type.
*
- * @todo This class is redundent with the {@link ChunkedConvertingIterator}
+ * @todo This class is redundant with the {@link ChunkedConvertingIterator}
*/
abstract public class ChunkedFilter<I extends IChunkedIterator<E>, E, F>
implements IFilter<I, E, F> {
/**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
* Caller's object.
*/
protected Object state;
@@ -72,7 +77,7 @@
}
- public ChunkedFilter(Object state) {
+ public ChunkedFilter(final Object state) {
this(state, null/* keyOrder */);
@@ -85,7 +90,7 @@
* @param keyOrder
* The natural sort order for the filtered iterator (optional).
*/
- public ChunkedFilter(Object state, IKeyOrder<F> keyOrder) {
+ public ChunkedFilter(final Object state, final IKeyOrder<F> keyOrder) {
this.state = state;
@@ -93,7 +98,7 @@
}
- public IChunkedOrderedIterator<F> filter(I src) {
+ public IChunkedOrderedIterator<F> filter(final I src) {
return new ChunkedFilteringIterator<I, E, F>(src, this);
@@ -103,7 +108,7 @@
* @todo the need for this variant is worrysome - it is required if you do
* NOT specify the generic types and then try to use this class.
*/
- public IChunkedOrderedIterator<F> filter(Iterator src) {
+ public IChunkedOrderedIterator<F> filter(final Iterator src) {
return new ChunkedFilteringIterator<I, E, F>((I) src, this);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/DistinctFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/DistinctFilter.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/DistinctFilter.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -32,6 +32,7 @@
import java.util.Arrays;
import java.util.UUID;
+import com.bigdata.bop.solutions.DistinctBindingSetOp;
import com.bigdata.btree.BTree;
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.IndexMetadata;
@@ -69,6 +70,8 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
+ *
+ * @deprecated by {@link DistinctBindingSetOp}
*/
abstract public class DistinctFilter<E> implements IChunkConverter<E, E> {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/Striterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/Striterator.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/Striterator.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -67,7 +67,7 @@
/**
* @param src The source iterator.
*/
- public Striterator(I src) {
+ public Striterator(final I src) {
if (src == null)
throw new IllegalArgumentException();
@@ -79,8 +79,8 @@
/**
* Wraps the enumeration as an iterator.
*
- * <strong>Ctor must be overriden for derived classes that specialize the
- * type of the iterator.</strong>
+ * <strong>The constructor must be overridden for derived classes that
+ * specialize the type of the iterator.</strong>
*
* @param srcEnum
* The source enumeration.
@@ -107,7 +107,7 @@
}
@SuppressWarnings("unchecked")
- public IStriterator<I, E> addFilter(IFilter<I, ?, E> filter) {
+ public IStriterator<I, E> addFilter(final IFilter<I, ?, E> filter) {
src = (I)filter.filter((I) src);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -1156,6 +1156,10 @@
/**
* FIXME Test the ability run a query reading on an access path using a
* DISTINCT filter (this is just stacking a striterator on the access path).
+ *
+ * @see DistinctSPOIterator, which should stack as a striterator. It might
+ * be best to break it up into something to strip off the context
+ * position and a distincterator.
*/
public void test_query_join1_distinctAccessPath() {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -69,6 +69,7 @@
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.ITx;
import com.bigdata.journal.Journal;
+import com.bigdata.rdf.spo.DistinctSPOIterator;
import com.bigdata.relation.accesspath.IAsynchronousIterator;
import com.bigdata.relation.accesspath.ThickAsynchronousIterator;
import com.bigdata.service.AbstractEmbeddedFederationTestCase;
@@ -1028,6 +1029,10 @@
* RMI access path, stacking a distinct filter on the access path, and
* marking the join evaluation context as ANY since we do not push the
* binding sets to the shards when using RMI access paths.
+ *
+ * @see DistinctSPOIterator, which should stack as a striterator. It might
+ * be best to break it up into something to strip off the context
+ * position and a distincterator.
*/
public void test_query_join1_distinctAccessPath() {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestRemoverator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestRemoverator.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestRemoverator.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -83,7 +83,7 @@
// all visited tuples will be removed.
return true;
}
- }.filter(newCursor(btree));
+ }.filterOnce(newCursor(btree), null/* context */);
assertTrue(itr.hasNext());
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java 2010-09-28 10:42:00 UTC (rev 3650)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java 2010-09-28 10:59:12 UTC (rev 3651)
@@ -34,7 +34,6 @@
import org.apache.log4j.Logger;
import com.bigdata.rdf.internal.IV;
-import com.bigdata.rdf.internal.TermId;
import com.bigdata.rdf.lexicon.ITermIVFilter;
import com.bigdata.rdf.model.StatementEnum;
import com.bigdata.rdf.rules.InferenceEngine;
@@ -44,7 +43,6 @@
import com.bigdata.rdf.spo.SPOKeyOrder;
import com.bigdata.rdf.spo.SPORelation;
import com.bigdata.rdf.store.AbstractTripleStore;
-import com.bigdata.rdf.store.IRawTripleStore;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.striterator.ChunkedArrayIterator;
import com.bigdata.striterator.IChunkedIterator;
@@ -53,7 +51,7 @@
import com.bigdata.striterator.IKeyOrder;
import cutthecrap.utils.striterators.Filter;
-import cutthecrap.utils.striterators.IFilter;
+import cutthecrap.utils.striterators.FilterBase;
import cutthecrap.utils.striterators.Resolver;
import cutthecrap.utils.striterators.Striterator;
@@ -814,7 +812,7 @@
* @version $Id$
* @param <E>
*/
- public static class PushbackFilter<E> implements IFilter {
+ public static class PushbackFilter<E> extends FilterBase {
/**
*
@@ -822,7 +820,7 @@
private static final long serialVersionUID = -8010263934867149205L;
@SuppressWarnings("unchecked")
- public PushbackIterator<E> filter(Iterator src) {
+ public PushbackIterator<E> filterOnce(Iterator src, Object context) {
return new PushbackIterator<E>((Iterator<E>) src);
@@ -853,7 +851,7 @@
*/
private E buffer;
- public PushbackIterator(Iterator<E> src) {
+ public PushbackIterator(final Iterator<E> src) {
if (src == null)
throw new IllegalArgumentException();
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
========================...
[truncated message content] |
|
From: <tho...@us...> - 2010-09-28 11:01:27
|
Revision: 3652
http://bigdata.svn.sourceforge.net/bigdata/?rev=3652&view=rev
Author: thompsonbry
Date: 2010-09-28 11:01:16 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Removed the ctc_utils.jar.
Removed references to that jar from build.xml.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/build.xml
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/lib/ctc_utils-5-4-2005.jar
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/lib/ctc_utils-5-4-2005.jar
===================================================================
(Binary files differ)
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2010-09-28 10:59:12 UTC (rev 3651)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2010-09-28 11:01:16 UTC (rev 3652)
@@ -155,7 +155,6 @@
<bndwrap jars="${build.dir}/lib/unimi/colt-1.2.0.jar" output="${build.dir}/bundles/colt-1.2.0.jar" definitions="${basedir}/osgi/" />
<bndwrap jars="${build.dir}/lib/unimi/fastutil-5.1.5.jar" output="${build.dir}/bundles/fastutil-5.1.5.jar" definitions="${basedir}/osgi/" />
- <bndwrap jars="${build.dir}/lib/ctc_utils-5-4-2005.jar" output="${build.dir}/bundles/ctc_utils-5-4-2005.jar" definitions="${basedir}/osgi/" />
<bndwrap jars="${build.dir}/lib/cweb-commons-1.1-b2-dev.jar" output="${build.dir}/bundles/cweb-commons-1.1.2.jar" definitions="${basedir}/osgi/" />
<bndwrap jars="${build.dir}/lib/cweb-extser-0.1-b2-dev.jar" output="${build.dir}/bundles/cweb-extser-1.1.2.jar" definitions="${basedir}/osgi/" />
<bndwrap jars="${build.dir}/lib/dsi-utils-1.0.6-020610.jar" output="${build.dir}/bundles/dsi-utils-1.0.6-020610.jar" definitions="${basedir}/osgi/" />
@@ -901,7 +900,6 @@
<!-- Utility libraries -->
<copy file="${bigdata.lib}/unimi/colt-1.2.0.jar" tofile="${dist.lib}/colt.jar" />
-<copy file="${bigdata.lib}/ctc_utils-5-4-2005.jar" tofile="${dist.lib}/ctc_utils.jar" />
<copy file="${bigdata.lib}/cweb-commons-1.1-b2-dev.jar" tofile="${dist.lib}/cweb-commons.jar" />
<copy file="${bigdata.lib}/cweb-extser-0.1-b2-dev.jar" tofile="${dist.lib}/cweb-extser.jar" />
<copy file="${bigdata.lib}/high-scale-lib-v1.1.2.jar" tofile="${dist.lib}/highscalelib.jar" />
@@ -1391,7 +1389,7 @@
<mkdir dir="${bigdata-test.lib}" />
<property name="bigdata-test.jar" location="${bigdata-test.lib}/bigdata-test.jar" />
-<property name="javac.test.classpath" value="${classes.dir}${path.separator}${junit.jar}${path.separator}${cweb-junit-ext.jar}${path.separator}${sesame-sparql-test.jar}${path.separator}${sesame-store-test.jar}${path.separator}${dist.lib}/classserver.jar${path.separator}${dist.lib}/ctc_utils.jar${path.separator}${dist.lib}/cweb-commons.jar${path.separator}${dist.lib}/cweb-extser.jar${path.separator}${dist.lib}/highscalelib.jar${path.separator}${dist.lib}/dsiutils.jar${path.separator}${dist.lib}/lgplutils.jar${path.separator}${dist.lib}/fastutil.jar${path.separator}${dist.lib}/icu4j.jar${path.separator}${dist.lib}/iris.jar${path.separator}${dist.lib}/jgrapht.jar${path.separator}${dist.lib}/log4j.jar${path.separator}${dist.lib}/openrdf-sesame.jar${path.separator}${dist.lib}/slf4j.jar${path.separator}${dist.lib}/jsk-lib.jar${path.separator}${dist.lib}/jsk-platform.jar${path.separator}${dist.lib}/nxparser.jar${path.separator}${dist.lib}/zookeeper.jar" />
+<property name="javac.test.classpath" value="${classes.dir}${path.separator}${junit.jar}${path.separator}${cweb-junit-ext.jar}${path.separator}${sesame-sparql-test.jar}${path.separator}${sesame-store-test.jar}${path.separator}${dist.lib}/classserver.jar${path.separator}${dist.lib}/cweb-commons.jar${path.separator}${dist.lib}/cweb-extser.jar${path.separator}${dist.lib}/highscalelib.jar${path.separator}${dist.lib}/dsiutils.jar${path.separator}${dist.lib}/lgplutils.jar${path.separator}${dist.lib}/fastutil.jar${path.separator}${dist.lib}/icu4j.jar${path.separator}${dist.lib}/iris.jar${path.separator}${dist.lib}/jgrapht.jar${path.separator}${dist.lib}/log4j.jar${path.separator}${dist.lib}/openrdf-sesame.jar${path.separator}${dist.lib}/slf4j.jar${path.separator}${dist.lib}/jsk-lib.jar${path.separator}${dist.lib}/jsk-platform.jar${path.separator}${dist.lib}/nxparser.jar${path.separator}${dist.lib}/zookeeper.jar" />
<echo>javac
</echo>
@@ -1605,7 +1603,6 @@
<pathelement location="${dist.lib}/colt.jar" />
<pathelement location="${dist.lib}/cweb-commons.jar" />
<pathelement location="${dist.lib}/cweb-extser.jar" />
- <pathelement location="${dist.lib}/ctc_utils.jar" />
<pathelement location="${dist.lib}/highscalelib.jar" />
<pathelement location="${dist.lib}/dsiutils.jar" />
<pathelement location="${dist.lib}/lgplutils.jar" />
@@ -1627,7 +1624,7 @@
<pathelement location="${dist.lib}/zookeeper.jar" />
</path>
-<property name="run.class.path" value="${junit.jar}${path.separator}${bigdata-test.jar}${path.separator}${cweb-junit-ext.jar}${path.separator}${sesame-sparql-test.jar}${path.separator}${sesame-store-test.jar}${path.separator}${dist.lib}/bigdata.jar${path.separator}${dist.lib}/colt.jar${path.separator}${dist.lib}/cweb-commons.jar${path.separator}${dist.lib}/cweb-extser.jar${path.separator}${dist.lib}/ctc_utils.jar${path.separator}${dist.lib}/highscalelib.jar${path.separator}${dist.lib}/dsiutils.jar${path.separator}${dist.lib}/lgplutils.jar${path.separator}${dist.lib}/fastutil.jar${path.separator}${dist.lib}/icu4j.jar${path.separator}${dist.lib}/iris.jar${path.separator}${dist.lib}/jgrapht.jar${path.separator}${dist.lib}/jsk-lib.jar${path.separator}${dist.lib}/jsk-platform.jar${path.separator}${dist.lib}/log4j.jar${path.separator}${dist.lib}/lucene-analyzer.jar${path.separator}${dist.lib}/lucene-core.jar${path.separator}${dist.lib}/openrdf-sesame.jar${path.separator}${dist.lib}/slf4j.jar${path.separator}${dist.lib}/slf4j-log4j.jar${path.separator}${dist.lib}/nxparser.jar${path.separator}${dist.lib}/zookeeper.jar" />
+<property name="run.class.path" value="${junit.jar}${path.separator}${bigdata-test.jar}${path.separator}${cweb-junit-ext.jar}${path.separator}${sesame-sparql-test.jar}${path.separator}${sesame-store-test.jar}${path.separator}${dist.lib}/bigdata.jar${path.separator}${dist.lib}/colt.jar${path.separator}${dist.lib}/cweb-commons.jar${path.separator}${dist.lib}/cweb-extser.jar${path.separator}${dist.lib}/highscalelib.jar${path.separator}${dist.lib}/dsiutils.jar${path.separator}${dist.lib}/lgplutils.jar${path.separator}${dist.lib}/fastutil.jar${path.separator}${dist.lib}/icu4j.jar${path.separator}${dist.lib}/iris.jar${path.separator}${dist.lib}/jgrapht.jar${path.separator}${dist.lib}/jsk-lib.jar${path.separator}${dist.lib}/jsk-platform.jar${path.separator}${dist.lib}/log4j.jar${path.separator}${dist.lib}/lucene-analyzer.jar${path.separator}${dist.lib}/lucene-core.jar${path.separator}${dist.lib}/openrdf-sesame.jar${path.separator}${dist.lib}/slf4j.jar${path.separator}${dist.lib}/slf4j-log4j.jar${path.separator}${dist.lib}/nxparser.jar${path.separator}${dist.lib}/zookeeper.jar" />
<echo> classpath: ${run.class.path}
</echo>
@@ -1719,7 +1716,6 @@
<sysproperty key="classserver.jar" value="${dist.lib}/classserver.jar" />
<sysproperty key="colt.jar" value="${dist.lib}/colt.jar" />
- <sysproperty key="ctc_utils.jar" value="${dist.lib}/ctc_utils.jar" />
<sysproperty key="cweb-commons.jar" value="${dist.lib}/cweb-commons.jar" />
<sysproperty key="cweb-extser.jar" value="${dist.lib}/cweb-extser.jar" />
<sysproperty key="highscalelib.jar" value="${dist.lib}/highscalelib.jar" />
@@ -1820,7 +1816,6 @@
<pathelement location="${dist.lib}/colt.jar" />
<pathelement location="${dist.lib}/cweb-commons.jar" />
<pathelement location="${dist.lib}/cweb-extser.jar" />
- <pathelement location="${dist.lib}/ctc_utils.jar" />
<pathelement location="${dist.lib}/highscalelib.jar" />
<pathelement location="${dist.lib}/dsiutils.jar" />
<pathelement location="${dist.lib}/lgplutils.jar" />
@@ -1839,7 +1834,7 @@
<pathelement location="${dist.lib}/nxparser.jar" />
<pathelement location="${dist.lib}/zookeeper.jar" />
</path>
-<property name="run.class.path" value="${junit.jar}${path.separator}${bigdata-test.jar}${path.separator}${cweb-junit-ext.jar}${path.separator}${sesame-sparql-test.jar}${path.separator}${sesame-store-test.jar}${path.separator}${dist.lib}/bigdata.jar${path.separator}${dist.lib}/colt.jar${path.separator}${dist.lib}/cweb-commons.jar${path.separator}${dist.lib}/cweb-extser.jar${path.separator}${dist.lib}/ctc_utils.jar${path.separator}${dist.lib}/highscalelib.jar${path.separator}${dist.lib}/dsiutils.jar${path.separator}${dist.lib}/lgplutils.jar${path.separator}${dist.lib}/fastutil.jar${path.separator}${dist.lib}/icu4j.jar${path.separator}${dist.lib}/iris.jar${path.separator}${dist.lib}/jgrapht.jar${path.separator}${dist.lib}/jsk-lib.jar${path.separator}${dist.lib}/jsk-platform.jar${path.separator}${dist.lib}/log4j.jar${path.separator}${dist.lib}/lucene-analyzer.jar${path.separator}${dist.lib}/lucene-core.jar${path.separator}${dist.lib}/openrdf-sesame.jar${path.separator}${dist.lib}/slf4j.jar${path.separator}${dist.lib}/slf4j-log4j.jar${path.separator}${dist.lib}/nxparser.jar${path.separator}${dist.lib}/zookeeper.jar" />
+<property name="run.class.path" value="${junit.jar}${path.separator}${bigdata-test.jar}${path.separator}${cweb-junit-ext.jar}${path.separator}${sesame-sparql-test.jar}${path.separator}${sesame-store-test.jar}${path.separator}${dist.lib}/bigdata.jar${path.separator}${dist.lib}/colt.jar${path.separator}${dist.lib}/cweb-commons.jar${path.separator}${dist.lib}/cweb-extser.jar${path.separator}${dist.lib}/highscalelib.jar${path.separator}${dist.lib}/dsiutils.jar${path.separator}${dist.lib}/lgplutils.jar${path.separator}${dist.lib}/fastutil.jar${path.separator}${dist.lib}/icu4j.jar${path.separator}${dist.lib}/iris.jar${path.separator}${dist.lib}/jgrapht.jar${path.separator}${dist.lib}/jsk-lib.jar${path.separator}${dist.lib}/jsk-platform.jar${path.separator}${dist.lib}/log4j.jar${path.separator}${dist.lib}/lucene-analyzer.jar${path.separator}${dist.lib}/lucene-core.jar${path.separator}${dist.lib}/openrdf-sesame.jar${path.separator}${dist.lib}/slf4j.jar${path.separator}${dist.lib}/slf4j-log4j.jar${path.separator}${dist.lib}/nxparser.jar${path.separator}${dist.lib}/zookeeper.jar" />
<!-- Generate the LUBM dataset.
<mkdir dir="${data}"/>
<java classname="edu.lehigh.swat.bench.uba.Generator" dir="${data}" fork="yes">
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-28 11:10:56
|
Revision: 3653
http://bigdata.svn.sourceforge.net/bigdata/?rev=3653&view=rev
Author: thompsonbry
Date: 2010-09-28 11:10:50 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Updated build.xml to notice ctc-striterators everywhere. Added the LICENSE.txt file.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/build.xml
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/ctc-striterators/LEGAL/
branches/QUADS_QUERY_BRANCH/ctc-striterators/LICENSE.txt
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2010-09-28 11:01:16 UTC (rev 3652)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2010-09-28 11:10:50 UTC (rev 3653)
@@ -76,6 +76,7 @@
<src path="${bigdata.dir}/bigdata-jini/src/java" />
<src path="${bigdata.dir}/bigdata-rdf/src/java" />
<src path="${bigdata.dir}/bigdata-sails/src/java" />
+ <src path="${bigdata.dir}/ctc-striterators/src/java" />
<!-- Do not include the unit tests @todo conditionally include?
<src path="${bigdata.dir}/bigdata/src/test"/>
<src path="${bigdata.dir}/bigdata-jini/src/test"/>
@@ -106,6 +107,10 @@
<fileset dir="${bigdata.dir}/bigdata-sails/src/resources/sesame-server">
<include name="META-INF/**" />
</fileset>
+ <fileset dir="${bigdata.dir}/ctc-striterators/src/java">
+ <exclude name="**/*.java" />
+ <exclude name="**/package.html" />
+ </fileset>
</copy>
</target>
@@ -150,6 +155,7 @@
<fileset dir="bigdata-jini/src/java" />
<fileset dir="bigdata-rdf/src/java" />
<fileset dir="bigdata-sails/src/java" />
+ <fileset dir="ctc-striterators/src/java" />
</jar>
<bnd output="${build.dir}/bundles/com.bigata-${osgi.version}.jar" classpath="${build.dir}/classes" eclipse="false" failok="false" exceptions="true" files="${basedir}/osgi/bigdata.bnd" />
@@ -183,6 +189,7 @@
<packageset dir="${bigdata.dir}/bigdata-rdf/src/java" />
<packageset dir="${bigdata.dir}/bigdata-sails/src/java" />
<packageset dir="${bigdata.dir}/bigdata-sails/src/samples" />
+ <packageset dir="${bigdata.dir}/ctc-striterators/src/java" />
<doctitle>
<![CDATA[<h1>bigdata®</h1>]]></doctitle>
<bottom>
@@ -234,6 +241,9 @@
<copy toDir="${build.dir}/bigdata-sails/src">
<fileset dir="${bigdata.dir}/bigdata-sails/src" />
</copy>
+<copy toDir="${build.dir}/ctc-striterators/src">
+ <fileset dir="${bigdata.dir}/ctc-striterators/src" />
+</copy>
<!-- common files for the root of the archive. -->
<copy tofile="${build.dir}/build.properties" file="build.properties" />
<copy tofile="${build.dir}/build.xml" file="build.xml" />
@@ -265,6 +275,7 @@
<exclude name="bigdata-jini/src/**" />
<exclude name="bigdata-rdf/src/**" />
<exclude name="bigdata-sails/src/**" />
+ <exclude name="ctc-striterators/src/**" />
<exclude name="classes/**" />
<include name="${version}.jar" />
<include name="lib/**" />
@@ -288,6 +299,7 @@
<include name="bigdata-jini/src/**" />
<include name="bigdata-rdf/src/**" />
<include name="bigdata-sails/src/**" />
+ <include name="ctc-striterators/src/**" />
<include name="lgpl-utils/src/**" />
<include name="src/**" />
<exclude name="classes/**" />
@@ -448,6 +460,7 @@
<include name="bigdata-jini/LEGAL/*" />
<include name="bigdata-rdf/LEGAL/*" />
<include name="bigdata-sails/LEGAL/*" />
+ <include name="ctc-striterators/LEGAL/*" />
</fileset>
</copy>
<!-- install JARs. -->
@@ -1058,6 +1071,9 @@
<copy toDir="${build.dir}/bigdata-sails/src">
<fileset dir="${bigdata.dir}/bigdata-sails/src" />
</copy>
+<copy toDir="${build.dir}/ctc-striterators/src">
+ <fileset dir="${bigdata.dir}/ctc-striterators/src" />
+</copy>
<copy toDir="${build.dir}/lgpl-utils/src">
<fileset dir="${bigdata.dir}/lgpl-utils/src" />
</copy>
@@ -1126,6 +1142,7 @@
<include name="bigdata-jini/src/**" />
<include name="bigdata-rdf/src/**" />
<include name="bigdata-sails/src/**" />
+ <include name="ctc-striterators/src/**" />
<include name="lgpl-utils/src/**" />
<include name="bigdata/lib/**" />
<include name="bigdata-jini/lib/**" />
@@ -1191,6 +1208,7 @@
<include name="bigdata-jini/LEGAL/*" />
<include name="bigdata-rdf/LEGAL/*" />
<include name="bigdata-sails/LEGAL/*" />
+ <include name="ctc-striterators/LEGAL/*" />
</fileset>
</copy>
@@ -1420,6 +1438,7 @@
<src path="${bigdata.dir}/bigdata-jini/src/test" />
<src path="${bigdata.dir}/bigdata-rdf/src/test" />
<src path="${bigdata.dir}/bigdata-sails/src/test" />
+ <src path="${bigdata.dir}/ctc-striterators/src/test" />
<!--
<src path="${bigdata.dir}/bigdata-gom/src/test"/>
-->
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/LICENSE.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/LICENSE.txt (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/LICENSE.txt 2010-09-28 11:10:50 UTC (rev 3653)
@@ -0,0 +1,354 @@
+The GNU General Public License (GPL)
+Version 2, June 1991
+
+Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+
+59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+Everyone is permitted to copy and distribute verbatim copies
+
+of this license document, but changing it is not allowed.
+
+
+Preamble
+
+
+The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+
+0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+a) You must cause the modified files to carry prominent notices
+stating that you changed the files and the date of any change.
+
+b) You must cause any work that you distribute or publish, that in
+whole or in part contains or is derived from the Program or any
+part thereof, to be licensed as a whole at no charge to all third
+parties under the terms of this License.
+
+c) If the modified program normally reads commands interactively
+when run, you must cause it, when started running for such
+interactive use in the most ordinary way, to print or display an
+announcement including an appropriate copyright notice and a
+notice that there is no warranty (or else, saying that you provide
+a warranty) and that users may redistribute the program under
+these conditions, and telling the user how to view a copy of this
+License. (Exception: if the Program itself is interactive but
+does not normally print such an announcement, your work based on
+the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+a) Accompany it with the complete corresponding machine-readable
+source code, which must be distributed under the terms of Sections
+1 and 2 above on a medium customarily used for software interchange; or,
+
+b) Accompany it with a written offer, valid for at least three
+years, to give any third party, for a charge no more than your
+cost of physically performing source distribution, a complete
+machine-readable copy of the corresponding source code, to be
+distributed under the terms of Sections 1 and 2 above on a medium
+customarily used for software interchange; or,
+
+c) Accompany it with the information you received as to the offer
+to distribute corresponding source code. (This alternative is
+allowed only for noncommercial distribution and only if you
+received the program in object code or executable form with such
+an offer, in accord with Subsection b above.)
+
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+END OF TERMS AND CONDITIONS
+
+
+How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+One line to give the program's name and a brief idea of what it does.
+
+Copyright (C) <year> <name of author>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+Gnomovision version 69, Copyright (C) year name of author
+Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+This is free software, and you are welcome to redistribute it
+under certain conditions; type `show c' for details.
+
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+`Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+signature of Ty Coon, 1 April 1989
+
+Ty Coon, President of Vice
+
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Library General
+Public License instead of this License.
+
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/LICENSE.txt
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-28 12:29:21
|
Revision: 3657
http://bigdata.svn.sourceforge.net/bigdata/?rev=3657&view=rev
Author: thompsonbry
Date: 2010-09-28 12:29:15 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Imposing my local settings on build.properties and bigdataStandalone.config for use in testing quads query on a single node.
The build.properties changes turn off snapshots and specify a specific machine name, etc.
The bigdataStandalone.config changes specifies two data services and no client services.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/build.properties
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
Modified: branches/QUADS_QUERY_BRANCH/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.properties 2010-09-28 12:18:37 UTC (rev 3656)
+++ branches/QUADS_QUERY_BRANCH/build.properties 2010-09-28 12:29:15 UTC (rev 3657)
@@ -41,7 +41,7 @@
# Set true to do a snapshot build. This changes the value of ${version} to
# include the date.
-snapshot=true
+snapshot=false
# Javadoc build may be disabled using this property. The javadoc target will
# not be executed unless this property is defined (its value does not matter).
@@ -84,7 +84,7 @@
# files on this path on each host. Therefore, if you are not installing as root
# this will need to be a file within the user's home directory or some directory
# which exists on each host and is writable by that user.
-LAS=/var/bigdata/${FED}
+LAS=/data/bigdata/${FED}
#LAS=~/bigdata/${FED}
# The location of the installed JDK that will be used to build / run bigdata.
@@ -94,7 +94,7 @@
# dumps with Sun jdk1.6.0_07 and FC6 on a 32-bit platform.
#
#JAVA_HOME=C:\\Program Files\\Java\\jdk1.6.0_10
-JAVA_HOME=/usr/java/jdk1.6.0_07
+JAVA_HOME=/usr/java/jdk1.6.0_17
#JAVA_HOME=/usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0.x86_64
#JAVA_HOME=/usr/java/jrockit-R27.3.0-jdk1.6.0_01
@@ -110,8 +110,8 @@
LOAD_BALANCER_PORT=8080
# Where the sysstat utilities are found (performance counter reporting for un*x).
-#SYSSTAT_HOME=/usr/local/bin
-SYSSTAT_HOME=/usr/bin
+SYSSTAT_HOME=/usr/local/bin
+#SYSSTAT_HOME=/usr/bin
# Specifies the value of com.sun.jini.jeri.tcp.useNIO. When true, use NIO for RMI.
USE_NIO=true
@@ -145,8 +145,9 @@
# This can be 'users' when trying to share across the hosts when running bigdata
# using a normal user login. It can be 'wheel' when trying to share across hosts
# using a root login.
-install.group=users
+#install.group=users
#install.group=wheel
+install.group=${user.name}
# The file permissions mask used for files that must be read/write for all hosts.
#
@@ -176,7 +177,7 @@
LOCK_FILE=${LAS}/lockFile
# The main bigdata configuration file.
-bigdata.config=${install.config.dir}/bigdataCluster.config
+bigdata.config=${install.config.dir}/bigdataStandalone.config
# The main jini configuration file.
jini.config=${install.config.dir}/jini/startAll.config
@@ -199,7 +200,7 @@
# Note: java.util.logging messages DO NOT get written onto this logger -- only
# log4j messages.
#
-LOG4J_SOCKET_LOGGER_HOST = XXX
+LOG4J_SOCKET_LOGGER_HOST = bigdata01.systap.com
LOG4J_SOCKET_LOGGER_PORT = 4445
# The socket logger uses a DailyRollingFileAppender by default and this
@@ -369,9 +370,9 @@
#
# The top-level directory above the various data files to be used.
-perf.data.dir=/usr/bigdata/data
+perf.data.dir=/nas/data
# The directory in which the performance tests will be run. This directory needs to be
# on a volume with a lot of room. The directory may be destroyed (by the test harness)
# after the performance tests have run their course.
-perf.run.dir=/usr/bigdata/runs
+perf.run.dir=/nas/perfruns
Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
===================================================================
--- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-09-28 12:18:37 UTC (rev 3656)
+++ branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-09-28 12:29:15 UTC (rev 3657)
@@ -168,7 +168,7 @@
};
// The target #of client servers.
- static private clientServiceCount = 1;
+ static private clientServiceCount = 0;
static private maxClientServicePerHost = 1;
// data servers
@@ -177,10 +177,10 @@
};
// The target #of data services.
- static private dataServiceCount = 1;
+ static private dataServiceCount = 2;
// Maximum #of data services per host.
- static private maxDataServicesPerHost = 1;
+ static private maxDataServicesPerHost = 2;
// @todo also specify k (replicationCount)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-28 12:43:46
|
Revision: 3658
http://bigdata.svn.sourceforge.net/bigdata/?rev=3658&view=rev
Author: thompsonbry
Date: 2010-09-28 12:43:35 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Merge trunk to branch [r3423:3657] for QUADS_QUERY_BRANCH.
Resolved conflicts in
build.xml (ctc_utils edit was reapplied in one section of the file).
AbstractEmbeddedFederationTestCase (accepted change were some fields were changed from public to private).
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/cache/RingBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/BufferMode.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/concurrent/StressTestNonBlockingLockManagerWithTxDag.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestMove.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/fastload.properties
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoins.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
branches/QUADS_QUERY_BRANCH/build.xml
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster.config
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataCluster16.config
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/cache/RingBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/cache/RingBuffer.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/cache/RingBuffer.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -154,7 +154,7 @@
public boolean add(final T ref) throws IllegalStateException {
if (ref == null)
- throw new IllegalArgumentException();
+ throw new NullPointerException();
beforeOffer( ref );
@@ -178,7 +178,7 @@
public boolean offer(final T ref) {
if (ref == null)
- throw new IllegalArgumentException();
+ throw new NullPointerException();
beforeOffer( ref );
@@ -387,12 +387,12 @@
if (index < 0 || index >= size)
throw new IllegalArgumentException();
- if (index + 1 == size) {
-
- // remove the LRU position.
- return remove();
-
- }
+// if (index + 1 == size) {
+//
+// // remove the LRU position.
+// return remove();
+//
+// }
/*
* Otherwise we are removing some non-LRU element.
@@ -409,7 +409,7 @@
for (;;) {
- int nexti = (i + 1) % capacity; // update index.
+ final int nexti = (i + 1) % capacity; // update index.
if (nexti != head) {
@@ -491,10 +491,9 @@
*/
final public boolean scanHead(final int nscan, final T ref) {
- assert nscan > 0;
-// if (nscan <= 0)
-// throw new IllegalArgumentException();
-//
+ if (nscan <= 0)
+ throw new IllegalArgumentException();
+
if (ref == null)
throw new IllegalArgumentException();
@@ -581,6 +580,9 @@
public boolean contains(final Object ref) {
+ if (ref == null)
+ throw new NullPointerException();
+
// MRU to LRU scan.
for (int n = 0, i = tail; n < size; n++) {
@@ -601,7 +603,8 @@
throw new NullPointerException();
if (c == this)
- throw new IllegalArgumentException();
+ return true;
+// throw new IllegalArgumentException();
for( Object e : c ) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/BufferMode.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/BufferMode.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/journal/BufferMode.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -59,15 +59,16 @@
Transient(false/* stable */, true/* fullyBuffered */),
/**
+ * <strong>This mode is not being actively developed and should not be used
+ * outside of unit tests.</strong>
* <p>
- * A direct buffer is allocated for the file image. Writes are applied
- * to the buffer. The buffer tracks dirty slots regardless of the
- * transaction that wrote them and periodically writes dirty slots
- * through to disk. On commit, any dirty index or allocation nodes are
- * written onto the buffer and all dirty slots on the buffer. Dirty
- * slots in the buffer are then synchronously written to disk, the
- * appropriate root block is updated, and the file is (optionally)
- * flushed to disk.
+ * A direct buffer is allocated for the file image. Writes are applied to
+ * the buffer. The buffer tracks dirty slots regardless of the transaction
+ * that wrote them and periodically writes dirty slots through to disk. On
+ * commit, any dirty index or allocation nodes are written onto the buffer
+ * and all dirty slots on the buffer. Dirty slots in the buffer are then
+ * synchronously written to disk, the appropriate root block is updated, and
+ * the file is (optionally) flushed to disk.
* </p>
* <p>
* This option offers wires an image of the journal file into memory and
@@ -79,6 +80,9 @@
Direct(true/* stable */, true/* fullyBuffered */),
/**
+ * <strong>This mode is not being actively developed and should not be used
+ * outside of unit tests. Memory mapped IO has the fatal weakness under Java
+ * that you can not reliably close or extend the backing file.</strong>
* <p>
* A memory-mapped buffer is allocated for the file image. Writes are
* applied to the buffer. Reads read from the buffer. On commit, the map is
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/mdi/PartitionLocator.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -185,8 +185,17 @@
}
- // Note: used by assertEquals in the test cases.
- public boolean equals(Object o) {
+ /*
+ * @todo There are some unit tests which depend on this implementation of
+ * equals. However, since the partition locator Id for a given scale out
+ * index SHOULD be immutable, running code can rely on partitionId ==
+ * o.partitionId. Therefore the unit tests should be modified to extract an
+ * "assertSamePartitionLocator" method and rely on that. We could then
+ * simplify this method to just test the partitionId. That would reduce the
+ * effort when maintaining hash tables based on the PartitionLocator since
+ * we would not be comparing the keys, UUIDs, etc.
+ */
+ public boolean equals(final Object o) {
if (this == o)
return true;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/resources/AsynchronousOverflowTask.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -632,7 +632,7 @@
if (vmd.mergePriority > 0 || forceCompactingMerges) {
- if(forceCompactingMerges && vmd.getAction().equals(OverflowActionEnum.Copy)) {
+ if(forceCompactingMerges && OverflowActionEnum.Copy.equals(vmd.getAction())) {
vmd.clearCopyAction();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/util/config/NicUtil.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -26,28 +26,20 @@
package com.bigdata.util.config;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.Inet4Address;
+import java.net.InetAddress;
import java.net.InterfaceAddress;
-import java.net.MalformedURLException;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
+import java.util.Collections;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Enumeration;
-import java.util.Collections;
-import java.util.logging.LogRecord;
import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import net.jini.config.Configuration;
-import net.jini.config.ConfigurationException;
-import com.sun.jini.config.Config;
-import com.sun.jini.logging.Levels;
-
/**
* Utility class that provides a set of static convenience methods
* related to processing information about the current node's Network
@@ -400,34 +392,34 @@
return macAddr;
}
- /**
- * Three-argument version of <code>getInetAddress</code> that retrieves
- * the desired interface name from the given <code>Configuration</code>
- * parameter.
- */
- public static InetAddress getInetAddress(Configuration config,
- String componentName,
- String nicNameEntry)
- {
- String nicName = "NoNetworkInterfaceName";
- try {
- nicName = (String)Config.getNonNullEntry(config,
- componentName,
- nicNameEntry,
- String.class,
- "eth0");
- } catch(ConfigurationException e) {
- jiniConfigLogger.log(WARNING, e
- +" - [componentName="+componentName
- +", nicNameEntry="+nicNameEntry+"]");
- utilLogger.log(Level.WARN, e
- +" - [componentName="+componentName
- +", nicNameEntry="+nicNameEntry+"]");
- e.printStackTrace();
- return null;
- }
- return ( getInetAddress(nicName, 0, null, false) );
- }
+// /**
+// * Three-argument version of <code>getInetAddress</code> that retrieves
+// * the desired interface name from the given <code>Configuration</code>
+// * parameter.
+// */
+// public static InetAddress getInetAddress(Configuration config,
+// String componentName,
+// String nicNameEntry)
+// {
+// String nicName = "NoNetworkInterfaceName";
+// try {
+// nicName = (String)Config.getNonNullEntry(config,
+// componentName,
+// nicNameEntry,
+// String.class,
+// "eth0");
+// } catch(ConfigurationException e) {
+// jiniConfigLogger.log(WARNING, e
+// +" - [componentName="+componentName
+// +", nicNameEntry="+nicNameEntry+"]");
+// utilLogger.log(Level.WARN, e
+// +" - [componentName="+componentName
+// +", nicNameEntry="+nicNameEntry+"]");
+// e.printStackTrace();
+// return null;
+// }
+// return ( getInetAddress(nicName, 0, null, false) );
+// }
// What follows are a number of versions of the getIpAddress method
// provided for convenience.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/cache/TestRingBuffer.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -28,9 +28,7 @@
package com.bigdata.cache;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -65,7 +63,7 @@
public void test_ctor() {
try {
- new RingBuffer(0);
+ new RingBuffer<String>(0);
fail("Expecting: " + IllegalArgumentException.class);
} catch (IllegalArgumentException ex) {
if (log.isInfoEnabled())
@@ -73,14 +71,14 @@
}
try {
- new RingBuffer(-1);
+ new RingBuffer<String>(-1);
fail("Expecting: " + IllegalArgumentException.class);
} catch (IllegalArgumentException ex) {
if (log.isInfoEnabled())
log.info("Ignoring excepted exception: " + ex);
}
- final RingBuffer b = new RingBuffer(1);
+ final RingBuffer<String> b = new RingBuffer<String>(1);
assertEquals("capacity", 1, b.capacity());
assertEquals("size", 0, b.size());
@@ -304,8 +302,6 @@
* remove(0) : [ _, _, _ ] : head=0; tail=0; size=0, returns [c] (empty, head==tail)
* </pre>
*
- * @todo must also test when remove not at the tail!
- *
* When removing the tail, head := (head-1) % capacity.
*/
public void test_removeNth() {
@@ -313,7 +309,7 @@
final String a = "a";
final String b = "b";
final String c = "c";
- final String d = "d";
+// final String d = "d";
final RingBuffer<String> buffer = new RingBuffer<String>(3);
@@ -429,8 +425,8 @@
try {
buffer.add(null);
- fail("Expecting: " + IllegalArgumentException.class);
- } catch (IllegalArgumentException ex) {
+ fail("Expecting: " + NullPointerException.class);
+ } catch (NullPointerException ex) {
if (log.isInfoEnabled())
log.info("Ignoring expected exception: " + ex);
}
@@ -442,8 +438,8 @@
try {
buffer.offer(null);
- fail("Expecting: " + IllegalArgumentException.class);
- } catch (IllegalArgumentException ex) {
+ fail("Expecting: " + NullPointerException.class);
+ } catch (NullPointerException ex) {
if (log.isInfoEnabled())
log.info("Ignoring expected exception: " + ex);
}
@@ -619,9 +615,9 @@
public void test_toArray1_nonempty() {
Object [] intArr = new Object[] {
- new Integer(1),
- new Integer(2),
- new Integer(3)
+ Integer.valueOf(1),
+ Integer.valueOf(2),
+ Integer.valueOf(3)
};
final RingBuffer<Object> buffer = new RingBuffer<Object>(intArr.length);
buffer.addAll(Arrays.asList(intArr));
@@ -631,9 +627,9 @@
public void test_toArray1_nonempty_oversized() {
Object [] intArr = new Object[] {
- new Integer(1),
- new Integer(2),
- new Integer(3)
+ Integer.valueOf(1),
+ Integer.valueOf(2),
+ Integer.valueOf(3)
};
final RingBuffer<Object> buffer = new RingBuffer<Object>(intArr.length);
buffer.addAll(Arrays.asList(intArr));
@@ -685,7 +681,7 @@
// see https://sourceforge.net/apps/trac/bigdata/ticket/101
public void test_remove_get_order() {
- String[] expected = new String[] {
+ final String[] expected = new String[] {
"a", "b", "c", "d"
};
final RingBuffer<String> b = new RingBuffer<String>(expected.length);
@@ -698,8 +694,8 @@
//Remove entries in MRU to LRU order -- differs from javadoc order
for (int i=(expected.length-1); i >= 0; i--) {
- String getString = b.get(i);
- String removeString = b.remove(i);
+ final String getString = b.get(i);
+ final String removeString = b.remove(i);
assertSame(getString, removeString);
}
assertTrue(b.isEmpty());
@@ -973,13 +969,10 @@
assertTrue(b.contains("c"));
}
- //TODO - check for exception on contains(null) once implemented
-
-
- public void test_contains_all_null() {
- final RingBuffer<String> b = new RingBuffer<String>(1);
+ public void test_contains_null() {
+ final RingBuffer<String> b = new RingBuffer<String>(1);
try {
- b.containsAll(null);
+ b.contains(null);
fail("Expecting: " + NullPointerException.class);
} catch (NullPointerException ex) {
if (log.isInfoEnabled())
@@ -987,16 +980,29 @@
}
}
- public void test_contains_all_this() {
+ public void test_contains_all_null() {
final RingBuffer<String> b = new RingBuffer<String>(1);
try {
- b.containsAll(b);
- fail("Expecting: " + IllegalArgumentException.class);
- } catch (IllegalArgumentException ex) {
+ b.containsAll(null);
+ fail("Expecting: " + NullPointerException.class);
+ } catch (NullPointerException ex) {
if (log.isInfoEnabled())
log.info("Ignoring excepted exception: " + ex);
}
}
+
+ public void test_contains_all_this() {
+ final RingBuffer<String> b = new RingBuffer<String>(1);
+ // Note: This is a tautology.
+ assertTrue(b.containsAll(b));
+// try {
+// b.containsAll(b);
+// fail("Expecting: " + IllegalArgumentException.class);
+// } catch (IllegalArgumentException ex) {
+// if (log.isInfoEnabled())
+// log.info("Ignoring excepted exception: " + ex);
+// }
+ }
public void test_contains_all_empty() {
final RingBuffer<String> b = new RingBuffer<String>(1);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/concurrent/StressTestNonBlockingLockManagerWithTxDag.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/concurrent/StressTestNonBlockingLockManagerWithTxDag.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/concurrent/StressTestNonBlockingLockManagerWithTxDag.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -234,7 +234,7 @@
final int nhorriddeath = Integer.parseInt(result.get("nhorriddeath"));
- // all tasks were either successfull or a died a horrid death.
+ // all tasks were either successful or a died a horrid death.
assertEquals(ntasks, nsuccess + nhorriddeath);
/*
@@ -243,9 +243,14 @@
* scheduled to die is random.
*/
final double actualErrorRate = nhorriddeath / (double) ntasks;
-
+
+ /*
+ * Note: I've increased the upper bound on the allowed error rate a bit
+ * since the CI builds were occasionally failing this with an actual
+ * error rate which was quite reasonable, e.g., .16.
+ */
if ((actualErrorRate < expectedErrorRate - .05)
- || (actualErrorRate > expectedErrorRate + .05)) {
+ || (actualErrorRate > expectedErrorRate + .1)) {
fail("error rate: expected=" + expectedErrorRate + ", actual="
+ actualErrorRate);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestAll.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestAll.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestAll.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -93,7 +93,25 @@
suite.addTest( TestTransientJournal.suite() );
- suite.addTest( TestDirectJournal.suite() );
+ /*
+ * Commented out since this mode is not used and there is an occasional
+ * test failure in:
+ *
+ * com.bigdata.journal.TestConcurrentJournal.test_concurrentReadersAreOk
+ *
+ * This error is stochastic and appears to be restricted to
+ * BufferMode#Direct. This is a journal mode based by a fixed capacity
+ * native ByteBuffer serving as a write through cache to the disk. Since
+ * the buffer can not be extended, that journal mode is not being
+ * excercised by anything. If you like, I can deprecate the Direct
+ * BufferMode and turn disable its test suite. (There is also a "Mapped"
+ * BufferMode whose tests we are not running due to problems with Java
+ * releasing native heap ByteBuffers and closing memory mapped files.
+ * Its use is strongly discouraged in the javadoc, but it has not been
+ * excised from the code since it might be appropriate for some
+ * applications.)
+ */
+// suite.addTest( TestDirectJournal.suite() );
/*
* Note: The mapped journal is somewhat problematic and its tests are
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestTransactionService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestTransactionService.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/journal/TestTransactionService.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -40,6 +40,7 @@
import com.bigdata.service.AbstractTransactionService;
import com.bigdata.service.CommitTimeIndex;
import com.bigdata.service.TxServiceRunState;
+import com.bigdata.util.MillisecondTimestampFactory;
/**
* Unit tests of the {@link AbstractTransactionService} using a mock client.
@@ -259,6 +260,24 @@
}
+ /**
+ * FIXME This currently waits until at least two milliseconds have
+ * elapsed. This is a workaround for
+ * {@link TestTransactionService#test_newTx_readOnly()} until <a href=
+ * "https://sourceforge.net/apps/trac/bigdata/ticket/145" >ISSUE#145
+ * </a> is resolved. This override of {@link #nextTimestamp()} should
+ * be removed once that issue is fixed.
+ */
+ @Override
+ public long nextTimestamp() {
+
+ // skip at least one millisecond.
+ MillisecondTimestampFactory.nextMillis();
+
+ return MillisecondTimestampFactory.nextMillis();
+
+ }
+
}
/**
@@ -596,17 +615,25 @@
* GT the lastCommitTime since that could allow data not yet committed to
* become visible during the transaction (breaking isolation).
* <p>
- * A commitTime is identified by looking up the callers timestamp in a log of
- * the historical commit times and returning the first historical commit
+ * A commitTime is identified by looking up the callers timestamp in a log
+ * of the historical commit times and returning the first historical commit
* time LTE the callers timestamp.
* <p>
* The transaction start time is then chosen from the half-open interval
* <i>commitTime</i> (inclusive lower bound) : <i>nextCommitTime</i>
* (exclusive upper bound).
*
- * @throws IOException
+ * @throws IOException
*
- * @todo This test fails occasionally. I have not figured out why yet. BBT
+ * @todo This test fails occasionally. This occurs if the timestamps
+ * assigned by the {@link MockTransactionService} are only 1 unit
+ * apart. When that happens, there are not enough distinct values
+ * available to allow 2 concurrent read-only transactions. See <a
+ * href=
+ * "https://sourceforge.net/apps/trac/bigdata/ticket/145">ISSUE#145
+ * </a>. Also see {@link MockTransactionService#nextTimestamp()}
+ * which has been overridden to guarantee that there are at least
+ * two distinct values such that this test will pass.
*/
public void test_newTx_readOnly() throws IOException {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/AbstractEmbeddedFederationTestCase.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -78,11 +78,11 @@
super(arg0);
}
- public IBigdataClient<?> client;
- public IBigdataFederation<?> fed;
- public IMetadataService metadataService;
- public IDataService dataService0;
- public IDataService dataService1;
+ protected IBigdataClient<?> client;
+ protected IBigdataFederation<?> fed;
+ protected IMetadataService metadataService;
+ protected IDataService dataService0;
+ protected IDataService dataService1;
public Properties getProperties() {
@@ -356,9 +356,10 @@
* FIXME You can change this constant if you are debugging so that
* the test will not terminate too soon, but change it back so that
* the test will terminate quickly when run automatically. The value
- * should be [2000] ms.
+ * should be only a few seconds. 2000 ms is sometimes to little, so
+ * I have raised this value to 5000 ms.
*/
- if (elapsed > 2000) {
+ if (elapsed > 5000) {
fail("No overflow after " + elapsed + "ms?");
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestMove.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestMove.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestMove.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -360,7 +360,7 @@
* Set flag to force overflow on group commit.
*/
dataService0
- .forceOverflow(false/* immediate */, false/* compactingMerge */);
+ .forceOverflow(false/* immediate */, true/* compactingMerge */);
// insert the data into the scale-out index.
fed.getIndex(name, ITx.UNISOLATED)
@@ -395,7 +395,7 @@
int ndataService0 = 0;// #of index partitions on data service 0.
int ndataService1 = 0;// #of index partitions on data service 1.
- final ITupleIterator itr = new RawDataServiceTupleIterator(
+ final ITupleIterator<?> itr = new RawDataServiceTupleIterator(
fed.getMetadataService(),//
MetadataService.getMetadataIndexName(name), //
ITx.READ_COMMITTED,//
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/ndx/pipeline/TestMasterTask.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -194,20 +194,34 @@
*
* @throws InterruptedException
* @throws ExecutionException
+ *
+ * @todo This test now logs a warning rather than failing pending resolution
+ * of https://sourceforge.net/apps/trac/bigdata/ticket/147
*/
public void test_stress_startWriteStop2() throws InterruptedException,
ExecutionException {
- for (int i = 0; i < 10000; i++) {
+ final int LIMIT = 10000;
+ int nerr = 0;
+ for (int i = 0; i < LIMIT; i++) {
try {
doStartWriteStop2Test();
} catch (Throwable t) {
- fail("Pass#=" + i, t);
+ // fail("Pass#=" + i, t);
+ log.warn("Would have failed: pass#=" + i + ", cause=" + t);
+ nerr++;
}
}
+ if (nerr > 0) {
+
+ log.error("Test would have failed: nerrs=" + nerr + " out of "
+ + LIMIT + " trials");
+
+ }
+
}
/**
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/jini/start/config/JiniServiceConfiguration.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -34,7 +34,6 @@
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.io.Writer;
-import java.net.InetAddress;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
@@ -131,8 +130,6 @@
public final Properties properties;
public final String[] jiniOptions;
- private final String serviceIpAddr;
-
protected void toString(StringBuilder sb) {
super.toString(sb);
@@ -178,12 +175,6 @@
} else {
log.warn("groups = " + Arrays.toString(this.groups));
}
-
- try {
- this.serviceIpAddr = NicUtil.getIpAddress("default.nic", "default", false);
- } catch(IOException e) {
- throw new ConfigurationException(e.getMessage(), e);
- }
}
/**
@@ -480,6 +471,9 @@
final ServiceDir serviceDir = new ServiceDir(this.serviceDir);
+ String serviceIpAddr = NicUtil.getIpAddress ( "default.nic", "default", false ) ;
+ if ( null == serviceIpAddr )
+ throw new IOException ( "Can't get a host ip address" ) ;
final Hostname hostName = new Hostname(serviceIpAddr);
final ServiceUUID serviceUUID = new ServiceUUID(this.serviceUUID);
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/src/java/edu/lehigh/swat/bench/ubt/Test.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -85,7 +85,7 @@
hostname = NicUtil.getIpAddress("default.nic", "default", false);
} catch(Throwable t) {//for now, maintain same failure logic as used previously
t.printStackTrace();
- s = NicUtil.getIpAddressByLocalHost();
+ hostname = NicUtil.getIpAddressByLocalHost();
}
QUERY_TEST_RESULT_FILE = hostname + "-result.txt";
} else {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -272,7 +272,18 @@
//
// /** {@value #DEFAULT_MAX_TRIES} */
// int DEFAULT_MAX_TRIES = 3;
-
+
+ /**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ String DEFAULT_GRAPH = "defaultGraph" ;
+
+ /**
+ * TODO Should we always enforce a real value? i.e. provide a real default
+ * or abort the load.
+ */
+ String DEFAULT_DEFAULT_GRAPH = null ;
}
/**
@@ -402,6 +413,12 @@
private transient RDFFormat rdfFormat;
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ public final String defaultGraph ;
+
+ /**
* Force the load of the NxParser integration class and its registration
* of the NQuadsParser#nquads RDFFormat.
*
@@ -496,6 +513,8 @@
sb.append(", " + ConfigurationOptions.RDF_FORMAT + "=" + rdfFormat);
+ sb.append(", " + ConfigurationOptions.DEFAULT_GRAPH + "=" + defaultGraph) ;
+
sb.append(", " + ConfigurationOptions.FORCE_OVERFLOW_BEFORE_CLOSURE + "="
+ forceOverflowBeforeClosure);
@@ -601,6 +620,10 @@
}
+ defaultGraph = (String) config.getEntry(component,
+ ConfigurationOptions.DEFAULT_GRAPH, String.class,
+ ConfigurationOptions.DEFAULT_DEFAULT_GRAPH);
+
rejectedExecutionDelay = (Long) config.getEntry(
component,
ConfigurationOptions.REJECTED_EXECUTION_DELAY, Long.TYPE,
@@ -979,6 +1002,7 @@
jobState.ontology,//file
jobState.ontology.getPath(),//baseURI
jobState.getRDFFormat(),//
+ jobState.defaultGraph,
jobState.ontologyFileFilter //
);
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFFileLoadTask.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -223,6 +223,7 @@
jobState.valuesInitialCapacity,//
jobState.bnodesInitialCapacity,//
jobState.getRDFFormat(), //
+ jobState.defaultGraph,
parserOptions,//
false, // deleteAfter is handled by the master!
jobState.parserPoolSize, //
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/SingleResourceReaderTask.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -186,7 +186,7 @@
// run the parser.
// @todo reuse the same underlying parser instance?
- loader.loadRdf(reader, baseURL, rdfFormat, parserOptions);
+ loader.loadRdf(reader, baseURL, rdfFormat, null, parserOptions);
success = true;
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -356,8 +356,14 @@
* The default {@link RDFFormat}.
*/
private final RDFFormat defaultFormat;
-
+
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ private final String defaultGraph;
+
+ /**
* Options for the {@link RDFParser}.
*/
private final RDFParserOptions parserOptions;
@@ -1423,7 +1429,7 @@
try {
// run the parser.
new PresortRioLoader(buffer).loadRdf(reader, baseURL,
- rdfFormat, parserOptions);
+ rdfFormat, defaultGraph, parserOptions);
} finally {
reader.close();
}
@@ -1490,6 +1496,9 @@
* {@link BNode}s parsed from a single document.
* @param defaultFormat
* The default {@link RDFFormat} which will be assumed.
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
* @param parserOptions
* Options for the {@link RDFParser}.
* @param deleteAfter
@@ -1529,6 +1538,7 @@
final int valuesInitialCapacity,//
final int bnodesInitialCapacity, //
final RDFFormat defaultFormat,//
+ final String defaultGraph,//
final RDFParserOptions parserOptions,//
final boolean deleteAfter,//
final int parserPoolSize,//
@@ -1566,6 +1576,8 @@
this.defaultFormat = defaultFormat;
+ this.defaultGraph = defaultGraph;
+
this.parserOptions = parserOptions;
this.deleteAfter = deleteAfter;
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/BasicRioLoader.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -37,6 +37,8 @@
import org.openrdf.rio.RDFParser;
import org.openrdf.rio.Rio;
+import com.bigdata.rdf.model.BigdataURI;
+
/**
* Parses data but does not load it into the indices.
*
@@ -74,6 +76,8 @@
private final ValueFactory valueFactory;
+ protected String defaultGraph;
+
public BasicRioLoader(final ValueFactory valueFactory) {
if (valueFactory == null)
@@ -153,18 +157,20 @@
}
final public void loadRdf(final InputStream is, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final RDFParserOptions options)
throws Exception {
- loadRdf2(is, baseURI, rdfFormat, options);
+ loadRdf2(is, baseURI, rdfFormat, defaultGraph, options);
}
final public void loadRdf(final Reader reader, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final RDFParserOptions options)
throws Exception {
- loadRdf2(reader, baseURI, rdfFormat, options);
+ loadRdf2(reader, baseURI, rdfFormat, defaultGraph, options);
}
@@ -180,7 +186,7 @@
* @throws Exception
*/
protected void loadRdf2(final Object source, final String baseURI,
- final RDFFormat rdfFormat, final RDFParserOptions options)
+ final RDFFormat rdfFormat, final String defaultGraph, final RDFParserOptions options)
throws Exception {
if (source == null)
@@ -198,6 +204,8 @@
if (log.isInfoEnabled())
log.info("format=" + rdfFormat + ", options=" + options);
+ this.defaultGraph = defaultGraph ;
+
final RDFParser parser = getParser(rdfFormat);
// apply options to the parser
@@ -212,7 +220,7 @@
// Note: reset so that rates are correct for each source loaded.
stmtsAdded = 0;
-
+
try {
before();
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/IRioLoader.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -72,12 +72,14 @@
* The base URL for those data.
* @param rdfFormat
* The interchange format.
+ * @param defaultGraph
+ * The default graph.
* @param options
* Options to be applied to the {@link RDFParser}.
* @throws Exception
*/
public void loadRdf(Reader reader, String baseURL, RDFFormat rdfFormat,
- RDFParserOptions options) throws Exception;
+ String defaultGraph, RDFParserOptions options) throws Exception;
/**
* Parse RDF data.
@@ -88,11 +90,13 @@
* The base URL for those data.
* @param rdfFormat
* The interchange format.
+ * @param defaultGraph
+ * The default graph.
* @param options
* Options to be applied to the {@link RDFParser}.
* @throws Exception
*/
public void loadRdf(InputStream is, String baseURI, RDFFormat rdfFormat,
- RDFParserOptions options) throws Exception;
+ String defaultGraph, RDFParserOptions options) throws Exception;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/PresortRioLoader.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -23,11 +23,14 @@
*/
package com.bigdata.rdf.rio;
+import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.Value;
import org.openrdf.rio.RDFHandler;
import org.openrdf.rio.RDFHandlerException;
+import com.bigdata.rdf.model.BigdataURI;
+
/**
* Statement handler for the RIO RDF Parser that writes on a
* {@link StatementBuffer}.
@@ -45,6 +48,12 @@
final protected IStatementBuffer<?> buffer;
/**
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
+ */
+ private BigdataURI defaultGraphURI = null ;
+
+ /**
* Sets up parser to load RDF.
*
* @param buffer
@@ -58,7 +67,7 @@
this.buffer = buffer;
}
-
+
/**
* bulk insert the buffered data into the store.
*/
@@ -87,8 +96,11 @@
public RDFHandler newRDFHandler() {
+ defaultGraphURI = null != defaultGraph && 4 == buffer.getDatabase ().getSPOKeyArity ()
+ ? buffer.getDatabase ().getValueFactory ().createURI ( defaultGraph )
+ : null
+ ;
return this;
-
}
public void handleStatement( final Statement stmt ) {
@@ -98,9 +110,13 @@
log.debug(stmt);
}
-
+
+ Resource graph = stmt.getContext() ;
+ if ( null == graph
+ && null != defaultGraphURI ) // only true when we know we are loading a quad store
+ graph = defaultGraphURI ;
// buffer the write (handles overflow).
- buffer.add( stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), stmt.getContext() );
+ buffer.add( stmt.getSubject(), stmt.getPredicate(), stmt.getObject(), graph );
stmtsAdded++;
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -558,11 +558,18 @@
final int p = this.p.hashCode();
final int o = this.o.hashCode();
-
- // Note: historical behavior was (s,p,o) based hash.
- hashCode = 961 * ((int) (s ^ (s >>> 32))) + 31
- * ((int) (p ^ (p >>> 32))) + ((int) (o ^ (o >>> 32)));
+ /*
+ * Note: The historical behavior was based on the int64 term
+ * identifiers. Since the hash code is now computed from the int32
+ * hash codes of the (s,p,o) IV objects, the original bit math was
+ * resulting in a hash code which was always zero (any 32 bit value
+ * shifted right by 32 bits is zero).
+ */
+ hashCode = 961 * s + 31 * p + o;
+// hashCode = 961 * ((int) (s ^ (s >>> 32))) + 31
+// * ((int) (p ^ (p >>> 32))) + ((int) (o ^ (o >>> 32)));
+
}
return hashCode;
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/DataLoader.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -640,7 +640,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, reader, baseURL, rdfFormat, true/*endOfBatch*/);
+ loadData3(totals, reader, baseURL, rdfFormat, null, true/*endOfBatch*/);
return totals;
@@ -668,7 +668,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, is, baseURL, rdfFormat, true/* endOfBatch */);
+ loadData3(totals, is, baseURL, rdfFormat, null, true/* endOfBatch */);
return totals;
@@ -704,7 +704,7 @@
final LoadStats totals = new LoadStats();
- loadData3(totals, is, baseURL, rdfFormat, true/*endOfBatch*/);
+ loadData3(totals, is, baseURL, rdfFormat, null, true/*endOfBatch*/);
return totals;
@@ -762,7 +762,7 @@
if(file.exists()) {
loadFiles(totals, 0/* depth */, file, baseURL,
- rdfFormat, filter, endOfBatch);
+ rdfFormat, null, filter, endOfBatch);
return;
@@ -789,7 +789,7 @@
try {
- loadData3(totals, reader, baseURL, rdfFormat, endOfBatch);
+ loadData3(totals, reader, baseURL, rdfFormat, null, endOfBatch);
} catch (Exception ex) {
@@ -817,6 +817,9 @@
* The format of the file (optional, when not specified the
* format is deduced for each file in turn using the
* {@link RDFFormat} static methods).
+ * @param defaultGraph
+ * The value that will be used for the graph/context co-ordinate when
+ * loading data represented in a triple format into a quad store.
* @param filter
* A filter selecting the file names that will be loaded
* (optional). When specified, the filter MUST accept directories
@@ -827,7 +830,8 @@
* @throws IOException
*/
public LoadStats loadFiles(final File file, final String baseURI,
- final RDFFormat rdfFormat, final FilenameFilter filter)
+ final RDFFormat rdfFormat, final String defaultGraph,
+ final FilenameFilter filter)
throws IOException {
if (file == null)
@@ -835,7 +839,7 @@
final LoadStats totals = new LoadStats();
- loadFiles(totals, 0/* depth */, file, baseURI, rdfFormat, filter, true/* endOfBatch */
+ loadFiles(totals, 0/* depth */, file, baseURI, rdfFormat, defaultGraph, filter, true/* endOfBatch */
);
return totals;
@@ -844,7 +848,8 @@
protected void loadFiles(final LoadStats totals, final int depth,
final File file, final String baseURI, final RDFFormat rdfFormat,
- final FilenameFilter filter, final boolean endOfBatch)
+ final String defaultGraph, final FilenameFilter filter,
+ final boolean endOfBatch)
throws IOException {
if (file.isDirectory()) {
@@ -864,7 +869,7 @@
// final RDFFormat fmt = RDFFormat.forFileName(f.toString(),
// rdfFormat);
- loadFiles(totals, depth + 1, f, baseURI, rdfFormat, filter,
+ loadFiles(totals, depth + 1, f, baseURI, rdfFormat, defaultGraph, filter,
(depth == 0 && i < files.length ? false : endOfBatch));
}
@@ -919,7 +924,7 @@
final String s = baseURI != null ? baseURI : file.toURI()
.toString();
- loadData3(totals, reader, s, fmt, endOfBatch);
+ loadData3(totals, reader, s, fmt, defaultGraph, endOfBatch);
return;
@@ -955,7 +960,7 @@
*/
protected void loadData3(final LoadStats totals, final Object source,
final String baseURL, final RDFFormat rdfFormat,
- final boolean endOfBatch) throws IOException {
+ final String defaultGraph, final boolean endOfBatch) throws IOException {
final long begin = System.currentTimeMillis();
@@ -978,11 +983,10 @@
}
// Setup the loader.
- final PresortRioLoader loader = new PresortRioLoader(buffer);
+ final PresortRioLoader loader = new PresortRioLoader ( buffer ) ;
// @todo review: disable auto-flush - caller will handle flush of the buffer.
// loader.setFlush(false);
-
// add listener to log progress.
loader.addRioLoaderListener( new RioLoaderListener() {
@@ -1006,12 +1010,12 @@
if(source instanceof Reader) {
- loader.loadRdf((Reader) source, baseURL, rdfFormat, parserOptions);
+ loader.loadRdf((Reader) source, baseURL, rdfFormat, defaultGraph, parserOptions);
} else if (source instanceof InputStream) {
loader.loadRdf((InputStream) source, baseURL, rdfFormat,
- parserOptions);
+ defaultGraph, parserOptions);
} else
throw new AssertionError();
@@ -1360,7 +1364,7 @@
// rdfFormat, filter);
dataLoader.loadFiles(totals, 0/* depth */, fileOrDir, baseURI,
- rdfFormat, filter, true/* endOfBatch */
+ rdfFormat, null, filter, true/* endOfBatch */
);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/Splitter.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -714,7 +714,7 @@
try {
// run the parser.
new MyLoader(buffer).loadRdf(reader, baseURL,
- defaultRDFFormat, s.parserOptions);
+ defaultRDFFormat, null, s.parserOptions);
} finally {
reader.close();
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/AbstractRIOTestCase.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -401,7 +401,7 @@
});
- loader.loadRdf((Reader) reader, baseURI, rdfFormat, options);
+ loader.loadRdf((Reader) reader, baseURI, rdfFormat, null, options);
if (log.isInfoEnabled())
log.info("Done: " + resource);
@@ -681,7 +681,7 @@
loader.loadRdf(new BufferedReader(new InputStreamReader(
new FileInputStream(resource))), baseURI, rdfFormat,
- options);
+ null, options);
if(log.isInfoEnabled())
log.info("End of reparse: nerrors=" + nerrs + ", file="
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/EDSAsyncLoader.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -161,6 +161,7 @@
valuesInitialCapacity,//
bnodesInitialCapacity,//
RDFFormat.RDFXML, // defaultFormat
+ null, // defaultGraph
parserOptions, // parserOptions
false, // deleteAfter
poolSize, // parserPoolSize,
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rio/TestAsynchronousStatementBufferFactory.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -400,6 +400,7 @@
valuesInitialCapacity,//
bnodesInitialCapacity,//
RDFFormat.RDFXML, // defaultFormat
+ null, // defaultGraph
parserOptions, //
false, // deleteAfter
parallel?5:1, // parserPoolSize,
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java 2010-09-28 12:43:35 UTC (rev 3658)
@@ -78,16 +78,16 @@
super(name);
}
- public void test_optionals_nextedSubquery()
- {
-
- final Properties p = new Properties(getProperties());
-
- p.setProperty(AbstractRelation.Options.NESTED_SUBQUERY, "true");
-
- doOptionalsTest(p);
-
- }
+// public void test_optionals_nextedSubquery()
+// {
+//
+// final Properties p = new Properties(getProperties());
+//
+// p.setProperty(AbstractRelation.Options.NESTED_SUBQUERY, "true");
+//
+// doOptionalsTest(p);
+//
+// }
public void test_optionals_pipeline()
{
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/fastload.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/fastload.properties 2010-09-28 12:29:15 UTC (rev 3657)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/samples/com/bigdata/samples/fastload.properties 2010-09-28 12:43:35 UTC (rev 3658)
@@ -1,7 +1,8 @@
-# Be very careful when you use this configuration! This turns off incremental
-# inference for load and retract, so you must explicitly force these operations,
-# which requires punching through the SAIL layer. Of course, if you are not
-# using inference then this is just the ticket and quite fast.
+# This configuration turns off incremental inference for load and retract, so
+# you must explicitly force these operations if you want to compute the closure
+# of the knowledge base. Forcing the closure requires punching through the SAIL
+# layer. Of course, if you are not using inference then this configuration is
+# just the ticket and is quite fast.
# set the initial and maximum extent o...
[truncated message content] |
|
From: <tho...@us...> - 2010-09-28 16:53:33
|
Revision: 3665
http://bigdata.svn.sourceforge.net/bigdata/?rev=3665&view=rev
Author: thompsonbry
Date: 2010-09-28 16:53:22 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Reorganized the pipeline operators, the index sampling operators, and the CTC style striterator operators.
Deprecated IPredicate#CONSTRAINTS. It will be replaced by INDEX_LOCAL_FILTERS and ACCESS_PATH_FILTERS. The former will always run locally at the index against the ITupleIterator. The latter will always run on the AccessPath side against the materialized elements. For example, the default graph "distinct spo" constraint will be imposed as a DistinctFilter wrapping a StripContextFilter. In scale-out, this will use a remote access path as necessary in order to have a total view of the sharded data. My next step is to implement that distinction.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/ConditionalRoutingOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/CopyBindingSetOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/Tee.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/Union.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryDecl.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryDecl.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngineTestAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/eval/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/mutation/InsertOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/DistinctBindingSetOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SliceOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SortOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/PipelineDelayOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestPipelineUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestRunState.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestNIOChunkMessage.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestThickChunkMessage.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ndx/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpResolver.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestSampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestSampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestDistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestResolver.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/filter/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/filter/StripContextFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/DataSetJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/filter/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/filter/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/filter/TestStripContextFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/join/
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/SampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/SampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/rdf/join/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestDistinctElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ndx/TestSampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ndx/TestSampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/rdf/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/rdf/join/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/DataSetJoin.java
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java (from rev 3630, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,116 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 18, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+
+import com.bigdata.relation.accesspath.IAccessPath;
+import com.bigdata.striterator.IChunkedOrderedIterator;
+
+/**
+ * Interface for evaluating operations producing chunks of elements (tuples
+ * materialized from some index of a relation).
+ *
+ * @see IAccessPath
+ * @see IChunkedOrderedIterator
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractAccessPathOp<E> extends BOpBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BOp.Annotations, BufferAnnotations {
+
+ }
+
+ /**
+ * Required shallow copy constructor.
+ *
+ * @param args
+ * @param annotations
+ */
+ public AbstractAccessPathOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ super(args, annotations);
+
+ }
+
+ /**
+ * Required deep copy constructor.
+ *
+ * @param op
+ */
+ public AbstractAccessPathOp(
+ final AbstractAccessPathOp<E> op) {
+ super(op);
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_CAPACITY
+ */
+ protected int getChunkCapacity() {
+
+ return getProperty(Annotations.CHUNK_CAPACITY,
+ Annotations.DEFAULT_CHUNK_CAPACITY);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_OF_CHUNKS_CAPACITY
+ */
+ protected int getChunkOfChunksCapacity() {
+
+ return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
+ Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+
+ }
+
+// protected int getFullyBufferedReadThreshold() {
+//
+// return getProperty(Annotations.FULLY_BUFFERED_READ_THRESHOLD,
+// Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
+//
+// }
+
+ /**
+ * @see BufferAnnotations#CHUNK_TIMEOUT
+ */
+ protected long getChunkTimeout() {
+
+ return getProperty(Annotations.CHUNK_TIMEOUT,
+ Annotations.DEFAULT_CHUNK_TIMEOUT);
+
+ }
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,112 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Aug 18, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class AbstractChunkedOrderedIteratorOp<E> extends BOpBase
- implements ChunkedOrderedIteratorOp<E> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends ChunkedOrderedIteratorOp.Annotations {
-
- }
-
-// /**
-// * @param args
-// */
-// protected AbstractChunkedOrderedIteratorOp(BOp[] args) {
-// super(args);
-// }
-
- /**
- * Required shallow copy constructor.
- *
- * @param args
- * @param annotations
- */
- protected AbstractChunkedOrderedIteratorOp(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- }
-
- /**
- * Required deep copy constructor.
- *
- * @param op
- */
- public AbstractChunkedOrderedIteratorOp(
- final AbstractChunkedOrderedIteratorOp<E> op) {
- super(op);
- }
-
- protected int getChunkCapacity() {
-
- return getProperty(Annotations.CHUNK_CAPACITY,
- Annotations.DEFAULT_CHUNK_CAPACITY);
-
- }
-
- protected int getChunkOfChunksCapacity() {
-
- return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
- Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
-
- }
-
-// protected int getFullyBufferedReadThreshold() {
-//
-// return getProperty(Annotations.FULLY_BUFFERED_READ_THRESHOLD,
-// Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
-//
-// }
-
- protected long getChunkTimeout() {
-
- return getProperty(Annotations.CHUNK_TIMEOUT,
- Annotations.DEFAULT_CHUNK_TIMEOUT);
-
- }
-
- /**
- * The {@link TimeUnit}s in which the {@link #chunkTimeout} is measured.
- */
- protected static transient final TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
-
-}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,88 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Sep 2, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.Map;
-
-/**
- * Abstract base class for pipeline operators where the data moving along the
- * pipeline is chunks of {@link IBindingSet}s.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class BindingSetPipelineOp extends
- PipelineOp<IBindingSet> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends PipelineOp.Annotations {
-
- /**
- * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
- * the ancestor in the operator tree which serves as the default sink
- * for binding sets (default is the parent).
- */
- String SINK_REF = BindingSetPipelineOp.class.getName() + ".sinkRef";
-
- /**
- * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
- * the ancestor in the operator tree which serves as the alternative
- * sink for binding sets (default is no alternative sink).
- */
- String ALT_SINK_REF = BindingSetPipelineOp.class.getName()
- + ".altSinkRef";
-
- }
-
- /**
- * Required deep copy constructor.
- *
- * @param op
- */
- protected BindingSetPipelineOp(PipelineOp<IBindingSet> op) {
- super(op);
- }
-
- /**
- * Shallow copy constructor.
- *
- * @param args
- * @param annotations
- */
- protected BindingSetPipelineOp(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- }
-
-}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -27,6 +27,8 @@
package com.bigdata.bop;
+import java.util.concurrent.TimeUnit;
+
import com.bigdata.relation.accesspath.BlockingBuffer;
import com.bigdata.relation.accesspath.IBuffer;
@@ -81,4 +83,9 @@
*/
int DEFAULT_CHUNK_TIMEOUT = 20;
+ /**
+ * The {@link TimeUnit}s in which the {@link #CHUNK_TIMEOUT} is measured.
+ */
+ TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
+
}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,25 +0,0 @@
-package com.bigdata.bop;
-
-import com.bigdata.relation.accesspath.IAccessPath;
-import com.bigdata.striterator.IChunkedOrderedIterator;
-
-/**
- * Interface for evaluating operations producing chunks of elements (tuples
- * materialized from some index of a relation).
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- *
- * @see IAccessPath
- * @see IChunkedOrderedIterator
- */
-public interface ChunkedOrderedIteratorOp<E> extends BOp {
-
- /**
- * Well known annotations.
- */
- public interface Annotations extends BOp.Annotations, BufferAnnotations {
-
-
- }
-
-}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,51 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Annotations for an operator using an internal concurrent hash map.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public interface ConcurrentHashMapAnnotations extends HashMapAnnotations {
+
+ /**
+ * The concurrency level of the {@link ConcurrentHashMap} used to impose the
+ * distinct constraint.
+ *
+ * @see #DEFAULT_CONCURRENCY_LEVEL
+ */
+ String CONCURRENCY_LEVEL = ConcurrentHashMapAnnotations.class.getName()
+ + ".concurrencyLevel";
+
+ int DEFAULT_CONCURRENCY_LEVEL = 16;
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,61 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Annotations for an operator using an internal hash map.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public interface HashMapAnnotations {
+
+ /**
+ * The initial capacity of the {@link ConcurrentHashMap} used to impose the
+ * distinct constraint.
+ *
+ * @see #DEFAULT_INITIAL_CAPACITY
+ */
+ String INITIAL_CAPACITY = HashMapAnnotations.class.getName()
+ + ".initialCapacity";
+
+ int DEFAULT_INITIAL_CAPACITY = 16;
+
+ /**
+ * The load factor of the {@link ConcurrentHashMap} used to impose the
+ * distinct constraint.
+ *
+ * @see #DEFAULT_LOAD_FACTOR
+ */
+ String LOAD_FACTOR = HashMapAnnotations.class.getName() + ".loadFactor";
+
+ float DEFAULT_LOAD_FACTOR = .75f;
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,89 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Sep 3, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.FutureTask;
-
-import com.bigdata.bop.engine.BOpStats;
-import com.bigdata.relation.accesspath.IBlockingBuffer;
-
-/**
- * An pipeline operator reads from a source and writes on a sink.
- *
- * @param <E>
- * The generic type of the objects processed by the operator.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-public interface IPipelineOp<E> extends BOp {
-
- /**
- * Return a new object which can be used to collect statistics on the
- * operator evaluation (this may be overridden to return a more specific
- * class depending on the operator).
- */
- BOpStats newStats();
-
- /**
- * Instantiate a buffer suitable as a sink for this operator. The buffer
- * will be provisioned based on the operator annotations.
- * <p>
- * Note: if the operation swallows binding sets from the pipeline (such as
- * operators which write on the database) then the operator MAY return an
- * immutable empty buffer.
- *
- * @param stats
- * The statistics on this object will automatically be updated as
- * elements and chunks are output onto the returned buffer.
- *
- * @return The buffer.
- */
- IBlockingBuffer<E[]> newBuffer(BOpStats stats);
-
- /**
- * Return a {@link FutureTask} which computes the operator against the
- * evaluation context. The caller is responsible for executing the
- * {@link FutureTask} (this gives them the ability to hook the completion of
- * the computation).
- *
- * @param context
- * The evaluation context.
- *
- * @return The {@link FutureTask} which will compute the operator's
- * evaluation.
- *
- * @todo Modify to return a {@link Callable}s for now since we must run each
- * task in its own thread until Java7 gives us fork/join pools and
- * asynchronous file I/O. For the fork/join model we will probably
- * return the ForkJoinTask.
- */
- FutureTask<Void> eval(BOpContext<E> context);
-
-}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -30,8 +30,13 @@
import java.io.Serializable;
+import com.bigdata.bop.ap.filter.BOpFilterBase;
+import com.bigdata.bop.ap.filter.DistinctFilter;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IRangeQuery;
+import com.bigdata.btree.ITuple;
+import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
@@ -88,10 +93,47 @@
/**
* Constraints on the elements read from the relation.
+ *
+ * @deprecated This is being replaced by two classes of filters. One
+ * which is always evaluated local to the index and one
+ * which is evaluated in the JVM in which the access path is
+ * evaluated once the {@link ITuple}s have been resolved to
+ * elements of the relation.
*/
String CONSTRAINT = "constraint";
/**
+ * An optional {@link BOpFilterBase} that will be evaluated local to the
+ * to the index. When the index is remote, the filter will be sent to
+ * the node on which the index resides and evaluated there. This makes
+ * it possible to efficiently filter out tuples which are not of
+ * interest for a given access path.
+ * <p>
+ * Note: The filter MUST NOT change the type of data visited by the
+ * iterator - it must remain an {@link ITupleIterator}. An attempt to
+ * change the type of the visited objects will result in a runtime
+ * exception.
+ * <p>
+ * You can chain {@link BOpFilterBase} filters by nesting them inside of
+ * one another.
+ *
+ * @see #ACCESS_PATH_FILTER
+ *
+ * @see IRangeQuery#rangeIterator(byte[], byte[], int, int,
+ * IFilterConstructor)
+ */
+ String INDEX_LOCAL_FILTER = "indexLocalFilter";
+
+ /**
+ * An optional {@link BOpFilterBase} to be applied to the elements of
+ * the relation as they are materialized from the index. Unlike
+ * {@link #INDEX_LOCAL_FILTER}, this an {@link #ACCESS_PATH_FILTER} is never
+ * sent to a remote index for evaluation. This makes it possible to
+ * impose {@link DistinctFilter} across a {@link #REMOTE_ACCESS_PATH}.
+ */
+ String ACCESS_PATH_FILTER = "accessPathFilter";
+
+ /**
* Expander pattern.
*/
String EXPANDER = "expander";
@@ -144,7 +186,7 @@
*
* @see #DEFAULT_FULLY_BUFFERED_READ_THRESHOLD
*/
- String FULLY_BUFFERED_READ_THRESHOLD = PipelineOp.class.getName()
+ String FULLY_BUFFERED_READ_THRESHOLD = IPredicate.class.getName()
+ ".fullyBufferedReadThreshold";
/**
@@ -168,7 +210,7 @@
*
* @see #DEFAULT_FLAGS
*/
- String FLAGS = PipelineOp.class.getName() + ".flags";
+ String FLAGS = IPredicate.class.getName() + ".flags";
/**
* The default flags will visit the keys and values of the non-deleted
@@ -278,13 +320,17 @@
* which is the only method declared by {@link ISolutionExpander}.
*/
public ISolutionExpander<E> getSolutionExpander();
-
+
/**
* An optional constraint on the visitable elements.
*
* @see Annotations#CONSTRAINT
*
- * @todo rename as get(Element)Filter().
+ * @deprecated This is being replaced by two classes of filters. One which
+ * is always evaluated local to the index and one which is
+ * evaluated in the JVM in which the access path is evaluated
+ * once the {@link ITuple}s have been resolved to elements of
+ * the relation.
*/
public IElementFilter<E> getConstraint();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -28,7 +28,7 @@
package com.bigdata.bop;
/**
- * An interface for {@link BindingSetPipelineOp}s which are mapped across
+ * An interface for {@link PipelineOp}s which are mapped across
* shards.
*
* @param <E>
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,230 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Aug 18, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import com.bigdata.bop.engine.BOpStats;
-import com.bigdata.bop.engine.QueryEngine;
-import com.bigdata.relation.accesspath.BlockingBuffer;
-import com.bigdata.relation.accesspath.IBlockingBuffer;
-
-/**
- * An pipeline operator reads from a source and writes on a sink. This is an
- * abstract base class for pipelined operators regardless of the type of data
- * moving along the pipeline.
- *
- * @param <E>
- * The generic type of the objects processed by the operator.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class PipelineOp<E> extends BOpBase implements IPipelineOp<E> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * Well known annotations pertaining to the binding set pipeline.
- */
- public interface Annotations extends BOp.Annotations, BufferAnnotations {
-
- }
-
- /**
- * Deep copy constructor.
- *
- * @param op
- */
- protected PipelineOp(final PipelineOp<E> op) {
-
- super(op);
-
- }
-
- /**
- * Shallow copy constructor.
- *
- * @param args
- * @param annotations
- */
- protected PipelineOp(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- }
-
- public int getChunkCapacity() {
-
- return getProperty(Annotations.CHUNK_CAPACITY,
- Annotations.DEFAULT_CHUNK_CAPACITY);
-
- }
-
- public int getChunkOfChunksCapacity() {
-
- return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
- Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
-
- }
-
- public long getChunkTimeout() {
-
- return getProperty(Annotations.CHUNK_TIMEOUT,
- Annotations.DEFAULT_CHUNK_TIMEOUT);
-
- }
-
- /**
- * The {@link TimeUnit}s in which the {@link #chunkTimeout} is measured.
- */
- protected static transient final TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
-
- /**
- * Return the {@link PipelineType} of the operator (default
- * {@link PipelineType#Vectored}).
- */
- public PipelineType getPipelineType() {
-
- return PipelineType.Vectored;
-
- }
-
- /**
- * Return <code>true</code> iff {@link #newStats()} must be shared across
- * all invocations of {@link #eval(BOpContext)} for this operator for a
- * given query (default <code>false</code>).
- * <p>
- * Note: {@link BOp#getEvaluationContext()} MUST be overridden to return
- * {@link BOpEvaluationContext#CONTROLLER} if this method is overridden to
- * return <code>true</code>.
- * <p>
- * When <code>true</code>, the {@link QueryEngine} will impose the necessary
- * constraints when the operator is evaluated.
- */
- public boolean isSharedState() {
-
- return false;
-
- }
-
- public BOpStats newStats() {
-
- return new BOpStats();
-
- }
-
- public IBlockingBuffer<E[]> newBuffer(final BOpStats stats) {
-
- if (stats == null)
- throw new IllegalArgumentException();
-
- return new BlockingBufferWithStats<E[]>(getChunkOfChunksCapacity(),
- getChunkCapacity(), getChunkTimeout(), chunkTimeoutUnit, stats);
-
- }
-
- private static class BlockingBufferWithStats<E> extends BlockingBuffer<E> {
-
- private final BOpStats stats;
-
- /**
- * @param chunkOfChunksCapacity
- * @param chunkCapacity
- * @param chunkTimeout
- * @param chunkTimeoutUnit
- * @param stats
- */
- public BlockingBufferWithStats(int chunkOfChunksCapacity,
- int chunkCapacity, long chunkTimeout,
- TimeUnit chunkTimeoutUnit, final BOpStats stats) {
-
- super(chunkOfChunksCapacity, chunkCapacity, chunkTimeout,
- chunkTimeoutUnit);
-
- this.stats = stats;
-
- }
-
- /**
- * Overridden to track {@link BOpStats#unitsOut} and
- * {@link BOpStats#chunksOut}.
- * <p>
- * Note: {@link BOpStats#chunksOut} will report the #of chunks added to
- * this buffer. However, the buffer MAY combine chunks either on add()
- * or when drained by the iterator so the actual #of chunks read back
- * from the iterator MAY differ.
- * <p>
- * {@inheritDoc}
- */
- @Override
- public boolean add(final E e, final long timeout, final TimeUnit unit)
- throws InterruptedException {
-
- final boolean ret = super.add(e, timeout, unit);
-
- if (e.getClass().getComponentType() != null) {
-
- stats.unitsOut.add(((Object[]) e).length);
-
- } else {
-
- stats.unitsOut.increment();
-
- }
-
- stats.chunksOut.increment();
-
- return ret;
-
- }
-
- /**
- * You can uncomment a line in this method to see who is closing the
- * buffer.
- * <p>
- * {@inheritDoc}
- */
- @Override
- public void close() {
-
-// if (isOpen())
-// log.error(toString(), new RuntimeException("STACK TRACE"));
-
- super.close();
-
- }
-
- }
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java (from rev 3614, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,282 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 2, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+
+import com.bigdata.bop.ap.E;
+import com.bigdata.bop.engine.BOpStats;
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.relation.accesspath.BlockingBuffer;
+import com.bigdata.relation.accesspath.IBlockingBuffer;
+
+/**
+ * Abstract base class for pipeline operators where the data moving along the
+ * pipeline is chunks of {@link IBindingSet}s.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class PipelineOp extends BOpBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BOp.Annotations, BufferAnnotations {
+
+ /**
+ * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
+ * the ancestor in the operator tree which serves as the default sink
+ * for binding sets (default is the parent).
+ */
+ String SINK_REF = PipelineOp.class.getName() + ".sinkRef";
+
+ /**
+ * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
+ * the ancestor in the operator tree which serves as the alternative
+ * sink for binding sets (default is no alternative sink).
+ */
+ String ALT_SINK_REF = PipelineOp.class.getName()
+ + ".altSinkRef";
+
+ }
+
+ /**
+ * Required deep copy constructor.
+ *
+ * @param op
+ */
+ protected PipelineOp(final PipelineOp op) {
+ super(op);
+ }
+
+ /**
+ * Shallow copy constructor.
+ *
+ * @param args
+ * @param annotations
+ */
+ protected PipelineOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ super(args, annotations);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_CAPACITY
+ */
+ public int getChunkCapacity() {
+
+ return getProperty(Annotations.CHUNK_CAPACITY,
+ Annotations.DEFAULT_CHUNK_CAPACITY);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_OF_CHUNKS_CAPACITY
+ */
+ public int getChunkOfChunksCapacity() {
+
+ return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
+ Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_TIMEOUT
+ */
+ public long getChunkTimeout() {
+
+ return getProperty(Annotations.CHUNK_TIMEOUT,
+ Annotations.DEFAULT_CHUNK_TIMEOUT);
+
+ }
+
+ /**
+ * Return the {@link PipelineType} of the operator (default
+ * {@link PipelineType#Vectored}).
+ */
+ public PipelineType getPipelineType() {
+
+ return PipelineType.Vectored;
+
+ }
+
+ /**
+ * Return <code>true</code> iff {@link #newStats()} must be shared across
+ * all invocations of {@link #eval(BOpContext)} for this operator for a
+ * given query (default <code>false</code>).
+ * <p>
+ * Note: {@link BOp#getEvaluationContext()} MUST be overridden to return
+ * {@link BOpEvaluationContext#CONTROLLER} if this method is overridden to
+ * return <code>true</code>.
+ * <p>
+ * When <code>true</code>, the {@link QueryEngine} will impose the necessary
+ * constraints when the operator is evaluated.
+ */
+ public boolean isSharedState() {
+
+ return false;
+
+ }
+
+ /**
+ * Return a new object which can be used to collect statistics on the
+ * operator evaluation (this may be overridden to return a more specific
+ * class depending on the operator).
+ */
+ public BOpStats newStats() {
+
+ return new BOpStats();
+
+ }
+
+ /**
+ * Instantiate a buffer suitable as a sink for this operator. The buffer
+ * will be provisioned based on the operator annotations.
+ * <p>
+ * Note: if the operation swallows binding sets from the pipeline (such as
+ * operators which write on the database) then the operator MAY return an
+ * immutable empty buffer.
+ *
+ * @param stats
+ * The statistics on this object will automatically be updated as
+ * elements and chunks are output onto the returned buffer.
+ *
+ * @return The buffer.
+ */
+ public IBlockingBuffer<IBindingSet[]> newBuffer(final BOpStats stats) {
+
+ if (stats == null)
+ throw new IllegalArgumentException();
+
+ return new BlockingBufferWithStats<IBindingSet[]>(
+ getChunkOfChunksCapacity(), getChunkCapacity(),
+ getChunkTimeout(), Annotations.chunkTimeoutUnit, stats);
+
+ }
+
+ /**
+ * Return a {@link FutureTask} which computes the operator against the
+ * evaluation context. The caller is responsible for executing the
+ * {@link FutureTask} (this gives them the ability to hook the completion of
+ * the computation).
+ *
+ * @param context
+ * The evaluation context.
+ *
+ * @return The {@link FutureTask} which will compute the operator's
+ * evaluation.
+ *
+ * @todo Modify to return a {@link Callable}s for now since we must run each
+ * task in its own thread until Java7 gives us fork/join pools and
+ * asynchronous file I/O. For the fork/join model we will probably
+ * return the ForkJoinTask.
+ */
+ abstract public FutureTask<Void> eval(BOpContext<IBindingSet> context);
+
+ private static class BlockingBufferWithStats<E> extends BlockingBuffer<E> {
+
+ private final BOpStats stats;
+
+ /**
+ * @param chunkOfChunksCapacity
+ * @param chunkCapacity
+ * @param chunkTimeout
+ * @param chunkTimeoutUnit
+ * @param stats
+ */
+ public BlockingBufferWithStats(int chunkOfChunksCapacity,
+ int chunkCapacity, long chunkTimeout,
+ TimeUnit chunkTimeoutUnit, final BOpStats stats) {
+
+ super(chunkOfChunksCapacity, chunkCapacity, chunkTimeout,
+ chunkTimeoutUnit);
+
+ this.stats = stats;
+
+ }
+
+ /**
+ * Overridden to track {@link BOpStats#unitsOut} and
+ * {@link BOpStats#chunksOut}.
+ * <p>
+ * Note: {@link BOpStats#chunksOut} will report the #of chunks added to
+ * this buffer. However, the buffer MAY combine chunks either on add()
+ * or when drained by the iterator so the actual #of chunks read back
+ * from the iterator MAY differ.
+ * <p>
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean add(final E e, final long timeout, final TimeUnit unit)
+ throws InterruptedException {
+
+ final boolean ret = super.add(e, timeout, unit);
+
+ if (e.getClass().getComponentType() != null) {
+
+ stats.unitsOut.add(((Object[]) e).length);
+
+ } else {
+
+ stats.unitsOut.increment();
+
+ }
+
+ stats.chunksOut.increment();
+
+ return ret;
+
+ }
+
+ /**
+ * You can uncomment a line in this method to see who is closing the
+ * buffer.
+ * <p>
+ * {@inheritDoc}
+ */
+ @Override
+ public void close() {
+
+// if (isOpen())
+// log.error(toString(), new RuntimeException("STACK TRACE"));
+
+ super.close();
+
+ }
+
+ }
+
+}
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java (from rev 3630, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/AbstractSampleIndex.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,125 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 16, 2010
+ */
+
+package com.bigdata.bop.ap;
+
+import com.bigdata.bop.AbstractAccessPathOp;
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.NV;
+import com.bigdata.btree.IIndex;
+import com.bigdata.relation.accesspath.IAccessPath;
+
+/**
+ * Abstract base class for sampling operator for an {@link IIndex}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ * @param <E>
+ * The generic type of the elements materialized from that index.
+ *
+ * @todo Implement sample operator. E.g., sampleRange(fromKey,toKey,limit). This
+ * could be on {@link IIndex} or on {@link IAccessPath}. For a shard view,
+ * it must proportionally select from among the ordered components of the
+ * view. For a hash table it would be sample(limit) since range based
+ * operations are not efficient.
+ * <p>
+ * This should accept an index, not a predicate (for RDF we determine the
+ * index an analysis of the bound and unbound arguments on the predicate
+ * and always have a good index, but this is not true in the general
+ * case). When the index is remote, it should be executed at the remote
+ * index.
+ *
+ * @todo This needs to operation on element chunks, not {@link IBindingSet}
+ * chunks. It also may not require pipelining.
+ */
+abstract public class AbstractSampleIndex<E> extends AbstractAccessPathOp<E> {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Known annotations.
+ */
+ public interface Annotations extends BOp.Annotations {
+ /**
+ * The sample limit.
+ */
+ String LIMIT = "limit";
+ }
+
+ protected AbstractSampleIndex(final IPredicate<E> pred, final int limit) {
+
+ super(new BOp[] { pred }, NV.asMap(new NV[] {//
+ new NV(Annotations.LIMIT, Integer.valueOf(limit)) //
+ }));
+
+ if (pred == null)
+ throw new IllegalArgumentException();
+
+ if (limit <= 0)
+ throw new IllegalArgumentException();
+
+ switch (getEvaluationContext()) {
+ case HASHED:
+ case SHARDED:
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ Annotations.EVALUATION_CONTEXT + "="
+ + getEvaluationContext());
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public IPredicate<E> pred() {
+
+ return (IPredicate<E>) get(0);
+
+ }
+
+ public int limit() {
+
+ return (Integer) annotations.get(Annotations.LIMIT);
+
+ }
+
+// /**
+// * This is a shard wise operator.
+// */
+// @Override
+// public BOpEvaluationContext getEvaluationContext() {
+//
+// return BOpEvaluationContext.SHARDED;
+//
+// }
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,320 +0,0 @@
-package com.bigdata.bop.ap;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.FutureTask;
-
-import com.bigdata.bop.BOp;
-import com.bigdata.bop.BOpBase;
-import com.bigdata.bop.BOpContext;
-import com.bigdata.bop.IBindingSet;
-import com.bigdata.rdf.spo.DistinctSPOIterator;
-import com.bigdata.rdf.spo.SPO;
-import com.bigdata.relation.accesspath.IElementFilter;
-import com.bigdata.striterator.DistinctFilter;
-import com.bigdata.striterator.IChunkConverter;
-import com.bigdata.striterator.MergeFilter;
-
-/**
- * A DISTINCT operator based for elements in a relation. The operator is based
- * on a hash table. New elements are constructed for each original element in
- * which only the distinct fields are preserved. If the new element is distinct
- * then it is passed by the filter.
- * <p>
- * The filter is capable of changing the type of the accepted elements.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: DistinctElementFilter.java 3466 2010-08-27 14:28:04Z
- * thompsonbry $
- *
- * @param <E>
- * The generic type of the source elements for the filter.
- * @param <F>
- * The generic type of the elements passed by the filter.
- *
- * @todo support changing the generic type as part of the filter. this is
- * similar to the {@link IChunkConverter}.
- *
- * @todo Reconcile with {@link IChunkConverter}, {@link DistinctFilter} (handles
- * solutions) and {@link MergeFilter} (handles comparables),
- * {@link DistinctSPOIterator}, etc.
- */
-public class DistinctElementFilter<E> extends BOpBase implements
- IElementFilter<E> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends BOp.Annotations {
-
- /**
- * The initial capacity of the {@link ConcurrentHashMap} used to impose
- * the distinct constraint.
- *
- * @see #DEFAULT_INITIAL_CAPACITY
- */
- String INITIAL_CAPACITY = DistinctElementFilter.class.getName()
- + ".initialCapacity";
-
- int DEFAULT_INITIAL_CAPACITY = 16;
-
- /**
- * The load factor of the {@link ConcurrentHashMap} used to impose the
- * distinct constraint.
- *
- * @see #DEFAULT_LOAD_FACTOR
- */
- String LOAD_FACTOR = DistinctElementFilter.class.getName()
- + ".loadFactor";
-
- float DEFAULT_LOAD_FACTOR = .75f;
-
- /**
- * The concurrency level of the {@link ConcurrentHashMap} used to impose
- * the distinct constraint.
- *
- * @see #DEFAULT_CONCURRENCY_LEVEL
- */
- String CONCURRENCY_LEVEL = DistinctElementFilter.class.getName()
- + ".concurrencyLevel";
-
- int DEFAULT_CONCURRENCY_LEVEL = 16;
-
- /**
- * The set of fields whose values must be distinct.
- *
- * @todo abstract base class to allow easy override for specific element
- * types such as {@link SPO}.
- */
- String FIELDS = DistinctElementFilter.class.getName() + ".fields";
-
- /**
- * An optional constraint on the runtime type of the elements which are
- * acceptable to this filter.
- *
- * @see IElementFilter#canAccept(Object)
- *
- * @todo I am not convinced that we need this. It parallels something
- * which was introduced into the {@link IElementFilter} interface,
- * but I suspect that we do not need that either.
- */
- String CLASS_CONSTRAINT = DistinctElementFilter.class.getName()
- + ".classConstraint";
-
- }
-
- /**
- * Required deep copy constructor.
- */
- public DistinctElementFilter(final DistinctElementFilter<E> op) {
- super(op);
- }
-
- /**
- * Required shallow copy constructor.
- */
- public DistinctElementFilter(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- final int[] fields = getFields();
-
- if (fields == null)
- throw new IllegalArgumentException();
-
- if (fields.length == 0)
- throw new IllegalArgumentException();
-
- }
-
- /**
- * @see Annotations#INITIAL_CAPACITY
- */
- public int getInitialCapacity() {
-
- return getProperty(Annotations.INITIAL_CAPACITY,
- Annotations.DEFAULT_INITIAL_CAPACITY);
-
- }
-
- /**
- * @see Annotations#LOAD_FACTOR
- */
- public float getLoadFactor() {
-
- return getProperty(Annotations.LOAD_FACTOR,
- Annotations.DEFAULT_LOAD_FACTOR);
-
- }
-
- /**
- * @see Annotations#CONCURRENCY_LEVEL
- */
- public int getConcurrencyLevel() {
-
- return getProperty(Annotations.CONCURRENCY_LEVEL,
- Annotations.DEFAULT_CONCURRENCY_LEVEL);
-
- }
-
- public int[] getFields() {
-
- return (int[]) getProperty(Annotations.FIELDS);
-
- }
-
- public FutureTask<Void> eval(final BOpContext<IBindingSet> context) {
-
-// return new FutureTask<Void>(new DistinctTask<E>(this, context));
- throw new UnsupportedOperationException();
-
- }
-
- public boolean accept(E e) {
- // TODO Auto-generated method stub
- return false;
- }
-
- public boolean canAccept(Object o) {
- // @todo by annotation giving an optional type constraint.
- return true;
- }
-
-// /**
-// * Task executing on the node.
-// */
-// static private class DistinctTask<E> implements Callable<Void> {
-//
-// private final BOpConte...
[truncated message content] |
|
From: <tho...@us...> - 2010-09-28 23:39:51
|
Revision: 3672
http://bigdata.svn.sourceforge.net/bigdata/?rev=3672&view=rev
Author: thompsonbry
Date: 2010-09-28 23:39:42 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Reconciled IFilterConstructor with IFilter.
Added BOpTupleFilter, which parallels TupleFilter and in fact uses the same underlying iterator to protect against reuse of the Tuple instance by the B+Tree iterators.
Fixed a problem in Filterator where eager initialization (in the constructor) caused the next element to be fetched before the filter stack was fully initialized by FilterBase.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IRangeQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadCommittedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadOnlyIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/view/FusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPathFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/CacheOnceMetadataIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/CachingMetadataIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IDataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/NoCacheMetadataIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/AbstractScaleOutClientIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/ClientIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/DataServiceTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/PartitionedTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedStriterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestPrefixFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestTupleFilters.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/view/TestFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/accesspath/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/WrappedTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestSameVariableConstraint.java
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/FilterConstructor.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/IFilterConstructor.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/accesspath/TestSameVariableConstraint.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -26,7 +26,6 @@
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
-import com.bigdata.btree.filter.FilterConstructor;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.btree.proc.ISimpleIndexProcedure;
@@ -52,7 +51,6 @@
import com.bigdata.sparse.LogicalRowSplitHandler;
import com.bigdata.sparse.Schema;
import com.bigdata.sparse.SparseRowStore;
-import com.bigdata.sparse.TPS.TPV;
import cutthecrap.utils.striterators.Resolver;
import cutthecrap.utils.striterators.Striterator;
@@ -375,7 +373,7 @@
/**
* The index in which the file metadata is stored (the index must exist).
*/
- public SparseRowStore getFilleMetadataIndex() {
+ public SparseRowStore getFileMetadataIndex() {
if (fileMetadataIndex == null) {
@@ -553,7 +551,7 @@
metadata.put(FileMetadataSchema.VERSION, AutoIncIntegerCounter.INSTANCE);
// write the metadata (atomic operation).
- final ITPS tps = getFilleMetadataIndex().write(metadataSchema, metadata,
+ final ITPS tps = getFileMetadataIndex().write(metadataSchema, metadata,
AUTO_TIMESTAMP_UNIQUE, null/* filter */, null/*precondition*/);
final int version = (Integer) tps.get(FileMetadataSchema.VERSION).getValue();
@@ -646,7 +644,7 @@
*/
public ITPS readMetadata(final String id, final long timestamp) {
- return getFilleMetadataIndex()
+ return getFileMetadataIndex()
.read(metadataSchema, id, timestamp/* fromTime */,
timestamp + 1/* toTime */, null/* filter */);
@@ -677,7 +675,7 @@
// remove the version identifier if any - we do not want this modified!
metadata.remove(FileMetadataSchema.VERSION);
- return getFilleMetadataIndex().write(metadataSchema, metadata,
+ return getFileMetadataIndex().write(metadataSchema, metadata,
AUTO_TIMESTAMP_UNIQUE, null/* filter */,null/*precondition*/).asMap();
}
@@ -752,7 +750,7 @@
// delete marker.
metadata.put(FileMetadataSchema.VERSION, null);
- getFilleMetadataIndex().write(metadataSchema, metadata, AUTO_TIMESTAMP_UNIQUE,
+ getFileMetadataIndex().write(metadataSchema, metadata, AUTO_TIMESTAMP_UNIQUE,
null/* filter */, null/*precondition*/);
}
@@ -905,7 +903,7 @@
public Iterator<? extends DocumentHeader> getDocumentHeaders(String fromId,
String toId) {
- return new Striterator(getFilleMetadataIndex().rangeIterator(metadataSchema,
+ return new Striterator(getFileMetadataIndex().rangeIterator(metadataSchema,
fromId, toId)).addFilter(new Resolver() {
private static final long serialVersionUID = 1L;
@@ -956,14 +954,13 @@
* range by replacing its VERSION column value with a null value
* (and updating the timestamp in the key).
*/
- getFilleMetadataIndex().getIndex().rangeIterator(
+ getFileMetadataIndex().getIndex().rangeIterator(
fromKey,
toKey,
0/* capacity */,
IRangeQuery.CURSOR,
- new FilterConstructor<TPV>()
- .addFilter(new FileVersionDeleter(
- IRowStoreConstants.AUTO_TIMESTAMP_UNIQUE)));
+ new FileVersionDeleter(
+ IRowStoreConstants.AUTO_TIMESTAMP_UNIQUE));
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -30,7 +30,7 @@
private final long timestamp;
/**
- * The timestamp choosen on the server (not serialized).
+ * The timestamp chosen on the server (not serialized).
*/
transient private long choosenTimestamp;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -144,7 +144,7 @@
*/
public RepositoryDocumentImpl(final BigdataFileSystem repo, final String id) {
- this(repo, id, repo.getFilleMetadataIndex().read(
+ this(repo, id, repo.getFileMetadataIndex().read(
BigdataFileSystem.metadataSchema, id,
IRowStoreConstants.MIN_TIMESTAMP,
IRowStoreConstants.CURRENT_ROW, null/* filter */));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -89,13 +89,19 @@
* arguments or annotations would have to explicitly overrides
* {@link #clone()} in order to set those fields based on the arguments on
* the cloned {@link BOpBase} class.
+ * <p>
+ * Note: This must be at least "effectively" final per the effectively
+ * immutable contract for {@link BOp}s.
*/
- protected final BOp[] args;
+ private final BOp[] args;
/**
* The operator annotations.
+ * <p>
+ * Note: This must be at least "effectively" final per the effectively
+ * immutable contract for {@link BOp}s.
*/
- protected final Map<String,Object> annotations;
+ private final Map<String,Object> annotations;
/**
* Check the operator argument.
@@ -134,6 +140,13 @@
}
}
+// /**
+// * Deserialization constructor (required).
+// */
+// public BOpBase() {
+//
+// }
+
/**
* Deep copy constructor (required).
* <p>
@@ -218,6 +231,29 @@
}
+ /**
+ * Set the value of an operand.
+ * <p>
+ * Note: This protected to facilitate copy-on-write patterns. It is not
+ * public to prevent arbitrary changes to operators outside of methods which
+ * clone the operator and return the modified version. This is part of the
+ * effectively immutable contract for {@link BOp}s.
+ *
+ * @param index
+ * The index.
+ * @param op
+ * The operand.
+ *
+ * @return The old value.
+ *
+ * @todo thread safety and visibility....
+ */
+ final protected void set(final int index, final BOp op) {
+
+ this.args[index] = op;
+
+ }
+
public int arity() {
return args.length;
@@ -341,6 +377,30 @@
}
+ /**
+ * Set an annotation.
+ * <p>
+ * Note: This protected to facilitate copy-on-write patterns. It is not
+ * public to prevent arbitrary changes to operators outside of methods which
+ * clone the operator and return the modified version. This is part of the
+ * effectively immutable contract for {@link BOp}s.
+ *
+ * @param name
+ * The name.
+ * @param value
+ * The value.
+ *
+ * @return The old value.
+ *
+ * @todo thread safety and visibility for concurrent access to and
+ * modifications of the annotations map.
+ */
+ protected void setProperty(final String name, final Object value) {
+
+ annotations.put(name,value);
+
+ }
+
public int getId() {
return (Integer) getRequiredProperty(Annotations.BOP_ID);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -31,12 +31,13 @@
import java.io.Serializable;
import com.bigdata.bop.ap.filter.BOpFilterBase;
+import com.bigdata.bop.ap.filter.BOpTupleFilter;
import com.bigdata.bop.ap.filter.DistinctFilter;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
-import com.bigdata.btree.filter.IFilterConstructor;
+import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
@@ -48,6 +49,9 @@
import com.bigdata.relation.rule.eval.pipeline.JoinMasterTask;
import com.bigdata.striterator.IKeyOrder;
+import cutthecrap.utils.striterators.FilterBase;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* An immutable constraint on the elements visited using an {@link IAccessPath}.
* The slots in the predicate corresponding to variables are named and those
@@ -114,22 +118,34 @@
* change the type of the visited objects will result in a runtime
* exception.
* <p>
+ * Note: This filter must be "aware" of the reuse of tuples within tuple
+ * iterators. See {@link BOpTupleFilter} and {@link TupleFilter} for
+ * starting points.
+ * <p>
* You can chain {@link BOpFilterBase} filters by nesting them inside of
- * one another.
+ * one another. You can chain {@link FilterBase} filters together as
+ * well.
*
* @see #ACCESS_PATH_FILTER
*
- * @see IRangeQuery#rangeIterator(byte[], byte[], int, int,
- * IFilterConstructor)
+ * @see IRangeQuery#rangeIterator(byte[], byte[], int, int, IFilter)
*/
String INDEX_LOCAL_FILTER = "indexLocalFilter";
/**
* An optional {@link BOpFilterBase} to be applied to the elements of
- * the relation as they are materialized from the index. Unlike
- * {@link #INDEX_LOCAL_FILTER}, this an {@link #ACCESS_PATH_FILTER} is never
- * sent to a remote index for evaluation. This makes it possible to
- * impose {@link DistinctFilter} across a {@link #REMOTE_ACCESS_PATH}.
+ * the relation as they are materialized from the index. {@link ITuple}s
+ * are automatically resolved into relation "elements" before this
+ * filter is applied.
+ * <p>
+ * Unlike {@link #INDEX_LOCAL_FILTER}, this an
+ * {@link #ACCESS_PATH_FILTER} is never sent to a remote index for
+ * evaluation. This makes it possible to impose {@link DistinctFilter}
+ * across a {@link #REMOTE_ACCESS_PATH}.
+ * <p>
+ * You can chain {@link BOpFilterBase} filters by nesting them inside of
+ * one another. You can chain {@link FilterBase} filters together as
+ * well.
*/
String ACCESS_PATH_FILTER = "accessPathFilter";
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -108,7 +108,7 @@
public int limit() {
- return (Integer) annotations.get(Annotations.LIMIT);
+ return (Integer) getRequiredProperty(Annotations.LIMIT);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -139,7 +139,7 @@
public String getOnlyRelationName() {
- final String[] relationName = (String[]) annotations.get(Annotations.RELATION_NAME);
+ final String[] relationName = (String[]) getRequiredProperty(Annotations.RELATION_NAME);
if (relationName.length != 1)
throw new IllegalStateException();
@@ -150,7 +150,7 @@
public String getRelationName(final int index) {
- final String[] relationName = (String[]) annotations.get(Annotations.RELATION_NAME);
+ final String[] relationName = (String[]) getRequiredProperty(Annotations.RELATION_NAME);
return relationName[index];
@@ -160,7 +160,7 @@
public int getRelationCount() {
- final String[] relationName = (String[]) annotations.get(Annotations.RELATION_NAME);
+ final String[] relationName = (String[]) getRequiredProperty(Annotations.RELATION_NAME);
return relationName.length;
@@ -171,7 +171,7 @@
// throw new UnsupportedOperationException();
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.RELATION_NAME, relationName);
+ tmp.setProperty(Annotations.RELATION_NAME, relationName);
return tmp;
@@ -200,7 +200,7 @@
final public boolean isOptional() {
- return (Boolean) annotations.get(Annotations.OPTIONAL);
+ return (Boolean) getProperty(Annotations.OPTIONAL,Boolean.FALSE);
}
@@ -215,14 +215,14 @@
@SuppressWarnings("unchecked")
final public IElementFilter<E> getConstraint() {
- return (IElementFilter<E>) annotations.get(Annotations.CONSTRAINT);
+ return (IElementFilter<E>) getProperty(Annotations.CONSTRAINT);
}
@SuppressWarnings("unchecked")
final public ISolutionExpander<E> getSolutionExpander() {
- return (ISolutionExpander<E>) annotations.get(Annotations.EXPANDER);
+ return (ISolutionExpander<E>) getProperty(Annotations.EXPANDER);
}
@@ -304,16 +304,7 @@
continue;
}
- // bound from the binding set.
-// try {
-// final Field f = tmp.getClass().getField("args");
-// f.setAccessible(true);
-// final BOp[] targs = (BOp[]) f.get(tmp);
-// targs[i] = val.clone();
-// } catch (Exception ex) {
-// throw new RuntimeException(ex);
-// }
- tmp.args[i] = val.clone();
+ tmp.set(i, val.clone());
}
@@ -325,30 +316,12 @@
return get(index).get(bindingSet);
-// if (bindingSet == null)
-// throw new IllegalArgumentException();
-//
-// final IVariableOrConstant<?> t = get(index);
-//
-// final IConstant<?> c;
-// if (t.isVar()) {
-//
-// c = bindingSet.get((IVariable<?>) t);
-//
-// } else {
-//
-// c = (IConstant<?>) t;
-//
-// }
-//
-// return c == null ? null : c.get();
-
}
@SuppressWarnings("unchecked")
public IKeyOrder<E> getKeyOrder() {
- return (IKeyOrder<E>) annotations.get(Annotations.KEY_ORDER);
+ return (IKeyOrder<E>) getProperty(Annotations.KEY_ORDER);
}
@@ -356,7 +329,7 @@
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.KEY_ORDER, keyOrder);
+ tmp.setProperty(Annotations.KEY_ORDER, keyOrder);
return tmp;
@@ -373,7 +346,7 @@
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.PARTITION_ID, partitionId);
+ tmp.setProperty(Annotations.PARTITION_ID, partitionId);
return tmp;
@@ -383,7 +356,7 @@
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.BOP_ID, bopId);
+ tmp.setProperty(Annotations.BOP_ID, bopId);
return tmp;
@@ -420,6 +393,7 @@
sb.append(")");
+ final Map<String,Object> annotations = annotations();
if (!annotations.isEmpty()) {
sb.append("[");
boolean first = true;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -31,12 +31,16 @@
import java.util.Map;
import com.bigdata.bop.BOp;
+import com.bigdata.btree.ITupleIterator;
import cutthecrap.utils.striterators.Filter;
import cutthecrap.utils.striterators.Filterator;
/**
* Used to filter for objects which satisfy some criteria.
+ * <p>
+ * <strong>WARNING : DO NOT USE THIS CLASS ON {@link ITupleIterator}s - use
+ * {@link BOpTupleFilter} instead.</strong>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -48,7 +52,16 @@
*/
private static final long serialVersionUID = 1L;
+// /**
+// * Deserialization.
+// */
+// public BOpFilter() {
+// super();
+// }
+
/**
+ * Deep copy.
+ *
* @param op
*/
public BOpFilter(BOpFilter op) {
@@ -56,6 +69,8 @@
}
/**
+ * Shallow copy.
+ *
* @param args
* @param annotations
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -62,6 +62,7 @@
}
/**
+ * Deep copy.
* @param op
*/
public BOpFilterBase(BOpFilterBase op) {
@@ -69,6 +70,7 @@
}
/**
+ * Shallow copy.
* @param args
* @param annotations
*/
@@ -82,7 +84,7 @@
src = filterOnce(src, context);
// wrap source with each additional filter from the filter chain.
- for (BOp arg : args) {
+ for (BOp arg : args()) {
src = ((BOpFilterBase) arg).filter(src, context);
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,80 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop.ap.filter;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+
+/**
+ * A filter which does nothing. This may be used to stack existing
+ * {@link BOpFilterBase} operators.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class BOpFilterNOP extends BOpFilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+// /**
+// * Deserialization.
+// */
+// public BOpFilterNOP() {
+// super();
+// }
+
+ /**
+ * Deep copy.
+ *
+ * @param op
+ */
+ public BOpFilterNOP(BOpFilterNOP op) {
+ super(op);
+ }
+
+ /**
+ * Shallow copy.
+ *
+ * @param args
+ * @param annotations
+ */
+ public BOpFilterNOP(BOp[] args, Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ @Override
+ final protected Iterator filterOnce(Iterator src, Object context) {
+ return src;
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,131 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop.ap.filter;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.btree.AbstractBTree;
+import com.bigdata.btree.BTree;
+import com.bigdata.btree.IRangeQuery;
+import com.bigdata.btree.ITuple;
+import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.Tuple;
+import com.bigdata.btree.filter.TupleFilter;
+import com.bigdata.btree.filter.TupleFilter.TupleFilterator;
+
+/**
+ * <p>
+ * Filter supporting {@link ITupleIterator}s.
+ * </p>
+ * <p>
+ * <strong>Warning: Unlike {@link BOpFilter}, this class correctly uses a second
+ * {@link Tuple} instance to perform filtering.<strong> This is necessary since
+ * the {@link Tuple} instance for the base {@link ITupleIterator}
+ * implementations for the {@link AbstractBTree} is reused by next() on each
+ * call and the {@link TupleFilter} uses one-step lookahead. Failure to use a
+ * second {@link Tuple} instance will result in <em>overwrite</em> of the
+ * current {@link Tuple} with data from the lookahead {@link Tuple}.
+ * </p>
+ * <p>
+ * Note: You must specify {@link IRangeQuery#KEYS} and/or
+ * {@link IRangeQuery#VALS} in order to filter on the keys and/or values
+ * associated with the visited tuples.
+ * </p>
+ * <p>
+ * Note: YOu must specify {@link IRangeQuery#CURSOR} to enabled
+ * {@link Iterator#remove()} for a <em>local</em> {@link BTree}
+ * </p>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class BOpTupleFilter<E> extends BOpFilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+// /**
+// * Deserialization.
+// */
+// public BOpFilter() {
+// super();
+// }
+
+ /**
+ * Deep copy.
+ *
+ * @param op
+ */
+ public BOpTupleFilter(BOpTupleFilter op) {
+ super(op);
+ }
+
+ /**
+ * Shallow copy.
+ *
+ * @param args
+ * @param annotations
+ */
+ public BOpTupleFilter(BOp[] args, Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ @Override
+ final protected Iterator filterOnce(Iterator src, final Object context) {
+
+ return new TupleFilterator((ITupleIterator<E>) src, context,
+ new FilterImpl());
+
+ }
+
+ /**
+ * Return <code>true</code> iff the object should be accepted.
+ *
+ * @param obj
+ * The object.
+ */
+ protected abstract boolean isValid(ITuple<E> obj);
+
+ private class FilterImpl extends TupleFilter<E> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected boolean isValid(ITuple<E> tuple) {
+
+ return BOpTupleFilter.this.isValid(tuple);
+
+ }
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java (from rev 3554, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,374 @@
+package com.bigdata.bop.ap.filter;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.CognitiveWeb.extser.LongPacker;
+
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IConstant;
+import com.bigdata.bop.IElement;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.relation.accesspath.IAccessPath;
+
+/**
+ * Filter imposes the "same variable" constraint on the elements visited by an
+ * {@link IAccessPath}. The filter is required IFF a {@link IVariable} appears
+ * in more than one position for the {@link IPredicate} associated with the
+ * {@link IAccessPath}. For example, in <code>spo(?g, p1, o1, ?g)</code>, the
+ * variable <i>g</i> shows up at both index ZERO (0) and index THREE (3).
+ * <p>
+ * An instance of the filter is created by passing in an {@link IPredicate}. The
+ * filter creates an array of variables which appear more than once and the
+ * index at which those variables appear in the predicate. The filter then does
+ * the minimum amount of work and just compares the values found in the
+ * different slots in which each variable appears for only those variables which
+ * appear more than once in the {@link IPredicate}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ * @param <E>
+ * The generic type of the elements that will be tested by the
+ * filter.
+ */
+public class SameVariableConstraint<E> implements Externalizable {
+
+ /**
+ * The predicate template.
+ */
+ private IPredicate<E> p;
+
+ /**
+ * An array containing one or more records, each of which has the form
+ *
+ * <pre>
+ * [n,index[0], index[1], ... index[n-1]]
+ * </pre>
+ *
+ * , where <i>n</i> is the number of occurrences of some variable and
+ * <i>index[i]</i>, for <i>i</i> in <code>[0,n-1]</code>, gives each index
+ * in the predicate at which that variable appears.
+ * <p>
+ * For example, given the predicate
+ *
+ * <pre>
+ * spo(?g,p1,o1,?g)
+ * </pre>
+ *
+ * the array would be coded as one record
+ *
+ * <pre>
+ * [2, 0, 3]
+ * </pre>
+ *
+ * Given the predicate
+ *
+ * <pre>
+ * foo(?g,?h,?h,?g,4,?h)
+ * </pre>
+ *
+ * the array would be coded as two records
+ *
+ * <pre>
+ * [[2, 0, 3], [3, 1, 2, 5]]
+ * </pre>
+ *
+ * where the records are indicated by the nested square brackets.
+ */
+ protected int[] indices;
+
+ /**
+ * De-serialization constructor.
+ */
+ public SameVariableConstraint() {
+
+ }
+
+ public SameVariableConstraint(final IPredicate<E> p, final int[] indices) {
+
+ if (p == null)
+ throw new IllegalArgumentException();
+
+ if (indices == null)
+ throw new IllegalArgumentException();
+
+ this.p = p;
+
+ this.indices = indices;
+
+ }
+
+// public boolean canAccept(final Object o) {
+// return true;
+// }
+
+ public boolean accept(final E e) {
+
+ int i = 0;
+ while (i < indices.length) {
+
+ // the #of slots at which that variable appears.
+ final int nslots = indices[i];
+
+ assert nslots >= 2 : "i=" + i + ", nslots=" + nslots + ", indices="
+ + Arrays.toString(indices);
+
+ assert i + nslots < indices.length;
+
+ i++;
+
+ final int firstIndex = indices[i];
+
+ @SuppressWarnings("unchecked")
+ final IConstant<?> c0 = new Constant(((IElement) e).get(firstIndex));
+// final IConstant<?> c0 = p.get(e, firstIndex);
+
+ for (int j = 0; j < nslots; j++, i++) {
+
+ if (j == firstIndex) {
+
+ // no need to compare with self.
+ continue;
+
+ }
+
+ final int thisIndex = indices[i];
+
+ @SuppressWarnings("unchecked")
+ final IConstant<?> c1 = new Constant(((IElement) e)
+ .get(thisIndex));
+// final IConstant<?> c1 = p.get(e, thisIndex);
+
+ // same reference (including null).
+ if (c0 == c1)
+ continue;
+
+ if (c0 != null && c1 == null) {
+
+ return false;
+
+ }
+
+ if (c1 == null && c0 != null) {
+
+ return false;
+
+ }
+
+ if (!c0.equals(c1)) {
+
+ // different constant.
+ return false;
+
+ }
+
+ }
+
+ }
+
+ return true;
+
+ }
+
+ public String toString() {
+
+ return super.toString() + "{p=" + p + ", indices="
+ + Arrays.toString(indices) + "}";
+
+ }
+
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
+ @SuppressWarnings("unchecked")
+ public void readExternal(ObjectInput in) throws IOException,
+ ClassNotFoundException {
+
+ final short version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
+ p = (IPredicate<E>) in.readObject();
+
+ final int len = (int) LongPacker.unpackLong(in);
+
+ indices = new int[len];
+
+ for (int i = 0; i < len; i++) {
+
+ indices[i] = (int) LongPacker.unpackLong(in);
+
+ }
+
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+
+ out.writeByte(VERSION);
+
+ out.writeObject(p);
+
+ LongPacker.packLong(out, indices.length);
+
+ for (int i = 0; i < indices.length; i++) {
+
+ LongPacker.packLong(out, indices[i]);
+
+ }
+
+ }
+
+ /**
+ * The filter is only created and populated for variables which appear more
+ * than once in the predicate. If there are no variables which appear more
+ * than once, then the filter IS NOT created.
+ *
+ * @param p
+ * The predicate.
+ *
+ * @param <E>
+ * The generic type of the elements to be tested by the filter.
+ *
+ * @return The filter -or- <code>null</code> iff no variables appear more
+ * than once in the {@link IPredicate}.
+ */
+ public static <E> SameVariableConstraint<E> newInstance(
+ final IPredicate<E> p) {
+
+ // #of slots in this predicate.
+ final int arity = p.arity();
+
+ // map exists iff variables are used more than once.
+ Map<IVariableOrConstant<?>, Integer> vars = null;
+
+ // #of occurrences across all variables which are used more than once.
+ int noccurs = 0;
+
+ // #of variables which are reused within the predicate.
+ int nreused = 0;
+
+ {
+
+ for (int i = 0; i < arity; i++) {
+
+ final IVariableOrConstant<?> t = p.get(i);
+
+ if (t != null && t.isVar()) {
+
+ // this slot is a variable.
+
+ if (vars == null) {
+
+ vars = new LinkedHashMap<IVariableOrConstant<?>, Integer>();
+
+ }
+
+ Integer cnt = vars.get(t);
+
+ if (cnt == null) {
+
+ vars.put(t, cnt = Integer.valueOf(1));
+
+ } else {
+
+ final int tmp = cnt.intValue() + 1;
+
+ vars.put(t, cnt = Integer.valueOf(tmp));
+
+ // note: add 2 for the 1st dup and 1 for each
+ // additional dup thereafter.
+ noccurs += (tmp == 2 ? 2 : 1);
+
+ if (tmp == 2) {
+
+ // this variable is reused.
+ nreused++;
+
+ }
+
+ }
+
+ }
+
+ }
+
+ if (nreused == 0) {
+
+ // there are no duplicate variables.
+ return null;
+
+ }
+
+ }
+
+ assert vars != null;
+
+ // allocate the array for the reused variable record(s).
+ final int[] indices = new int[nreused + noccurs];
+
+ // populate that array.
+ {
+
+ int i = 0;
+
+ final Iterator<Map.Entry<IVariableOrConstant<?>, Integer>> itr = vars
+ .entrySet().iterator();
+
+ while (itr.hasNext()) {
+
+ final Map.Entry<IVariableOrConstant<?>, Integer> e = itr.next();
+
+ final int nused = e.getValue().intValue();
+
+ if (nused < 2)
+ continue;
+
+ final IVariable<?> var = (IVariable<?>)e.getKey();
+
+ indices[i++] = nused;
+
+ for (int j = 0; j < arity; j++) {
+
+ final IVariableOrConstant<?> t = p.get(j);
+
+ if (t == var) {
+
+ // variable appears at this index.
+ indices[i++] = j;
+
+ }
+
+ }
+
+ }
+
+ assert i == indices.length;
+
+ }
+
+ return new SameVariableConstraint<E>(p, indices);
+
+ } // factory method.
+
+}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,105 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop.ap.filter;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+
+import cutthecrap.utils.striterators.Filter;
+import cutthecrap.utils.striterators.Filterator;
+
+/**
+ * Operator imposing a {@link SameVariableConstraint} filter.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class SameVariableConstraintFilter extends BOpFilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BOpFilterBase.Annotations {
+
+ /**
+ * The constraint to be imposed (required).
+ */
+ String FILTER = SameVariableConstraintFilter.class.getName()
+ + ".filter";
+
+ }
+
+ /**
+ * @param op
+ */
+ public SameVariableConstraintFilter(SameVariableConstraintFilter op) {
+ super(op);
+ }
+
+ /**
+ * @param args
+ * @param annotations
+ */
+ public SameVariableConstraintFilter(BOp[] args,
+ Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ @Override
+ final protected Iterator filterOnce(Iterator src, final Object context) {
+
+ final SameVariableConstraint filter = (SameVariableConstraint) getRequiredProperty(Annotations.FILTER);
+
+ return new Filterator(src, context, new FilterImpl(filter));
+
+ }
+
+ static private class FilterImpl extends Filter {
+
+ private static final long serialVersionUID = 1L;
+
+ private final SameVariableConstraint filter;
+
+ public FilterImpl(final SameVariableConstraint filter) {
+ this.filter = filter;
+ }
+
+ @Override
+ protected boolean isValid(Object obj) {
+
+ return filter.accept(obj);
+
+ }
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-28 23:39:42 UTC (rev 3672)
@@ -492,25 +492,8 @@
* <li></li>
* </ul>
+------------------------------------------------------------
-Martyn,
-
-Rationalizing CTC striterators, bigdata striterators, and access paths:
-
-Dropped the ctc_utils jar and added the source files to the classpath.
-
-I've added an IPropertySet interface:
-
-Object getProperty(String name).
-
-There are currently several varieties of IFilter. Each one knows how to create its corresponding implementation class. This exactly parallels the bops, where a concrete bop such as MemorySortOp, knows how to create the corresponding runtime class, MemorySortOp.SortTask. I've added a filterOnce() method to provide the semantics of wrapping an iterator with just that IFilter. The existing filter() method now wraps the chain, starting with _this_ IFilter and then wrapping anything which was added to the chain. The chain is empty if nothing was added. I've modified the various IFilter implementations to conform with this.
-
-I've modified all of the implementation classes (Appenderator, etc.) to accept the new context argument regardless of whether they use it. I have also modified them to always have the reference to the IFilter object so the IPropertySet will be accessible.
-
-Modified Uniqueness filter to use a linked hash map.
-
-Modified Sorter uses Arrays.sort() rather than TreeSort.
-
Some problems:
- Should Appender simply pass the other iterator as m_state to the FilterBase?
@@ -527,21 +510,43 @@
Write unit tests to sanity check these changes.
-Write BOp versions of Append, Expand, etc. These will extend BOpBase.
+done. Replace (I)FilterConstructor with IFilter. FilterConstructor
+implemented clone. I am not sure that I can recall why. Presumably
+because a cloned would have a distinct filter chain. We can not
+easily override clone() on FilterBase because of all the derived
+versions. We could have a deep copy constructor pattern, which is
+what I did for the BOpBase class and then use that for clone.
-The following places directly invoke filterOnce(src,ctx). The first two should be modified to pass in the correct context object. That might require raising a context object into an IRangeIterator parameter.
+Write unit tests at the IPredicate/AccessPath level to use stackable
+filters (for the LOCAL and REMOTE access paths). Write unit tests for
+correct range counts with and without local/remote filters. Write
+unit tests for caching of those range counts.
-AbstractBTree#2933
-FusedView#1281
-TestRemoverator#86
+- The stacking of the BOpFilters may be counter intuitive or just
+ wrong.
-Replace (I)FilterConstructor with IFilter. FilterConstructor implemented clone. I am not sure that I can recall why. Presumably because a cloned would have a distinct filter chain. We can not easily override clone() on FilterBase because of all the derived versions. We could have a deep copy constructor pattern, which is what I did for the BOpBase class and then use that for clone.
+- TestPredicateAccessPath#test_localFilter() does not report the
+ correct tuples. This is weirdness with reuse of the same backing
+ Tuple object. See TupleFilter and BOpFilter. The filter that is
+ being applied in the unit test does not protect against the tuple
+ reuse. It would be best to automatically wrap such asserts. E.g.,
+ BOpTupleFilter. It would also be good to catch runtime type
+ problems with this. E.g., if the source iterator is ITupleFilter
+ then it is an error to wrap with BOpFilter. You must use
+ BOpTupleFilter instead.
-Replace the IPredicate "CONSTRAINT" with "REMOTE_FILTERS" (an IFilter which is evaluated at the BTree) and "LOCAL_FILTERS" (an IFilter which is evaluated against the range iterator once it is materialized on the local node). If the index is local, then the same concepts apply but we do not have to serialize the REMOTE_FILTERS. For the moment, the AccessPath automatically resolves ITuples to relation elements and adds the SameVariableConstraint if necessary. We might shift the burden of responsibility for this to the query planner in the future.
+- Write BOp versions for TupleResolver, ...
-Write unit tests at the IPredicate/AccessPath level to use stackable filters.
+- All of the Filterators need to use deferred prefetch. Prefetch
+ during the constructor causes problems when we are stacking filters
+ using FilterBase.
-Fixed the build.xml script to accomodate the class path changes.
+IPredicate#getConstraint() must go. This will require touching the
+backchainer classes. The (SPO|Magic)Predicate contructors should also
+be touched up since they will otherwise DROP the local and remote
+filters.
-Thanks,
-Bryan
+Reconcile IElementFilter and implementations.
+
+Reconcile the com.bigdata.striterators package. Can we get rid of it?
+Incrementally?
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -54,9 +54,9 @@
import com.bigdata.btree.data.IAbstractNodeData;
import com.bigdata.btree.data.ILeafData;
import com.bigdata.btree.data.INodeData;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.filter.Reverserator;
import com.bigdata.btree.filter.TupleRemover;
+import com.bigdata.btree.filter.WrappedTupleIterator;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor;
@@ -92,6 +92,8 @@
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* <p>
* Base class for mutable and immutable B+-Tree implementations.
@@ -2699,7 +2701,7 @@
final public ITupleIterator rangeIterator(Object fromKey, Object toKey,
final int capacity,//
final int flags,//
- final IFilterConstructor filter//
+ final IFilter filter//
) {
fromKey = fromKey == null ? null : metadata.getTupleSerializer()
@@ -2728,7 +2730,7 @@
* iterator.
* <p>
* Note:
- * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)}
+ * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilter)}
* is also responsible for constructing an {@link ITupleIterator} in a
* manner similar to this method. If you are updating the logic here, then
* check the logic in that method as well!
@@ -2741,7 +2743,7 @@
final byte[] toKey,//
final int capacityIsIgnored,//
final int flags,//
- final IFilterConstructor filter//
+ final IFilter filter//
) {
// btreeCounters.nrangeIterator.incrementAndGet();
@@ -2911,8 +2913,9 @@
* REMOVEALL (those are the assumptions for the flags).
*/
- src = filter.newInstance(src);
-
+ src = new WrappedTupleIterator(filter
+ .filter(src, null/* context */));
+
}
if ((flags & REMOVEALL) != 0) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -34,7 +34,6 @@
import org.apache.log4j.Logger;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.keys.SuccessorUtil;
import com.bigdata.io.ByteArrayBuffer;
import com.bigdata.io.DataInputBuffer;
@@ -44,6 +43,8 @@
import com.bigdata.journal.TimestampUtility;
import com.bigdata.rawstore.IBlock;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* A chunked iterator that proceeds a {@link ResultSet} at a time. This
* introduces the concept of a {@link #continuationQuery()} so that the iterator
@@ -99,7 +100,7 @@
/**
* Optional filter.
*/
- protected final IFilterConstructor filter;
+ protected final IFilter filter;
/**
* The #of range query operations executed.
@@ -253,7 +254,7 @@
public AbstractChunkedTupleIterator(final byte[] fromKey,
final byte[] toKey, int capacity, final int flags,
- final IFilterConstructor filter) {
+ final IFilter filter) {
if (capacity < 0) {
@@ -286,7 +287,7 @@
* @return
*/
abstract protected ResultSet getResultSet(long timestamp,byte[] fromKey, byte[] toKey,
- int capacity, int flags, IFilterConstructor filter);
+ int capacity, int flags, IFilter filter);
/**
* Issues the original range query.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -37,7 +37,6 @@
import java.util.Set;
import java.util.SortedMap;
-import com.bigdata.btree.filter.FilterConstructor;
import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.journal.ConcurrencyManager;
@@ -381,7 +380,7 @@
final ITupleIterator itr = ndx.rangeIterator(fromKey, toKey,
0/* capacity */, IRangeQuery.VALS/* flags */,
- new FilterConstructor().addFilter(new TupleFilter() {
+ new TupleFilter() {
private static final long serialVersionUID = 1L;
@@ -391,7 +390,7 @@
return BytesUtil.bytesEqual(tuple.getValue(), val);
}
- }));
+ });
while(itr.hasNext()) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -32,12 +32,13 @@
import java.nio.ByteBuffer;
import java.util.Iterator;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.filter.ITupleFilter;
import com.bigdata.io.ByteBufferInputStream;
import com.bigdata.rawstore.IBlock;
import com.bigdata.rawstore.IRawStore;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* Chunked range iterator running against a local index or index view.
* <p>
@@ -70,7 +71,7 @@
* @param filter
*/
public ChunkedLocalRangeIterator(IIndex ndx, byte[] fromKey, byte[] toKey,
- int capacity, int flags, IFilterConstructor filter) {
+ int capacity, int flags, IFilter filter) {
super(fromKey, toKey, capacity, flags, filter);
@@ -92,7 +93,7 @@
@Override
protected ResultSet getResultSet(final long timestamp,
final byte[] fromKey, final byte[] toKey, final int capacity,
- final int flags, final IFilterConstructor filter) {
+ final int flags, final IFilter filter) {
/*
* Note: This turns off the REMOVEALL flag unless CURSOR was also
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -28,7 +28,6 @@
package com.bigdata.btree;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor;
import com.bigdata.btree.proc.IKeyRangeIndexProcedure;
import com.bigdata.btree.proc.IResultHandler;
@@ -36,6 +35,8 @@
import com.bigdata.counters.ICounterSet;
import com.bigdata.mdi.IResourceMetadata;
+import cutthecrap.utils.striterators.IFilter;
+
/**
...
[truncated message content] |
|
From: <mar...@us...> - 2010-09-29 11:16:22
|
Revision: 3673
http://bigdata.svn.sourceforge.net/bigdata/?rev=3673&view=rev
Author: martyncutcher
Date: 2010-09-29 11:16:15 +0000 (Wed, 29 Sep 2010)
Log Message:
-----------
Process child filters prior to calling filterOnce for more intuitive initialization
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/E.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java 2010-09-28 23:39:42 UTC (rev 3672)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java 2010-09-29 11:16:15 UTC (rev 3673)
@@ -80,9 +80,6 @@
final public Iterator filter(Iterator src, final Object context) {
- // wrap src with _this_ filter.
- src = filterOnce(src, context);
-
// wrap source with each additional filter from the filter chain.
for (BOp arg : args()) {
@@ -90,6 +87,9 @@
}
+ // wrap src with _this_ filter.
+ src = filterOnce(src, context);
+
return src;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 11:16:15 UTC (rev 3673)
@@ -82,9 +82,15 @@
return new TupleFilter.TupleFilterator((ITupleIterator) src, context, this);
}
+/*
+ public Iterator filterOnce(Iterator src, Object context) {
- abstract protected boolean isValid(ITuple<E> tuple);
+ return new TupleFilter.Filterator((ITupleIterator) src);
+ }
+*/
+ abstract protected boolean isValid(ITuple<E> tuple);
+
/**
* Implementation class knows how to avoid side-effects from the reuse of
* the same {@link Tuple} instance by the base {@link ITupleIterator} impls.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/E.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/E.java 2010-09-28 23:39:42 UTC (rev 3672)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/E.java 2010-09-29 11:16:15 UTC (rev 3673)
@@ -89,4 +89,8 @@
return true;
}
+ public int hashCode() {
+ return name.hashCode() + value.hashCode();
+ }
+
}
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java 2010-09-28 23:39:42 UTC (rev 3672)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java 2010-09-29 11:16:15 UTC (rev 3673)
@@ -108,14 +108,17 @@
}
final public Iterator filter(Iterator src, final Object context) {
- // wrap src with _this_ filter.
- src = filterOnce(src, context);
+ // makes most sense to consider the filterchain as preprocessing the
+ // src prior to application of this filter.
if (filterChain != null) {
// wrap source with each additional filter from the filter chain.
for (IFilter filter : filterChain) {
src = filter.filter(src, context);
}
}
+
+ // wrap src with _this_ filter.
+ src = filterOnce(src, context);
return src;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mar...@us...> - 2010-09-29 15:37:02
|
Revision: 3677
http://bigdata.svn.sourceforge.net/bigdata/?rev=3677&view=rev
Author: martyncutcher
Date: 2010-09-29 15:36:55 +0000 (Wed, 29 Sep 2010)
Log Message:
-----------
Remove references to m_state Filter member field
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -94,14 +94,7 @@
private final LinkedHashSet members;
public DistinctFilterImpl() {
- this(null);
- }
- @SuppressWarnings("unchecked")
- public DistinctFilterImpl(final Object state) {
-
- super(state);
-
members = new LinkedHashSet(getInitialCapacity(), getLoadFactor());
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -64,17 +64,8 @@
public TupleFilter() {
- this(null/* state */);
-
}
- public TupleFilter(final Object state) {
-
-// this.state = state;
- super(state);
-
- }
-
@SuppressWarnings("unchecked")
@Override
public ITupleIterator<E> filterOnce(final Iterator src,Object context) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -25,12 +25,6 @@
}
- public TupleUpdater(Object state) {
-
- super(state);
-
- }
-
@SuppressWarnings("unchecked")
@Override
public ITupleIterator<E> filterOnce(Iterator src, Object context) {
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -29,14 +29,16 @@
* Derived from Filter, and excludes a single object from the iteration.
*/
public class ExclusionFilter extends Filter {
- public ExclusionFilter(Object state) {
- super(state);
+ private Object m_exclude;
+
+ public ExclusionFilter(Object exclude) {
+ m_exclude = exclude;
}
/***********************************************************************
* Just make sure that the current object is not the one to be excluded.
**/
protected boolean isValid(Object obj) {
- return obj != m_state;
+ return obj != m_exclude;
}
}
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Expander.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -39,10 +39,6 @@
public Expander() { }
- public Expander(Object state) {
- super(state);
- }
-
//-------------------------------------------------------------
@Override
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -39,10 +39,6 @@
public Filter() {}
- public Filter(Object state) {
- super(state);
- }
-
//-------------------------------------------------------------
@Override
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -57,28 +57,10 @@
*/
/*private*/ volatile Map<String, Object> annotations;
- /**
- * State from the constructor (optional).
- * <p>
- * Note: Striterators should not have a side-effect on state object since
- * that can have unexpected consequences if the {@link IFilter} is reused.
- */
- final protected Object m_state;
-
public FilterBase() {
- m_state = null;
}
/**
- *
- * @param state
- * State (optional).
- */
- public FilterBase(final Object state) {
- m_state = state;
- }
-
- /**
* Add a filter to the end of this filter chain.
*
* @param filter
@@ -202,8 +184,7 @@
public String toString() {
final StringBuilder sb = new StringBuilder();
sb.append(super.toString());
- sb.append("{state=" + m_state);
- sb.append(",annotations=" + annotations);
+ sb.append("{annotations=" + annotations);
sb.append(",filterChain=" + filterChain);
sb.append("}");
return sb.toString();
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Resolver.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -39,10 +39,6 @@
public Resolver() {}
- public Resolver(Object state) {
- super(state);
- }
-
//-------------------------------------------------------------
@Override
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -136,10 +136,10 @@
}
/** check each object against cls.isInstance(object) **/
- public IStriterator addTypeFilter(Class cls) {
- addFilter(new Filter(cls) {
+ public IStriterator addTypeFilter(final Class cls) {
+ addFilter(new Filter() {
protected boolean isValid(Object obj) {
- boolean ret = ((Class) m_state).isInstance(obj);
+ boolean ret = cls.isInstance(obj);
return ret;
}
@@ -149,10 +149,10 @@
}
/** check each object against cls.isInstance(object) **/
- public IStriterator addInstanceOfFilter(Class cls) {
- addFilter(new Filter(cls) {
+ public IStriterator addInstanceOfFilter(final Class cls) {
+ addFilter(new Filter() {
protected boolean isValid(Object obj) {
- return obj.getClass() == m_state;
+ return obj == null ? false : obj.getClass() == obj;
}
} );
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java 2010-09-29 15:29:12 UTC (rev 3676)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
@@ -43,7 +43,6 @@
private final LinkedHashSet<Object> m_members = new LinkedHashSet<Object>();
public UniquenessFilter() {
- super(null);
}
/***********************************************************************
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-29 15:48:43
|
Revision: 3678
http://bigdata.svn.sourceforge.net/bigdata/?rev=3678&view=rev
Author: thompsonbry
Date: 2010-09-29 15:48:34 +0000 (Wed, 29 Sep 2010)
Log Message:
-----------
Cleaned up the use of the "constraint" annotation. The code should now use INDEX_LOCAL_FILTER or ACCESS_PATH_FILTER everywhere.
Cleaned up a lot of the SPOPredicate and MagicPredicate constructors.
Reconciled some changes in the striterators package.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/query-cost-model.xls
branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/segment math.xls
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/CounterSet.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractArrayBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractUnsynchronizedArrayBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/SolutionFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsynchronizedSolutionBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedWrappedIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestDefaultEvaluationPlan.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/AbstractSPOBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/RdfTypeRdfsResourceFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/IRISUtils.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicIndexWriter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/BackchainAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/DoNotAddFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/MatchRule.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ExplicitSPOFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphBinarySearchFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InGraphHashSetFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/InferredSPOFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NoAxiomFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOArrayIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestIRIS.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/magic/TestMagicStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOStarJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/ExclusionFilter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/IFilterTest.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/query-cost-model.xls
===================================================================
(Binary files differ)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/segment math.xls
===================================================================
(Binary files differ)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOp.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -175,12 +175,15 @@
* @see Annotations#MUTATION
*/
boolean isMutation();
-
+
/**
* The timestamp or transaction identifier on which the operator will read
* or write.
*
* @see Annotations#TIMESTAMP
+ *
+ * @throws IllegalStateException
+ * if {@link Annotations#TIMESTAMP} was not specified.
*/
long getTimestamp();
@@ -238,7 +241,7 @@
/**
* The timestamp (or transaction identifier) used by this operator if it
- * reads or writes on the database.
+ * reads or writes on the database (no default).
*
* @see #MUTATION
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -287,7 +287,7 @@
}
/** deep copy the arguments. */
- static private BOp[] deepCopy(final BOp[] a) {
+ static protected BOp[] deepCopy(final BOp[] a) {
final BOp[] t = new BOp[a.length];
for (int i = 0; i < a.length; i++) {
t[i] = a[i] == null ? null : a[i].clone();
@@ -311,7 +311,7 @@
* containing an ontology or some conditional assertions with a query
* plan.
*/
- static private Map<String,Object> deepCopy(final Map<String,Object> a) {
+ static protected Map<String,Object> deepCopy(final Map<String,Object> a) {
// allocate map.
final Map<String, Object> t = new LinkedHashMap<String, Object>(a
.size());
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -211,7 +211,7 @@
private static final long serialVersionUID = 1L;
@Override
- protected boolean isValid(Object arg0) {
+ public boolean isValid(Object arg0) {
return arg0 instanceof BOp;
}
});
@@ -314,7 +314,7 @@
private static final long serialVersionUID = 1L;
@Override
- protected boolean isValid(Object arg0) {
+ public boolean isValid(Object arg0) {
return arg0 instanceof IVariable<?>;
}
});
@@ -338,7 +338,7 @@
private static final long serialVersionUID = 1L;
@Override
- protected boolean isValid(final Object arg0) {
+ public boolean isValid(final Object arg0) {
return arg0 instanceof IVariable<?>;
}
});
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -41,6 +41,7 @@
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
+import com.bigdata.relation.accesspath.ElementFilter;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IElementFilter;
import com.bigdata.relation.rule.IRule;
@@ -95,33 +96,36 @@
*/
String OPTIONAL = "optional";
- /**
- * Constraints on the elements read from the relation.
- *
- * @deprecated This is being replaced by two classes of filters. One
- * which is always evaluated local to the index and one
- * which is evaluated in the JVM in which the access path is
- * evaluated once the {@link ITuple}s have been resolved to
- * elements of the relation.
- */
- String CONSTRAINT = "constraint";
+// /**
+// * Constraints on the elements read from the relation.
+// *
+// * @deprecated This is being replaced by two classes of filters. One
+// * which is always evaluated local to the index and one
+// * which is evaluated in the JVM in which the access path is
+// * evaluated once the {@link ITuple}s have been resolved to
+// * elements of the relation.
+// * <p>
+// * This was historically an {@link IElementFilter}, which is
+// * an {@link IFilterTest}. {@link #INDEX_LOCAL_FILTER} is an
+// * {@link IFilter}. You can wrap the {@link IFilterTest} as
+// * an {@link IFilter} using {@link ElementFilter}.
+// */
+// String CONSTRAINT = "constraint";
/**
- * An optional {@link BOpFilterBase} that will be evaluated local to the
- * to the index. When the index is remote, the filter will be sent to
- * the node on which the index resides and evaluated there. This makes
- * it possible to efficiently filter out tuples which are not of
- * interest for a given access path.
+ * An optional {@link IFilter} that will be evaluated local to the to
+ * the index. When the index is remote, the filter will be sent to the
+ * node on which the index resides and evaluated there. This makes it
+ * possible to efficiently filter out tuples which are not of interest
+ * for a given access path.
* <p>
* Note: The filter MUST NOT change the type of data visited by the
* iterator - it must remain an {@link ITupleIterator}. An attempt to
* change the type of the visited objects will result in a runtime
- * exception.
+ * exception. This filter must be "aware" of the reuse of tuples within
+ * tuple iterators. See {@link BOpTupleFilter}, {@link TupleFilter} and
+ * {@link ElementFilter} for starting points.
* <p>
- * Note: This filter must be "aware" of the reuse of tuples within tuple
- * iterators. See {@link BOpTupleFilter} and {@link TupleFilter} for
- * starting points.
- * <p>
* You can chain {@link BOpFilterBase} filters by nesting them inside of
* one another. You can chain {@link FilterBase} filters together as
* well.
@@ -337,20 +341,36 @@
*/
public ISolutionExpander<E> getSolutionExpander();
+// /**
+// * An optional constraint on the visitable elements.
+// *
+// * @see Annotations#CONSTRAINT
+// *
+// * @deprecated This is being replaced by two classes of filters. One which
+// * is always evaluated local to the index and one which is
+// * evaluated in the JVM in which the access path is evaluated
+// * once the {@link ITuple}s have been resolved to elements of
+// * the relation.
+// */
+// public IElementFilter<E> getConstraint();
+
/**
- * An optional constraint on the visitable elements.
+ * Return the optional filter to be evaluated local to the index.
*
- * @see Annotations#CONSTRAINT
- *
- * @deprecated This is being replaced by two classes of filters. One which
- * is always evaluated local to the index and one which is
- * evaluated in the JVM in which the access path is evaluated
- * once the {@link ITuple}s have been resolved to elements of
- * the relation.
+ * @see Annotations#INDEX_LOCAL_FILTER
*/
- public IElementFilter<E> getConstraint();
+ public IFilter getIndexLocalFilter();
/**
+ * Return the optional filter to be evaluated once tuples have been
+ * converted into relation elements by the access path (local to the
+ * caller).
+ *
+ * @see Annotations#ACCESS_PATH_FILTER
+ */
+ public IFilter getAccessPathFilter();
+
+ /**
* Return the {@link IKeyOrder} assigned to this {@link IPredicate} by the
* query optimizer.
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -28,6 +28,7 @@
package com.bigdata.bop.ap;
+import java.util.Iterator;
import java.util.Map;
import cern.colt.Arrays;
@@ -42,12 +43,14 @@
import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.NV;
-import com.bigdata.btree.ITuple;
-import com.bigdata.journal.ITx;
+import com.bigdata.relation.accesspath.ElementFilter;
import com.bigdata.relation.accesspath.IElementFilter;
import com.bigdata.relation.rule.ISolutionExpander;
import com.bigdata.striterator.IKeyOrder;
+import cutthecrap.utils.striterators.FilterBase;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* A generic implementation of an immutable {@link IPredicate}.
*
@@ -81,20 +84,31 @@
}
/**
- * Simplified ctor.
- *
- * @param values
- * The values (order is important!).
- * @param relationName
- * Identifies the relation to be queried.
+ * Variable argument version of the shallow constructor.
+ * @param vars
+ * @param annotations
*/
- public Predicate(final IVariableOrConstant<?>[] values,
- final String relationName) {
-
- this(values, relationName, -1/* partitionId */, false/* optional */,
- null/* constraint */, null/* expander */, ITx.READ_COMMITTED);
-
+ public Predicate(final BOp[] vars, final NV... annotations) {
+
+ super(vars, NV.asMap(annotations));
+
}
+
+// /**
+// * Simplified ctor.
+// *
+// * @param values
+// * The values (order is important!).
+// * @param relationName
+// * Identifies the relation to be queried.
+// */
+// public Predicate(final IVariableOrConstant<?>[] values,
+// final String relationName) {
+//
+// this(values, relationName, -1/* partitionId */, false/* optional */,
+// null/* constraint */, null/* expander */, ITx.READ_COMMITTED);
+//
+// }
/**
*
@@ -121,7 +135,8 @@
new NV(Annotations.RELATION_NAME,new String[]{relationName}),//
new NV(Annotations.PARTITION_ID,partitionId),//
new NV(Annotations.OPTIONAL,optional),//
- new NV(Annotations.CONSTRAINT,constraint),//
+ new NV(Annotations.INDEX_LOCAL_FILTER,
+ ElementFilter.newInstance(constraint)),//
new NV(Annotations.EXPANDER,expander),//
new NV(Annotations.TIMESTAMP, timestamp)
}));
@@ -204,21 +219,33 @@
}
- /**
- *
- * @deprecated This is being replaced by two classes of filters. One which
- * is always evaluated local to the index and one which is
- * evaluated in the JVM in which the access path is evaluated
- * once the {@link ITuple}s have been resolved to elements of
- * the relation.
- */
- @SuppressWarnings("unchecked")
- final public IElementFilter<E> getConstraint() {
+// /**
+// *
+// * @deprecated This is being replaced by two classes of filters. One which
+// * is always evaluated local to the index and one which is
+// * evaluated in the JVM in which the access path is evaluated
+// * once the {@link ITuple}s have been resolved to elements of
+// * the relation.
+// */
+// @SuppressWarnings("unchecked")
+// final public IElementFilter<E> getConstraint() {
+//
+// return (IElementFilter<E>) getProperty(Annotations.CONSTRAINT);
+//
+// }
- return (IElementFilter<E>) getProperty(Annotations.CONSTRAINT);
+ final public IFilter getIndexLocalFilter() {
+ return (IFilter) getProperty(Annotations.INDEX_LOCAL_FILTER);
+
}
+ final public IFilter getAccessPathFilter() {
+
+ return (IFilter) getProperty(Annotations.ACCESS_PATH_FILTER);
+
+ }
+
@SuppressWarnings("unchecked")
final public ISolutionExpander<E> getSolutionExpander() {
@@ -362,6 +389,96 @@
}
+ public Predicate<E> setTimestamp(final long timestamp) {
+
+ final Predicate<E> tmp = this.clone();
+
+ tmp.setProperty(Annotations.TIMESTAMP, timestamp);
+
+ return tmp;
+
+ }
+
+ /**
+ * Add an {@link Annotations#INDEX_LOCAL_FILTER}. When there is a filter for
+ * the named property, the filters are combined. Otherwise the filter is
+ * set.
+ *
+ * @param filter
+ * The filter.
+ *
+ * @return The new predicate to which that filter was added.
+ */
+ public Predicate<E> addIndexLocalFilter(final IFilter filter) {
+
+ final Predicate<E> tmp = this.clone();
+
+ tmp.addFilter(Annotations.INDEX_LOCAL_FILTER, filter);
+
+ return tmp;
+
+ }
+
+ /**
+ * Add an {@link Annotations#INDEX_LOCAL_FILTER}. When there is a filter for
+ * the named property, the filters are combined. Otherwise the filter is
+ * set.
+ *
+ * @param filter
+ * The filter.
+ *
+ * @return The new predicate to which that filter was added.
+ */
+ public Predicate<E> addAccessPathFilter(final IFilter filter) {
+
+ final Predicate<E> tmp = this.clone();
+
+ tmp.addFilter(Annotations.ACCESS_PATH_FILTER, filter);
+
+ return tmp;
+
+ }
+
+ /**
+ * Private method used to add a filter. When there is a filter for the named
+ * property, the filters are combined. Otherwise the filter is set. DO NOT
+ * use this outside of the copy-on-write helper methods.
+ *
+ * @param name
+ * The property name.
+ * @param filter
+ * The filter.
+ */
+ private void addFilter(final String name, final IFilter filter) {
+
+ if (filter == null)
+ throw new IllegalArgumentException();
+
+ final IFilter current = (IFilter) getProperty(name);
+
+ if (current == null) {
+
+ /*
+ * Set the filter.
+ */
+ setProperty(Annotations.INDEX_LOCAL_FILTER, filter);
+
+ } else {
+
+ /*
+ * Wrap the filter.
+ */
+ setProperty(Annotations.INDEX_LOCAL_FILTER, new FilterBase() {
+
+ @Override
+ protected Iterator filterOnce(Iterator src, Object context) {
+ return src;
+ }
+ }.addFilter(current).addFilter(filter));
+ }
+
+ }
+
public String toString() {
return toString(null/* bindingSet */);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -98,7 +98,7 @@
private static final long serialVersionUID = 1L;
@Override
- protected boolean isValid(Object obj) {
+ public boolean isValid(Object obj) {
return BOpFilter.this.isValid(obj);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -40,6 +40,8 @@
import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.btree.filter.TupleFilter.TupleFilterator;
+import cutthecrap.utils.striterators.IFilterTest;
+
/**
* <p>
* Filter supporting {@link ITupleIterator}s.
@@ -99,6 +101,34 @@
super(args, annotations);
}
+// /**
+// * Convenience method wraps the {@link IFilterTest} as a
+// * {@link BOpTupleFilter}.
+// *
+// * @param test
+// * The test (optional).
+// *
+// * @return The filter wrapping the test -or- <code>null</code> if the
+// * argument is <code>null</code>.
+// */
+// @SuppressWarnings("unchecked")
+// public static BOpFilterBase newInstance(final IFilterTest test) {
+//
+// if (test == null) {
+// // No test. No filter.
+// return null;
+// }
+//
+// return new BOpTupleFilter(new BOp[0]/*filters*/,null/*annotations*/) {
+// private static final long serialVersionUID = 1L;
+// @Override
+// protected boolean isValid(ITuple obj) {
+// return test.isValid(obj);
+// }
+// };
+//
+// }
+
@Override
final protected Iterator filterOnce(Iterator src, final Object context) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -94,14 +94,14 @@
private final LinkedHashSet members;
public DistinctFilterImpl() {
-
+
members = new LinkedHashSet(getInitialCapacity(), getLoadFactor());
}
@SuppressWarnings("unchecked")
@Override
- protected boolean isValid(Object obj) {
+ public boolean isValid(Object obj) {
return members.add(obj);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -94,7 +94,7 @@
}
@Override
- protected boolean isValid(Object obj) {
+ public boolean isValid(Object obj) {
return filter.accept(obj);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -69,8 +69,6 @@
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
- *
- * FIXME Implement Rule2BOpUtility.
*/
public class Rule2BOpUtility {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-29 15:48:34 UTC (rev 3678)
@@ -508,45 +508,64 @@
Here is what I have not done yet:
-Write unit tests to sanity check these changes.
+- Striterator unit tests.
-done. Replace (I)FilterConstructor with IFilter. FilterConstructor
-implemented clone. I am not sure that I can recall why. Presumably
-because a cloned would have a distinct filter chain. We can not
-easily override clone() on FilterBase because of all the derived
-versions. We could have a deep copy constructor pattern, which is
-what I did for the BOpBase class and then use that for clone.
+- Bop-erator tests.
-Write unit tests at the IPredicate/AccessPath level to use stackable
-filters (for the LOCAL and REMOTE access paths). Write unit tests for
-correct range counts with and without local/remote filters. Write
-unit tests for caching of those range counts.
+- MGC: All of the Filterators need to use deferred prefetch. Prefetch
+ during the constructor causes problems when we are stacking filters
+ using FilterBase. (This is also true for the ITuple filters).
-- The stacking of the BOpFilters may be counter intuitive or just
- wrong.
+- BT: Write unit tests at the IPredicate/AccessPath level to use
+ stackable filters (for the LOCAL and REMOTE access paths). Write
+ unit tests for correct range counts with and without local/remote
+ filters. Write unit tests for caching of those range counts.
-- TestPredicateAccessPath#test_localFilter() does not report the
- correct tuples. This is weirdness with reuse of the same backing
- Tuple object. See TupleFilter and BOpFilter. The filter that is
- being applied in the unit test does not protect against the tuple
- reuse. It would be best to automatically wrap such asserts. E.g.,
- BOpTupleFilter. It would also be good to catch runtime type
- problems with this. E.g., if the source iterator is ITupleFilter
- then it is an error to wrap with BOpFilter. You must use
- BOpTupleFilter instead.
-
- Write BOp versions for TupleResolver, ...
-- All of the Filterators need to use deferred prefetch. Prefetch
- during the constructor causes problems when we are stacking filters
- using FilterBase.
+- done. IPredicate#getConstraint() must go (issues remain with
+ BackchainAccessPath).
-IPredicate#getConstraint() must go. This will require touching the
-backchainer classes. The (SPO|Magic)Predicate contructors should also
-be touched up since they will otherwise DROP the local and remote
-filters.
+- The (SPO|Magic)Predicate contructors must be touched up.
-Reconcile IElementFilter and implementations.
+ - They assume an IElementFilter (IFilterTest). However, the filters
+ are now specified with IFilter.
+ - Those constructor will DROP the local and remote filters unless
+ the logic is modified.
+
+ - When used remotely, IElementFilter must do ITuple.getObject() and
+ MUST be wrapped using ElementFilter to be "Tuple" aware (problem
+ with Tuple reference reuse inside the ITupleIterators).
+
+ - Reconcile IElementFilter and implementations. IElementFilter was
+ transparently wrapped by AccessPath to resolve the ITuple to the
+ element before applying the filter. If IElementFilter is used in
+ contexts other than the index then all implementations must be
+ modified to conditionally resolve the element while filtering.
+
+ - IPredicates created from a Relation MUST use the timestamp of that
+ relation NOT READ_COMMITTED. Remove the default timestamp
+ annotation. Make this required where it is used so we can track
+ all of this down.
+
+MikeP:
+
+ - BackchainAccessPath. Are the index local or access path filters
+ ever used here? If they are then this code needs to be modified.
+
+ - The timestamp on which the backchain access path will read MUST be
+ passed through to the IPredicate constructors.
+
+ - Leave C as an anonymous variable when it will not be used, not
+ null.
+
+ - Replace NamedGraph and DefaultGraph access paths per the decision
+ tree and cost model.
+
+ - Set the TIMESTAMP on the predicate.
+
+ - Additional SPOPredicate and Predicate constructor cleanup.
+
Reconcile the com.bigdata.striterators package. Can we get rid of it?
Incrementally?
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -56,16 +56,19 @@
private static final long serialVersionUID = 1L;
protected static transient final Logger log = Logger.getLogger(TupleFilter.class);
-
-// /**
-// * Optional state specified by the ctor.
-// */
-// protected Object state;
-
+
public TupleFilter() {
+ this(null/* state */);
+
}
+ public TupleFilter(final Object state) {
+
+ super(state);
+
+ }
+
@SuppressWarnings("unchecked")
@Override
public ITupleIterator<E> filterOnce(final Iterator src,Object context) {
@@ -73,21 +76,14 @@
return new TupleFilter.TupleFilterator((ITupleIterator) src, context, this);
}
-/*
- public Iterator filterOnce(Iterator src, Object context) {
- return new TupleFilter.Filterator((ITupleIterator) src);
+ abstract protected boolean isValid(ITuple<E> tuple);
- }
-*/
- abstract protected boolean isValid(ITuple<E> tuple);
-
/**
* Implementation class knows how to avoid side-effects from the reuse of
* the same {@link Tuple} instance by the base {@link ITupleIterator} impls.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
* @param <E>
*/
static public class TupleFilterator<E> implements ITupleIterator<E> {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/CounterSet.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/CounterSet.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/CounterSet.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -361,7 +361,7 @@
private static final long serialVersionUID = 1L;
@Override
- protected boolean isValid(Object val) {
+ public boolean isValid(Object val) {
final ICounter counter = (ICounter) val;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractArrayBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractArrayBuffer.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractArrayBuffer.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -119,7 +119,7 @@
if (filter != null) {
- if(!filter.accept(e)) {
+ if(!filter.isValid(e)) {
// rejected by the filter.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractUnsynchronizedArrayBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractUnsynchronizedArrayBuffer.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AbstractUnsynchronizedArrayBuffer.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -139,7 +139,7 @@
if (filter != null && filter.canAccept(e)) {
- return filter.accept(e);
+ return filter.isValid(e);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -124,7 +124,16 @@
protected final int chunkCapacity;
protected final int fullyBufferedReadThreshold;
+ /**
+ * <code>true</code> iff the {@link IPredicate}is fully bound.
+ */
private final boolean isFullyBoundForKey;
+
+ /**
+ * <code>true</code> iff there is a filter for the access path (either local
+ * or remote).
+ */
+ private final boolean hasFilter;
/**
* <code>true</code> iff all elements in the predicate which are required
@@ -399,6 +408,9 @@
this.accessPathFilter = (BOpFilterBase) predicate
.getProperty(IPredicate.Annotations.ACCESS_PATH_FILTER);
+ // true iff there is a filter (either local or remote).
+ this.hasFilter = (indexLocalFilter != null || accessPathFilter != null);
+
final IKeyBuilder keyBuilder = ndx.getIndexMetadata()
.getTupleSerializer().getKeyBuilder();
@@ -408,41 +420,6 @@
}
-// /**
-// * Align the predicate's {@link IElementFilter} constraint with
-// * {@link ITupleFilter} so that the {@link IElementFilter} can be evaluated
-// * close to the data by an {@link ITupleIterator}.
-// *
-// * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
-// * @version $Id$
-// * @param <R>
-// * The generic type of the elements presented to the filter.
-// */
-// public static class ElementFilter<R> extends TupleFilter<R> {
-//
-// private static final long serialVersionUID = 1L;
-//
-// private final IElementFilter<R> constraint;
-//
-// public ElementFilter(final IElementFilter<R> constraint) {
-//
-// if (constraint == null)
-// throw new IllegalArgumentException();
-//
-// this.constraint = constraint;
-//
-// }
-//
-// public boolean isValid(final ITuple<R> tuple) {
-//
-// final R obj = (R) tuple.getObject();
-//
-// return constraint.accept( obj );
-//
-// }
-//
-// }
-
public String toString() {
return getClass().getName()
@@ -1148,19 +1125,12 @@
}
- /**
- * Note: When there is a {@link IPredicate#getConstraint()} on the
- * {@link IPredicate} the exact range count will apply that constraint as a
- * filter during traversal. However, the constraint is ignored for the fast
- * range count.
- */
final public long rangeCount(final boolean exact) {
assertInitialized();
long n = 0L;
- final boolean hasFilter = (indexLocalFilter != null || accessPathFilter != null);
if (exact) {
/*
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -0,0 +1,89 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 29, 2010
+ */
+
+package com.bigdata.relation.accesspath;
+
+import com.bigdata.btree.ITuple;
+import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.filter.ITupleFilter;
+import com.bigdata.btree.filter.TupleFilter;
+
+import cutthecrap.utils.striterators.IFilter;
+
+/**
+ * Align the predicate's {@link IElementFilter} constraint with
+ * {@link ITupleFilter} so that the {@link IElementFilter} can be evaluated
+ * close to the data by an {@link ITupleIterator}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ * @param <R>
+ * The generic type of the elements presented to the filter.
+ */
+public class ElementFilter<R> extends TupleFilter<R> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final IElementFilter<R> test;
+
+ /**
+ * Helper method conditionally wraps the <i>test</i>.
+ *
+ * @param <R>
+ * @param test
+ * The test.
+ *
+ * @return The wrapper test -or- <code>null</code> iff the <i>test</i> is
+ * <code>null</code>.
+ */
+ public static <R> IFilter newInstance(final IElementFilter<R> test) {
+
+ if (test == null)
+ return null;
+
+ return new ElementFilter<R>(test);
+
+ }
+
+ public ElementFilter(final IElementFilter<R> test) {
+
+ if (test == null)
+ throw new IllegalArgumentException();
+
+ this.test = test;
+
+ }
+
+ public boolean isValid(final ITuple<R> tuple) {
+
+ final R obj = (R) tuple.getObject();
+
+ return test.isValid(obj);
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IAccessPath.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -39,6 +39,8 @@
import com.bigdata.striterator.IChunkedOrderedIterator;
import com.bigdata.striterator.IKeyOrder;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* An abstraction for efficient reads on an {@link IRelation} using the index
* selected by an {@link IPredicate} constraint. Like their {@link #iterator()},
@@ -71,14 +73,20 @@
* empty (there may be "deleted" index entries within the key range).
*/
public boolean isEmpty();
-
+
/**
* Return the maximum #of elements spanned by the {@link IPredicate}.
+ * <p>
+ * Note: When there is an {@link IFilter} on the {@link IPredicate} the
+ * exact range count MUST apply that {@link IFilter}, which means that it
+ * will be required to traverse the index counting tuples which pass the
+ * {@link IFilter}. However, {@link IFilter}s are ignored for the fast
+ * range count.
*
* @param exact
- * When <code>true</code>, the result will be an exact count
- * and may require a key-range scan. When <code>false</code>,
- * the result will be an upper bound IFF delete markers are
+ * When <code>true</code>, the result will be an exact count and
+ * may require a key-range scan. When <code>false</code>, the
+ * result will be an upper bound IFF delete markers are
* provisioned for the backing index (delete markers are required
* for transactions and for scale-out indices).
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -30,30 +30,25 @@
import java.io.Serializable;
-import cutthecrap.utils.striterators.IFilter;
+import cutthecrap.utils.striterators.IFilterTest;
/**
* Filter for accepting or rejecting visited elements.
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
- *
- * @todo Can we replace this with an interface declared by CTC with isValid() or
- * just the CTC {@link IFilter} so we allow non-filtering striterators to
- * be applied as well?
*/
-public interface IElementFilter<E> extends // BOp,
- Serializable {
+public interface IElementFilter<E> extends IFilterTest, Serializable {
- /**
- * True iff the argument is matched by the filter.
- *
- * @param e
- * An element.
- *
- * @return true iff the element is accepted by the filter.
- */
- public boolean accept(E e);
+// /**
+// * True iff the argument is matched by the filter.
+// *
+// * @param e
+// * An element.
+// *
+// * @return true iff the element is accepted by the filter.
+// */
+// public boolean isValid(E e);
/**
* Return true iff this this filter can be used on the specified object
@@ -61,7 +56,7 @@
* <p>
* Note: This was added to make it possible filter out cases where the
* runtime type system was throwing a {@link ClassCastException} in the
- * {@link #accept(Object)} implementation.
+ * {@link #isValid(Object)} implementation.
*
* @param o
* An object of some type.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/SolutionFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/SolutionFilter.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/SolutionFilter.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -26,14 +26,25 @@
this.delegate = delegate;
}
-
- public boolean accept(final ISolution<E> solution) {
- final E e = solution.get();
+ public boolean isValid(final Object o) {
- return delegate.accept( e );
+ return delegate.isValid(o);
+
}
+
+ /*
+ * Note: The old implementation is below. Based on it, the canAccept()
+ * method was not (and still is not) being invoked for SolutionFilter.
+ */
+// public boolean accept(final ISolution<E> solution) {
+//
+// final E e = solution.get();
+//
+// return delegate.accept( e );
+// }
+
public boolean canAccept(final Object o) {
if (!(o instanceof ISolution<?>)) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinTask.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -1128,7 +1128,7 @@
return new Striterator(cache.values().iterator()).addFilter(new Filter(){
private static final long serialVersionUID = 1L;
@Override
- protected boolean isValid(final Object e) {
+ public boolean isValid(final Object e) {
/*
* Filter out any tasks which are not done or which had an
* error.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsynchronizedSolutionBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsynchronizedSolutionBuffer.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/UnsynchronizedSolutionBuffer.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -90,7 +90,7 @@
final ISolution solution = joinNexus.newSolution(rule,
bindingSet);
- if (solutionFilter == null || solutionFilter.accept(solution)) {
+ if (solutionFilter == null || solutionFilter.isValid(solution)) {
a[naccepted++] = solution;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedWrappedIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedWrappedIterator.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedWrappedIterator.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -121,11 +121,6 @@
* @param filter
* Optional filter. When non-<code>null</code> only elements
* accepted by the filter will be visited by this iterator.
- * <p>
- * Note: This filter will be applied on the client side after
- * tuples have already been resolved to relation elements. Where
- * possible, you are better off passing the filter into the
- * source iterator running on the server(s).
*/
@SuppressWarnings("unchecked")
public ChunkedWrappedIterator(final Iterator<E> src, final int chunkSize,
@@ -153,9 +148,9 @@
private static final long serialVersionUID = 1L;
- protected boolean isValid(final Object arg0) {
+ public boolean isValid(final Object arg0) {
- return filter.accept((E) arg0);
+ return filter.isValid((E) arg0);
}}
)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicate.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicate.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -38,6 +38,7 @@
import com.bigdata.bop.IPredicate;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.bop.NV;
import com.bigdata.bop.Var;
/**
@@ -77,9 +78,10 @@
final Var<Long> u = Var.var("u");
- final IPredicate<?> p1 = new Predicate(
- new IVariableOrConstant[] { u, c1, c2 }, relation);
-
+ final IPredicate<?> p1 = new Predicate(new IVariableOrConstant[] {
+ u, c1, c2 }, new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }));
+
if (log.isInfoEnabled())
log.info(p1.toString());
@@ -103,7 +105,9 @@
final Var<Long> v = Var.var("v");
final IPredicate<?> p1 = new Predicate(
- new IVariableOrConstant[] { u, c1, v }, relation);
+ new IVariableOrConstant[] { u, c1, v },
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }));
if (log.isInfoEnabled())
log.info(p1.toString());
@@ -131,10 +135,12 @@
final Var<Long> u = Var.var("u");
final IPredicate<?> p1 = new Predicate(new IVariableOrConstant[] { u, c1, c2 },
- relation);
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }));
final IPredicate<?> p2 = new Predicate(new IVariableOrConstant[] { u, c3, c4 },
- relation);
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }));
if (log.isInfoEnabled()) {
@@ -157,9 +163,10 @@
final Var<Long> u = Var.var("u");
- final IPredicate<?> p1 = new Predicate(new IVariableOrConstant[] { u, c1, c2 },
- relation);
-
+ final IPredicate<?> p1 = new Predicate(new IVariableOrConstant[] { u,
+ c1, c2 }, new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }));
+
assertEquals("arity", 3, p1.arity());
// verify variables versus constants.
@@ -246,9 +253,10 @@
final Var<Long> u = Var.var("u");
- final IPredicate<?> p1 = new Predicate(new IVariableOrConstant[] { u, c1, c2 },
- relation);
-
+ final IPredicate<?> p1 = new Predicate(new IVariableOrConstant[] { u,
+ c1, c2 }, new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }));
+
assertEquals("arity", 3, p1.arity());
// verify variables versus constants.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -172,9 +172,10 @@
public void test_nothingBound() {
// nothing bound.
- final IAccessPath<E> accessPath = rel
- .getAccessPath(new Predicate<E>(new IVariableOrConstant[] {
- Var.var("name"), Var.var("value") }, namespace));
+ final IAccessPath<E> accessPath = rel.getAccessPath(new Predicate<E>(
+ new BOp[] { Var.var("name"), Var.var("value") }, new NV(
+ IPredicate.Annotations.RELATION_NAME,
+ new String[] { namespace })));
// verify the range count.
assertEquals(5, accessPath.rangeCount(true/* exact */));
@@ -215,7 +216,9 @@
final IAccessPath<E> accessPath = rel.getAccessPath(new Predicate<E>(
new IVariableOrConstant[] { new Constant<String>("Mary"),
- Var.var("value") }, namespace));
+ Var.var("value") }, new NV(
+ Predicate.Annotations.RELATION_NAME,
+ new String[] { namespace })));
// verify the range count.
assertEquals(2, accessPath.rangeCount(true/* exact */));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestDefaultEvaluationPlan.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestDefaultEvaluationPlan.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/eval/TestDefaultEvaluationPlan.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -41,12 +41,14 @@
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.bop.NV;
import com.bigdata.bop.Var;
import com.bigdata.bop.ap.Predicate;
import com.bigdata.btree.keys.ISortKeyBuilder;
import com.bigdata.config.IValidator;
import com.bigdata.io.IStreamSerializer;
import com.bigdata.journal.IIndexManager;
+import com.bigdata.journal.ITx;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.IMutableRelation;
import com.bigdata.relation.IRelation;
@@ -105,6 +107,7 @@
public void test_lubmQuery8() {
final String relation = "spo";
+ final long timestamp = ITx.READ_COMMITTED;
final Constant<?> rdfType = new Constant<String>("rdfType");
final Constant<?> Department = new Constant<String>("Department");
@@ -117,31 +120,41 @@
final IPredicate<?> pred0 = new Predicate(//
new IVariableOrConstant[] {//
Var.var("y"), rdfType, Department },//
- relation//
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }),//
+ new NV(Predicate.Annotations.TIMESTAMP,timestamp)//
);
final IPredicate<?> pred1 = new Predicate(//
new IVariableOrConstant[] {//
Var.var("x"), rdfType, Student },//
- relation//
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }),//
+ new NV(Predicate.Annotations.TIMESTAMP,timestamp)//
);
final IPredicate<?> pred2 = new Predicate( //
new IVariableOrConstant[] {//
Var.var("x"), memberOf, Var.var("y") },//
- relation//
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }),//
+ new NV(Predicate.Annotations.TIMESTAMP,timestamp)//
);
final IPredicate<?> pred3 = new Predicate(//
new IVariableOrConstant[] {//
Var.var("y"), subOrganizationOf, University0 },//
- relation//
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }),//
+ new NV(Predicate.Annotations.TIMESTAMP,timestamp)//
);
final IPredicate<?> pred4 = new Predicate(//
new IVariableOrConstant[] {//
Var.var("x"), emailAddress, Var.var("z") },
- relation//
+ new NV(Predicate.Annotations.RELATION_NAME,
+ new String[] { relation }),//
+ new NV(Predicate.Annotations.TIMESTAMP,timestamp)//
);
final IRule rule = new Rule(getName(), null/* head */,
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/AbstractSPOBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/AbstractSPOBuffer.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/AbstractSPOBuffer.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -186,7 +186,7 @@
assert stmt != null;
- if (filter != null && filter.accept(stmt)) {
+ if (filter != null && filter.isValid(stmt)) {
/*
* Note: Do not store statements (or justifications) matched by the
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java 2010-09-29 15:36:55 UTC (rev 3677)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/BackchainTypeResourceIterator.java 2010-09-29 15:48:34 UTC (rev 3678)
@@ -274,7 +274,7 @@
private static final long serialVersionUID = 1L;
- protected boolean isValid(Object arg0) {
+ public boolean isValid(Object arg0) {
final SPO o = (SPO) arg0;
Modified: branches/QUADS_QUERY_BRANCH/bigdat...
[truncated message content] |
|
From: <tho...@us...> - 2010-09-29 16:28:54
|
Revision: 3680
http://bigdata.svn.sourceforge.net/bigdata/?rev=3680&view=rev
Author: thompsonbry
Date: 2010-09-29 16:28:48 +0000 (Wed, 29 Sep 2010)
Log Message:
-----------
Working around a compiler error in a test suite.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/test/cutthecrap/utils/striterators/TestFilterBase.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 16:03:00 UTC (rev 3679)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-09-29 16:28:48 UTC (rev 3680)
@@ -59,15 +59,15 @@
public TupleFilter() {
- this(null/* state */);
+// this(null/* state */);
+//
+// }
+//
+// public TupleFilter(final Object state) {
+//
+// super(state);
}
-
- public TupleFilter(final Object state) {
-
- super(state);
-
- }
@SuppressWarnings("unchecked")
@Override
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java 2010-09-29 16:03:00 UTC (rev 3679)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java 2010-09-29 16:28:48 UTC (rev 3680)
@@ -95,6 +95,9 @@
* @param s
* @param p
* @param o
+ *
+ * @deprecated Only used by the unit tests. They should use the shallow
+ * copy constructor form.
*/
public SPOPredicate(final String relationName,
final IVariableOrConstant<IV> s,
@@ -128,8 +131,7 @@
// this(new String[] { relationName }, -1/* partitionId */, s, p, o, c,
// false/* optional */, null/* constraint */, null/* expander */);
- super((c == null ? new IVariableOrConstant[] { s, p, o }
- : new IVariableOrConstant[] { s, p, o, c }), //
+ super(new IVariableOrConstant[] { s, p, o, c }, //
new NV(Annotations.RELATION_NAME, relationName) //
);
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java 2010-09-29 16:03:00 UTC (rev 3679)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOAccessPath.java 2010-09-29 16:28:48 UTC (rev 3680)
@@ -29,7 +29,10 @@
import org.openrdf.model.Statement;
+import com.bigdata.bop.BOp;
import com.bigdata.bop.Constant;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.NV;
import com.bigdata.bop.Var;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.TermId;
@@ -321,12 +324,14 @@
{
final SPOPredicate predicate = new SPOPredicate(//
- store.getSPORelation().getNamespace(),//
- Var.var("g"), // s
- new Constant<IV>(p1.getIV()), // p
- new Constant<IV>(o1.getIV()), // o
- Var.var("h") // c
- );
+ new BOp[] { Var.var("g"), // s
+ new Constant<IV>(p1.getIV()), // p
+ new Constant<IV>(o1.getIV()), // o
+ Var.var("h") // c
+ }, new NV(IPredicate.Annotations.RELATION_NAME,
+ new String[] { store.getSPORelation()
+ .getNamespace() //
+ }));
final IAccessPath<ISPO> accessPath = store.getSPORelation()
.getAccessPath(predicate);
@@ -344,12 +349,13 @@
// shared 'g' variable (?g, p1, o1, ?g)
{
final SPOPredicate predicate = new SPOPredicate(//
- store.getSPORelation().getNamespace(),//
- Var.var("g"), // s
- new Constant<IV>(p1.getIV()), // p
- new Constant<IV>(o1.getIV()), // o
- Var.var("g") // c
- );
+ new BOp[] { Var.var("g"), // s
+ new Constant<IV>(p1.getIV()), // p
+ new Constant<IV>(o1.getIV()), // o
+ Var.var("g") // c
+ }, new NV(IPredicate.Annotations.RELATION_NAME,
+ new String[] { store.getSPORelation()
+ .getNamespace() }));
final IAccessPath<ISPO> accessPath = store.getSPORelation()
.getAccessPath(predicate);
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java 2010-09-29 16:03:00 UTC (rev 3679)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPOKeyOrder.java 2010-09-29 16:28:48 UTC (rev 3680)
@@ -29,9 +29,13 @@
package com.bigdata.rdf.spo;
import java.util.Iterator;
+
import junit.framework.TestCase2;
+import com.bigdata.bop.BOp;
import com.bigdata.bop.Constant;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.NV;
import com.bigdata.bop.Var;
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.keys.KeyBuilder;
@@ -184,8 +188,9 @@
final IV P = tid(1);
- final SPOPredicate pred = new SPOPredicate("testRelation",
- Var.var("s"), new Constant<IV>(P), Var.var("o"), Var.var("c"));
+ final SPOPredicate pred = new SPOPredicate(new BOp[]{
+ Var.var("s"), new Constant<IV>(P), Var.var("o"), Var.var("c")},
+ new NV(IPredicate.Annotations.RELATION_NAME,new String[]{"testRelation"}));
final KeyBuilder keyBuilder = new KeyBuilder();
@@ -224,8 +229,10 @@
final IV P = tid(1);
- final SPOPredicate pred = new SPOPredicate("testRelation",
- Var.var("s"), new Constant<IV>(P), Var.var("o"), Var.var("c"));
+ final SPOPredicate pred = new SPOPredicate(new BOp[] { Var.var("s"),
+ new Constant<IV>(P), Var.var("o") }, new NV(
+ IPredicate.Annotations.RELATION_NAME,
+ new String[] { "testRelation" }));// , Var.var("c"));
final KeyBuilder keyBuilder = new KeyBuilder();
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/test/cutthecrap/utils/striterators/TestFilterBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/test/cutthecrap/utils/striterators/TestFilterBase.java 2010-09-29 16:03:00 UTC (rev 3679)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/test/cutthecrap/utils/striterators/TestFilterBase.java 2010-09-29 16:28:48 UTC (rev 3680)
@@ -52,228 +52,228 @@
super(name);
}
- public void test_filterBase_ctor() {
-
- final FilterBase fixture = new MockFilterBase();
-
- assertEquals("state", null, fixture.m_state);
- assertEquals("annotations", null, fixture.annotations);
- assertEquals("filterChain", null, fixture.filterChain);
-
- }
-
- public void test_filterBase_ctor2() {
-
- final Object state = new Object();
-
- final FilterBase fixture = new MockFilterBase(state);
-
- assertEquals("state", state, fixture.m_state);
- assertEquals("annotations", null, fixture.annotations);
- assertEquals("filterChain", null, fixture.filterChain);
-
- }
-
- public void test_filterBase_annotations() {
-
- final FilterBase fixture = new MockFilterBase();
-
- assertEquals("annotations", null, fixture.annotations);
-
- final String name = "name";
- final Object value = Integer.valueOf(0);
- final Object value2 = Integer.valueOf(1);
-
- try {
- fixture.getRequiredProperty(name);
- fail("Expecting: " + IllegalStateException.class);
- } catch (IllegalStateException ex) {
-
- }
-
- assertNull(fixture.getProperty(name));
- assertNull(fixture.setProperty(name, value));
- assertNotNull("annotations", fixture.annotations);
- assertEquals("annotations", 1, fixture.annotations.size());
- assertEquals(value, fixture.getProperty(name));
- assertEquals(value, fixture.getRequiredProperty(name));
- assertEquals(value, fixture.setProperty(name, value2));
- assertEquals(value2, fixture.getProperty(name));
- assertEquals(value2, fixture.getRequiredProperty(name));
-
- }
-
- public void test_filterBase_filterChain() {
-
- final Object s1 = "s1";
- final Object s2 = "s2";
- final Object s3 = "s3";
-
- final FilterBase f1, f2, f3;
-
- final FilterBase fixture = f1 = new MockFilterBase(s1);
-
- assertNull("filterChain", fixture.filterChain);
-
- fixture.addFilter(f2 = new MockFilterBase(s2));
-
- assertNotNull("filterChain", fixture.filterChain);
-
- fixture.addFilter(f3 = new MockFilterBase(s3));
-
- final IFilter[] expected = new IFilter[] { f2, f3 };
-
- final IFilter[] actual = f1.filterChain.toArray(new IFilter[] {});
-
- assertEquals("#filters", expected.length, actual.length);
-
- for (int i = 0; i < expected.length; i++) {
-
- assertEquals("filter[" + i + "]", expected[i], actual[i]);
-
- }
-
- }
-
- /**
- * Test creation of iterator without filter chain. make sure that the
- * context is passed through.
- */
- public void test_filter() {
-
- final FilterBase fixture = new MockFilterBase();
-
- final Object context = new Object();
-
- final Iterator src = EmptyIterator.DEFAULT;
-
- final MockIterator actual = (MockIterator) fixture.filter(src, context);
-
- assertNotNull(actual);
- assertTrue("src", actual.src == src);
- assertTrue("context", actual.context == context);
- assertTrue("filter", actual.filter == fixture);
-
- }
-
- /**
- * Test creation of iterator with filter chain. Make sure that the create
- * order is correct and that the context is passed through to each iterator.
- * The iterators are assembled in FIFO order, so the iterator stack winds up
- * being LIFO.
- */
- public void test_filter2() {
-
- // create filters w/ state objects (helps visual inspection in debugger).
- final Object s1 = "s1";
- final Object s2 = "s2";
- final Object s3 = "s3";
- final FilterBase fixture1 = new MockFilterBase(s1);
- final FilterBase fixture2 = new MockFilterBase(s2);
- final FilterBase fixture3 = new MockFilterBase(s3);
-
- // chain 2 filters to the first.
- fixture1.addFilter(fixture2);
- fixture1.addFilter(fixture3);
-
- // verify the filter chain.
- assertNotNull(fixture1.filterChain);
- assertEquals(2, fixture1.filterChain.size());
- assertTrue(fixture2 == fixture1.filterChain.get(0));
- assertTrue(fixture3 == fixture1.filterChain.get(1));
-
- // verify other filter chains are empty.
- assertNull(fixture2.filterChain);
- assertNull(fixture3.filterChain);
-
- final Object context = "context";
-
- final Iterator src = EmptyIterator.DEFAULT;
-
- /*
- * Create and verify the iterator stack.
- *
- * Note: The iterator are created in the order in filter chain order,
- * but each iterator wraps the previous iterator. This has the effect of
- * building an iterator stack which is the reverse of the filter chain.
- *
- * logical filter chain: filter1, filter2, filter3
- *
- * logical iterator stack: itr1(filter3), itr2(filter2), itr3(filter1).
- */
- final MockIterator actual1 = (MockIterator) fixture1.filter(src, context);
- final MockIterator actual2 = (MockIterator) actual1.src;
- final MockIterator actual3 = (MockIterator) actual2.src;
- // itr3 (bottom of the stack)
- assertTrue("src", actual3.src == src);
- assertTrue("context", actual3.context == context);
- assertTrue("filter", actual3.filter == fixture1);
- // itr2
- assertTrue("src", actual2.src == actual3);
- assertTrue("context", actual2.context == context);
- assertTrue("filter", actual2.filter == fixture2);
- // itr1 (top of the stack)
- assertTrue("src", actual1.src == actual2);
- assertTrue("context", actual1.context == context);
- assertTrue("filter", actual1.filter == fixture3);
- }
-
- /**
- * Mock object.
- */
- private static class MockFilterBase extends FilterBase {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public MockFilterBase() {
- super();
- }
-
- public MockFilterBase(Object o) {
- super(o);
- }
-
- @Override
- protected Iterator filterOnce(Iterator src, Object context) {
-
- return new MockIterator(src, context, this);
-
- }
-
- }
-
- private static class MockIterator<E> implements Iterator<E> {
-
- final Iterator src;
-
- final Object context;
-
- final MockFilterBase filter;
-
- public MockIterator(Iterator src, Object context, MockFilterBase filter) {
-
- this.src = src;
-
- this.context = context;
-
- this.filter = filter;
-
- }
-
- public boolean hasNext() {
- return false;
- }
-
- public E next() {
- return null;
- }
-
- public void remove() {
- }
-
- }
+// public void test_filterBase_ctor() {
+//
+// final FilterBase fixture = new MockFilterBase();
+//
+// assertEquals("state", null, fixture.m_state);
+// assertEquals("annotations", null, fixture.annotations);
+// assertEquals("filterChain", null, fixture.filterChain);
+//
+// }
+//
+// public void test_filterBase_ctor2() {
+//
+// final Object state = new Object();
+//
+// final FilterBase fixture = new MockFilterBase(state);
+//
+// assertEquals("state", state, fixture.m_state);
+// assertEquals("annotations", null, fixture.annotations);
+// assertEquals("filterChain", null, fixture.filterChain);
+//
+// }
+//
+// public void test_filterBase_annotations() {
+//
+// final FilterBase fixture = new MockFilterBase();
+//
+// assertEquals("annotations", null, fixture.annotations);
+//
+// final String name = "name";
+// final Object value = Integer.valueOf(0);
+// final Object value2 = Integer.valueOf(1);
+//
+// try {
+// fixture.getRequiredProperty(name);
+// fail("Expecting: " + IllegalStateException.class);
+// } catch (IllegalStateException ex) {
+//
+// }
+//
+// assertNull(fixture.getProperty(name));
+// assertNull(fixture.setProperty(name, value));
+// assertNotNull("annotations", fixture.annotations);
+// assertEquals("annotations", 1, fixture.annotations.size());
+// assertEquals(value, fixture.getProperty(name));
+// assertEquals(value, fixture.getRequiredProperty(name));
+// assertEquals(value, fixture.setProperty(name, value2));
+// assertEquals(value2, fixture.getProperty(name));
+// assertEquals(value2, fixture.getRequiredProperty(name));
+//
+// }
+//
+// public void test_filterBase_filterChain() {
+//
+// final Object s1 = "s1";
+// final Object s2 = "s2";
+// final Object s3 = "s3";
+//
+// final FilterBase f1, f2, f3;
+//
+// final FilterBase fixture = f1 = new MockFilterBase(s1);
+//
+// assertNull("filterChain", fixture.filterChain);
+//
+// fixture.addFilter(f2 = new MockFilterBase(s2));
+//
+// assertNotNull("filterChain", fixture.filterChain);
+//
+// fixture.addFilter(f3 = new MockFilterBase(s3));
+//
+// final IFilter[] expected = new IFilter[] { f2, f3 };
+//
+// final IFilter[] actual = f1.filterChain.toArray(new IFilter[] {});
+//
+// assertEquals("#filters", expected.length, actual.length);
+//
+// for (int i = 0; i < expected.length; i++) {
+//
+// assertEquals("filter[" + i + "]", expected[i], actual[i]);
+//
+// }
+//
+// }
+//
+// /**
+// * Test creation of iterator without filter chain. make sure that the
+// * context is passed through.
+// */
+// public void test_filter() {
+//
+// final FilterBase fixture = new MockFilterBase();
+//
+// final Object context = new Object();
+//
+// final Iterator src = EmptyIterator.DEFAULT;
+//
+// final MockIterator actual = (MockIterator) fixture.filter(src, context);
+//
+// assertNotNull(actual);
+// assertTrue("src", actual.src == src);
+// assertTrue("context", actual.context == context);
+// assertTrue("filter", actual.filter == fixture);
+//
+// }
+//
+// /**
+// * Test creation of iterator with filter chain. Make sure that the create
+// * order is correct and that the context is passed through to each iterator.
+// * The iterators are assembled in FIFO order, so the iterator stack winds up
+// * being LIFO.
+// */
+// public void test_filter2() {
+//
+// // create filters w/ state objects (helps visual inspection in debugger).
+// final Object s1 = "s1";
+// final Object s2 = "s2";
+// final Object s3 = "s3";
+// final FilterBase fixture1 = new MockFilterBase(s1);
+// final FilterBase fixture2 = new MockFilterBase(s2);
+// final FilterBase fixture3 = new MockFilterBase(s3);
+//
+// // chain 2 filters to the first.
+// fixture1.addFilter(fixture2);
+// fixture1.addFilter(fixture3);
+//
+// // verify the filter chain.
+// assertNotNull(fixture1.filterChain);
+// assertEquals(2, fixture1.filterChain.size());
+// assertTrue(fixture2 == fixture1.filterChain.get(0));
+// assertTrue(fixture3 == fixture1.filterChain.get(1));
+//
+// // verify other filter chains are empty.
+// assertNull(fixture2.filterChain);
+// assertNull(fixture3.filterChain);
+//
+// final Object context = "context";
+//
+// final Iterator src = EmptyIterator.DEFAULT;
+//
+// /*
+// * Create and verify the iterator stack.
+// *
+// * Note: The iterator are created in the order in filter chain order,
+// * but each iterator wraps the previous iterator. This has the effect of
+// * building an iterator stack which is the reverse of the filter chain.
+// *
+// * logical filter chain: filter1, filter2, filter3
+// *
+// * logical iterator stack: itr1(filter3), itr2(filter2), itr3(filter1).
+// */
+// final MockIterator actual1 = (MockIterator) fixture1.filter(src, context);
+// final MockIterator actual2 = (MockIterator) actual1.src;
+// final MockIterator actual3 = (MockIterator) actual2.src;
+// // itr3 (bottom of the stack)
+// assertTrue("src", actual3.src == src);
+// assertTrue("context", actual3.context == context);
+// assertTrue("filter", actual3.filter == fixture1);
+// // itr2
+// assertTrue("src", actual2.src == actual3);
+// assertTrue("context", actual2.context == context);
+// assertTrue("filter", actual2.filter == fixture2);
+// // itr1 (top of the stack)
+// assertTrue("src", actual1.src == actual2);
+// assertTrue("context", actual1.context == context);
+// assertTrue("filter", actual1.filter == fixture3);
+// }
+//
+// /**
+// * Mock object.
+// */
+// private static class MockFilterBase extends FilterBase {
+//
+// /**
+// *
+// */
+// private static final long serialVersionUID = 1L;
+//
+// public MockFilterBase() {
+// super();
+// }
+//
+// public MockFilterBase(Object o) {
+// super(o);
+// }
+//
+// @Override
+// protected Iterator filterOnce(Iterator src, Object context) {
+//
+// return new MockIterator(src, context, this);
+//
+// }
+//
+// }
+//
+// private static class MockIterator<E> implements Iterator<E> {
+//
+// final Iterator src;
+//
+// final Object context;
+//
+// final MockFilterBase filter;
+//
+// public MockIterator(Iterator src, Object context, MockFilterBase filter) {
+//
+// this.src = src;
+//
+// this.context = context;
+//
+// this.filter = filter;
+//
+// }
+//
+// public boolean hasNext() {
+// return false;
+// }
+//
+// public E next() {
+// return null;
+// }
+//
+// public void remove() {
+// }
+//
+// }
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-29 16:57:27
|
Revision: 3682
http://bigdata.svn.sourceforge.net/bigdata/?rev=3682&view=rev
Author: thompsonbry
Date: 2010-09-29 16:57:20 +0000 (Wed, 29 Sep 2010)
Log Message:
-----------
Added a NOPFilter for stacking filters when you don't have one to start with.
Modified AccessPath to not assume BOpTupleFilter for the filters.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/NOPFilter.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-29 16:37:18 UTC (rev 3681)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-29 16:57:20 UTC (rev 3682)
@@ -40,8 +40,6 @@
import com.bigdata.bop.BOp;
import com.bigdata.bop.IPredicate;
import com.bigdata.bop.NV;
-import com.bigdata.bop.ap.filter.BOpFilterBase;
-import com.bigdata.bop.ap.filter.BOpTupleFilter;
import com.bigdata.bop.ap.filter.SameVariableConstraint;
import com.bigdata.bop.ap.filter.SameVariableConstraintFilter;
import com.bigdata.btree.BytesUtil;
@@ -65,7 +63,9 @@
import com.bigdata.striterator.IChunkedOrderedIterator;
import com.bigdata.striterator.IKeyOrder;
+import cutthecrap.utils.striterators.FilterBase;
import cutthecrap.utils.striterators.IFilter;
+import cutthecrap.utils.striterators.NOPFilter;
import cutthecrap.utils.striterators.Striterator;
/**
@@ -205,13 +205,13 @@
* is added regardless of whether the {@link IPredicate} specified a filter
* or not.
*/
- final protected BOpFilterBase indexLocalFilter;
+ final protected IFilter indexLocalFilter;
/**
* The filter derived from optional
* {@link IPredicate.Annotations#ACCESS_PATH_FILTER}.
*/
- final protected BOpFilterBase accessPathFilter;
+ final protected IFilter accessPathFilter;
/**
* Used to detect failure to call {@link #init()}.
@@ -373,8 +373,7 @@
* reuse of tuples within tuple iterators. That is why it is being
* cast to a BOpTupleIterator.
*/
- final BOpTupleFilter indexLocalFilter = (BOpTupleFilter<?>) predicate
- .getProperty(IPredicate.Annotations.INDEX_LOCAL_FILTER);
+ final IFilter indexLocalFilter = predicate.getIndexLocalFilter();
/*
* Optional constraint enforces the "same variable" constraint. The
@@ -389,13 +388,19 @@
/*
* Stack filters.
*/
- this.indexLocalFilter = new SameVariableConstraintFilter(
- (indexLocalFilter == null ? new BOp[0]
- : new BOp[] { indexLocalFilter }),
+ final FilterBase tmp = new NOPFilter();
+
+ if (indexLocalFilter != null)
+ tmp.addFilter(indexLocalFilter);
+
+ tmp.addFilter(new SameVariableConstraintFilter(
+ new BOp[] {},
NV.asMap(new NV[] { new NV(
- SameVariableConstraintFilter.Annotations.FILTER,
- sameVarConstraint) }));
+ SameVariableConstraintFilter.Annotations.FILTER,
+ sameVarConstraint) })));
+ this.indexLocalFilter = indexLocalFilter;
+
} else {
this.indexLocalFilter = indexLocalFilter;
@@ -405,8 +410,7 @@
}
// optional filter to be evaluated by the AccessPath.
- this.accessPathFilter = (BOpFilterBase) predicate
- .getProperty(IPredicate.Annotations.ACCESS_PATH_FILTER);
+ this.accessPathFilter = predicate.getAccessPathFilter();;
// true iff there is a filter (either local or remote).
this.hasFilter = (indexLocalFilter != null || accessPathFilter != null);
Added: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/NOPFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/NOPFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/NOPFilter.java 2010-09-29 16:57:20 UTC (rev 3682)
@@ -0,0 +1,56 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 29, 2010
+ */
+
+package cutthecrap.utils.striterators;
+
+import java.util.Iterator;
+
+/**
+ * A filter which is initially a NOP. You can add other filters to this to stack
+ * them.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+public class NOPFilter extends FilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ *
+ */
+ public NOPFilter() {
+ }
+
+ @Override
+ protected Iterator filterOnce(Iterator src, Object context) {
+ return src;
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/NOPFilter.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-29 19:14:40
|
Revision: 3683
http://bigdata.svn.sourceforge.net/bigdata/?rev=3683&view=rev
Author: thompsonbry
Date: 2010-09-29 19:14:33 +0000 (Wed, 29 Sep 2010)
Log Message:
-----------
The SameVariableConstraint was not being applied successfully because we were failing to create an SPOAccessPath in BOpContextBase. I've added a newAccessPath(...) method on AbstractRelation. All access path creates now delegate through that method to a sole constructor on AccessPath or SPOAccessPath.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractRelation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -353,11 +353,10 @@
}
// Obtain the access path for that relation and index.
- final IAccessPath<E> accessPath = new AccessPath<E>(
- relation, indexManager, timestamp,
- predicate, keyOrder, ndx, flags,
- chunkOfChunksCapacity, chunkCapacity,
- fullyBufferedReadThreshold).init();
+ final IAccessPath<E> accessPath = ((AbstractRelation<E>) relation)
+ .newAccessPath(relation, indexManager, timestamp, predicate,
+ keyOrder, ndx, flags, chunkOfChunksCapacity,
+ chunkCapacity, fullyBufferedReadThreshold);
// optionally wrap with an expander pattern.
return expander(predicate, accessPath);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -40,8 +40,6 @@
import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.btree.filter.TupleFilter.TupleFilterator;
-import cutthecrap.utils.striterators.IFilterTest;
-
/**
* <p>
* Filter supporting {@link ITupleIterator}s.
@@ -101,34 +99,6 @@
super(args, annotations);
}
-// /**
-// * Convenience method wraps the {@link IFilterTest} as a
-// * {@link BOpTupleFilter}.
-// *
-// * @param test
-// * The test (optional).
-// *
-// * @return The filter wrapping the test -or- <code>null</code> if the
-// * argument is <code>null</code>.
-// */
-// @SuppressWarnings("unchecked")
-// public static BOpFilterBase newInstance(final IFilterTest test) {
-//
-// if (test == null) {
-// // No test. No filter.
-// return null;
-// }
-//
-// return new BOpTupleFilter(new BOp[0]/*filters*/,null/*annotations*/) {
-// private static final long serialVersionUID = 1L;
-// @Override
-// protected boolean isValid(ITuple obj) {
-// return test.isValid(obj);
-// }
-// };
-//
-// }
-
@Override
final protected Iterator filterOnce(Iterator src, final Object context) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -19,6 +19,8 @@
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.relation.accesspath.IAccessPath;
+import cutthecrap.utils.striterators.IFilterTest;
+
/**
* Filter imposes the "same variable" constraint on the elements visited by an
* {@link IAccessPath}. The filter is required IFF a {@link IVariable} appears
@@ -39,7 +41,7 @@
* The generic type of the elements that will be tested by the
* filter.
*/
-public class SameVariableConstraint<E> implements Externalizable {
+public class SameVariableConstraint<E> implements IFilterTest, Externalizable {
/**
* The predicate template.
@@ -110,8 +112,15 @@
// return true;
// }
- public boolean accept(final E e) {
+ @SuppressWarnings("unchecked")
+ public boolean isValid(final Object obj) {
+ return accept((E) obj);
+
+ }
+
+ private boolean accept(final E e) {
+
int i = 0;
while (i < indices.length) {
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -1,105 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Sep 28, 2010
- */
-
-package com.bigdata.bop.ap.filter;
-
-import java.util.Iterator;
-import java.util.Map;
-
-import com.bigdata.bop.BOp;
-
-import cutthecrap.utils.striterators.Filter;
-import cutthecrap.utils.striterators.Filterator;
-
-/**
- * Operator imposing a {@link SameVariableConstraint} filter.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-public class SameVariableConstraintFilter extends BOpFilterBase {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends BOpFilterBase.Annotations {
-
- /**
- * The constraint to be imposed (required).
- */
- String FILTER = SameVariableConstraintFilter.class.getName()
- + ".filter";
-
- }
-
- /**
- * @param op
- */
- public SameVariableConstraintFilter(SameVariableConstraintFilter op) {
- super(op);
- }
-
- /**
- * @param args
- * @param annotations
- */
- public SameVariableConstraintFilter(BOp[] args,
- Map<String, Object> annotations) {
- super(args, annotations);
- }
-
- @Override
- final protected Iterator filterOnce(Iterator src, final Object context) {
-
- final SameVariableConstraint filter = (SameVariableConstraint) getRequiredProperty(Annotations.FILTER);
-
- return new Filterator(src, context, new FilterImpl(filter));
-
- }
-
- static private class FilterImpl extends Filter {
-
- private static final long serialVersionUID = 1L;
-
- private final SameVariableConstraint filter;
-
- public FilterImpl(final SameVariableConstraint filter) {
- this.filter = filter;
- }
-
- @Override
- public boolean isValid(Object obj) {
-
- return filter.accept(obj);
-
- }
-
- }
-
-}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractRelation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractRelation.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractRelation.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -44,6 +44,8 @@
import com.bigdata.journal.Journal;
import com.bigdata.journal.TemporaryRawStore;
import com.bigdata.journal.TemporaryStore;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.spo.SPORelation;
import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.service.DataService;
@@ -232,6 +234,35 @@
}
+ /**
+ * Core impl.
+ *
+ * @param relation
+ * @param indexManager
+ * @param timestamp
+ * @param predicate
+ * @param keyOrder
+ * @param ndx
+ * @param flags
+ * @param chunkOfChunksCapacity
+ * @param chunkCapacity
+ * @param fullyBufferedReadThreshold
+ * @return
+ *
+ * @todo Raise into interface?
+ */
+ public IAccessPath<E> newAccessPath(final IRelation<E> relation,
+ final IIndexManager indexManager, final long timestamp,
+ final IPredicate<E> predicate, final IKeyOrder<E> keyOrder,
+ final IIndex ndx, final int flags, final int chunkOfChunksCapacity,
+ final int chunkCapacity, final int fullyBufferedReadThreshold) {
+
+ return new AccessPath<E>(this/* relation */, indexManager, timestamp,
+ predicate, keyOrder, ndx, flags, chunkOfChunksCapacity,
+ chunkCapacity, fullyBufferedReadThreshold).init();
+
+ }
+
public IAccessPath<E> getAccessPath(final IPredicate<E> predicate) {
// find the best key order.
@@ -243,10 +274,10 @@
// default flags.
final int flags = IRangeQuery.DEFAULT;
- return new AccessPath<E>(this/* relation */, getIndexManager(),
+ return newAccessPath(this/*relation*/, getIndexManager(),
getTimestamp(), predicate, keyOrder, ndx, flags,
getChunkOfChunksCapacity(), getChunkCapacity(),
- getFullyBufferedReadThreshold()).init();
+ getFullyBufferedReadThreshold());
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -37,18 +37,18 @@
import org.apache.log4j.Logger;
-import com.bigdata.bop.BOp;
import com.bigdata.bop.IPredicate;
-import com.bigdata.bop.NV;
+import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.ap.filter.SameVariableConstraint;
-import com.bigdata.bop.ap.filter.SameVariableConstraintFilter;
import com.bigdata.btree.BytesUtil;
import com.bigdata.btree.IBloomFilter;
import com.bigdata.btree.IIndex;
import com.bigdata.btree.ILocalBTreeView;
import com.bigdata.btree.IRangeQuery;
+import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.Tuple;
+import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.journal.IIndexManager;
import com.bigdata.journal.TimestampUtility;
@@ -393,13 +393,14 @@
if (indexLocalFilter != null)
tmp.addFilter(indexLocalFilter);
- tmp.addFilter(new SameVariableConstraintFilter(
- new BOp[] {},
- NV.asMap(new NV[] { new NV(
- SameVariableConstraintFilter.Annotations.FILTER,
- sameVarConstraint) })));
+ tmp.addFilter(new TupleFilter(){
+ private static final long serialVersionUID = 1L;
+ @Override
+ protected boolean isValid(ITuple tuple) {
+ return sameVarConstraint.isValid(tuple.getObject());
+ }});
- this.indexLocalFilter = indexLocalFilter;
+ this.indexLocalFilter = tmp;
} else {
@@ -410,7 +411,7 @@
}
// optional filter to be evaluated by the AccessPath.
- this.accessPathFilter = predicate.getAccessPathFilter();;
+ this.accessPathFilter = predicate.getAccessPathFilter();
// true iff there is a filter (either local or remote).
this.hasFilter = (indexLocalFilter != null || accessPathFilter != null);
@@ -436,12 +437,12 @@
+ ", fromKey="
+ (fromKey == null ? "n/a" : BytesUtil.toString(fromKey))
+ ", toKey="
- + (toKey == null ? "n/a" : BytesUtil.toString(toKey)
- + ", indexLocalFilter="
- + (indexLocalFilter == null ? "n/a" : indexLocalFilter)
- + ", accessPathFilter="
- + (accessPathFilter == null ? "n/a" : accessPathFilter)
- + "}");
+ + (toKey == null ? "n/a" : BytesUtil.toString(toKey))
+ + ", indexLocalFilter="
+ + (indexLocalFilter == null ? "n/a" : indexLocalFilter)
+ + ", accessPathFilter="
+ + (accessPathFilter == null ? "n/a" : accessPathFilter)
+ + "}";
}
@@ -1284,4 +1285,23 @@
}
+ /**
+ * Return the constant bound on the {@link #getPredicate() predicate} for
+ * this access path at the specified index -or- {@link #NULL} iff the
+ * predicate is not bound at that position.
+ *
+ * @param index
+ * The index.
+ *
+ * @return Either the bound value -or- {@link #NULL} iff the index is
+ * unbound for the predicate for this access path.
+ */
+ public Object get(final int index) {
+
+ final IVariableOrConstant<?> t = predicate.get(index);
+
+ return t == null || t.isVar() ? null : t.get();
+
+ }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/AbstractRuleDistinctTermScan.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -35,7 +35,6 @@
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IVariable;
import com.bigdata.rdf.internal.IV;
-import com.bigdata.rdf.spo.SPOAccessPath;
import com.bigdata.rdf.spo.SPOKeyOrder;
import com.bigdata.rdf.spo.SPOPredicate;
import com.bigdata.rdf.spo.SPORelation;
@@ -51,9 +50,8 @@
/**
* Base class for rules having a single predicate that is none bound in the tail
- * and a single variable in the head. These rules can be evaluated using
- * {@link SPOAccessPath#distinctTermScan()} rather than a full index scan. For
- * example:
+ * and a single variable in the head. These rules can be evaluated using a
+ * distinctTermScan rather than a full index scan. For example:
*
* <pre>
* rdf1: (?u ?a ?y) -> (?a rdf:type rdf:Property)
@@ -62,7 +60,8 @@
* </pre>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
+ * @version $Id: AbstractRuleDistinctTermScan.java 3448 2010-08-18 20:55:58Z
+ * thompsonbry $
*/
abstract public class AbstractRuleDistinctTermScan extends Rule {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctTermAdvancer.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -35,10 +35,11 @@
import com.bigdata.btree.keys.SuccessorUtil;
import com.bigdata.rdf.internal.IVUtility;
import com.bigdata.rdf.store.IRawTripleStore;
+import com.bigdata.relation.accesspath.AccessPath;
/**
* Advances the source {@link ITupleCursor} through the distinct term
- * identifiers for some {@link SPOAccessPath}. Each time a new
+ * identifiers for some {@link AccessPath}. Each time a new
* {@link ITuple} is visited, the term identifier for the first position in
* that tuple is decoded and its successor is formed. The source
* {@link ITupleCursor} is then advanced to the key having that term
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -305,8 +305,7 @@
* statement identifier are not specified. This may be used as a convenience
* to extract the {s, p, o, c} from an {@link IPredicate} or from an
* {@link IAccessPath} when the predicate is not known to be an
- * {@link SPOPredicate} or the {@link IAccessPath} is not known to be an
- * {@link SPOAccessPath}.
+ * {@link SPOPredicate}.
*
* @param predicate
* The predicate.
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -46,94 +46,57 @@
*/
public class SPOAccessPath extends AccessPath<ISPO> {
-// private SPOTupleSerializer tupleSer;
-
-// /** Relation (resolved lazily if not specified to the ctor). */
-// private SPORelation relation;
-
/**
- * Variant when the {@link SPORelation} has already been materialized.
- * <p>
- * Note: Filters should be specified when the {@link IAccessPath} is
- * constructed so that they will be evaluated on the data service rather
- * than materializing the elements and then filtering them. This can be
- * accomplished by adding the filter as a constraint on the predicate when
- * specifying the access path.
*
+ * @param relation (optional).
+ * @param indexManager (required)
+ * @param timestamp
* @param predicate
* @param keyOrder
* @param ndx
* @param flags
+ * @param chunkOfChunksCapacity
+ * @param chunkCapacity
+ * @param fullyBufferedReadThreshold
*/
public SPOAccessPath(final SPORelation relation,
+ final IIndexManager indexManager, final long timestamp,
final IPredicate<ISPO> predicate, final IKeyOrder<ISPO> keyOrder,
final IIndex ndx, final int flags, final int chunkOfChunksCapacity,
final int chunkCapacity, final int fullyBufferedReadThreshold) {
- super(relation, relation.getIndexManager(), relation.getTimestamp(),
+ super(relation, indexManager, timestamp,
+// relation.getIndexManager(), relation.getTimestamp(),
predicate, keyOrder, ndx, flags, chunkOfChunksCapacity,
chunkCapacity, fullyBufferedReadThreshold);
}
- /**
- * Variant does not require the {@link SPORelation} to have been
- * materialized. This is useful when you want an {@link IAccessPath} for a
- * specific index partition.
- *
- * @param indexManager
- * @param timestamp
- * @param predicate
- * @param keyOrder
- * @param ndx
- * @param flags
- * @param chunkOfChunksCapacity
- * @param chunkCapacity
- * @param fullyBufferedReadThreshold
- */
- public SPOAccessPath(final IIndexManager indexManager,
- final long timestamp, final IPredicate<ISPO> predicate,
- final IKeyOrder<ISPO> keyOrder, final IIndex ndx, final int flags,
- final int chunkOfChunksCapacity, final int chunkCapacity,
- final int fullyBufferedReadThreshold) {
-
- super(null/* relation */, indexManager, timestamp, predicate, keyOrder,
- ndx, flags, chunkOfChunksCapacity, chunkCapacity,
- fullyBufferedReadThreshold);
-
- }
-
- /**
- * Return the constant bound on the {@link #getPredicate() predicate} for
- * this access path at the specified index -or- {@link #NULL} iff the
- * predicate is not bound at that position.
- *
- * @param index
- * The index.
- *
- * @return Either the bound value -or- {@link #NULL} iff the index is
- * unbound for the predicate for this access path.
- */
- @SuppressWarnings("unchecked")
- public IV get(final int index) {
-
- final IVariableOrConstant<IV> t = predicate.get(index);
-
- return t == null || t.isVar() ? null : t.get();
-
- }
-
-// protected SPOTupleSerializer getTupleSerializer() {
+// /**
+// * Variant does not require the {@link SPORelation} to have been
+// * materialized. This is useful when you want an {@link IAccessPath} for a
+// * specific index partition.
+// *
+// * @param indexManager
+// * @param timestamp
+// * @param predicate
+// * @param keyOrder
+// * @param ndx
+// * @param flags
+// * @param chunkOfChunksCapacity
+// * @param chunkCapacity
+// * @param fullyBufferedReadThreshold
+// */
+// public SPOAccessPath(final IIndexManager indexManager,
+// final long timestamp, final IPredicate<ISPO> predicate,
+// final IKeyOrder<ISPO> keyOrder, final IIndex ndx, final int flags,
+// final int chunkOfChunksCapacity, final int chunkCapacity,
+// final int fullyBufferedReadThreshold) {
//
-// if (tupleSer == null) {
+// super(null/* relation */, indexManager, timestamp, predicate, keyOrder,
+// ndx, flags, chunkOfChunksCapacity, chunkCapacity,
+// fullyBufferedReadThreshold);
//
-// tupleSer = (SPOTupleSerializer) ndx.getIndexMetadata()
-// .getTupleSerializer();
-//
-// }
-//
-// return tupleSer;
-//
// }
/**
@@ -143,35 +106,12 @@
*/
public SPOAccessPath init() {
-// final IKeyBuilder keyBuilder = getTupleSerializer().getKeyBuilder();
-//
-// setFromKey(((SPOKeyOrder) keyOrder).getFromKey(keyBuilder, predicate));
-//
-// setToKey(((SPOKeyOrder) keyOrder).getToKey(keyBuilder, predicate));
-
super.init();
return this;
}
-// /**
-// * Resolved lazily if not specified to the ctor.
-// */
-// synchronized
-// public SPORelation getRelation() {
-//
-// if (relation == null) {
-//
-// relation = (SPORelation) indexManager.getResourceLocator().locate(
-// predicate.getOnlyRelationName(), timestamp);
-//
-// }
-//
-// return relation;
-//
-// }
-
/**
* Strengthened return type.
* <p>
@@ -185,18 +125,14 @@
}
/**
- * Overridden to delegate to
- * {@link AbstractTripleStore#removeStatements(IChunkedOrderedIterator)} in
- * order to (a) write on all access paths; (b) handle statement identifiers,
- * including truth maintenance for statement identifiers; and (c) if
- * justifications are being maintained, then retract justifications having
- * no support once the statements visitable by this access path have been
- * retracted.
+ * Strengthened return type.
+ * <p>
+ * {@inheritDoc}
*/
@Override
- public long removeAll() {
+ public SPOPredicate getPredicate() {
- return getRelation().getContainer().removeStatements(iterator());
+ return (SPOPredicate) super.getPredicate();
}
@@ -206,13 +142,30 @@
* {@inheritDoc}
*/
@Override
- public SPOPredicate getPredicate() {
+ @SuppressWarnings("unchecked")
+ public IV get(final int index) {
- return (SPOPredicate) super.getPredicate();
+ return (IV) super.get(index);
}
/**
+ * Overridden to delegate to
+ * {@link AbstractTripleStore#removeStatements(IChunkedOrderedIterator)} in
+ * order to (a) write on all access paths; (b) handle statement identifiers,
+ * including truth maintenance for statement identifiers; and (c) if
+ * justifications are being maintained, then retract justifications having
+ * no support once the statements visitable by this access path have been
+ * retracted.
+ */
+ @Override
+ public long removeAll() {
+
+ return getRelation().getContainer().removeStatements(iterator());
+
+ }
+
+ /**
* Return a new {@link SPOAccessPath} where the context position has been
* bound to the specified constant. The context position MUST be a variable.
* All instances of that variable will be replaced by the specified
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -87,6 +87,7 @@
import com.bigdata.rdf.store.LocalTripleStore;
import com.bigdata.relation.AbstractRelation;
import com.bigdata.relation.IRelation;
+import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.ElementFilter;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IElementFilter;
@@ -1221,9 +1222,9 @@
final ILocalBTreeView ndx = (ILocalBTreeView) indexManager
.getIndex(name, timestamp);
- return new SPOAccessPath(indexManager, timestamp, predicate,
- keyOrder, ndx, flags, getChunkOfChunksCapacity(),
- getChunkCapacity(), getFullyBufferedReadThreshold()).init();
+ return newAccessPath(this/* relation */, indexManager, timestamp,
+ predicate, keyOrder, ndx, flags, getChunkOfChunksCapacity(),
+ getChunkCapacity(), getFullyBufferedReadThreshold());
}
@@ -1291,12 +1292,25 @@
final int fullyBufferedReadThreshold = container.getFullyBufferedReadThreshold();
- return new SPOAccessPath(this, predicate, keyOrder, ndx, flags,
- chunkOfChunksCapacity, chunkCapacity,
- fullyBufferedReadThreshold).init();
-
+ return newAccessPath(this, getIndexManager(), getTimestamp(),
+ predicate, keyOrder, ndx, flags, chunkOfChunksCapacity,
+ chunkCapacity, fullyBufferedReadThreshold);
+
}
-
+
+ @Override
+ public SPOAccessPath newAccessPath(final IRelation<ISPO> relation,
+ final IIndexManager indexManager, final long timestamp,
+ final IPredicate<ISPO> predicate, final IKeyOrder<ISPO> keyOrder,
+ final IIndex ndx, final int flags, final int chunkOfChunksCapacity,
+ final int chunkCapacity, final int fullyBufferedReadThreshold) {
+
+ return new SPOAccessPath(this/* relation */, indexManager, timestamp,
+ predicate, keyOrder, ndx, flags, chunkOfChunksCapacity,
+ chunkCapacity, fullyBufferedReadThreshold).init();
+
+ }
+
// public long getElementCount(boolean exact) {
//
// final IIndex ndx = getIndex(SPOKeyOrder.SPO);
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -1714,6 +1714,13 @@
new Bigdata2Sesame2BindingSetIterator<QueryEvaluationException>(
new BigdataBindingSetResolverator(database, it2).start(database
.getExecutorService()));
+
+ try {
+ // Wait for the Future (checks for errors).
+ runningQuery.get();
+ } catch (Exception ex) {
+ throw new QueryEvaluationException(ex);
+ }
// final boolean backchain = //
// tripleSource.getDatabase().getAxioms().isRdfSchema()
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -133,7 +133,6 @@
import com.bigdata.rdf.spo.ISPO;
import com.bigdata.rdf.spo.InferredSPOFilter;
import com.bigdata.rdf.spo.SPO;
-import com.bigdata.rdf.spo.SPOAccessPath;
import com.bigdata.rdf.spo.SPOKeyOrder;
import com.bigdata.rdf.spo.SPORelation;
import com.bigdata.rdf.store.AbstractTripleStore;
@@ -148,6 +147,7 @@
import com.bigdata.rdf.store.ScaleOutTripleStore;
import com.bigdata.rdf.store.TempTripleStore;
import com.bigdata.rdf.vocab.NoVocabulary;
+import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.EmptyAccessPath;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.accesspath.IElementFilter;
@@ -2972,7 +2972,7 @@
/**
* Note: The <i>includeInferred</i> argument is applied in two ways.
- * First, inferences are stripped out of the {@link SPOAccessPath}.
+ * First, inferences are stripped out of the {@link AccessPath}.
* Second, query time expansion of
* <code>foo rdf:type rdfs:Resource</code>, owl:sameAs, etc.
* <p>
Modified: branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java 2010-09-29 16:57:20 UTC (rev 3682)
+++ branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java 2010-09-29 19:14:33 UTC (rev 3683)
@@ -42,8 +42,8 @@
*/
public class Striterator implements IStriterator {
volatile List<IFilter> filters = null; // Note: NOT serializable.
- private transient Iterator realSource;
- private transient Iterator m_src = null;
+ private volatile Iterator realSource;
+ private volatile Iterator m_src = null;
/**
* Deserialization constructor.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-09-30 16:58:52
|
Revision: 3700
http://bigdata.svn.sourceforge.net/bigdata/?rev=3700&view=rev
Author: thompsonbry
Date: 2010-09-30 16:58:46 +0000 (Thu, 30 Sep 2010)
Log Message:
-----------
Modified the QueryEngine to have a concept of its RMI "proxy" and modified the FederatedQueryEngine to use a real RMI proxy for itself as the queryController field to be sent around with the query.
Fixed build.xml. It was not staging out the ctc-striterators.
Made R.KeyOrder Serializable.
However, we still can not unit test the federated query engine because the test classes (R, KeyOrder) are not being packaged with the deployment....
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/R.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/build.xml
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2010-09-30 16:14:43 UTC (rev 3699)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2010-09-30 16:58:46 UTC (rev 3700)
@@ -271,6 +271,16 @@
}
/**
+ * The RMI proxy for this {@link QueryEngine} when used as a query controller.
+ * The default implementation returns <i>this</i>.
+ */
+ public IQueryClient getProxy() {
+
+ return this;
+
+ }
+
+ /**
* Return <code>true</code> iff running against an
* {@link IBigdataFederation}.
*/
@@ -655,7 +665,7 @@
throw new IllegalArgumentException();
final RunningQuery runningQuery = newRunningQuery(this, queryId,
- true/* controller */, this/* clientProxy */, query);
+ true/* controller */, getProxy()/*queryController*/, query);
final long timeout = query.getProperty(BOp.Annotations.TIMEOUT,
BOp.Annotations.DEFAULT_TIMEOUT);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java 2010-09-30 16:14:43 UTC (rev 3699)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java 2010-09-30 16:58:46 UTC (rev 3700)
@@ -41,8 +41,8 @@
import org.apache.log4j.Logger;
+import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.PipelineOp;
-import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.engine.IChunkMessage;
import com.bigdata.bop.engine.IQueryClient;
import com.bigdata.bop.engine.IQueryDecl;
@@ -55,6 +55,7 @@
import com.bigdata.service.IDataService;
import com.bigdata.service.ManagedResourceService;
import com.bigdata.service.ResourceService;
+import com.bigdata.service.jini.JiniFederation;
import com.bigdata.util.InnerCause;
/**
@@ -91,6 +92,11 @@
private final ManagedResourceService resourceService;
/**
+ * The proxy for this query engine when used as a query controller.
+ */
+ private final IQueryClient clientProxy;
+
+ /**
* A queue of {@link IChunkMessage}s which needs to have their data
* materialized so an operator can consume those data on this node.
*/
@@ -126,7 +132,19 @@
}
+ /**
+ * Overridden to return an RMI proxy for this {@link FederatedQueryEngine}.
+ * <p>
+ * {@inheritDoc}
+ */
@Override
+ public IQueryClient getProxy() {
+
+ return clientProxy;
+
+ }
+
+ @Override
final public boolean isScaleOut() {
return true;
@@ -196,6 +214,9 @@
this.resourceService = resourceService;
+ // the proxy for this query engine when used as a query controller.
+ this.clientProxy = (IQueryClient) ((JiniFederation<?>)fed).getProxy(this, false/*enableDGC*/);
+
}
/**
@@ -475,7 +496,7 @@
final PipelineOp query) {
return new FederatedRunningQuery(this, queryId, controller,
- clientProxy, query);
+ this.clientProxy, query);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/R.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/R.java 2010-09-30 16:14:43 UTC (rev 3699)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/R.java 2010-09-30 16:58:46 UTC (rev 3700)
@@ -27,6 +27,7 @@
package com.bigdata.bop.ap;
+import java.io.Serializable;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
@@ -64,9 +65,11 @@
/**
* Metadata about the index orders for this relation.
*/
- private static class KeyOrder extends AbstractKeyOrder<E> {
+ public static class KeyOrder extends AbstractKeyOrder<E> implements Serializable {
- public Comparator<E> getComparator() {
+ private static final long serialVersionUID = 1L;
+
+ public Comparator<E> getComparator() {
return new EComparator();
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java 2010-09-30 16:14:43 UTC (rev 3699)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java 2010-09-30 16:58:46 UTC (rev 3700)
@@ -48,6 +48,7 @@
import com.bigdata.bop.engine.BOpStats;
import com.bigdata.bop.engine.MockRunningQuery;
import com.bigdata.btree.ITuple;
+import com.bigdata.io.SerializerUtil;
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.ITx;
import com.bigdata.journal.Journal;
@@ -164,6 +165,12 @@
}
+ public void test_keyOrderSerializable() {
+
+ SerializerUtil.serialize(R.primaryKeyOrder);
+
+ }
+
/**
* Using a predicate with nothing bound, verify that we get the
* right range count on the relation and that we read the correct elements
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java 2010-09-30 16:14:43 UTC (rev 3699)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java 2010-09-30 16:58:46 UTC (rev 3700)
@@ -217,8 +217,8 @@
super.setUp();
-// dataService0 = fed.getDataService(dataServices[0]);
-// dataService1 = fed.getDataService(dataServices[1]);
+ dataService0 = fed.getDataService(dataServices[0]);
+ dataService1 = fed.getDataService(dataServices[1]);
// {
//
// // @todo need to wait for the dataService to be running.
Modified: branches/QUADS_QUERY_BRANCH/build.xml
===================================================================
--- branches/QUADS_QUERY_BRANCH/build.xml 2010-09-30 16:14:43 UTC (rev 3699)
+++ branches/QUADS_QUERY_BRANCH/build.xml 2010-09-30 16:58:46 UTC (rev 3700)
@@ -1124,6 +1124,9 @@
<copy toDir="${build.dir}/bigdata-sails/src">
<fileset dir="${bigdata.dir}/bigdata-sails/src" />
</copy>
+ <copy toDir="${build.dir}/ctc-striterators/src">
+ <fileset dir="${bigdata.dir}/ctc-striterators/src" />
+ </copy>
<copy toDir="${build.dir}/lgpl-utils/src">
<fileset dir="${bigdata.dir}/lgpl-utils/src" />
</copy>
@@ -1194,6 +1197,7 @@
<include name="bigdata-jini/src/**" />
<include name="bigdata-rdf/src/**" />
<include name="bigdata-sails/src/**" />
+ <include name="ctc-striterators/src/**" />
<include name="lgpl-utils/src/**" />
<include name="bigdata/lib/**" />
<include name="bigdata-jini/lib/**" />
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-10-05 19:57:35
|
Revision: 3730
http://bigdata.svn.sourceforge.net/bigdata/?rev=3730&view=rev
Author: thompsonbry
Date: 2010-10-05 19:57:25 +0000 (Tue, 05 Oct 2010)
Log Message:
-----------
UNION is still not implemented. It is the only remaining feature to be
code complete for scale-out quads query. The decision tree rewrites
in Rule2BOpUtility are still disabled by default. However, if you
enable it and then run TestBigdataSailWithQuads, only one additional
test fails ("Basic - Var 2") due to a "duplicate bop exception". The
problem goes back to the hashCode() and equals() implementations of
Predicate. I will address those in a second commit.
I also plan to look at some optimizations, including: deferring some
aspects of default graph and named graph rewrites to run time;
improved pipelining in the QueryEngine (chaining BlockingBuffers
together in pipes).
- Constants are no longer cloned during deep copy of operator trees.
Also, both Constant and Var now extend a common base class which
disallows copy-on-write mutations to their annotations.
- Hooked in cost model for SCALE-OUT. This requires a mixture of
models to approximate the shard views. Changed from a scalar 'cost'
into a ScanCostReport and a SubqueryCostReport. The cost reports
are attached as annotations to the query plan by Rule2BOpUtility.
- Added a shard split handler for the xxxC indices. Added a test suite
for that split handler. Refactored the existing split handler test
suites to have one test suite per split handler implementation.
- Fixed a bug in DataSetJoin. It was not constraining the output
solutions correctly.
- Removed the "NESTED_SUBQUERY" option, the implementation class
(NestedSubqueryWithJoinThreadsTask), and all of the logic to test
that join throughout the various test suites.
- Cleaned up logic to obtain an AccessPath (all code paths). There was
a broken assumption built into the refactor that the IKeyOrder could
be specified on the IPredicate as an annotation. This does not work
because we select the IKeyOrder, and hence the IAccessPath, based on
the asBound predicate during joins in order to always read on the
most efficient access path. This is also true in scale-out where we
map each binding set across the predicate for the target join in
order to obtain the asBound state for that predicate and then push
the binding set to the shard on which that asBound predicate will
read (or write).
- Fixed mapping over shards. It assumed that the target IKeyOrder was
fixed in advance. In fact, it should be determined at runtime for
normal pipeline joins.
- Moved the Rule2BOpUtility and associated classes to com.bigdata.sail
since they are SPARQL specific.
- Renamed ISolutionExpander to IAccessPathExpander
- An occasional test failure appears in
TestQueryEngine#test_queryJoin2_concurrentStressTest() where it
reports one failure ("nerror=1, ncancel=0, ntimeout=0, ninterrupt=0,
nsuccess=999"). I have not tracked this down, but I have noticed
this off and on for several days.
- TestFederatedQueryEngine is hanging / having controller identity
issues. This shows up on
test_query_join_2shards_nothingBoundOnAccessPath(). Once it happens
the query engine does not tear down correctly and CI will hang.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/query-cost-model.xls
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Constant.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Var.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_FullyBoundPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Bundle.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/MapBindingSetsOverShardsBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractTuple.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeStatistics.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BTreeUtilizationReport.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ILocalBTreeView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IndexSegment.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractRelation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/AbstractResource.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/IRelation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/RelationFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/BlockingBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/EmptyAccessPathExpander.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/AbstractJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultEvaluationPlan2.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRangeCountFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/DefaultRuleTaskFactory.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/IJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/NoReorderEvaluationPlan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/pipeline/DistributedJoinMasterTask.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/search/FullTextIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractScaleOutFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IBigdataClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedWrappedIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/IChunkedIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestBOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/shards/TestMapBindingSetsOverShards.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/locator/TestDefaultResourceLocator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/rule/TestRule.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestSegSplitter.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/DataSetJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/OwlSameAsPropertiesExpandingIterator.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/load/MappedRDFDataLoadMaster.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/magic/MagicRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rules/RDFJoinNexus.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/NamedGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/join/TestDataSetJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestContextAdvancer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestDatabaseAtOnceClosure.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestOptionals.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestRuleExpansion.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/rules/TestSlice.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestSPORelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/store/StressTestCentos.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/FreeTextSearchExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNamedGraphs.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestOptionals.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataConnectionTest.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataSparqlTest.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataStoreTest.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/stress/LoadClosureAndQueryTest.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/NOPFilter.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ImmutableBOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/IAccessPathExpander.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractTestSegSplitter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestFixedLengthPrefixShardSplits.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/TestSparseRowStoreSplitHandler.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/XXXCShardSplitHandler.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/spo/TestXXXCShardSplitHandler.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/DGExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/DataSetSummary.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsWithoutInlining.java
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/ISolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/rule/eval/NestedSubqueryWithJoinThreadsTask.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoins.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuadsAndPipelineJoinsWithoutInlining.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/architecture/query-cost-model.xls
===================================================================
(Binary files differ)
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -37,6 +37,7 @@
import java.util.Map;
import com.bigdata.bop.constraint.EQ;
+import com.bigdata.btree.Tuple;
/**
* Abstract base class for {@link BOp}s.
@@ -411,10 +412,12 @@
* The name.
* @param value
* The value.
+ *
+ * @return The old value.
*/
- protected void _setProperty(final String name, final Object value) {
+ protected Object _setProperty(final String name, final Object value) {
- annotations.put(name,value);
+ return annotations.put(name,value);
}
@@ -472,7 +475,7 @@
final BOpBase tmp = this.clone();
- if (tmp.annotations.put(name, value) != null)
+ if (tmp._setProperty(name, value) != null)
throw new IllegalStateException("Already set: name=" + name
+ ", value=" + value);
@@ -480,6 +483,49 @@
}
+ /**
+ * Clear the named annotation.
+ *
+ * @param name
+ * The annotation.
+ * @return A copy of this {@link BOp} in which the named annotation has been
+ * removed.
+ */
+ public BOpBase clearProperty(final String name) {
+
+ if (name == null)
+ throw new IllegalArgumentException();
+
+ final BOpBase tmp = this.clone();
+
+ tmp._clearProperty(name);
+
+ return tmp;
+
+ }
+
+ /**
+ * Strips off the named annotations.
+ *
+ * @param names
+ * The annotations to be removed.
+ *
+ * @return A copy of this {@link BOp} in which the specified annotations do not appear.
+ */
+ public BOp clearAnnotations(final String[] names) {
+
+ final BOpBase tmp = this.clone();
+
+ for(String name : names) {
+
+ tmp._clearProperty(name);
+
+ }
+
+ return tmp;
+
+ }
+
public int getId() {
return (Integer) getRequiredProperty(Annotations.BOP_ID);
@@ -489,14 +535,23 @@
public String toString() {
final StringBuilder sb = new StringBuilder();
-// sb.append(getClass().getName());
- sb.append(super.toString());
+ sb.append(getClass().getName());
+// sb.append(super.toString());
+ final Integer bopId = (Integer) getProperty(Annotations.BOP_ID);
+ if (bopId != null) {
+ sb.append("[" + bopId + "]");
+ }
sb.append("(");
- for (int i = 0; i < args.length; i++) {
- final BOp t = args[i];
- if (i > 0)
+ int nwritten = 0;
+ for (BOp t : args) {
+ if (nwritten > 0)
sb.append(',');
sb.append(t.getClass().getSimpleName());
+ final Integer tid = (Integer) t.getProperty(Annotations.BOP_ID);
+ if (tid != null) {
+ sb.append("[" + tid + "]");
+ }
+ nwritten++;
}
sb.append(")");
annotationsToString(sb);
@@ -510,11 +565,16 @@
sb.append("[");
boolean first = true;
for (Map.Entry<String, Object> e : annotations.entrySet()) {
- if (!first)
+ if (first)
+ sb.append(" ");
+ else
sb.append(", ");
if (e.getValue() != null && e.getValue().getClass().isArray()) {
sb.append(e.getKey() + "="
+ Arrays.toString((Object[]) e.getValue()));
+ } else if (e.getKey() == IPredicate.Annotations.FLAGS) {
+ sb.append(e.getKey() + "="
+ + Tuple.flagString((Integer) e.getValue()));
} else {
sb.append(e.getKey() + "=" + e.getValue());
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpContextBase.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -30,25 +30,15 @@
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
-import org.apache.log4j.Logger;
-
import com.bigdata.bop.engine.QueryEngine;
-import com.bigdata.btree.IIndex;
import com.bigdata.btree.ILocalBTreeView;
-import com.bigdata.btree.IRangeQuery;
import com.bigdata.journal.IIndexManager;
-import com.bigdata.journal.TimestampUtility;
-import com.bigdata.relation.AbstractRelation;
import com.bigdata.relation.IRelation;
-import com.bigdata.relation.accesspath.AccessPath;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.locator.IResourceLocator;
import com.bigdata.relation.rule.IRule;
-import com.bigdata.relation.rule.ISolutionExpander;
import com.bigdata.relation.rule.eval.IJoinNexus;
-import com.bigdata.service.DataService;
import com.bigdata.service.IBigdataFederation;
-import com.bigdata.striterator.IKeyOrder;
/**
* Base class for the bigdata operation evaluation context (NOT serializable).
@@ -58,11 +48,16 @@
*/
public class BOpContextBase {
- static private final transient Logger log = Logger.getLogger(BOpContextBase.class);
+// static private final transient Logger log = Logger.getLogger(BOpContextBase.class);
-// private final QueryEngine queryEngine;
-
+ /**
+ * The federation iff running in scale-out.
+ */
private final IBigdataFederation<?> fed;
+
+ /**
+ * The <strong>local</strong> index manager.
+ */
private final IIndexManager indexManager;
/**
@@ -104,8 +99,11 @@
/**
* Core constructor.
+ *
* @param fed
+ * The federation iff running in scale-out.
* @param indexManager
+ * The <strong>local</strong> index manager.
*/
public BOpContextBase(final IBigdataFederation<?> fed,
final IIndexManager indexManager) {
@@ -232,7 +230,7 @@
* conversion is done. It has much of the same logic (this also
* handles remote access paths now).
*
- * @todo Support mutable relation views.
+ * @todo Support mutable relation views (no - just fix truth maintenance).
*/
// @SuppressWarnings("unchecked")
public <E> IAccessPath<E> getAccessPath(final IRelation<E> relation,
@@ -244,147 +242,149 @@
if (predicate == null)
throw new IllegalArgumentException();
- /*
- * FIXME This should be as assigned by the query planner so the query is
- * fully declarative.
- */
- final IKeyOrder<E> keyOrder;
- {
- final IKeyOrder<E> tmp = predicate.getKeyOrder();
- if (tmp != null) {
- // use the specified index.
- keyOrder = tmp;
- } else {
- // ask the relation for the best index.
- keyOrder = relation.getKeyOrder(predicate);
- }
- }
+ return relation.getAccessPath(indexManager/* localIndexManager */,
+ relation.getKeyOrder(predicate), predicate);
- if (keyOrder == null)
- throw new RuntimeException("No access path: " + predicate);
+// /*
+// * Note: ALWAYS use the "perfect" index.
+// */
+// final IKeyOrder<E> keyOrder = relation.getKeyOrder(predicate);
+//// {
+//// final IKeyOrder<E> tmp = predicate.getKeyOrder();
+//// if (tmp != null) {
+//// // use the specified index.
+//// keyOrder = tmp;
+//// } else {
+//// // ask the relation for the best index.
+//// keyOrder = relation.getKeyOrder(predicate);
+//// }
+//// }
+////
+//// if (keyOrder == null)
+//// throw new RuntimeException("No access path: " + predicate);
+//
+// final int partitionId = predicate.getPartitionId();
+//
+// final long timestamp = (Long) predicate
+// .getRequiredProperty(BOp.Annotations.TIMESTAMP);
+//
+// final int flags = predicate.getProperty(
+// IPredicate.Annotations.FLAGS,
+// IPredicate.Annotations.DEFAULT_FLAGS)
+// | (TimestampUtility.isReadOnly(timestamp) ? IRangeQuery.READONLY
+// : 0);
+//
+// final int chunkOfChunksCapacity = predicate.getProperty(
+// BufferAnnotations.CHUNK_OF_CHUNKS_CAPACITY,
+// BufferAnnotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+//
+// final int chunkCapacity = predicate.getProperty(
+// BufferAnnotations.CHUNK_CAPACITY,
+// BufferAnnotations.DEFAULT_CHUNK_CAPACITY);
+//
+// final int fullyBufferedReadThreshold = predicate.getProperty(
+// IPredicate.Annotations.FULLY_BUFFERED_READ_THRESHOLD,
+// IPredicate.Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
+//
+// if (partitionId != -1) {
+//
+// /*
+// * Note: This handles a read against a local index partition. For
+// * scale-out, the [indexManager] will be the data service's local
+// * index manager.
+// *
+// * Note: Expanders ARE NOT applied in this code path. Expanders
+// * require a total view of the relation, which is not available
+// * during scale-out pipeline joins. Likewise, the [backchain]
+// * property will be ignored since it is handled by an expander.
+// *
+// * @todo Replace this with IRelation#getAccessPathForIndexPartition()
+// */
+//// return ((AbstractRelation<?>) relation)
+//// .getAccessPathForIndexPartition(indexManager,
+//// (IPredicate) predicate);
+//
+// /*
+// * @todo This is an error since expanders are currently ignored on
+// * shard-wise access paths. While it is possible to enable expanders
+// * for shard-wise access paths.
+// */
+// if (predicate.getSolutionExpander() != null)
+// throw new IllegalArgumentException();
+//
+// final String namespace = relation.getNamespace();//predicate.getOnlyRelationName();
+//
+// // The name of the desired index partition.
+// final String name = DataService.getIndexPartitionName(namespace
+// + "." + keyOrder.getIndexName(), partitionId);
+//
+// // MUST be a local index view.
+// final ILocalBTreeView ndx = (ILocalBTreeView) indexManager
+// .getIndex(name, timestamp);
+//
+// return new AccessPath<E>(relation, indexManager, timestamp,
+// predicate, keyOrder, ndx, flags, chunkOfChunksCapacity,
+// chunkCapacity, fullyBufferedReadThreshold).init();
+//
+// }
+//
+//// accessPath = relation.getAccessPath((IPredicate) predicate);
+//
+// // Decide on a local or remote view of the index.
+// final IIndexManager indexManager;
+// if (predicate.isRemoteAccessPath()) {
+// // use federation in scale-out for a remote access path.
+// indexManager = fed != null ? fed : this.indexManager;
+// } else {
+// indexManager = this.indexManager;
+// }
+//
+// // Obtain the index.
+// final String fqn = AbstractRelation.getFQN(relation, keyOrder);
+// final IIndex ndx = AbstractRelation.getIndex(indexManager, fqn, timestamp);
+//
+// if (ndx == null) {
+//
+// throw new IllegalArgumentException("no index? relation="
+// + relation.getNamespace() + ", timestamp=" + timestamp
+// + ", keyOrder=" + keyOrder + ", pred=" + predicate
+// + ", indexManager=" + getIndexManager());
+//
+// }
+//
+// // Obtain the access path for that relation and index.
+// final IAccessPath<E> accessPath = ((AbstractRelation<E>) relation)
+// .newAccessPath(relation, indexManager, timestamp, predicate,
+// keyOrder, ndx, flags, chunkOfChunksCapacity,
+// chunkCapacity, fullyBufferedReadThreshold);
+//
+// // optionally wrap with an expander pattern.
+// return expander(predicate, accessPath);
- final int partitionId = predicate.getPartitionId();
-
- final long timestamp = (Long) predicate
- .getRequiredProperty(BOp.Annotations.TIMESTAMP);
-
- final int flags = predicate.getProperty(
- IPredicate.Annotations.FLAGS,
- IPredicate.Annotations.DEFAULT_FLAGS)
- | (TimestampUtility.isReadOnly(timestamp) ? IRangeQuery.READONLY
- : 0);
-
- final int chunkOfChunksCapacity = predicate.getProperty(
- BufferAnnotations.CHUNK_OF_CHUNKS_CAPACITY,
- BufferAnnotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
-
- final int chunkCapacity = predicate.getProperty(
- BufferAnnotations.CHUNK_CAPACITY,
- BufferAnnotations.DEFAULT_CHUNK_CAPACITY);
-
- final int fullyBufferedReadThreshold = predicate.getProperty(
- IPredicate.Annotations.FULLY_BUFFERED_READ_THRESHOLD,
- IPredicate.Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
-
- if (partitionId != -1) {
-
- /*
- * Note: This handles a read against a local index partition. For
- * scale-out, the [indexManager] will be the data service's local
- * index manager.
- *
- * Note: Expanders ARE NOT applied in this code path. Expanders
- * require a total view of the relation, which is not available
- * during scale-out pipeline joins. Likewise, the [backchain]
- * property will be ignored since it is handled by an expander.
- *
- * @todo Replace this with IRelation#getAccessPathForIndexPartition()
- */
-// return ((AbstractRelation<?>) relation)
-// .getAccessPathForIndexPartition(indexManager,
-// (IPredicate) predicate);
-
- /*
- * @todo This is an error since expanders are currently ignored on
- * shard-wise access paths. While it is possible to enable expanders
- * for shard-wise access paths.
- */
- if (predicate.getSolutionExpander() != null)
- throw new IllegalArgumentException();
-
- final String namespace = relation.getNamespace();//predicate.getOnlyRelationName();
-
- // The name of the desired index partition.
- final String name = DataService.getIndexPartitionName(namespace
- + "." + keyOrder.getIndexName(), partitionId);
-
- // MUST be a local index view.
- final ILocalBTreeView ndx = (ILocalBTreeView) indexManager
- .getIndex(name, timestamp);
-
- return new AccessPath<E>(relation, indexManager, timestamp,
- predicate, keyOrder, ndx, flags, chunkOfChunksCapacity,
- chunkCapacity, fullyBufferedReadThreshold).init();
-
- }
-
-// accessPath = relation.getAccessPath((IPredicate) predicate);
-
- // Decide on a local or remote view of the index.
- final IIndexManager indexManager;
- if (predicate.isRemoteAccessPath()) {
- // use federation in scale-out for a remote access path.
- indexManager = fed != null ? fed : this.indexManager;
- } else {
- indexManager = this.indexManager;
- }
-
- // Obtain the index.
- final String fqn = AbstractRelation.getFQN(relation, keyOrder);
- final IIndex ndx = AbstractRelation.getIndex(indexManager, fqn, timestamp);
-
- if (ndx == null) {
-
- throw new IllegalArgumentException("no index? relation="
- + relation.getNamespace() + ", timestamp=" + timestamp
- + ", keyOrder=" + keyOrder + ", pred=" + predicate
- + ", indexManager=" + getIndexManager());
-
- }
-
- // Obtain the access path for that relation and index.
- final IAccessPath<E> accessPath = ((AbstractRelation<E>) relation)
- .newAccessPath(relation, indexManager, timestamp, predicate,
- keyOrder, ndx, flags, chunkOfChunksCapacity,
- chunkCapacity, fullyBufferedReadThreshold);
-
- // optionally wrap with an expander pattern.
- return expander(predicate, accessPath);
-
}
- /**
- * Optionally wrap with an expander pattern.
- *
- * @param predicate
- * @param accessPath
- * @return
- * @param <E>
- */
- private <E> IAccessPath<E> expander(final IPredicate<E> predicate,
- final IAccessPath<E> accessPath) {
+// /**
+// * Optionally wrap with an expander pattern.
+// *
+// * @param predicate
+// * @param accessPath
+// * @return
+// * @param <E>
+// */
+// private <E> IAccessPath<E> expander(final IPredicate<E> predicate,
+// final IAccessPath<E> accessPath) {
+//
+// final ISolutionExpander<E> expander = predicate.getSolutionExpander();
+//
+// if (expander != null) {
+//
+// // allow the predicate to wrap the access path
+// return expander.getAccessPath(accessPath);
+//
+// }
+//
+// return accessPath;
+//
+// }
- final ISolutionExpander<E> expander = predicate.getSolutionExpander();
-
- if (expander != null) {
-
- // allow the predicate to wrap the access path
- return expander.getAccessPath(accessPath);
-
- }
-
- return accessPath;
-
- }
-
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpUtility.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -413,7 +413,8 @@
* trees whose sinks target a descendant, which is another way
* to create a loop.
*/
- throw new DuplicateBOpException(t.toString());
+ throw new DuplicateBOpException("id=" + t.getId() + ", root="
+ + toString(op));
}
}
// wrap to ensure immutable and thread-safe.
@@ -528,4 +529,93 @@
} // toArray()
+ /**
+ * Pretty print a bop.
+ *
+ * @param bop
+ * The bop.
+ *
+ * @return The formatted representation.
+ */
+ public static String toString(final BOp bop) {
+
+ final StringBuilder sb = new StringBuilder();
+
+ toString(bop, sb, 0);
+
+ // chop off the last \n
+ sb.setLength(sb.length() - 1);
+
+ return sb.toString();
+
+ }
+
+ private static void toString(final BOp bop, final StringBuilder sb,
+ final int indent) {
+
+ sb.append(indent(indent)).append(
+ bop == null ? "<null>" : bop.toString()).append('\n');
+
+ if (bop == null)
+ return;
+
+ for (BOp arg : bop.args()) {
+
+ if (arg.arity() > 0) {
+
+ toString(arg, sb, indent+1);
+
+ }
+
+ }
+
+ }
+
+ /**
+ * Returns a string that may be used to indent a dump of the nodes in
+ * the tree.
+ *
+ * @param height
+ * The height.
+ *
+ * @return A string suitable for indent at that height.
+ */
+ private static String indent(final int height) {
+
+ if( height == -1 ) {
+
+ // The height is not defined.
+
+ return "";
+
+ }
+
+ return ws.substring(0, height * 4);
+
+ }
+
+ private static final transient String ws = " ";
+
+// /**
+// * Verify that all bops from the identified <i>startId</i> to the root are
+// * {@link PipelineOp}s and have an assigned {@link BOp.Annotations#BOP_ID}.
+// * This is required in order for us to be able to target messages to those
+// * operators.
+// *
+// * @param startId
+// * The {@link BOp.Annotations#BOP_ID} at which the query will
+// * start. This is typically the left-most descendant.
+// * @param root
+// * The root of the operator tree.
+// *
+// * @throws RuntimeException
+// * if the operator tree does not meet any of the criteria for
+// * pipeline evaluation.
+// */
+// public static void verifyPipline(final int startId, final BOp root) {
+//
+// throw new UnsupportedOperationException();
+//
+// }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -55,7 +55,7 @@
int DEFAULT_CHUNK_OF_CHUNKS_CAPACITY = 100;
/**
- * Sets the capacity of the {@link IBuffer}s used to accumulate a chunk of
+ * Sets the capacity of the {@link IBuffer}[]s used to accumulate a chunk of
* {@link IBindingSet}s (default {@value #CHUNK_CAPACITY}). Partial chunks
* may be automatically combined into full chunks.
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Constant.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Constant.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Constant.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -29,7 +29,7 @@
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
-final public class Constant<E> extends BOpBase implements IConstant<E> {
+final public class Constant<E> extends ImmutableBOp implements IConstant<E> {
/**
*
@@ -84,6 +84,15 @@
}
+ /**
+ * Clone is overridden to reduce heap churn.
+ */
+ final public Constant<E> clone() {
+
+ return this;
+
+ }
+
public String toString() {
return value.toString();
@@ -150,5 +159,5 @@
throw new UnsupportedOperationException();
}
-
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -45,7 +45,7 @@
import com.bigdata.relation.accesspath.ElementFilter;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.rule.IRule;
-import com.bigdata.relation.rule.ISolutionExpander;
+import com.bigdata.relation.rule.IAccessPathExpander;
import com.bigdata.relation.rule.eval.IEvaluationPlan;
import com.bigdata.relation.rule.eval.pipeline.JoinMasterTask;
import com.bigdata.striterator.IKeyOrder;
@@ -81,15 +81,15 @@
*/
String RELATION_NAME = "relationName";
- /**
- * The {@link IKeyOrder} which will be used to read on the relation.
- * <p>
- * Note: This is generally assigned by the query optimizer. The query
- * optimizer considers the possible indices for each {@link IPredicate}
- * in a query and the possible join orders and then specifies the order
- * of the joins and the index to use for each join.
- */
- String KEY_ORDER = "keyOrder";
+// /**
+// * The {@link IKeyOrder} which will be used to read on the relation.
+// * <p>
+// * Note: This is generally assigned by the query optimizer. The query
+// * optimizer considers the possible indices for each {@link IPredicate}
+// * in a query and the possible join orders and then specifies the order
+// * of the joins and the index to use for each join.
+// */
+// String KEY_ORDER = "keyOrder";
/**
* <code>true</code> iff the predicate is optional (the right operand of
@@ -160,17 +160,28 @@
* Access path expander pattern. This allows you to wrap or replace the
* {@link IAccessPath}.
* <p>
- * Note: You MUST be extremely careful when using this feature in
- * scale-out. Access path expanders in scale-out are logically
- * consistent with used with a {@link #REMOTE_ACCESS_PATH}, but remote
- * access paths often lack the performance of a local access path.
+ * Note: Access path expanders in scale-out are logically consistent
+ * when used with a {@link #REMOTE_ACCESS_PATH}. However, remote access
+ * paths often lack the performance of a local access path. In order for
+ * the expander to be consistent with a local access path it MUST NOT
+ * rewrite the predicate in such a manner as to read on data onto found
+ * on the shard onto which the predicate was mapped during query
+ * evaluation.
* <p>
- * In order for the expander to be consistent with a local access path
- * it MUST NOT rewrite the predicate in such a manner as to read on data
- * onto found on the shard onto which the predicate was mapped during
- * query evaluation.
+ * In general, scale-out query depends on binding sets being mapped onto
+ * the shards against which they will read or write. When an expander
+ * changes the bindings on an {@link IPredicate}, it typically changes
+ * the access path which will be used. This is only supported when the
+ * access path is {@link #REMOTE_ACCESS_PATH}.
+ * <p>
+ * Note: If an expander generates nested {@link IAccessPath}s then it
+ * typically must strip off both the {@link #ACCESS_PATH_EXPANDER} and
+ * the {@link #ACCESS_PATH_EXPANDER} from the {@link IPredicate} before
+ * generating the inner {@link IAccessPath} and then layer the
+ * {@link #ACCESS_PATH_FILTER} back onto the expanded visitation
+ * pattern.
*
- * @see ISolutionExpander
+ * @see IAccessPathExpander
*/
String ACCESS_PATH_EXPANDER = "accessPathExpander";
@@ -335,7 +346,7 @@
* enforced using {@link IConstraint}s on the {@link IRule}.
* <p>
* More control over the behavior of optionals may be gained through the use
- * of an {@link ISolutionExpander} pattern.
+ * of an {@link IAccessPathExpander} pattern.
*
* @return <code>true</code> iff this predicate is optional when evaluating
* a JOIN.
@@ -348,14 +359,14 @@
* Returns the object that may be used to selectively override the
* evaluation of the predicate.
*
- * @return The {@link ISolutionExpander}.
+ * @return The {@link IAccessPathExpander}.
*
* @see Annotations#ACCESS_PATH_EXPANDER
*
- * @todo replace with {@link ISolutionExpander#getAccessPath(IAccessPath)},
- * which is the only method declared by {@link ISolutionExpander}.
+ * @todo Replace with {@link IAccessPathExpander#getAccessPath(IAccessPath)}
+ * , which is the only method declared by {@link IAccessPathExpander}?
*/
- public ISolutionExpander<E> getSolutionExpander();
+ public IAccessPathExpander<E> getAccessPathExpander();
// /**
// * An optional constraint on the visitable elements.
@@ -386,32 +397,32 @@
*/
public IFilter getAccessPathFilter();
- /**
- * Return the {@link IKeyOrder} assigned to this {@link IPredicate} by the
- * query optimizer.
- *
- * @return The assigned {@link IKeyOrder} or <code>null</code> if the query
- * optimizer has not assigned the {@link IKeyOrder} yet.
- *
- * @see Annotations#KEY_ORDER
- */
- public IKeyOrder<E> getKeyOrder();
+// /**
+// * Return the {@link IKeyOrder} assigned to this {@link IPredicate} by the
+// * query optimizer.
+// *
+// * @return The assigned {@link IKeyOrder} or <code>null</code> if the query
+// * optimizer has not assigned the {@link IKeyOrder} yet.
+// *
+// * @see Annotations#KEY_ORDER
+// */
+// public IKeyOrder<E> getKeyOrder();
+//
+// /**
+// * Set the {@link IKeyOrder} annotation on the {@link IPredicate}, returning
+// * a new {@link IPredicate} in which the annotation takes on the given
+// * binding.
+// *
+// * @param keyOrder
+// * The {@link IKeyOrder}.
+// *
+// * @return The new {@link IPredicate}.
+// *
+// * @see Annotations#KEY_ORDER
+// */
+// public IPredicate<E> setKeyOrder(final IKeyOrder<E> keyOrder);
/**
- * Set the {@link IKeyOrder} annotation on the {@link IPredicate}, returning
- * a new {@link IPredicate} in which the annotation takes on the given
- * binding.
- *
- * @param keyOrder
- * The {@link IKeyOrder}.
- *
- * @return The new {@link IPredicate}.
- *
- * @see Annotations#KEY_ORDER
- */
- public IPredicate<E> setKeyOrder(final IKeyOrder<E> keyOrder);
-
- /**
* Figure out if all positions in the predicate which are required to form
* the key for this access path are bound in the predicate.
*/
@@ -487,6 +498,9 @@
/**
* Return a new instance in which all occurrences of the given variable have
* been replaced by the specified constant.
+ * <p>
+ * Note: The optimal {@link IKeyOrder} often changes when binding a variable
+ * on a predicate.
*
* @param var
* The variable.
@@ -504,6 +518,9 @@
/**
* Return a new instance in which all occurrences of the variable appearing
* in the binding set have been replaced by their bound values.
+ * <p>
+ * Note: The optimal {@link IKeyOrder} often changes when binding a variable
+ * on a predicate.
*
* @param bindingSet
* The binding set.
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ImmutableBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ImmutableBOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ImmutableBOp.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -0,0 +1,76 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Oct 5, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+
+/**
+ * Base class for immutable operators such as {@link Var} and {@link Constant}.
+ * These operators do not deep copy their data and do not permit decoration with
+ * annotations.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class ImmutableBOp extends BOpBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * @param op
+ */
+ public ImmutableBOp(BOpBase op) {
+ super(op);
+ }
+
+ /**
+ * @param args
+ * @param annotations
+ */
+ public ImmutableBOp(BOp[] args, Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ /*
+ * Overrides for the copy-on-write mutation API.
+ */
+
+ @Override
+ protected Object _setProperty(String name,Object value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void _clearProperty(String name) {
+ throw new UnsupportedOperationException();
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ImmutableBOp.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Var.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Var.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/Var.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -32,7 +32,7 @@
* limit the binding patterns across the rule. (In fact, variables can be
* "named" by their index into the binding set for most purposes.)
*/
-final public class Var<E> extends BOpBase implements IVariable<E>,
+final public class Var<E> extends ImmutableBOp implements IVariable<E>,
Comparable<IVariable<E>> {
private static final long serialVersionUID = -7100443208125002485L;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -28,7 +28,6 @@
package com.bigdata.bop.ap;
-import java.util.Iterator;
import java.util.Map;
import com.bigdata.bop.AbstractAccessPathOp;
@@ -44,11 +43,11 @@
import com.bigdata.bop.NV;
import com.bigdata.relation.accesspath.ElementFilter;
import com.bigdata.relation.accesspath.IElementFilter;
-import com.bigdata.relation.rule.ISolutionExpander;
+import com.bigdata.relation.rule.IAccessPathExpander;
import com.bigdata.striterator.IKeyOrder;
-import cutthecrap.utils.striterators.FilterBase;
import cutthecrap.utils.striterators.IFilter;
+import cutthecrap.utils.striterators.NOPFilter;
/**
* A generic implementation of an immutable {@link IPredicate}.
@@ -140,7 +139,7 @@
public Predicate(final IVariableOrConstant<?>[] values,
final String relationName, final int partitionId,
final boolean optional, final IElementFilter<E> constraint,
- final ISolutionExpander<E> expander, final long timestamp) {
+ final IAccessPathExpander<E> expander, final long timestamp) {
this(values, NV.asMap(new NV[] {//
new NV(Annotations.RELATION_NAME,new String[]{relationName}),//
@@ -258,9 +257,9 @@
}
@SuppressWarnings("unchecked")
- final public ISolutionExpander<E> getSolutionExpander() {
+ final public IAccessPathExpander<E> getAccessPathExpander() {
- return (ISolutionExpander<E>) getProperty(Annotations.ACCESS_PATH_EXPANDER);
+ return (IAccessPathExpander<E>) getProperty(Annotations.ACCESS_PATH_EXPANDER);
}
@@ -325,18 +324,12 @@
final int arity = arity();
+// boolean modified = false;
+
for (int i = 0; i < arity; i++) {
final IVariableOrConstant<?> t = (IVariableOrConstant<?>) get(i);
-// if (t == null) {
-// /*
-// * Note: t != null handles the case where the [c] position of an
-// * SPO is allowed to be null.
-// */
-// continue;
-// }
-
if (t.isConstant())
continue;
@@ -350,6 +343,8 @@
}
tmp.set(i, val.clone());
+
+// modified = true;
}
@@ -363,22 +358,22 @@
}
- @SuppressWarnings("unchecked")
- public IKeyOrder<E> getKeyOrder() {
-
- return (IKeyOrder<E>) getProperty(Annotations.KEY_ORDER);
-
- }
-
- public Predicate<E> setKeyOrder(final IKeyOrder<E> keyOrder) {
-
- final Predicate<E> tmp = this.clone();
-
- tmp._setProperty(Annotations.KEY_ORDER, keyOrder);
-
- return tmp;
-
- }
+// @SuppressWarnings("unchecked")
+// public IKeyOrder<E> getKeyOrder() {
+//
+// return (IKeyOrder<E>) getProperty(Annotations.KEY_ORDER);
+//
+// }
+//
+// public Predicate<E> setKeyOrder(final IKeyOrder<E> keyOrder) {
+//
+// final Predicate<E> tmp = this.clone();
+//
+// tmp._setProperty(Annotations.KEY_ORDER, keyOrder);
+//
+// return tmp;
+//
+// }
@SuppressWarnings("unchecked")
public Predicate<E> clone() {
@@ -486,38 +481,23 @@
/*
* Wrap the filter.
*/
- _setProperty(name, new FilterBase() {
-
- @Override
- protected Iterator filterOnce(Iterator src, Object context) {
- return src;
- }
-
- }.addFilter(current).addFilter(filter));
+ _setProperty(name, new NOPFilter().addFilter(current)
+ .addFilter(filter));
}
}
-
+
/**
- * Strips off the named annotations.
- *
- * @param names
- * The annotations to be removed.
- *
- * @return A new predicate in which the specified annotations do not appear.
+ * Strengthened return type.
+ * <p>
+ * {@inheritDoc}
*/
+ @SuppressWarnings("unchecked")
+ @Override
public Predicate<E> clearAnnotations(final String[] names) {
-
- final Predicate<E> tmp = this.clone();
-
- for(String name : names) {
-
- tmp._clearProperty(name);
-
- }
-
- return tmp;
+ return (Predicate<E>) super.clearAnnotations(names);
+
}
public String toString() {
@@ -533,7 +513,15 @@
final StringBuilder sb = new StringBuilder();
sb.append(getClass().getName());
+
+ final Integer bopId = (Integer) getProperty(Annotations.BOP_ID);
+ if (bopId != null) {
+
+ sb.append("[" + bopId + "]");
+
+ }
+
sb.append("(");
for (int i = 0; i < arity; i++) {
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -0,0 +1,76 @@
+package com.bigdata.bop.cost;
+
+
+/**
+ * A report on the expected cost of an index key range scan.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ * @version $Id$
+ */
+public class ScanCostReport {
+
+ /**
+ * The fast range count.
+ */
+ public final long rangeCount;
+
+ /**
+ * The #of index partitions in the scan.
+ */
+ public final long shardCount;
+
+ /**
+ * The expected cost of the scan (milliseconds).
+ */
+ public final double cost;
+
+ /**
+ *
+ * @param rangeCount
+ * The fast range count.
+ * @param cost
+ * The expected cost for the scan (milliseconds).
+ */
+ public ScanCostReport(final long rangeCount, final double cost) {
+
+ this.rangeCount = rangeCount;
+
+ this.shardCount = 1;
+
+ this.cost = cost;
+
+ }
+
+ /**
+ *
+ * @param rangeCount
+ * The fast range count.
+ * @param shardCount
+ * The #of index partitions in the scan.
+ * @param cost
+ * The expected cost for the scan (milliseconds).
+ */
+ public ScanCostReport(final long rangeCount, final long shardCount,
+ final double cost) {
+
+ this.rangeCount = rangeCount;
+
+ this.shardCount = shardCount;
+
+ this.cost = cost;
+
+ }
+
+ /**
+ * Human readable representation.
+ */
+ public String toString() {
+ return super.toString() + //
+ "{rangeCount=" + rangeCount + //
+ ",shardCount=" + shardCount + //
+ ",cost=" + cost + //
+ "}";
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/ScanCostReport.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -0,0 +1,74 @@
+package com.bigdata.bop.cost;
+
+import java.io.Serializable;
+
+/**
+ * Subquery cost report.
+ */
+public class SubqueryCostReport implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /** The #of graphs against which subqueries will be issued. */
+ public final int ngraphs;
+
+ /** The #of samples to be taken. */
+ public final int limit;
+
+ /** The #of samples taken. */
+ public final int nsamples;
+
+ /**
+ * An estimated range count based on the samples and adjusted for the
+ * #of graphs.
+ */
+ public final long rangeCount;
+
+ /**
+ * An estimated cost (latency in milliseconds) based on the samples and
+ * adjusted for the #of graphs.
+ */
+ public final double subqueryCost;
+
+ /**
+ *
+ * @param ngraphs
+ * The #of graphs against which subqueries will be issued.
+ * @param limit
+ * The #of samples to be taken.
+ * @param nsamples
+ * The #of samples taken.
+ * @param rangeCount
+ * An estimated range count based on the samples and adjusted
+ * for the #of graphs.
+ * @param subqueryCost
+ * An estimated cost (latency in milliseconds) based on the
+ * samples and adjusted for the #of graphs.
+ */
+ public SubqueryCostReport(final int ngraphs, final int limit,
+ final int nsamples, final long rangeCount,
+ final double subqueryCost) {
+ this.ngraphs = ngraphs;
+ this.limit = limit;
+ this.nsamples = nsamples;
+ this.rangeCount = rangeCount;
+ this.subqueryCost = subqueryCost;
+ }
+
+ /**
+ * Human readable representation.
+ */
+ public String toString() {
+ return super.toString() + //
+ "{ngraphs=" + ngraphs + //
+ ",limit=" + limit + //
+ ",nsamples=" + nsamples + //
+ ",rangeCount=" + rangeCount + //
+ ",subqueryCost=" + subqueryCost + //
+ "}";
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/cost/SubqueryCostReport.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2010-10-05 02:33:11 UTC (rev 3729)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java 2010-10-05 19:57:25 UTC (rev 3730)
@@ -46,8 +46,8 @@
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.PipelineOp;
import com.bigdata.bop.ap.Predicate;
-import com.bigdata.bop.cost.BTreeCostModel;
import com.bigdata.bop.cost.DiskCostModel;
+import com.bigdata.bop.cost.ScanCostReport;
import com.bigdata.btree.AbstractBTree;
import com.bigdata.btree.BTree;
import com.bigdata.btree.IBTreeStatistics;
@@ -443,6 +443,10 @@
// pool).
log.warn("Dropping chunk: queryId=" + queryId);
continue;
+ } catch (Throwable ex) {
+ // halt that query.
+ q.halt(ex);
+ continue;
}
} catch (InterruptedException e) {
...
[truncated message content] |
|
From: <tho...@us...> - 2010-10-13 19:39:56
|
Revision: 3791
http://bigdata.svn.sourceforge.net/bigdata/?rev=3791&view=rev
Author: thompsonbry
Date: 2010-10-13 19:39:49 +0000 (Wed, 13 Oct 2010)
Log Message:
-----------
1. Added toString() to DelegateIndexManager to report the DataService
UUID in log messages.
2. Added putIfAbsent() pattern to FederatedQueryEngine's proxy cache.
3. Added some log messages to FederatedQueryEngine and
FederatedRunningQuery. They are at trace and have been commmented
out now that I have found the source problem.
3. Added some logging and a query optimization to
Algorithm_NestedLocatorScan.
4. Modified Bundle to pass in the asBound predicate.
5. Bug fix to MapBindingSetsOverShards. This was the root cause of
the failure with 2 data services. The code was using the predicate
before the binding set had been applied to decide the shard against
which the next join would occur. In fact, it needed to use the
asBound version of the predicate for that purpose.
Note: While this fixes the problem, we do not yet have a unit test
which detects the error.
6. Moved the sort of the Bundles out of MapBindingSetsOverShards.
This can be done by the individual algorithm implementations.
7. Moved rangeCheck() out of AbstractBTreeTupleCursor into BytesUtil
so it can be reused by MapBindingSetsOverShards (it was used to
optimize Algorithm_NestedLocatorScan by deciding when a predicate
lies entirely within a given shard).
8. Restored shutdown of various things to BigdataFederationSparqlTest
and SPARQLQueryTest. This issue was a red-herring. The actual
problem was in MapBindingSetsOverShards.
9. Moved the RELAY_DELAY option into the properties NV[] (it was being
set on the command line using -D but the code did not check System.
getProperties() for a default so that value was being ignored).
With this change set, I am getting 229 out of 230 tests to pass against two data services. "Basic - List 4" is the one test which is failing on repeat trials.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/DelegateIndexManager.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Bundle.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/MapBindingSetsOverShardsBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTreeTupleCursor.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BytesUtil.java
branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties
branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/DelegateIndexManager.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/DelegateIndexManager.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/DelegateIndexManager.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -129,4 +129,11 @@
}
+ public String toString() {
+
+ return super.toString() + "{dataServiceUUID="
+ + dataService.getServiceUUID() + "}";
+
+ }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -540,8 +540,14 @@
*/
protected IQueryPeer getQueryPeer(final UUID serviceUUID) {
+ if (serviceUUID == null)
+ throw new IllegalArgumentException();
+
IQueryPeer proxy = proxyMap.get(serviceUUID);
+// if(log.isTraceEnabled()) log.trace("serviceUUID=" + serviceUUID
+// + (proxy != null ? "cached=" + proxy : " not cached."));
+
if (proxy == null) {
final IDataService dataService = getFederation().getDataService(
@@ -556,8 +562,15 @@
throw new RuntimeException(e);
}
- proxyMap.put(serviceUUID, proxy);
+ IQueryPeer tmp = proxyMap.putIfAbsent(serviceUUID, proxy);
+
+ if (tmp != null) {
+ proxy = tmp;
+ }
+// if(log.isTraceEnabled()) log.trace("serviceUUID=" + serviceUUID + ", addedToCache="
+// + (tmp == null) + ", proxy=" + proxy);
+
}
return proxy;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -347,14 +347,21 @@
*/
queryPeer = getQueryEngine();
+// if(log.isTraceEnabled()) log.trace("Target is self: "+serviceUUID);
+
} else if (serviceUUID.equals(queryControllerUUID)) {
// The target is the query controller.
queryPeer = getQueryController();
-
+
+// if(log.isTraceEnabled()) log.trace("Target is controller: "+serviceUUID);
+
} else {
// The target is some data service.
+
+// if(log.isTraceEnabled()) log.trace("Target is peer: "+serviceUUID);
+
queryPeer = getQueryEngine().getQueryPeer(serviceUUID);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Algorithm_NestedLocatorScan.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -1,10 +1,12 @@
package com.bigdata.bop.fed.shards;
+import java.util.Arrays;
import java.util.Iterator;
import org.apache.log4j.Logger;
import com.bigdata.bop.IBindingSet;
+import com.bigdata.btree.BytesUtil;
import com.bigdata.mdi.IMetadataIndex;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.accesspath.IBuffer;
@@ -31,8 +33,48 @@
public void mapOverShards(final Bundle<F>[] bundles) {
- for (Bundle<F> bundle : bundles) {
+ /*
+ * Sort the binding sets in the chunk by the fromKey associated with
+ * each asBound predicate.
+ */
+ Arrays.sort(bundles);
+ // The most recently discovered locator.
+ PartitionLocator current = null;
+
+// // The list of binding sets which are bound for the current locator.
+// List<IBindingSet> list = new LinkedList<IBindingSet>();
+
+ final Iterator<Bundle<F>> bitr = Arrays.asList(bundles).iterator();
+
+ while(bitr.hasNext()) {
+
+ final Bundle<F> bundle = bitr.next();
+
+ if (current != null
+ && BytesUtil.rangeCheck(bundle.fromKey, current
+ .getLeftSeparatorKey(), current
+ .getRightSeparatorKey())
+ && BytesUtil.rangeCheck(bundle.toKey, current
+ .getLeftSeparatorKey(), current
+ .getRightSeparatorKey())) {
+
+ /*
+ * Optimization when the bundle fits inside of the last index
+ * partition scanned (this optimization is only possible when
+ * the asBound predicate will be mapped onto a single index
+ * partition, but this is a very common case since we try to
+ * choose selective indices for access paths).
+ */
+
+ final IBuffer<IBindingSet[]> sink = op.getBuffer(current);
+
+ sink.add(new IBindingSet[] { bundle.bindingSet });
+
+ continue;
+
+ }
+
/*
* Locator scan for the index partitions for that predicate as
* bound.
@@ -42,11 +84,12 @@
while (itr.hasNext()) {
- final PartitionLocator locator = itr.next();
+ final PartitionLocator locator = current = itr.next();
if (log.isTraceEnabled())
- log.trace("adding bindingSet to buffer" + ": partitionId="
- + locator.getPartitionId() + "dataService="
+ log.trace("adding bindingSet to buffer" + ": asBound="
+ + bundle.asBound + ", partitionId="
+ + locator.getPartitionId() + ", dataService="
+ locator.getDataServiceUUID() + ", bindingSet="
+ bundle.bindingSet);
@@ -57,6 +100,7 @@
}
}
+
}
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Bundle.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Bundle.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/Bundle.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -30,12 +30,12 @@
/** The toKey generated from that asBound predicate. */
final byte[] toKey;
- public Bundle(final IKeyBuilder keyBuilder, final IPredicate<F> pred,
+ public Bundle(final IKeyBuilder keyBuilder, final IPredicate<F> asBound,
final IKeyOrder<F> keyOrder, final IBindingSet bindingSet) {
this.bindingSet = bindingSet;
- this.asBound = pred.asBound(bindingSet);
+ this.asBound = asBound;
this.keyOrder = keyOrder;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/MapBindingSetsOverShardsBuffer.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/MapBindingSetsOverShardsBuffer.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/shards/MapBindingSetsOverShardsBuffer.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -292,27 +292,37 @@
@SuppressWarnings("unchecked")
final Bundle<F>[] bundles = new Bundle[chunk.length];
- /*
- * Create the asBound version of the predicate and the associated
- * fromKey for each bindingSet in the chunk.
- */
- for (int i = 0; i < chunk.length; i++) {
+ /*
+ * Create the asBound version of the predicate and the associated
+ * fromKey for each bindingSet in the chunk.
+ */
+ for (int i = 0; i < chunk.length; i++) {
- final IKeyOrder<F> keyOrder = relation.getKeyOrder(pred);
+ // an intermediate solution.
+ final IBindingSet bindingSet = chunk[i];
- final IKeyBuilder keyBuilder = relation.getIndex(keyOrder)
- .getIndexMetadata().getKeyBuilder();
-
- bundles[i] = new Bundle<F>(keyBuilder, pred, keyOrder, chunk[i]);
-
- }
+ // the asBound version of the predicate.
+ final IPredicate<F> asBound = pred.asBound(bindingSet);
- /*
- * Sort the binding sets in the chunk by the fromKey associated with
- * each asBound predicate.
- */
- Arrays.sort(bundles);
+ // the index which will be used for that asBound predicate.
+ final IKeyOrder<F> keyOrder = relation.getKeyOrder(asBound);
+ // the key builder associated with that index.
+ final IKeyBuilder keyBuilder = relation.getIndex(keyOrder)
+ .getIndexMetadata().getKeyBuilder();
+
+ // save the bundle for processing.
+ bundles[i] = new Bundle<F>(keyBuilder, asBound, keyOrder,
+ bindingSet);
+
+ }
+
+// /*
+// * Sort the binding sets in the chunk by the fromKey associated with
+// * each asBound predicate. [Sort is moved into the implementation.]
+// */
+// Arrays.sort(bundles);
+
/*
* Map the bundles over the shards.
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTreeTupleCursor.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTreeTupleCursor.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTreeTupleCursor.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -254,50 +254,52 @@
*/
final protected boolean rangeCheck(final byte[] key) {
- if (fromKey == null && toKey == null) {
-
- // no range constraint.
- return true;
-
- }
-
- if (fromKey != null) {
-
- if (BytesUtil.compareBytes(key, fromKey) < 0) {
-
- if (DEBUG) {
-
- log.debug("key=" + BytesUtil.toString(key) + " LT fromKey"
- + BytesUtil.toString(fromKey));
-
- }
-
- // key is LT then the optional inclusive lower bound.
- return false;
-
- }
-
- }
-
- if (toKey != null) {
-
- if (BytesUtil.compareBytes(key, toKey) >= 0) {
-
- if (DEBUG) {
-
- log.debug("key=" + BytesUtil.toString(key) + " GTE toKey"
- + BytesUtil.toString(toKey));
-
- }
-
- // key is GTE the optional exclusive upper bound
- return false;
-
- }
-
- }
-
- return true;
+ return BytesUtil.rangeCheck(key, fromKey, toKey);
+
+// if (fromKey == null && toKey == null) {
+//
+// // no range constraint.
+// return true;
+//
+// }
+//
+// if (fromKey != null) {
+//
+// if (BytesUtil.compareBytes(key, fromKey) < 0) {
+//
+// if (DEBUG) {
+//
+// log.debug("key=" + BytesUtil.toString(key) + " LT fromKey"
+// + BytesUtil.toString(fromKey));
+//
+// }
+//
+// // key is LT then the optional inclusive lower bound.
+// return false;
+//
+// }
+//
+// }
+//
+// if (toKey != null) {
+//
+// if (BytesUtil.compareBytes(key, toKey) >= 0) {
+//
+// if (DEBUG) {
+//
+// log.debug("key=" + BytesUtil.toString(key) + " GTE toKey"
+// + BytesUtil.toString(toKey));
+//
+// }
+//
+// // key is GTE the optional exclusive upper bound
+// return false;
+//
+// }
+//
+// }
+//
+// return true;
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BytesUtil.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BytesUtil.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BytesUtil.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -740,6 +740,72 @@
}
+ /**
+ * Return <code>true</code> if the <i>key</i> lies inside of the optional
+ * half-open range constraint.
+ *
+ * @param key
+ * The key.
+ * @param fromKey
+ * The inclusive lower bound -or- <code>null</code> if there is
+ * no lower bound.
+ * @param toKey
+ * The exclusive upper bound -or- <code>null</code> if there is
+ * no upper bound.
+ *
+ * @return <code>true</code> unless the <i>key</i> is LT [fromKey] or GTE
+ * [toKey].
+ */
+ final static public boolean rangeCheck(final byte[] key,
+ final byte[] fromKey, final byte[] toKey) {
+
+ if (fromKey == null && toKey == null) {
+
+ // no range constraint.
+ return true;
+
+ }
+
+ if (fromKey != null) {
+
+ if (BytesUtil.compareBytes(key, fromKey) < 0) {
+
+ if (log.isDebugEnabled()) {
+
+ log.debug("key=" + BytesUtil.toString(key) + " LT fromKey"
+ + BytesUtil.toString(fromKey));
+
+ }
+
+ // key is LT then the optional inclusive lower bound.
+ return false;
+
+ }
+
+ }
+
+ if (toKey != null) {
+
+ if (BytesUtil.compareBytes(key, toKey) >= 0) {
+
+ if (log.isDebugEnabled()) {
+
+ log.debug("key=" + BytesUtil.toString(key) + " GTE toKey"
+ + BytesUtil.toString(toKey));
+
+ }
+
+ // key is GTE the optional exclusive upper bound
+ return false;
+
+ }
+
+ }
+
+ return true;
+
+ }
+
/**
* This method forces the load of the JNI library and tries to execute the
* JNI methods.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/resources/logging/log4j.properties 2010-10-13 19:39:49 UTC (rev 3791)
@@ -89,7 +89,7 @@
#log4j.logger.com.bigdata.service=ALL
-log4j.logger.com.bigdata.util.concurrent.Haltable=ALL
+#log4j.logger.com.bigdata.util.concurrent.Haltable=ALL
#log4j.logger.com.bigdata.bop=ALL
#log4j.logger.com.bigdata.bop.join.PipelineJoin=ALL
@@ -97,7 +97,7 @@
#log4j.logger.com.bigdata.bop.engine=ALL
#log4j.logger.com.bigdata.bop.engine.QueryEngine=ALL
#log4j.logger.com.bigdata.bop.engine.RunningQuery=ALL
-log4j.logger.com.bigdata.bop.engine.RunState=INFO
+#log4j.logger.com.bigdata.bop.engine.RunState=INFO
#log4j.logger.com.bigdata.bop.engine.RunningQuery$ChunkTask=ALL
#log4j.logger.com.bigdata.bop.fed.FederatedQueryEngine=ALL
#log4j.logger.com.bigdata.bop.fed.FederatedRunningQuery=ALL
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/lubm/build.properties 2010-10-13 19:39:49 UTC (rev 3791)
@@ -48,9 +48,9 @@
# Laptop benchmark data directory.
#lubm.baseDir=d:/bigdata-perf-analysis/lubm/U${lubm.univ}
# Server benchmark directory.
-#lubm.baseDir=/nas/data/lubm/U${lubm.univ}
+lubm.baseDir=/nas/data/lubm/U${lubm.univ}
# Windows Server 2008 benchmark data directory.
-lubm.baseDir=c:/usr/local/data/lubm/lubm_${lubm.univ}
+#lubm.baseDir=c:/usr/local/data/lubm/lubm_${lubm.univ}
## Where to put the XML results files.
#bsbm.resultsDir=${bsbm.baseDir}/..
@@ -80,11 +80,11 @@
# The name of the file used for the journal.
#lubm.journalFile=${lubm.baseDir}/bigdata-lubm.${journalMode}.jnl
# Note: This is on the large volume.
-#lubm.journalFile=/data/lubm/U${lubm.univ}/bigdata-lubm.${journalMode}.jnl
+lubm.journalFile=/data/lubm/U${lubm.univ}/bigdata-lubm.${journalMode}.jnl
# SSD.
#lubm.journalFile=e:/data/lubm/U${lubm.univ}/bigdata-lubm.${journalMode}.jnl
# SAS
-lubm.journalFile=f:/data/lubm/U${lubm.univ}/bigdata-lubm.${journalMode}.jnl
+#lubm.journalFile=f:/data/lubm/U${lubm.univ}/bigdata-lubm.${journalMode}.jnl
# The database to test.
lubm.configFile=${lubm.dir}/src/resources/config/config.kb.sparql
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -138,13 +138,13 @@
* test suite is done. Again, that should be Ok.)
*/
if (_ts != null) {
-// _ts.destroy();
+ _ts.destroy();
_ts = null;
}
-// if (_fed != null) {
-// _fed.shutdownNow();
-// _fed = null;
-// }
+ if (_fed != null) {
+ _fed.shutdownNow();
+ _fed = null;
+ }
}
@Override protected Repository newRepository ()
@@ -188,7 +188,7 @@
return _fed ;
}
- private String getConfiguration ()
+ static private String getConfiguration ()
throws Exception
{
String c = System.getProperty ( CONFIG_PROPERTY ) ;
@@ -318,4 +318,62 @@
private ScaleOutTripleStore _ts = null ;
private BigdataSail _sail = null;
+// /**
+// * Dumps the locators for an index of a relation.
+// *
+// * @param fed
+// * @param namespace
+// * The relation namespace.
+// * @param timestamp
+// * The timestamp of the view.
+// * @param keyOrder
+// * The index.
+// */
+// private static void dumpMDI(AbstractScaleOutFederation<?> fed,
+// final String namespace, final long timestamp,
+// final IKeyOrder<?> keyOrder) {
+//
+// final String name = namespace + "." + keyOrder.getIndexName();
+//
+// final Iterator<PartitionLocator> itr = fed
+// .locatorScan(name, timestamp, new byte[] {}/* fromKey */,
+// null/* toKey */, false/* reverseScan */);
+//
+// System.err.println("name=" + name + " @ "
+// + TimestampUtility.toString(timestamp));
+//
+// while (itr.hasNext()) {
+//
+// System.err.println(itr.next());
+//
+// }
+//
+// }
+//
+// public static void main(String[] args) throws ConfigurationException,
+// Exception {
+//
+// String namespace = "SPARQLTest_14e097f5-eba9-4ff8-a250-a5f9f19b9c66.spo";
+//
+// long timestamp=1287001362979L;
+//
+// IKeyOrder<?> keyOrder = SPOKeyOrder.SPOC;
+//
+// JiniClient<Object> jc = new JiniClient<Object>(
+// new String[] { getConfiguration() });
+//
+// try {
+//
+// final JiniFederation<?> fed = jc.connect();
+//
+// dumpMDI(fed, namespace, timestamp, keyOrder);
+//
+// } finally {
+//
+// jc.disconnect(true/* immediate */);
+//
+// }
+//
+// }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/org/openrdf/query/parser/sparql/SPARQLQueryTest.java 2010-10-13 19:39:49 UTC (rev 3791)
@@ -8,6 +8,7 @@
import info.aduna.io.IOUtil;
import info.aduna.iteration.Iterations;
import info.aduna.text.StringUtil;
+
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -17,8 +18,10 @@
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
+
import junit.framework.TestCase;
import junit.framework.TestSuite;
+
import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
@@ -61,8 +64,8 @@
import org.openrdf.sail.memory.MemoryStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import com.bigdata.rdf.sail.BigdataSailQuery;
-import com.bigdata.rdf.sail.BigdataSailRepository;
public abstract class SPARQLQueryTest extends TestCase {
@@ -295,7 +298,7 @@
}
RepositoryConnection con = ((DatasetRepository)dataRep).getDelegate().getConnection();
- System.err.println(con.getClass());
+// System.err.println(con.getClass());
try {
String queryString = readQueryString();
Query query = con.prepareQuery(QueryLanguage.SPARQL, queryString, queryFileURL);
Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
===================================================================
--- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-10-13 19:12:52 UTC (rev 3790)
+++ branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-10-13 19:39:49 UTC (rev 3791)
@@ -303,8 +303,6 @@
"-Djava.security.policy="+bigdata.policy,
"-Djava.util.logging.config.file="+bigdata.logging,
"-Dcom.bigdata.counters.linux.sysstat.path=@SYSSTAT_HOME@",
- // Turn off periodic performance counter reporting.
- "-Dcom.bigdata.service.IBigdataClient.reportDelay=0",
//bigdata.profilerAgent,
};
@@ -368,6 +366,13 @@
*/
new NV(IBigdataClient.Options.CLIENT_MAX_STALE_LOCATOR_RETRIES,"1000"),
+ /*
+ * Turn off periodic performance counter reporting to reduce the memory
+ * demand on the LBS (performance counters are used for moves on a multi-
+ * machine cluster and provide detailed reporting on the bigdata internals).
+ */
+ new NV(IBigdataClient.Options.REPORT_DELAY,"0"),
+
};
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-10-13 20:44:34
|
Revision: 3792
http://bigdata.svn.sourceforge.net/bigdata/?rev=3792&view=rev
Author: thompsonbry
Date: 2010-10-13 20:44:28 +0000 (Wed, 13 Oct 2010)
Log Message:
-----------
1. Added a cache miss handler to AbstractServiceCache#getService(UUID serviceUUID). This parallels the existing cache miss handling logic for other methods on this class. This addresses a problem where a test in BigdataFederationSparqlTest would sometimes fail due to a service not yet being in the JiniFederation's local cache.
2. Made _fed an instance field in BigdataFederationSparqlTest.
All tests are now passing for me against two data services.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java 2010-10-13 19:39:49 UTC (rev 3791)
+++ branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/service/jini/lookup/AbstractCachingServiceClient.java 2010-10-13 20:44:28 UTC (rev 3792)
@@ -341,28 +341,102 @@
}
- /**
- * Return the {@link ServiceItem} associated with the {@link UUID}.
- *
- * @param serviceUUID
- * The service {@link UUID}.
- *
- * @return The service item iff it is found in the cache and
- * <code>null</code> otherwise.
- */
+ /**
+ * Return the {@link ServiceItem} associated with the {@link UUID}. -or-
+ * <code>null</code> if there is no such service in the cache and a remote
+ * lookup times out.
+ *
+ * @param serviceUUID
+ * The service {@link UUID}.
+ *
+ * @return The service item iff it is found in the cache and
+ * <code>null</code> otherwise.
+ */
final public ServiceItem getServiceItem(final UUID serviceUUID) {
if (serviceUUID == null)
throw new IllegalArgumentException();
- final ServiceItem serviceItem = serviceCache
- .getServiceItemByID(JiniUtil.uuid2ServiceID(serviceUUID));
+ final ServiceID serviceId = JiniUtil.uuid2ServiceID(serviceUUID);
+ ServiceItem serviceItem = serviceCache
+ .getServiceItemByID(serviceId);
+
+ if (serviceItem == null) {
+
+ if (log.isInfoEnabled())
+ log.info("Cache miss.");
+
+ serviceItem = handleCacheMiss(serviceId);
+
+ if (serviceItem == null) {
+
+ log.warn("No matching service.");
+
+ return null;
+
+ }
+
+ }
+
return serviceItem;
}
/**
+ * Handles a cache miss by a remote query on the managed set of service
+ * registrars.
+ *
+ * @param filter
+ * The specific filter to be applied.
+ */
+ final private ServiceItem handleCacheMiss(final ServiceID serviceId) {
+
+ ServiceItem item = null;
+
+ try {
+
+ final ServiceTemplate template = new ServiceTemplate(serviceId,
+ null/* serviceTypes */, null/* attrSetTemplates */);
+
+ item = serviceDiscoveryManager.lookup(template, filter,
+ cacheMissTimeout);
+
+ } catch (RemoteException ex) {
+
+ log.error(ex);
+
+ return null;
+
+ } catch (InterruptedException ex) {
+
+ if (log.isInfoEnabled())
+ log.info("Interrupted - no match.");
+
+ return null;
+
+ }
+
+ if (item == null) {
+
+ // Could not discover a matching service.
+
+ log.warn("Could not discover matching service: serviceID="
+ + serviceId + ", template=" + template + ", filter="
+ + filter + ", timeout=" + cacheMissTimeout);
+
+ return null;
+
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Found: " + item);
+
+ return item;
+
+ }
+
+ /**
* Return an array {@link ServiceItem}s for up to <i>maxCount</i>
* discovered services which satisify the optional <i>filter</i>.
*
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-13 19:39:49 UTC (rev 3791)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/tck/BigdataFederationSparqlTest.java 2010-10-13 20:44:28 UTC (rev 3792)
@@ -128,15 +128,6 @@
throws Exception
{
super.tearDown () ;
- /*
- * @todo We should destroy the triple store here, but this is causing
- * problems with tear down of the query while it is still running. Once
- * that issue has been fixed, uncomment both the line to destroy the
- * triple store and the line to shutdown the federation (the latter is
- * really optional - it should be Ok to leave the federation up across
- * the test runs, but then we will never take it down cleanly when the
- * test suite is done. Again, that should be Ok.)
- */
if (_ts != null) {
_ts.destroy();
_ts = null;
@@ -313,8 +304,11 @@
private static final Logger _logger = Logger.getLogger ( BigdataFederationSparqlTest.class ) ;
- private static JiniFederation<Object> _fed = null ;
-
+ /*
+ * Instance fields for the current test.
+ */
+
+ private JiniFederation<Object> _fed = null ;
private ScaleOutTripleStore _ts = null ;
private BigdataSail _sail = null;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2010-10-14 12:33:44
|
Revision: 3796
http://bigdata.svn.sourceforge.net/bigdata/?rev=3796&view=rev
Author: thompsonbry
Date: 2010-10-14 12:33:37 +0000 (Thu, 14 Oct 2010)
Log Message:
-----------
Added IBigdataClient.Options.REPORT_ALL, which controls whether or not only the minimum set of performance counters are reported to the load balancer for its proper functioning (default true) or whether all collected performance counters are reported. This may be overridden in the configuration file to collect more information on a cluster.
This change was made to offer a short-term option for reducing the memory burden imposed by collecting all performance counters at the load balancer.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/query/QueryUtil.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IBigdataClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/LoadBalancerService.java
branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/query/QueryUtil.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/query/QueryUtil.java 2010-10-14 11:33:31 UTC (rev 3795)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/counters/query/QueryUtil.java 2010-10-14 12:33:37 UTC (rev 3796)
@@ -56,12 +56,20 @@
import com.bigdata.counters.HistoryInstrument;
import com.bigdata.counters.ICounter;
import com.bigdata.counters.ICounterNode;
+import com.bigdata.counters.ICounterSet;
+import com.bigdata.counters.IHostCounters;
+import com.bigdata.counters.IRequiredHostCounters;
import com.bigdata.counters.PeriodEnum;
import com.bigdata.counters.History.SampleIterator;
import com.bigdata.counters.ICounterSet.IInstrumentFactory;
import com.bigdata.counters.httpd.DummyEventReportingService;
+import com.bigdata.journal.ConcurrencyManager.IConcurrencyManagerCounters;
import com.bigdata.rawstore.Bytes;
+import com.bigdata.resources.ResourceManager.IResourceManagerCounters;
+import com.bigdata.resources.StoreManager.IStoreManagerCounters;
import com.bigdata.service.Event;
+import com.bigdata.service.DataService.IDataServiceCounters;
+import com.bigdata.util.concurrent.IQueueCounters.IThreadPoolExecutorTaskCounters;
/**
* Some static utility methods.
@@ -578,5 +586,79 @@
return out;
}
-
+
+ /**
+ * Return a {@link Pattern} which will match the minimum set of performance
+ * counters required by the load balancer to perform its function.
+ */
+ public static Pattern getRequiredPerformanceCountersFilter() {
+
+ return requiredPerformanceCountersFilter;
+
+ }
+
+ private static final String[] requiredPerformanceCounterPaths = new String[] {
+
+ IRequiredHostCounters.Memory_majorFaultsPerSecond,
+
+ IRequiredHostCounters.LogicalDisk_PercentFreeSpace,
+
+ IRequiredHostCounters.CPU_PercentProcessorTime,
+
+ IHostCounters.CPU_PercentIOWait,
+
+ IDataServiceCounters.concurrencyManager + ICounterSet.pathSeparator
+ + IConcurrencyManagerCounters.writeService
+ + ICounterSet.pathSeparator
+ + IThreadPoolExecutorTaskCounters.AverageQueuingTime,
+
+ IDataServiceCounters.resourceManager + ICounterSet.pathSeparator
+ + IResourceManagerCounters.StoreManager
+ + ICounterSet.pathSeparator
+ + IStoreManagerCounters.DataDirBytesAvailable,
+
+ IDataServiceCounters.resourceManager + ICounterSet.pathSeparator
+ + IResourceManagerCounters.StoreManager
+ + ICounterSet.pathSeparator
+ + IStoreManagerCounters.TmpDirBytesAvailable, };
+
+ private static final Pattern requiredPerformanceCountersFilter = QueryUtil
+ .getPattern(
+ Arrays.asList(requiredPerformanceCounterPaths)/* strings */,
+ null/* regex */);
+
+ /**
+ * Utility may be used to read the required performance counters for the
+ * load balancer from zero or more files specified on the command line. The
+ * results are written using the XML interchange format on stdout.
+ *
+ * @param args The file(s).
+ *
+ * @throws IOException
+ * @throws SAXException
+ * @throws ParserConfigurationException
+ */
+ public static void main(String[] args) throws IOException, SAXException,
+ ParserConfigurationException {
+
+ final Pattern filter = getRequiredPerformanceCountersFilter();
+
+ System.err.println("required counter pattern: " + filter);
+
+ final CounterSet counterSet = new CounterSet();
+
+ for (String s : args) {
+
+ final File file = new File(s);
+
+ readCountersFromFile(file, counterSet, filter,
+ new DefaultInstrumentFactory(60/* slots */,
+ PeriodEnum.Minutes, false/* overwrite */));
+
+ }
+
+ System.out.println("counters: " + counterSet.asXML(null/* filter */));
+
+ }
+
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-10-14 11:33:31 UTC (rev 3795)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/AbstractFederation.java 2010-10-14 12:33:37 UTC (rev 3796)
@@ -32,6 +32,7 @@
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
@@ -45,6 +46,7 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
import org.apache.log4j.Logger;
@@ -57,12 +59,19 @@
import com.bigdata.counters.CounterSet;
import com.bigdata.counters.ICounter;
import com.bigdata.counters.ICounterSet;
+import com.bigdata.counters.IHostCounters;
+import com.bigdata.counters.IRequiredHostCounters;
import com.bigdata.counters.IServiceCounters;
import com.bigdata.counters.OneShotInstrument;
+import com.bigdata.counters.query.QueryUtil;
import com.bigdata.journal.TemporaryStore;
import com.bigdata.journal.TemporaryStoreFactory;
+import com.bigdata.journal.ConcurrencyManager.IConcurrencyManagerCounters;
import com.bigdata.rawstore.Bytes;
import com.bigdata.relation.locator.DefaultResourceLocator;
+import com.bigdata.resources.ResourceManager.IResourceManagerCounters;
+import com.bigdata.resources.StoreManager.IStoreManagerCounters;
+import com.bigdata.service.DataService.IDataServiceCounters;
import com.bigdata.service.IBigdataClient.Options;
import com.bigdata.service.ndx.IClientIndex;
import com.bigdata.service.ndx.ScaleOutIndexCounters;
@@ -72,6 +81,7 @@
import com.bigdata.util.concurrent.ShutdownHelper;
import com.bigdata.util.concurrent.TaskCounters;
import com.bigdata.util.concurrent.ThreadPoolExecutorStatisticsTask;
+import com.bigdata.util.concurrent.IQueueCounters.IThreadPoolExecutorTaskCounters;
import com.bigdata.util.httpd.AbstractHTTPD;
/**
@@ -1391,26 +1401,39 @@
}
- /*
- * @todo this is probably worth compressing as there will be a lot
- * of redundency.
- *
- * @todo allow filter on what gets sent to the load balancer?
- */
- final ByteArrayOutputStream baos = new ByteArrayOutputStream(
- Bytes.kilobyte32 * 2);
+ /*
+ * @todo When sending all counters, this is probably worth
+ * compressing as there will be a lot of redundancy and a lot of
+ * data.
+ */
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream(
+ Bytes.kilobyte32 * 2);
- fed.getCounterSet().asXML(baos, "UTF-8", null/* filter */);
+ final Properties p = fed.getClient().getProperties();
+ final boolean reportAll = Boolean.valueOf(p.getProperty(
+ Options.REPORT_ALL, Options.DEFAULT_REPORT_ALL));
+
+ fed.getCounterSet().asXML(
+ baos,
+ "UTF-8",
+ reportAll ? null/* filter */: QueryUtil
+ .getRequiredPerformanceCountersFilter());
+
+ if (log.isInfoEnabled())
+ log.info("reportAll=" + reportAll + ", service="
+ + fed.getServiceName() + ", #bytesReported="
+ + baos.size());
+
loadBalancerService.notify(serviceUUID, baos.toByteArray());
if (log.isInfoEnabled())
log.info("Notified the load balancer.");
-
- }
- }
+ }
+ }
+
/**
* @todo it may be possible to optimize this for the jini case.
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IBigdataClient.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IBigdataClient.java 2010-10-14 11:33:31 UTC (rev 3795)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IBigdataClient.java 2010-10-14 12:33:37 UTC (rev 3796)
@@ -34,6 +34,7 @@
import com.bigdata.cache.HardReferenceQueue;
import com.bigdata.counters.AbstractStatisticsCollector;
import com.bigdata.counters.CounterSet;
+import com.bigdata.counters.query.QueryUtil;
import com.bigdata.journal.ITx;
import com.bigdata.relation.accesspath.IAccessPath;
import com.bigdata.relation.locator.ILocatableResource;
@@ -456,6 +457,19 @@
* The default {@link #REPORT_DELAY}.
*/
String DEFAULT_REPORT_DELAY = ""+(60*1000);
+
+ /**
+ * When <code>true</code>, all collected performance counters are
+ * reported (default {@value #DEFAULT_REPORT_ALL)}. Otherwise only the
+ * {@link QueryUtil#getRequiredPerformanceCountersFilter()} will be
+ * reported. Reporting all performance counters is useful when
+ * diagnosing the services in a cluster. However, only a small number of
+ * performance counters are actually necessary for the functioning of
+ * the {@link ILoadBalancerService}.
+ */
+ String REPORT_ALL = IBigdataClient.class.getName() + ".reportAll";
+
+ String DEFAULT_REPORT_ALL = "false";
/**
* Integer option specifies the port on which an httpd service will be
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/LoadBalancerService.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/LoadBalancerService.java 2010-10-14 11:33:31 UTC (rev 3795)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/LoadBalancerService.java 2010-10-14 12:33:37 UTC (rev 3796)
@@ -36,6 +36,7 @@
import com.bigdata.counters.IRequiredHostCounters;
import com.bigdata.counters.PeriodEnum;
import com.bigdata.counters.ICounterSet.IInstrumentFactory;
+import com.bigdata.counters.query.QueryUtil;
import com.bigdata.journal.BufferMode;
import com.bigdata.journal.Journal;
import com.bigdata.journal.ConcurrencyManager.IConcurrencyManagerCounters;
@@ -932,6 +933,8 @@
* service and report the largest value for lastModified() on any
* counter for that service.
*
+ * @see QueryUtil#getRequiredPerformanceCountersFilter()
+ *
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
*/
Modified: branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config
===================================================================
--- branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-10-14 11:33:31 UTC (rev 3795)
+++ branches/QUADS_QUERY_BRANCH/src/resources/config/bigdataStandalone.config 2010-10-14 12:33:37 UTC (rev 3796)
@@ -370,8 +370,16 @@
* Turn off periodic performance counter reporting to reduce the memory
* demand on the LBS (performance counters are used for moves on a multi-
* machine cluster and provide detailed reporting on the bigdata internals).
+ * (default 60 seconds).
*/
- new NV(IBigdataClient.Options.REPORT_DELAY,"0"),
+ //new NV(IBigdataClient.Options.REPORT_DELAY,"0"),
+
+ /*
+ * When true, report all performance counters. When false, report just the
+ * performance counters required by the load balancer for its correct
+ * functioning (default false).
+ */
+ //new NV(IBigdataClient.Options.REPORT_ALL,"false"),
};
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2010-10-19 16:02:24
|
Revision: 3821
http://bigdata.svn.sourceforge.net/bigdata/?rev=3821&view=rev
Author: mrpersonick
Date: 2010-10-19 16:02:18 +0000 (Tue, 19 Oct 2010)
Log Message:
-----------
testing single tail rules
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepository.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMultiGraphs.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java 2010-10-19 15:56:02 UTC (rev 3820)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/DefaultGraphSolutionExpander.java 2010-10-19 16:02:18 UTC (rev 3821)
@@ -271,7 +271,7 @@
if(!(accessPath1 instanceof SPOAccessPath)) {
// The logic relies on wrapping an SPOAccessPath, at least for now.
- throw new IllegalArgumentException();
+ throw new IllegalArgumentException(accessPath1.getClass().toString());
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-19 15:56:02 UTC (rev 3820)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2010-10-19 16:02:18 UTC (rev 3821)
@@ -497,31 +497,6 @@
}
- /**
- * Override evaluation of StatementPatterns to recognize magic search
- * predicate.
- @Override
- public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(
- final StatementPattern sp, final BindingSet bindings)
- throws QueryEvaluationException {
-
- // no check against the nativeJoins property here because we are simply
- // using the native execution model to take care of magic searches.
-
- if (log.isDebugEnabled()) {
- log.debug("evaluating statement pattern:\n" + sp);
- }
-
- IStep query = createNativeQuery(sp);
-
- if (query == null) {
- return new EmptyIteration<BindingSet, QueryEvaluationException>();
- }
-
- return execute(query);
-
- }
- */
/**
* Translate top-level JOINs into native bigdata programs for execution.
@@ -953,20 +928,20 @@
// }
// }
- if (log.isDebugEnabled()) {
- for (IPredicate<ISPO> tail : tails) {
- IAccessPathExpander<ISPO> expander = tail.getAccessPathExpander();
- if (expander != null) {
- IAccessPath<ISPO> accessPath = database.getSPORelation()
- .getAccessPath(tail);
- accessPath = expander.getAccessPath(accessPath);
- IChunkedOrderedIterator<ISPO> it = accessPath.iterator();
- while (it.hasNext()) {
- log.debug(it.next().toString(database));
- }
- }
- }
- }
+// if (log.isDebugEnabled()) {
+// for (IPredicate<ISPO> tail : tails) {
+// IAccessPathExpander<ISPO> expander = tail.getAccessPathExpander();
+// if (expander != null) {
+// IAccessPath<ISPO> accessPath = database.getSPORelation()
+// .getAccessPath(tail);
+// accessPath = expander.getAccessPath(accessPath);
+// IChunkedOrderedIterator<ISPO> it = accessPath.iterator();
+// while (it.hasNext()) {
+// log.debug(it.next().toString(database));
+// }
+// }
+// }
+// }
/*
* Collect a set of variables required beyond just the join (i.e.
@@ -1843,6 +1818,46 @@
}
}
+// /**
+// * Override evaluation of StatementPatterns to recognize magic search
+// * predicate.
+// */
+// @Override
+// public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(
+// final StatementPattern sp, final BindingSet bindings)
+// throws QueryEvaluationException {
+//
+// if (log.isDebugEnabled()) {
+// log.debug("evaluating statement pattern:\n" + sp);
+// }
+//
+// // check for magic search
+// final Var predVar = sp.getPredicateVar();
+// final Value predValue = getVarValue(predVar, bindings);
+// if (BD.SEARCH.equals(predValue)) {
+// final Var ovar = sp.getObjectVar();
+// final Value oval = getVarValue(ovar, bindings);
+// if (oval == null) {
+// throw new QueryEvaluationException(BD.SEARCH
+// + " : object must be bound.");
+// }
+// if (!(oval instanceof Literal)) {
+// throw new QueryEvaluationException(BD.SEARCH
+// + " : object must be literal.");
+// }
+// final Literal lit = (Literal) oval;
+// if (lit.getDatatype() != null) {
+// throw new QueryEvaluationException(BD.SEARCH
+// + " : object is datatype literal.");
+// }
+// return search(sp.getSubjectVar(), lit.getLanguage(),
+// lit.getLabel(), bindings, sp.getScope());
+// }
+//
+// return super.evaluate(sp, bindings);
+//
+// }
+
/**
* Override evaluation of StatementPatterns to recognize magic search
* predicate.
@@ -1852,34 +1867,28 @@
final StatementPattern sp, final BindingSet bindings)
throws QueryEvaluationException {
+ // no check against the nativeJoins property here because we are simply
+ // using the native execution model to take care of magic searches.
+
if (log.isDebugEnabled()) {
log.debug("evaluating statement pattern:\n" + sp);
}
- // check for magic search
- final Var predVar = sp.getPredicateVar();
- final Value predValue = getVarValue(predVar, bindings);
- if (BD.SEARCH.equals(predValue)) {
- final Var ovar = sp.getObjectVar();
- final Value oval = getVarValue(ovar, bindings);
- if (oval == null) {
- throw new QueryEvaluationException(BD.SEARCH
- + " : object must be bound.");
- }
- if (!(oval instanceof Literal)) {
- throw new QueryEvaluationException(BD.SEARCH
- + " : object must be literal.");
- }
- final Literal lit = (Literal) oval;
- if (lit.getDatatype() != null) {
- throw new QueryEvaluationException(BD.SEARCH
- + " : object is datatype literal.");
- }
- return search(sp.getSubjectVar(), lit.getLanguage(),
- lit.getLabel(), bindings, sp.getScope());
+ final IStep query = createNativeQuery(sp);
+
+ if (query == null) {
+ return new EmptyIteration<BindingSet, QueryEvaluationException>();
}
+
+ try {
- return super.evaluate(sp, bindings);
+ return execute(query);
+
+ } catch (Exception ex) {
+
+ throw new QueryEvaluationException(ex);
+
+ }
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepository.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepository.java 2010-10-19 15:56:02 UTC (rev 3820)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepository.java 2010-10-19 16:02:18 UTC (rev 3821)
@@ -34,7 +34,8 @@
// }
@Override
- public SailRepositoryConnection getConnection() throws RepositoryException {
+ public BigdataSailRepositoryConnection getConnection()
+ throws RepositoryException {
try {
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java 2010-10-19 15:56:02 UTC (rev 3820)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestBigdataSailWithQuads.java 2010-10-19 16:02:18 UTC (rev 3821)
@@ -97,6 +97,8 @@
suite.addTestSuite(TestDescribe.class);
+ suite.addTestSuite(TestMultiGraphs.class);
+
// The Sesame TCK, including the SPARQL test suite.
{
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMultiGraphs.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMultiGraphs.java 2010-10-19 15:56:02 UTC (rev 3820)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestMultiGraphs.java 2010-10-19 16:02:18 UTC (rev 3821)
@@ -99,8 +99,17 @@
* You'll get the same answer whether you run with Bigdata or Sesame.
*/
if (true) {
- sail = getSail();
- repo = new BigdataSailRepository((BigdataSail) sail);
+
+ final BigdataSail bdSail = getSail();
+ sail = bdSail;
+
+ if (bdSail.getDatabase().isQuads() == false) {
+ bdSail.__tearDownUnitTest();
+ return;
+ }
+
+ repo = new BigdataSailRepository(bdSail);
+
} else {
sail = new MemoryStore();
repo = new SailRepository(sail);
@@ -156,17 +165,17 @@
"select ?p ?o " +
"WHERE { " +
- " ?s rdf:type ns:Person . " +
- " ?s ?p ?o . " +
+// " ?s rdf:type ns:Person . " +
+ " ns:Mike ?p ?o . " +
"}";
final TupleQuery tupleQuery =
cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
TupleQueryResult result = tupleQuery.evaluate();
- while (result.hasNext()) {
- System.err.println(result.next());
- }
+// while (result.hasNext()) {
+// System.err.println(result.next());
+// }
Collection<BindingSet> solution = new LinkedList<BindingSet>();
solution.add(createBindingSet(new Binding[] {
@@ -181,20 +190,8 @@
new BindingImpl("p", RDFS.LABEL),
new BindingImpl("o", l1),
}));
- solution.add(createBindingSet(new Binding[] {
- new BindingImpl("p", RDF.TYPE),
- new BindingImpl("o", person),
- }));
- solution.add(createBindingSet(new Binding[] {
- new BindingImpl("p", likes),
- new BindingImpl("o", rdf),
- }));
- solution.add(createBindingSet(new Binding[] {
- new BindingImpl("p", RDFS.LABEL),
- new BindingImpl("o", l1),
- }));
-// compare(result, solution);
+ compare(result, solution);
}
@@ -216,9 +213,9 @@
cxn.prepareTupleQuery(QueryLanguage.SPARQL, query);
TupleQueryResult result = tupleQuery.evaluate();
- while (result.hasNext()) {
- System.err.println(result.next());
- }
+// while (result.hasNext()) {
+// System.err.println(result.next());
+// }
Collection<BindingSet> solution = new LinkedList<BindingSet>();
solution.add(createBindingSet(new Binding[] {
@@ -233,20 +230,8 @@
new BindingImpl("p", RDFS.LABEL),
new BindingImpl("o", l1),
}));
- solution.add(createBindingSet(new Binding[] {
- new BindingImpl("p", RDF.TYPE),
- new BindingImpl("o", person),
- }));
- solution.add(createBindingSet(new Binding[] {
- new BindingImpl("p", likes),
- new BindingImpl("o", rdf),
- }));
- solution.add(createBindingSet(new Binding[] {
- new BindingImpl("p", RDFS.LABEL),
- new BindingImpl("o", l1),
- }));
-// compare(result, solution);
+ compare(result, solution);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <mrp...@us...> - 2011-01-05 22:42:08
|
Revision: 4059
http://bigdata.svn.sourceforge.net/bigdata/?rev=4059&view=rev
Author: mrpersonick
Date: 2011-01-05 22:42:01 +0000 (Wed, 05 Jan 2011)
Log Message:
-----------
added support for inline mathematical operators
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestInlineConstraints.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedOptionals.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2011-01-05 18:31:58 UTC (rev 4058)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/IVUtility.java 2011-01-05 22:42:01 UTC (rev 4059)
@@ -31,6 +31,9 @@
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.UUID;
+
+import org.openrdf.query.algebra.MathExpr.MathOp;
+
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.rawstore.Bytes;
@@ -162,6 +165,174 @@
}
+ public static final IV numericalMath(final IV iv1, final IV iv2,
+ final MathOp op) {
+
+ if (!iv1.isInline())
+ throw new IllegalArgumentException(
+ "left term is not inline: left=" + iv1 + ", right=" + iv2);
+
+ if (!iv2.isInline())
+ throw new IllegalArgumentException(
+ "right term is not inline: left=" + iv1 + ", right=" + iv2);
+
+ if (!iv1.isLiteral())
+ throw new IllegalArgumentException(
+ "left term is not literal: left=" + iv1 + ", right=" + iv2);
+
+ if (!iv2.isLiteral())
+ throw new IllegalArgumentException(
+ "right term is not literal: left=" + iv1 + ", right=" + iv2);
+
+ final DTE dte1 = iv1.getDTE();
+ final DTE dte2 = iv2.getDTE();
+
+ if (!dte1.isNumeric())
+ throw new IllegalArgumentException(
+ "right term is not numeric: left=" + iv1 + ", right=" + iv2);
+
+ if (!dte2.isNumeric())
+ throw new IllegalArgumentException(
+ "left term is not numeric: left=" + iv1 + ", right=" + iv2);
+
+ final AbstractLiteralIV num1 = (AbstractLiteralIV) iv1;
+ final AbstractLiteralIV num2 = (AbstractLiteralIV) iv2;
+
+ // if one's a BigDecimal we should use the BigDecimal comparator for both
+ if (dte1 == DTE.XSDDecimal || dte2 == DTE.XSDDecimal) {
+ return numericalMath(num1.decimalValue(), num2.decimalValue(), op);
+ }
+
+ // same for BigInteger
+ if (dte1 == DTE.XSDInteger || dte2 == DTE.XSDInteger) {
+ return numericalMath(num1.integerValue(), num2.integerValue(), op);
+ }
+
+ // fixed length numerics
+ if (dte1.isFloatingPointNumeric() || dte2.isFloatingPointNumeric()) {
+ // non-BigDecimal floating points
+ if (dte1 == DTE.XSDFloat && dte2 == DTE.XSDFloat)
+ return numericalMath(num1.floatValue(), num2.floatValue(), op);
+ else
+ return numericalMath(num1.doubleValue(), num2.doubleValue(), op);
+ } else {
+ // non-BigInteger integers
+ if (dte1 == DTE.XSDInt && dte2 == DTE.XSDInt)
+ return numericalMath(num1.intValue(), num2.intValue(), op);
+ else
+ return numericalMath(num1.longValue(), num2.longValue(), op);
+ }
+
+ }
+
+ private static final IV numericalMath(final BigDecimal left,
+ final BigDecimal right, final MathOp op) {
+
+ switch(op) {
+ case PLUS:
+ return new XSDDecimalIV(left.add(right));
+ case MINUS:
+ return new XSDDecimalIV(left.subtract(right));
+ case MULTIPLY:
+ return new XSDDecimalIV(left.multiply(right));
+ case DIVIDE:
+ return new XSDDecimalIV(left.divide(right));
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private static final IV numericalMath(final BigInteger left,
+ final BigInteger right, final MathOp op) {
+
+ switch(op) {
+ case PLUS:
+ return new XSDIntegerIV(left.add(right));
+ case MINUS:
+ return new XSDIntegerIV(left.subtract(right));
+ case MULTIPLY:
+ return new XSDIntegerIV(left.multiply(right));
+ case DIVIDE:
+ return new XSDIntegerIV(left.divide(right));
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private static final IV numericalMath(final float left,
+ final float right, final MathOp op) {
+
+ switch(op) {
+ case PLUS:
+ return new XSDFloatIV(left+right);
+ case MINUS:
+ return new XSDFloatIV(left-right);
+ case MULTIPLY:
+ return new XSDFloatIV(left*right);
+ case DIVIDE:
+ return new XSDFloatIV(left/right);
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private static final IV numericalMath(final double left,
+ final double right, final MathOp op) {
+
+ switch(op) {
+ case PLUS:
+ return new XSDDoubleIV(left+right);
+ case MINUS:
+ return new XSDDoubleIV(left-right);
+ case MULTIPLY:
+ return new XSDDoubleIV(left*right);
+ case DIVIDE:
+ return new XSDDoubleIV(left/right);
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private static final IV numericalMath(final int left,
+ final int right, final MathOp op) {
+
+ switch(op) {
+ case PLUS:
+ return new XSDIntIV(left+right);
+ case MINUS:
+ return new XSDIntIV(left-right);
+ case MULTIPLY:
+ return new XSDIntIV(left*right);
+ case DIVIDE:
+ return new XSDIntIV(left/right);
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private static final IV numericalMath(final long left,
+ final long right, final MathOp op) {
+
+ switch(op) {
+ case PLUS:
+ return new XSDLongIV(left+right);
+ case MINUS:
+ return new XSDLongIV(left-right);
+ case MULTIPLY:
+ return new XSDLongIV(left*right);
+ case DIVIDE:
+ return new XSDLongIV(left/right);
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
/**
* Used to test whether a given value constant can be used in an inline
* filter or not. If so, we can use one of the inline constraints
Added: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/CompareBOp.java 2011-01-05 22:42:01 UTC (rev 4059)
@@ -0,0 +1,134 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+
+package com.bigdata.rdf.internal.constraints;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.openrdf.query.algebra.Compare.CompareOp;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IConstant;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.PipelineOp;
+import com.bigdata.bop.constraint.BOpConstraint;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.IVUtility;
+import com.bigdata.rdf.internal.constraints.MathBOp.Annotations;
+
+/**
+ * Use inline terms to perform numerical comparison operations.
+ *
+ * @see IVUtility#numericalCompare(IV, IV)
+ */
+public class CompareBOp extends BOpConstraint {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends PipelineOp.Annotations {
+
+ /**
+ * The compare operator
+ */
+ String OP = CompareBOp.class.getName() + ".op";
+
+ }
+
+ /**
+ * Required shallow copy constructor.
+ */
+ public CompareBOp(final BOp[] values,
+ final Map<String, Object> annotations) {
+ super(values, annotations);
+ }
+
+ /**
+ * Required deep copy constructor.
+ */
+ public CompareBOp(final CompareBOp op) {
+ super(op);
+ }
+
+ public CompareBOp(final IVariableOrConstant<IV> left,
+ final IVariableOrConstant<IV> right, final CompareOp op) {
+
+ super(new BOp[] { left, right }, NV.asMap(new NV(Annotations.OP, op)));
+
+ if (left == null || right == null || op == null)
+ throw new IllegalArgumentException();
+
+ }
+
+ public boolean accept(final IBindingSet s) {
+
+ final IV left = ((IVariableOrConstant<IV>) get(0)).get(s);
+ final IV right = ((IVariableOrConstant<IV>) get(1)).get(s);
+
+ if (left == null || right == null)
+ return true; // not yet bound.
+
+ if (IVUtility.canNumericalCompare(left) &&
+ IVUtility.canNumericalCompare(right)) {
+
+ return _accept(IVUtility.numericalCompare(left, right));
+
+ } else {
+
+ return _accept(left.compareTo(right));
+
+ }
+
+ }
+
+ protected boolean _accept(final int compare) {
+
+ final CompareOp op = (CompareOp) getProperty(Annotations.OP);
+
+ switch(op) {
+ case EQ:
+ return compare == 0;
+ case NE:
+ return compare != 0;
+ case GT:
+ return compare > 0;
+ case GE:
+ return compare >= 0;
+ case LT:
+ return compare < 0;
+ case LE:
+ return compare <= 0;
+ default:
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+}
Added: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/internal/constraints/MathBOp.java 2011-01-05 22:42:01 UTC (rev 4059)
@@ -0,0 +1,216 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2011. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+package com.bigdata.rdf.internal.constraints;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.openrdf.query.algebra.Compare.CompareOp;
+import org.openrdf.query.algebra.MathExpr.MathOp;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IConstant;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.bop.ImmutableBOp;
+import com.bigdata.bop.NV;
+import com.bigdata.bop.Constant.Annotations;
+import com.bigdata.rdf.internal.IV;
+import com.bigdata.rdf.internal.IVUtility;
+
+/**
+ * A math expression involving a left and right IVariableOrConstant operand.
+ */
+final public class MathBOp extends ImmutableBOp
+ implements IVariableOrConstant<IV> {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 9136864442064392445L;
+
+ public interface Annotations extends ImmutableBOp.Annotations {
+
+ /**
+ * The {@link IVariable} which is bound to that constant value
+ * (optional).
+ */
+ String OP = MathBOp.class.getName() + ".op";
+
+ }
+
+ final public boolean isVar() {
+
+ return ((IVariableOrConstant) get(0)).isVar() ||
+ ((IVariableOrConstant) get(1)).isVar();
+
+ }
+
+ final public boolean isConstant() {
+
+ return !isVar();
+
+ }
+
+ /**
+ * Required deep copy constructor.
+ *
+ * @param op
+ */
+ public MathBOp(final MathBOp op) {
+
+ super(op);
+
+ }
+
+ public MathBOp(final IVariableOrConstant<IV> left,
+ final IVariableOrConstant<IV> right, final MathOp op) {
+
+ super(new BOp[] { left, right }, NV.asMap(new NV(Annotations.OP, op)));
+
+ if (left == null || right == null || op == null)
+ throw new IllegalArgumentException();
+
+ }
+
+// /**
+// * Clone is overridden to reduce heap churn.
+// */
+// final public Math clone() {
+//
+// return this;
+//
+// }
+
+ public IVariableOrConstant<IV> left() {
+ return (IVariableOrConstant<IV>) get(0);
+ }
+
+ public IVariableOrConstant<IV> right() {
+ return (IVariableOrConstant<IV>) get(1);
+ }
+
+ public MathOp op() {
+ return (MathOp) getRequiredProperty(Annotations.OP);
+ }
+
+ public String toString() {
+
+ final StringBuilder sb = new StringBuilder();
+ sb.append(op());
+ sb.append("(").append(left()).append(", ").append(right()).append(")");
+ return sb.toString();
+
+ }
+
+ final public boolean equals(final MathBOp m) {
+
+ if (m == null)
+ return false;
+
+ if (this == m)
+ return true;
+
+ return op().equals(m.op()) &&
+ left().equals(m.left()) &&
+ right().equals(m.right());
+
+ }
+
+ final public boolean equals(final IVariableOrConstant<IV> o) {
+
+ if(!(o instanceof MathBOp)) {
+
+ // incomparable types.
+ return false;
+
+ }
+
+ return equals((MathBOp) o);
+
+ }
+
+
+ /**
+ * Caches the hash code.
+ */
+ private int hash = 0;
+
+ public int hashCode() {
+
+ int h = hash;
+
+ if (h == 0) {
+
+ final int n = arity();
+
+ for (int i = 0; i < n; i++) {
+
+ h = 31 * h + get(i).hashCode();
+
+ }
+
+ h = 31 * h + op().hashCode();
+
+ hash = h;
+
+ }
+
+ return h;
+
+ }
+
+ final public IV get() {
+
+ final IV left = left().get();
+ final IV right = right().get();
+
+ if (left == null || right == null)
+ return null;
+
+ return IVUtility.numericalMath(left, right, op());
+
+ }
+
+ final public IV get(final IBindingSet bindingSet) {
+
+ final IV left = left().get(bindingSet);
+ final IV right = right().get(bindingSet);
+
+ if (left == null || right == null)
+ return null;
+
+ return IVUtility.numericalMath(left, right, op());
+
+ }
+
+ final public String getName() {
+
+ throw new UnsupportedOperationException();
+
+ }
+
+}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestInlineConstraints.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestInlineConstraints.java 2011-01-05 18:31:58 UTC (rev 4058)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/internal/constraints/TestInlineConstraints.java 2011-01-05 22:42:01 UTC (rev 4059)
@@ -27,8 +27,9 @@
package com.bigdata.rdf.internal.constraints;
-import java.util.Properties;
import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.query.algebra.Compare.CompareOp;
+import org.openrdf.query.algebra.MathExpr.MathOp;
import com.bigdata.bop.Constant;
import com.bigdata.bop.IBindingSet;
@@ -140,7 +141,7 @@
},
// constraints on the rule.
new IConstraint[] {
- new InlineGT(a, _35.getIV())
+ new CompareBOp(a, new Constant<IV>(_35.getIV()), CompareOp.GT)
});
try {
@@ -242,7 +243,7 @@
},
// constraints on the rule.
new IConstraint[] {
- new InlineGE(a, _35.getIV())
+ new CompareBOp(a, new Constant<IV>(_35.getIV()), CompareOp.GE)
});
try {
@@ -346,7 +347,7 @@
},
// constraints on the rule.
new IConstraint[] {
- new InlineLT(a, _35.getIV())
+ new CompareBOp(a, new Constant<IV>(_35.getIV()), CompareOp.LT)
});
if (log.isInfoEnabled())
@@ -454,7 +455,7 @@
},
// constraints on the rule.
new IConstraint[] {
- new InlineLE(a, _35.getIV())
+ new CompareBOp(a, new Constant<IV>(_35.getIV()), CompareOp.LE)
});
if (log.isInfoEnabled())
@@ -502,6 +503,117 @@
}
+ public void testMath() {
+
+ // store with no owl:sameAs closure
+ AbstractTripleStore db = getStore();
+
+ // do not run if we are not inlining
+ if (!db.getLexiconRelation().isInlineLiterals()) {
+ return;
+ }
+
+ try {
+
+ BigdataValueFactory vf = db.getValueFactory();
+
+ final BigdataURI A = vf.createURI("http://www.bigdata.com/A");
+ final BigdataURI B = vf.createURI("http://www.bigdata.com/B");
+ final BigdataURI C = vf.createURI("http://www.bigdata.com/C");
+ final BigdataURI D = vf.createURI("http://www.bigdata.com/D");
+ final BigdataURI X = vf.createURI("http://www.bigdata.com/X");
+ final BigdataURI AGE = vf.createURI("http://www.bigdata.com/AGE");
+ final BigdataLiteral _5 = vf.createLiteral((double) 5);
+ final BigdataLiteral _30 = vf.createLiteral((double) 30);
+ final BigdataLiteral _25 = vf.createLiteral((double) 25);
+ final BigdataLiteral _35 = vf.createLiteral((long) 35);
+ final BigdataLiteral _45 = vf.createLiteral((long) 45);
+
+ db.addTerms( new BigdataValue[] { A, B, C, X, AGE, _25, _35, _45, D, _5, _30 } );
+
+ {
+ StatementBuffer buffer = new StatementBuffer
+ ( db, 100/* capacity */
+ );
+
+ buffer.add(A, RDF.TYPE, X);
+ buffer.add(A, AGE, _25);
+ buffer.add(B, RDF.TYPE, X);
+ buffer.add(B, AGE, _45);
+ buffer.add(C, RDF.TYPE, X);
+ buffer.add(C, AGE, _35);
+ buffer.add(D, AGE, _30);
+
+ // write statements on the database.
+ buffer.flush();
+
+ }
+
+ if (log.isInfoEnabled())
+ log.info("\n" +db.dumpStore(true, true, false));
+
+ { // works great
+
+ final String SPO = db.getSPORelation().getNamespace();
+ final IVariable<IV> s = Var.var("s");
+ final IConstant<IV> type = new Constant<IV>(db.getIV(RDF.TYPE));
+ final IConstant<IV> x = new Constant<IV>(X.getIV());
+ final IConstant<IV> age = new Constant<IV>(AGE.getIV());
+ final IVariable<IV> a = Var.var("a");
+ final IConstant<IV> d = new Constant<IV>(D.getIV());
+ final IVariable<IV> dAge = Var.var("dAge");
+
+ final IRule rule =
+ new Rule("test_math", null, // head
+ new IPredicate[] {
+ new SPOPredicate(SPO, d, age, dAge),
+ new SPOPredicate(SPO, s, type, x),
+ new SPOPredicate(SPO, s, age, a)
+ },
+ // constraints on the rule.
+ new IConstraint[] {
+ new CompareBOp(a, new MathBOp(dAge, new Constant<IV>(_5.getIV()), MathOp.PLUS), CompareOp.GT)
+ });
+
+ try {
+
+ int numSolutions = 0;
+
+ IChunkedOrderedIterator<ISolution> solutions = runQuery(db, rule);
+
+ while (solutions.hasNext()) {
+
+ ISolution solution = solutions.next();
+
+ IBindingSet bs = solution.getBindingSet();
+
+ System.err.println(bs);
+
+ assertEquals(bs.get(s).get(), B.getIV());
+ assertEquals(bs.get(a).get(), _45.getIV());
+
+ numSolutions++;
+
+ }
+
+ assertEquals("wrong # of solutions", 1, numSolutions);
+
+ } catch(Exception ex) {
+
+ ex.printStackTrace();
+
+ }
+
+ }
+
+ } finally {
+
+ db.__tearDownUnitTest();
+
+ }
+
+ }
+
private IChunkedOrderedIterator<ISolution> runQuery(AbstractTripleStore db, IRule rule)
throws Exception {
// run the query as a native rule.
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2011-01-05 18:31:58 UTC (rev 4058)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java 2011-01-05 22:42:01 UTC (rev 4059)
@@ -25,10 +25,13 @@
import org.openrdf.query.Dataset;
import org.openrdf.query.QueryEvaluationException;
import org.openrdf.query.algebra.Compare;
+import org.openrdf.query.algebra.Compare.CompareOp;
import org.openrdf.query.algebra.Filter;
import org.openrdf.query.algebra.Group;
import org.openrdf.query.algebra.Join;
import org.openrdf.query.algebra.LeftJoin;
+import org.openrdf.query.algebra.MathExpr;
+import org.openrdf.query.algebra.MathExpr.MathOp;
import org.openrdf.query.algebra.MultiProjection;
import org.openrdf.query.algebra.Or;
import org.openrdf.query.algebra.Order;
@@ -39,14 +42,13 @@
import org.openrdf.query.algebra.QueryRoot;
import org.openrdf.query.algebra.SameTerm;
import org.openrdf.query.algebra.StatementPattern;
+import org.openrdf.query.algebra.StatementPattern.Scope;
import org.openrdf.query.algebra.TupleExpr;
import org.openrdf.query.algebra.UnaryTupleOperator;
import org.openrdf.query.algebra.Union;
import org.openrdf.query.algebra.ValueConstant;
import org.openrdf.query.algebra.ValueExpr;
import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.Compare.CompareOp;
-import org.openrdf.query.algebra.StatementPattern.Scope;
import org.openrdf.query.algebra.evaluation.impl.EvaluationStrategyImpl;
import org.openrdf.query.algebra.evaluation.iterator.FilterIterator;
import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
@@ -57,11 +59,11 @@
import com.bigdata.bop.IBindingSet;
import com.bigdata.bop.IConstraint;
import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.IVariable;
import com.bigdata.bop.IVariableOrConstant;
import com.bigdata.bop.NV;
import com.bigdata.bop.PipelineOp;
-import com.bigdata.bop.IPredicate.Annotations;
import com.bigdata.bop.ap.Predicate;
import com.bigdata.bop.constraint.EQ;
import com.bigdata.bop.constraint.EQConstant;
@@ -77,12 +79,14 @@
import com.bigdata.rdf.internal.DummyIV;
import com.bigdata.rdf.internal.IV;
import com.bigdata.rdf.internal.IVUtility;
+import com.bigdata.rdf.internal.constraints.CompareBOp;
import com.bigdata.rdf.internal.constraints.InlineEQ;
import com.bigdata.rdf.internal.constraints.InlineGE;
import com.bigdata.rdf.internal.constraints.InlineGT;
import com.bigdata.rdf.internal.constraints.InlineLE;
import com.bigdata.rdf.internal.constraints.InlineLT;
import com.bigdata.rdf.internal.constraints.InlineNE;
+import com.bigdata.rdf.internal.constraints.MathBOp;
import com.bigdata.rdf.lexicon.LexiconRelation;
import com.bigdata.rdf.model.BigdataValue;
import com.bigdata.rdf.sail.BigdataSail.Options;
@@ -1589,70 +1593,85 @@
compare.getOperator());
}
+ /**
+ * FIXME: implement compare two variables and compare MathExpr
+ */
private IConstraint generateConstraint(ValueExpr left, ValueExpr right,
CompareOp operator) {
- IVariable<IV> var = null;
- BigdataValue constant = null;
+ IVariableOrConstant<IV> iv1, iv2;
+
if (left instanceof Var) {
- var = com.bigdata.bop.Var.var(((Var) left).getName());
+ iv1 = com.bigdata.bop.Var.var(((Var) left).getName());
} else if (left instanceof ValueConstant) {
- constant = (BigdataValue) ((ValueConstant) left).getValue();
+ final IV iv = ((BigdataValue) ((ValueConstant) left).getValue()).getIV();
+ if (iv == null)
+ return null;
+ iv1 = new Constant<IV>(iv);
+ } else if (left instanceof MathExpr) {
+ iv1 = generateMath((MathExpr) left);
+ if (iv1 == null)
+ return null;
} else {
return null;
}
+
if (right instanceof Var) {
- var = com.bigdata.bop.Var.var(((Var) right).getName());
+ iv2 = com.bigdata.bop.Var.var(((Var) right).getName());
} else if (right instanceof ValueConstant) {
- constant = (BigdataValue) ((ValueConstant) right).getValue();
+ final IV iv = ((BigdataValue) ((ValueConstant) right).getValue()).getIV();
+ if (iv == null)
+ return null;
+ iv2 = new Constant<IV>(iv);
+ } else if (right instanceof MathExpr) {
+ iv2 = generateMath((MathExpr) right);
+ if (iv2 == null)
+ return null;
} else {
return null;
}
- if (log.isDebugEnabled()) {
- log.debug("var: " + var);
- log.debug("constant: " + constant);
- log.debug("constant.getIV(): " + constant.getIV());
- }
- if (var == null || constant == null || constant.getIV() == null) {
- if (log.isDebugEnabled()) {
- log.debug("left: " + left);
- log.debug("right: " + right);
- }
+
+ return new CompareBOp(iv1, iv2, operator);
+
+ }
+
+ private MathBOp generateMath(final MathExpr mathExpr) {
+ final ValueExpr left = mathExpr.getLeftArg();
+ final ValueExpr right = mathExpr.getRightArg();
+ final MathOp op = mathExpr.getOperator();
+
+ IVariableOrConstant<IV> iv1, iv2;
+
+ if (left instanceof Var) {
+ iv1 = com.bigdata.bop.Var.var(((Var) left).getName());
+ } else if (left instanceof ValueConstant) {
+ final IV iv = ((BigdataValue) ((ValueConstant) left).getValue()).getIV();
+ if (iv == null)
+ return null;
+ iv1 = new Constant<IV>(iv);
+ } else if (left instanceof MathExpr) {
+ iv1 = generateMath((MathExpr) left);
+ if (iv1 == null)
+ return null;
+ } else {
return null;
}
- final IV iv = constant.getIV();
- // we can do equals, not equals
- if (inlineTerms && IVUtility.canNumericalCompare(iv)) {
- if (log.isInfoEnabled()) {
- log.debug("inline constant, using inline numerical comparison: "
- + iv);
- }
- try {
- switch (operator) {
- case GT:
- return new InlineGT(var, iv);
- case GE:
- return new InlineGE(var, iv);
- case LT:
- return new InlineLT(var, iv);
- case LE:
- return new InlineLE(var, iv);
- case EQ:
- return new InlineEQ(var, iv);
- case NE:
- return new InlineNE(var, iv);
- default:
- return null;
- }
- } catch (Exception ex) {
- return null;
- }
- } else if (operator == CompareOp.EQ) {
- return new EQConstant(var, new Constant(iv));
- } else if (operator == CompareOp.NE) {
- return new NEConstant(var, new Constant(iv));
+
+ if (right instanceof Var) {
+ iv2 = com.bigdata.bop.Var.var(((Var) right).getName());
+ } else if (right instanceof ValueConstant) {
+ final IV iv = ((BigdataValue) ((ValueConstant) right).getValue()).getIV();
+ if (iv == null)
+ return null;
+ iv2 = new Constant<IV>(iv);
+ } else if (right instanceof MathExpr) {
+ iv2 = generateMath((MathExpr) right);
+ if (iv2 == null)
+ return null;
} else {
return null;
}
+
+ return new MathBOp(iv1, iv2, op);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedOptionals.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedOptionals.java 2011-01-05 18:31:58 UTC (rev 4058)
+++ branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestNestedOptionals.java 2011-01-05 22:42:01 UTC (rev 4059)
@@ -251,7 +251,7 @@
}
- private void __testNestedOptionals1() throws Exception {
+ public void testNestedOptionals1() throws Exception {
final BigdataSail sail = getSail();
sail.initialize();
@@ -333,11 +333,11 @@
final Projection p = (Projection) root.getArg();
final LeftJoin leftJoin = (LeftJoin) p.getArg();
- final List<Tail> tails = collectTails(leftJoin);
+ final List<Op> tails = collectTails(leftJoin);
if (INFO) {
System.err.println(query);
- for (Tail t : tails) {
+ for (Op t : tails) {
System.err.println(t);
}
}
@@ -355,7 +355,7 @@
}
- private void __testNestedOptionals2() throws Exception {
+ public void testNestedOptionals2() throws Exception {
final BigdataSail sail = getSail();
sail.initialize();
@@ -446,7 +446,7 @@
" ?s <"+RDF.TYPE+"> <"+T4+"> .\n" + // tail=A, group=1, parent=0
" ?s <"+RDF.TYPE+"> <"+T5+"> .\n" + // tail=B, group=1, parent=0
" ?s <"+P4+"> ?p4 .\n" + // tail=C, group=1, parent=0
- " FILTER ( ?p4 > 30 ) .\n" +
+ " FILTER ( ?p4 > (?p1*?p0+10+20) ) .\n" +
" OPTIONAL { ?s <"+P5+"> ?p5 . }\n" + // tail=D, group=2, parent=1
" OPTIONAL { ?s <"+P6+"> ?p6 . }\n" + // tail=E, group=3, parent=1
" }\n" +
@@ -490,11 +490,11 @@
final Projection p = (Projection) root.getArg();
final LeftJoin leftJoin = (LeftJoin) p.getArg();
- final List<Tail> tails = collectTails(leftJoin);
+ final List<Op> tails = collectTails(leftJoin);
if (INFO) {
System.err.println(query);
- for (Tail t : tails) {
+ for (Op t : tails) {
System.err.println(t);
}
}
@@ -513,9 +513,9 @@
}
- private List<Tail> collectTails(final LeftJoin root) {
+ private List<Op> collectTails(final LeftJoin root) {
- final List<Tail> tails = new LinkedList<Tail>();
+ final List<Op> tails = new LinkedList<Op>();
log.info("\n"+root);
@@ -530,11 +530,17 @@
return ++group;
}
- private void collectTails(final List<Tail> tails, final LeftJoin leftJoin,
+ private void collectTails(final List<Op> tails, final LeftJoin leftJoin,
final boolean rslj, final int g, final int pg) {
- final ValueExpr condition = leftJoin.getCondition();
+ final ValueExpr ve = leftJoin.getCondition();
// conditional for tails in this group
+ if (ve != null) {
+ final Constraint c = new Constraint(ve);
+ c.setGroup(g);
+ c.setParentGroup(pg);
+ tails.add(c);
+ }
final TupleExpr left = leftJoin.getLeftArg();
@@ -571,7 +577,7 @@
}
- private void collectTails(final List<Tail> tails, final Join join,
+ private void collectTails(final List<Op> tails, final Join join,
final boolean rslj, final int g, final int pg) {
final TupleExpr left = join.getLeftArg();
@@ -604,11 +610,17 @@
}
- private void collectTails(final List<Tail> tails, final Filter filter,
+ private void collectTails(final List<Op> tails, final Filter filter,
final boolean rslj, final int g, final int pg) {
final ValueExpr ve = filter.getCondition();
// make a constraint, attach it to the rule
+ if (ve != null) {
+ final Constraint c = new Constraint(ve);
+ c.setGroup(g);
+ c.setParentGroup(pg);
+ tails.add(c);
+ }
final TupleExpr arg = filter.getArg();
@@ -626,7 +638,7 @@
}
- private void collectTails(final List<Tail> tails, final StatementPattern sp,
+ private void collectTails(final List<Op> tails, final StatementPattern sp,
final boolean rslj, final int g, final int pg) {
final Tail t = new Tail(sp);
@@ -637,7 +649,19 @@
}
- private static class Tail {
+ private static interface Op {
+
+ void setGroup(int g);
+
+ int getGroup();
+
+ void setParentGroup(int pg);
+
+ int getParentGroup();
+
+ }
+
+ private static class Tail implements Op {
private StatementPattern sp;
@@ -713,4 +737,54 @@
}
+ private static class Constraint implements Op {
+
+ private ValueExpr ve;
+
+ private int group, parent;
+
+ public Constraint(ValueExpr ve) {
+
+ this.ve = ve;
+
+ }
+
+ public void setGroup(final int group) {
+
+ this.group = group;
+
+ }
+
+ public int getGroup() {
+
+ return group;
+
+ }
+
+ public void setParentGroup(final int parent) {
+
+ this.parent = parent;
+
+ }
+
+ public int getParentGroup() {
+
+ return parent;
+
+ }
+
+ public String toString() {
+
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("Constraint: group=").append(group);
+ sb.append(", parent=").append(parent);
+ sb.append(", filter=").append(ve);
+
+ return sb.toString();
+
+ }
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <tho...@us...> - 2011-01-06 21:02:54
|
Revision: 4065
http://bigdata.svn.sourceforge.net/bigdata/?rev=4065&view=rev
Author: thompsonbry
Date: 2011-01-06 21:02:44 +0000 (Thu, 06 Jan 2011)
Log Message:
-----------
Merge trunk to QUADS_QUERY_BRANCH [r3658:r4061].
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java
branches/QUADS_QUERY_BRANCH/bigdata-perf/README.txt
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPOAssertionBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/SPORetractionBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/inf/TruthMaintenance.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/model/BigdataStatementImpl.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/rio/StatementBuffer.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/ISPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexRemover.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriteProc.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexWriter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/StatementWriter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/vocab/BaseVocabulary.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSail.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataSailRepositoryConnection.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/ProxyBigdataSailTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestSids.java
branches/QUADS_QUERY_BRANCH/build.xml
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOIndexMutation.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/changesets/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/test/com/bigdata/rdf/sail/TestChangeSets.java
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/
branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java
Property Changed:
----------------
branches/QUADS_QUERY_BRANCH/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco/
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config/
branches/QUADS_QUERY_BRANCH/bigdata-perf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/util/
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/bench/
branches/QUADS_QUERY_BRANCH/dsi-utils/LEGAL/
branches/QUADS_QUERY_BRANCH/dsi-utils/lib/
branches/QUADS_QUERY_BRANCH/dsi-utils/src/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/java/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/lgpl-utils/src/test/it/unimi/dsi/fastutil/bytes/custom/
branches/QUADS_QUERY_BRANCH/osgi/
Property changes on: branches/QUADS_QUERY_BRANCH
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/dev-btm:2574-2730
/branches/fko:3150-3194
+ /branches/BTREE_BUFFER_BRANCH:2004-2045
/branches/DEV_BRANCH_27_OCT_2009:2270-2546,2548-2782
/branches/LEXICON_REFACTOR_BRANCH:2633-3304
/branches/bugfix-btm:2594-3237
/branches/dev-btm:2574-2730
/branches/fko:3150-3194
/trunk:3659-4061
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java 2011-01-06 20:52:15 UTC (rev 4064)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/proc/AbstractKeyArrayIndexProcedure.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -39,6 +39,7 @@
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.io.OutputStream;
+import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -795,18 +796,34 @@
IResultHandler<ResultBitBuffer, ResultBitBuffer> {
private final boolean[] results;
+
+ /**
+ * I added this so I could encode information about tuple modification
+ * that takes more than one boolean to encode. For example, SPOs can
+ * be: INSERTED, REMOVED, UPDATED, NO_OP (2 bits).
+ */
+ private final int multiplier;
+
private final AtomicInteger onCount = new AtomicInteger();
public ResultBitBufferHandler(final int nkeys) {
+
+ this(nkeys, 1);
+
+ }
+
+ public ResultBitBufferHandler(final int nkeys, final int multiplier) {
- results = new boolean[nkeys];
+ results = new boolean[nkeys*multiplier];
+ this.multiplier = multiplier;
}
public void aggregate(final ResultBitBuffer result, final Split split) {
- System.arraycopy(result.getResult(), 0, results, split.fromIndex,
- split.ntuples);
+ System.arraycopy(result.getResult(), 0, results,
+ split.fromIndex*multiplier,
+ split.ntuples*multiplier);
onCount.addAndGet(result.getOnCount());
Deleted: branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java
===================================================================
--- trunk/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java 2011-01-06 20:13:43 UTC (rev 4061)
+++ branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,276 +0,0 @@
-/*
-
-Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-*/
-/*
- * Created on Nov 19, 2010
- */
-package com.bigdata.journal;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Properties;
-import java.util.UUID;
-
-import junit.framework.TestCase2;
-
-import com.bigdata.Banner;
-import com.bigdata.btree.IIndex;
-import com.bigdata.btree.IndexMetadata;
-
-/**
- * Test suite for binary compatibility, portability, and forward compatibility
- * or automated migration of persistent stores and persistence or serialization
- * capable objects across different bigdata releases. The tests in this suite
- * rely on artifacts which are archived within SVN.
- *
- * @todo create w/ small extent and truncate (RW store does not support
- * truncate).
- *
- * @todo test binary migration and forward compatibility.
- *
- * @todo stubs to create and organize artifacts,etc.
- *
- * @todo data driven test suite?
- *
- * @todo create artifact for each release, name the artifacts systematically,
- * e.g., test.release.(RW|WORM).jnl or test.release.seg. Collect a list of
- * the created artifacts and run each test against each of the versions of
- * the artifact.
- *
- * @todo Force artifact file name case for file system compatibility?
- *
- * @todo test journal (WORM and RW), btree, index segment, row store, persistent
- * data structures (checkpoints, index metadata, tuple serializers, etc.),
- * RDF layer, RMI message formats, etc.
- *
- * @todo Specific tests for
- * <p>
- * Name2Addr and DefaultKeyBuilderFactory portability problem. See
- * https://sourceforge.net/apps/trac/bigdata/ticket/193
- * <p>
- * WORM global row store resolution problem introduced in the
- * JOURNAL_HA_BRANCH. See
- * https://sourceforge.net/apps/trac/bigdata/ticket/171#comment:5
- * <p>
- * Sparse row store JDK encoding problem:
- * https://sourceforge.net/apps/trac/bigdata/ticket/107
- */
-public class TestBinaryCompatibility extends TestCase2 {
-
- /**
- *
- */
- public TestBinaryCompatibility() {
- }
-
- /**
- * @param name
- */
- public TestBinaryCompatibility(String name) {
- super(name);
- }
-
- /**
- * @todo munge the release version into a name that is compatibility with
- * the file system ("." to "_"). Store artifacts at each release? At
- * each release in which an incompatibility is introduced? At each
- * release in which a persistence capable data structure or change is
- * introduced?
- */
- static protected final File artifactDir = new File(
- "bigdata-compatibility/src/resources/artifacts");
-
- protected static class Version {
- private final String version;
- private final String revision;
- public Version(String version,String revision) {
- this.version = version;
- this.revision = revision;
- }
-
- /**
- * The bigdata version number associated with the release. This is in
- * the form <code>xx.yy.zz</code>
- */
- public String getVersion() {
- return version;
- }
-
- /**
- * The SVN repository revision associated with the release. This is in
- * the form <code>####</code>.
- */
- public String getRevision() {
- return revision;
- }
- }
-
- /**
- * Known release versions.
- */
- protected static Version V_0_83_2 = new Version("0.83.2", "3349");
-
- /**
- * Tested Versions.
- */
- protected Version[] versions = new Version[] {
- V_0_83_2
- };
-
- protected void setUp() throws Exception {
-
- Banner.banner();
-
- super.setUp();
-
- if (!artifactDir.exists()) {
-
- if (!artifactDir.mkdirs()) {
-
- throw new IOException("Could not create: " + artifactDir);
-
- }
-
- }
-
- for (Version version : versions) {
-
- final File versionDir = new File(artifactDir, version.getVersion());
-
- if (!versionDir.exists()) {
-
- if (!versionDir.mkdirs()) {
-
- throw new IOException("Could not create: " + versionDir);
-
- }
-
- }
-
- }
-
- }
-
- protected void tearDown() throws Exception {
-
- super.tearDown();
-
- }
-
- /**
- * @throws Throwable
- *
- * @todo Each 'test' should run an instance of a class which knows how to
- * create the appropriate artifacts and how to test them.
- */
- public void test_WORM_compatibility_with_JOURNAL_HA_BRANCH()
- throws Throwable {
-
- final Version version = V_0_83_2;
-
- final File versionDir = new File(artifactDir, version.getVersion());
-
- final File artifactFile = new File(versionDir, getName()
- + BufferMode.DiskWORM + Journal.Options.JNL);
-
- if (!artifactFile.exists()) {
-
- createArtifact(artifactFile);
-
- }
-
- verifyArtifact(artifactFile);
-
- }
-
- protected void createArtifact(final File artifactFile) throws Throwable {
-
- if (log.isInfoEnabled())
- log.info("Creating: " + artifactFile);
-
- final Properties properties = new Properties();
-
- properties.setProperty(Journal.Options.FILE, artifactFile.toString());
-
- properties.setProperty(Journal.Options.INITIAL_EXTENT, ""
- + Journal.Options.minimumInitialExtent);
-
- final Journal journal = new Journal(properties);
-
- try {
-
- final IndexMetadata md = new IndexMetadata(UUID.randomUUID());
-
- final IIndex ndx = journal.registerIndex("kb.spo.SPO", md);
-
- ndx.insert(1,1);
-
- journal.commit();
-
- // reduce to minimum footprint.
- journal.truncate();
-
- } catch (Throwable t) {
-
- journal.destroy();
-
- throw new RuntimeException(t);
-
- } finally {
-
- if (journal.isOpen())
- journal.close();
-
- }
-
- }
-
- protected void verifyArtifact(final File artifactFile) throws Throwable {
-
- if (log.isInfoEnabled())
- log.info("Verifying: " + artifactFile);
-
- final Properties properties = new Properties();
-
- properties.setProperty(Journal.Options.FILE, artifactFile.toString());
-
- final Journal journal = new Journal(properties);
-
- try {
-
- final IIndex ndx = journal.getIndex("kb.spo.SPO");
-
- assertNotNull(ndx);
-
- assertEquals(1,ndx.lookup(1));
-
- } finally {
-
- journal.close();
-
- }
-
- }
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java (from rev 4061, trunk/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-compatibility/src/test/com/bigdata/journal/TestBinaryCompatibility.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -0,0 +1,276 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Nov 19, 2010
+ */
+package com.bigdata.journal;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.UUID;
+
+import junit.framework.TestCase2;
+
+import com.bigdata.Banner;
+import com.bigdata.btree.IIndex;
+import com.bigdata.btree.IndexMetadata;
+
+/**
+ * Test suite for binary compatibility, portability, and forward compatibility
+ * or automated migration of persistent stores and persistence or serialization
+ * capable objects across different bigdata releases. The tests in this suite
+ * rely on artifacts which are archived within SVN.
+ *
+ * @todo create w/ small extent and truncate (RW store does not support
+ * truncate).
+ *
+ * @todo test binary migration and forward compatibility.
+ *
+ * @todo stubs to create and organize artifacts,etc.
+ *
+ * @todo data driven test suite?
+ *
+ * @todo create artifact for each release, name the artifacts systematically,
+ * e.g., test.release.(RW|WORM).jnl or test.release.seg. Collect a list of
+ * the created artifacts and run each test against each of the versions of
+ * the artifact.
+ *
+ * @todo Force artifact file name case for file system compatibility?
+ *
+ * @todo test journal (WORM and RW), btree, index segment, row store, persistent
+ * data structures (checkpoints, index metadata, tuple serializers, etc.),
+ * RDF layer, RMI message formats, etc.
+ *
+ * @todo Specific tests for
+ * <p>
+ * Name2Addr and DefaultKeyBuilderFactory portability problem. See
+ * https://sourceforge.net/apps/trac/bigdata/ticket/193
+ * <p>
+ * WORM global row store resolution problem introduced in the
+ * JOURNAL_HA_BRANCH. See
+ * https://sourceforge.net/apps/trac/bigdata/ticket/171#comment:5
+ * <p>
+ * Sparse row store JDK encoding problem:
+ * https://sourceforge.net/apps/trac/bigdata/ticket/107
+ */
+public class TestBinaryCompatibility extends TestCase2 {
+
+ /**
+ *
+ */
+ public TestBinaryCompatibility() {
+ }
+
+ /**
+ * @param name
+ */
+ public TestBinaryCompatibility(String name) {
+ super(name);
+ }
+
+ /**
+ * @todo munge the release version into a name that is compatibility with
+ * the file system ("." to "_"). Store artifacts at each release? At
+ * each release in which an incompatibility is introduced? At each
+ * release in which a persistence capable data structure or change is
+ * introduced?
+ */
+ static protected final File artifactDir = new File(
+ "bigdata-compatibility/src/resources/artifacts");
+
+ protected static class Version {
+ private final String version;
+ private final String revision;
+ public Version(String version,String revision) {
+ this.version = version;
+ this.revision = revision;
+ }
+
+ /**
+ * The bigdata version number associated with the release. This is in
+ * the form <code>xx.yy.zz</code>
+ */
+ public String getVersion() {
+ return version;
+ }
+
+ /**
+ * The SVN repository revision associated with the release. This is in
+ * the form <code>####</code>.
+ */
+ public String getRevision() {
+ return revision;
+ }
+ }
+
+ /**
+ * Known release versions.
+ */
+ protected static Version V_0_83_2 = new Version("0.83.2", "3349");
+
+ /**
+ * Tested Versions.
+ */
+ protected Version[] versions = new Version[] {
+ V_0_83_2
+ };
+
+ protected void setUp() throws Exception {
+
+ Banner.banner();
+
+ super.setUp();
+
+ if (!artifactDir.exists()) {
+
+ if (!artifactDir.mkdirs()) {
+
+ throw new IOException("Could not create: " + artifactDir);
+
+ }
+
+ }
+
+ for (Version version : versions) {
+
+ final File versionDir = new File(artifactDir, version.getVersion());
+
+ if (!versionDir.exists()) {
+
+ if (!versionDir.mkdirs()) {
+
+ throw new IOException("Could not create: " + versionDir);
+
+ }
+
+ }
+
+ }
+
+ }
+
+ protected void tearDown() throws Exception {
+
+ super.tearDown();
+
+ }
+
+ /**
+ * @throws Throwable
+ *
+ * @todo Each 'test' should run an instance of a class which knows how to
+ * create the appropriate artifacts and how to test them.
+ */
+ public void test_WORM_compatibility_with_JOURNAL_HA_BRANCH()
+ throws Throwable {
+
+ final Version version = V_0_83_2;
+
+ final File versionDir = new File(artifactDir, version.getVersion());
+
+ final File artifactFile = new File(versionDir, getName()
+ + BufferMode.DiskWORM + Journal.Options.JNL);
+
+ if (!artifactFile.exists()) {
+
+ createArtifact(artifactFile);
+
+ }
+
+ verifyArtifact(artifactFile);
+
+ }
+
+ protected void createArtifact(final File artifactFile) throws Throwable {
+
+ if (log.isInfoEnabled())
+ log.info("Creating: " + artifactFile);
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Journal.Options.FILE, artifactFile.toString());
+
+ properties.setProperty(Journal.Options.INITIAL_EXTENT, ""
+ + Journal.Options.minimumInitialExtent);
+
+ final Journal journal = new Journal(properties);
+
+ try {
+
+ final IndexMetadata md = new IndexMetadata(UUID.randomUUID());
+
+ final IIndex ndx = journal.registerIndex("kb.spo.SPO", md);
+
+ ndx.insert(1,1);
+
+ journal.commit();
+
+ // reduce to minimum footprint.
+ journal.truncate();
+
+ } catch (Throwable t) {
+
+ journal.destroy();
+
+ throw new RuntimeException(t);
+
+ } finally {
+
+ if (journal.isOpen())
+ journal.close();
+
+ }
+
+ }
+
+ protected void verifyArtifact(final File artifactFile) throws Throwable {
+
+ if (log.isInfoEnabled())
+ log.info("Verifying: " + artifactFile);
+
+ final Properties properties = new Properties();
+
+ properties.setProperty(Journal.Options.FILE, artifactFile.toString());
+
+ final Journal journal = new Journal(properties);
+
+ try {
+
+ final IIndex ndx = journal.getIndex("kb.spo.SPO");
+
+ assertNotNull(ndx);
+
+ assertEquals(1,ndx.lookup(1));
+
+ } finally {
+
+ journal.close();
+
+ }
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/attr
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/attr:3369-3423
+ /trunk/bigdata-jini/src/java/com/bigdata/attr:3369-3423,3659-4061
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/disco
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/disco:3369-3423
+ /trunk/bigdata-jini/src/java/com/bigdata/disco:3369-3423,3659-4061
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata-jini/src/java/com/bigdata/util/config
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-jini/src/java/com/bigdata/util/config:3369-3423
+ /trunk/bigdata-jini/src/java/com/bigdata/util/config:3369-3423,3659-4061
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata-perf
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/bigdata-perf:3369-3423
+ /trunk/bigdata-perf:3369-3423,3659-4061
Modified: branches/QUADS_QUERY_BRANCH/bigdata-perf/README.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-perf/README.txt 2011-01-06 20:52:15 UTC (rev 4064)
+++ branches/QUADS_QUERY_BRANCH/bigdata-perf/README.txt 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,2 +1,6 @@
This module contains drivers for a variety of data sets and benchmarks used as
-part of a performance test suite.
\ No newline at end of file
+part of a performance test suite.
+
+Note: You must run "ant bundleJar" in the top-level directory first. This will
+build the bigdata code base and bundle together the various dependencies so they
+will be available for the ant scripts in this module.
Deleted: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java 2011-01-06 20:13:43 UTC (rev 4061)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,98 +0,0 @@
-package com.bigdata.rdf.changesets;
-
-import java.util.Comparator;
-import com.bigdata.rdf.spo.ISPO;
-import com.bigdata.rdf.spo.SPOComparator;
-
-public class ChangeRecord implements IChangeRecord {
-
- private final ISPO stmt;
-
- private final ChangeAction action;
-
-// private final StatementEnum oldType;
-
- public ChangeRecord(final ISPO stmt, final ChangeAction action) {
-
-// this(stmt, action, null);
-//
-// }
-//
-// public ChangeRecord(final BigdataStatement stmt, final ChangeAction action,
-// final StatementEnum oldType) {
-//
- this.stmt = stmt;
- this.action = action;
-// this.oldType = oldType;
-
- }
-
- public ChangeAction getChangeAction() {
-
- return action;
-
- }
-
-// public StatementEnum getOldStatementType() {
-//
-// return oldType;
-//
-// }
-
- public ISPO getStatement() {
-
- return stmt;
-
- }
-
- @Override
- public boolean equals(Object o) {
-
- if (o == this)
- return true;
-
- if (o == null || o instanceof IChangeRecord == false)
- return false;
-
- final IChangeRecord rec = (IChangeRecord) o;
-
- final ISPO stmt2 = rec.getStatement();
-
- // statements are equal
- if (stmt == stmt2 ||
- (stmt != null && stmt2 != null && stmt.equals(stmt2))) {
-
- // actions are equal
- return action == rec.getChangeAction();
-
- }
-
- return false;
-
- }
-
- public String toString() {
-
- StringBuilder sb = new StringBuilder();
-
- sb.append(action).append(": ").append(stmt);
-
- return sb.toString();
-
- }
-
- public static final Comparator<IChangeRecord> COMPARATOR =
- new Comparator<IChangeRecord>() {
-
- public int compare(final IChangeRecord r1, final IChangeRecord r2) {
-
- final ISPO spo1 = r1.getStatement();
- final ISPO spo2 = r2.getStatement();
-
- return SPOComparator.INSTANCE.compare(spo1, spo2);
-
- }
-
- };
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java (from rev 4061, trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/ChangeRecord.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -0,0 +1,98 @@
+package com.bigdata.rdf.changesets;
+
+import java.util.Comparator;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.spo.SPOComparator;
+
+public class ChangeRecord implements IChangeRecord {
+
+ private final ISPO stmt;
+
+ private final ChangeAction action;
+
+// private final StatementEnum oldType;
+
+ public ChangeRecord(final ISPO stmt, final ChangeAction action) {
+
+// this(stmt, action, null);
+//
+// }
+//
+// public ChangeRecord(final BigdataStatement stmt, final ChangeAction action,
+// final StatementEnum oldType) {
+//
+ this.stmt = stmt;
+ this.action = action;
+// this.oldType = oldType;
+
+ }
+
+ public ChangeAction getChangeAction() {
+
+ return action;
+
+ }
+
+// public StatementEnum getOldStatementType() {
+//
+// return oldType;
+//
+// }
+
+ public ISPO getStatement() {
+
+ return stmt;
+
+ }
+
+ @Override
+ public boolean equals(Object o) {
+
+ if (o == this)
+ return true;
+
+ if (o == null || o instanceof IChangeRecord == false)
+ return false;
+
+ final IChangeRecord rec = (IChangeRecord) o;
+
+ final ISPO stmt2 = rec.getStatement();
+
+ // statements are equal
+ if (stmt == stmt2 ||
+ (stmt != null && stmt2 != null && stmt.equals(stmt2))) {
+
+ // actions are equal
+ return action == rec.getChangeAction();
+
+ }
+
+ return false;
+
+ }
+
+ public String toString() {
+
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(action).append(": ").append(stmt);
+
+ return sb.toString();
+
+ }
+
+ public static final Comparator<IChangeRecord> COMPARATOR =
+ new Comparator<IChangeRecord>() {
+
+ public int compare(final IChangeRecord r1, final IChangeRecord r2) {
+
+ final ISPO spo1 = r1.getStatement();
+ final ISPO spo2 = r2.getStatement();
+
+ return SPOComparator.INSTANCE.compare(spo1, spo2);
+
+ }
+
+ };
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java 2011-01-06 20:13:43 UTC (rev 4061)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,38 +0,0 @@
-package com.bigdata.rdf.changesets;
-
-/**
- * Provides detailed information on changes made to statements in the database.
- * Change records are generated for any statements that are used in
- * addStatement() or removeStatements() operations on the SAIL connection, as
- * well as any inferred statements that are added or removed as a result of
- * truth maintenance when the database has inference enabled. Change records
- * will be sent to an instance of this class via the
- * {@link #changeEvent(IChangeRecord)} method. These events will
- * occur on an ongoing basis as statements are added to or removed from the
- * indices. It is the change log's responsibility to collect change records.
- * When the transaction is actually committed (or aborted), the change log will
- * receive notification via {@link #transactionCommited()} or
- * {@link #transactionAborted()}.
- */
-public interface IChangeLog {
-
- /**
- * Occurs when a statement add or remove is flushed to the indices (but
- * not yet committed).
- *
- * @param record
- * the {@link IChangeRecord}
- */
- void changeEvent(final IChangeRecord record);
-
- /**
- * Occurs when the current SAIL transaction is committed.
- */
- void transactionCommited();
-
- /**
- * Occurs if the current SAIL transaction is aborted.
- */
- void transactionAborted();
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java (from rev 4061, trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeLog.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -0,0 +1,38 @@
+package com.bigdata.rdf.changesets;
+
+/**
+ * Provides detailed information on changes made to statements in the database.
+ * Change records are generated for any statements that are used in
+ * addStatement() or removeStatements() operations on the SAIL connection, as
+ * well as any inferred statements that are added or removed as a result of
+ * truth maintenance when the database has inference enabled. Change records
+ * will be sent to an instance of this class via the
+ * {@link #changeEvent(IChangeRecord)} method. These events will
+ * occur on an ongoing basis as statements are added to or removed from the
+ * indices. It is the change log's responsibility to collect change records.
+ * When the transaction is actually committed (or aborted), the change log will
+ * receive notification via {@link #transactionCommited()} or
+ * {@link #transactionAborted()}.
+ */
+public interface IChangeLog {
+
+ /**
+ * Occurs when a statement add or remove is flushed to the indices (but
+ * not yet committed).
+ *
+ * @param record
+ * the {@link IChangeRecord}
+ */
+ void changeEvent(final IChangeRecord record);
+
+ /**
+ * Occurs when the current SAIL transaction is committed.
+ */
+ void transactionCommited();
+
+ /**
+ * Occurs if the current SAIL transaction is aborted.
+ */
+ void transactionAborted();
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java 2011-01-06 20:13:43 UTC (rev 4061)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,120 +0,0 @@
-package com.bigdata.rdf.changesets;
-
-import com.bigdata.rdf.model.BigdataStatement;
-import com.bigdata.rdf.model.StatementEnum;
-import com.bigdata.rdf.spo.ISPO;
-
-/**
- * Provides detailed information on changes made to statements in the database.
- * Change records are generated for any statements that are used in
- * addStatement() or removeStatements() operations on the SAIL connection, as
- * well as any inferred statements that are added or removed as a result of
- * truth maintenance when the database has inference enabled.
- * <p>
- * See {@link IChangeLog}.
- */
-public interface IChangeRecord {
-
- /**
- * Attempting to add or remove statements can have a number of different
- * effects. This enum captures the different actions that can take place as
- * a result of trying to add or remove a statement from the database.
- */
- public enum ChangeAction {
-
- /**
- * The focus statement was not in the database before and will be
- * in the database after the commit. This can be the result of either
- * explicit addStatement() operations on the SAIL connection, or from
- * new inferences being generated via truth maintenance when the
- * database has inference enabled. If the focus statement has a
- * statement type of explicit then it was added via an addStatement()
- * operation. If the focus statement has a statement type of inferred
- * then it was added via truth maintenance.
- */
- INSERTED,
-
- /**
- * The focus statement was in the database before and will not
- * be in the database after the commit. When the database has inference
- * and truth maintenance enabled, the statement that is the focus of
- * this change record was either an explicit statement that was the
- * subject of a removeStatements() operation on the connection, or it
- * was an inferred statement that was removed as a result of truth
- * maintenance. Either way, the statement is no longer provable as an
- * inference using other statements still in the database after the
- * commit. If it were still provable, the explicit statement would have
- * had its type changed to inferred, and the inferred statement would
- * have remained untouched by truth maintenance. If an inferred
- * statement was the subject of a removeStatement() operation on the
- * connection it would have resulted in a no-op, since inferences can
- * only be removed via truth maintenance.
- */
- REMOVED,
-
- /**
- * This change action can only occur when inference and truth
- * maintenance are enabled on the database. Sometimes an attempt at
- * statement addition or removal via an addStatement() or
- * removeStatements() operation on the connection will result in a type
- * change rather than an actual assertion or deletion. When in
- * inference mode, statements can have one of three statement types:
- * explicit, inferred, or axiom (see {@link StatementEnum}). There are
- * several reasons why a statement will change type rather than be
- * asserted or deleted:
- * <p>
- * <ul>
- * <li> A statement is asserted, but already exists in the database as
- * an inference or an axiom. The existing statement will have its type
- * changed from inference or axiom to explicit. </li>
- * <li> An explicit statement is retracted, but is still provable by
- * other means. It will have its type changed from explicit to
- * inference. </li>
- * <li> An explicit statement is retracted, but is one of the axioms
- * needed for inference. It will have its type changed from explicit to
- * axiom. </li>
- * </ul>
- */
- UPDATED,
-
-// /**
-// * This change action can occur for one of two reasons:
-// * <p>
-// * <ul>
-// * <li> A statement is asserted, but already exists in the database as
-// * an explicit statement. </li>
-// * <li> An inferred statement or an axiom is retracted. Only explicit
-// * statements can be retracted via removeStatements() operations. </li>
-// * </ul>
-// */
-// NO_OP
-
- }
-
- /**
- * Return the ISPO that is the focus of this change record.
- *
- * @return
- * the {@link ISPO}
- */
- ISPO getStatement();
-
- /**
- * Return the change action for this change record.
- *
- * @return
- * the {@link ChangeAction}
- */
- ChangeAction getChangeAction();
-
-// /**
-// * If the change action is {@link ChangeAction#TYPE_CHANGE}, this method
-// * will return the old statement type of the focus statement. The
-// * new statement type is available on the focus statement itself.
-// *
-// * @return
-// * the old statement type of the focus statement
-// */
-// StatementEnum getOldStatementType();
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java (from rev 4061, trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/IChangeRecord.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -0,0 +1,120 @@
+package com.bigdata.rdf.changesets;
+
+import com.bigdata.rdf.model.BigdataStatement;
+import com.bigdata.rdf.model.StatementEnum;
+import com.bigdata.rdf.spo.ISPO;
+
+/**
+ * Provides detailed information on changes made to statements in the database.
+ * Change records are generated for any statements that are used in
+ * addStatement() or removeStatements() operations on the SAIL connection, as
+ * well as any inferred statements that are added or removed as a result of
+ * truth maintenance when the database has inference enabled.
+ * <p>
+ * See {@link IChangeLog}.
+ */
+public interface IChangeRecord {
+
+ /**
+ * Attempting to add or remove statements can have a number of different
+ * effects. This enum captures the different actions that can take place as
+ * a result of trying to add or remove a statement from the database.
+ */
+ public enum ChangeAction {
+
+ /**
+ * The focus statement was not in the database before and will be
+ * in the database after the commit. This can be the result of either
+ * explicit addStatement() operations on the SAIL connection, or from
+ * new inferences being generated via truth maintenance when the
+ * database has inference enabled. If the focus statement has a
+ * statement type of explicit then it was added via an addStatement()
+ * operation. If the focus statement has a statement type of inferred
+ * then it was added via truth maintenance.
+ */
+ INSERTED,
+
+ /**
+ * The focus statement was in the database before and will not
+ * be in the database after the commit. When the database has inference
+ * and truth maintenance enabled, the statement that is the focus of
+ * this change record was either an explicit statement that was the
+ * subject of a removeStatements() operation on the connection, or it
+ * was an inferred statement that was removed as a result of truth
+ * maintenance. Either way, the statement is no longer provable as an
+ * inference using other statements still in the database after the
+ * commit. If it were still provable, the explicit statement would have
+ * had its type changed to inferred, and the inferred statement would
+ * have remained untouched by truth maintenance. If an inferred
+ * statement was the subject of a removeStatement() operation on the
+ * connection it would have resulted in a no-op, since inferences can
+ * only be removed via truth maintenance.
+ */
+ REMOVED,
+
+ /**
+ * This change action can only occur when inference and truth
+ * maintenance are enabled on the database. Sometimes an attempt at
+ * statement addition or removal via an addStatement() or
+ * removeStatements() operation on the connection will result in a type
+ * change rather than an actual assertion or deletion. When in
+ * inference mode, statements can have one of three statement types:
+ * explicit, inferred, or axiom (see {@link StatementEnum}). There are
+ * several reasons why a statement will change type rather than be
+ * asserted or deleted:
+ * <p>
+ * <ul>
+ * <li> A statement is asserted, but already exists in the database as
+ * an inference or an axiom. The existing statement will have its type
+ * changed from inference or axiom to explicit. </li>
+ * <li> An explicit statement is retracted, but is still provable by
+ * other means. It will have its type changed from explicit to
+ * inference. </li>
+ * <li> An explicit statement is retracted, but is one of the axioms
+ * needed for inference. It will have its type changed from explicit to
+ * axiom. </li>
+ * </ul>
+ */
+ UPDATED,
+
+// /**
+// * This change action can occur for one of two reasons:
+// * <p>
+// * <ul>
+// * <li> A statement is asserted, but already exists in the database as
+// * an explicit statement. </li>
+// * <li> An inferred statement or an axiom is retracted. Only explicit
+// * statements can be retracted via removeStatements() operations. </li>
+// * </ul>
+// */
+// NO_OP
+
+ }
+
+ /**
+ * Return the ISPO that is the focus of this change record.
+ *
+ * @return
+ * the {@link ISPO}
+ */
+ ISPO getStatement();
+
+ /**
+ * Return the change action for this change record.
+ *
+ * @return
+ * the {@link ChangeAction}
+ */
+ ChangeAction getChangeAction();
+
+// /**
+// * If the change action is {@link ChangeAction#TYPE_CHANGE}, this method
+// * will return the old statement type of the focus statement. The
+// * new statement type is available on the focus statement itself.
+// *
+// * @return
+// * the old statement type of the focus statement
+// */
+// StatementEnum getOldStatementType();
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java 2011-01-06 20:13:43 UTC (rev 4061)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,163 +0,0 @@
-package com.bigdata.rdf.changesets;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import org.apache.log4j.Logger;
-import com.bigdata.rdf.model.BigdataStatement;
-import com.bigdata.rdf.spo.ISPO;
-import com.bigdata.rdf.store.AbstractTripleStore;
-import com.bigdata.rdf.store.BigdataStatementIterator;
-import com.bigdata.striterator.ChunkedArrayIterator;
-
-/**
- * This is a very simple implementation of a change log. NOTE: This is not
- * a particularly great implementation. First of all it ends up storing
- * two copies of the change set. Secondly it needs to be smarter about
- * concurrency, or maybe we can be smart about it when we do the
- * implementation on the other side (the SAIL connection can just write
- * change events to a buffer and then the buffer can be drained by
- * another thread that doesn't block the actual read/write operations,
- * although then we need to be careful not to issue the committed()
- * notification before the buffer is drained).
- *
- * @author mike
- *
- */
-public class InMemChangeLog implements IChangeLog {
-
- protected static final Logger log = Logger.getLogger(InMemChangeLog.class);
-
- /**
- * Running tally of new changes since the last commit notification.
- */
- private final Map<ISPO,IChangeRecord> changeSet =
- new HashMap<ISPO, IChangeRecord>();
-
- /**
- * Keep a record of the change set as of the last commit.
- */
- private final Map<ISPO,IChangeRecord> committed =
- new HashMap<ISPO, IChangeRecord>();
-
- /**
- * See {@link IChangeLog#changeEvent(IChangeRecord)}.
- */
- public synchronized void changeEvent(final IChangeRecord record) {
-
- if (log.isInfoEnabled())
- log.info(record);
-
- changeSet.put(record.getStatement(), record);
-
- }
-
- /**
- * See {@link IChangeLog#transactionCommited()}.
- */
- public synchronized void transactionCommited() {
-
- if (log.isInfoEnabled())
- log.info("transaction committed");
-
- committed.clear();
-
- committed.putAll(changeSet);
-
- changeSet.clear();
-
- }
-
- /**
- * See {@link IChangeLog#transactionAborted()}.
- */
- public synchronized void transactionAborted() {
-
- if (log.isInfoEnabled())
- log.info("transaction aborted");
-
- changeSet.clear();
-
- }
-
- /**
- * Return the change set as of the last commmit point.
- *
- * @return
- * a collection of {@link IChangeRecord}s as of the last commit
- * point
- */
- public Collection<IChangeRecord> getLastCommit() {
-
- return committed.values();
-
- }
-
- /**
- * Return the change set as of the last commmit point, using the supplied
- * database to resolve ISPOs to BigdataStatements.
- *
- * @return
- * a collection of {@link IChangeRecord}s as of the last commit
- * point
- */
- public Collection<IChangeRecord> getLastCommit(final AbstractTripleStore db) {
-
- return resolve(db, committed.values());
-
- }
-
- /**
- * Use the supplied database to turn a set of ISPO change records into
- * BigdataStatement change records. BigdataStatements also implement
- * ISPO, the difference being that BigdataStatements also contain
- * materialized RDF terms for the 3 (or 4) positions, in addition to just
- * the internal identifiers (IVs) for those terms.
- *
- * @param db
- * the database containing the lexicon needed to materialize
- * the BigdataStatement objects
- * @param unresolved
- * the ISPO change records that came from IChangeLog notification
- * events
- * @return
- * the fully resolves BigdataStatement change records
- */
- private Collection<IChangeRecord> resolve(final AbstractTripleStore db,
- final Collection<IChangeRecord> unresolved) {
-
- final Collection<IChangeRecord> resolved =
- new LinkedList<IChangeRecord>();
-
- // collect up the ISPOs out of the unresolved change records
- final ISPO[] spos = new ISPO[unresolved.size()];
- int i = 0;
- for (IChangeRecord rec : unresolved) {
- spos[i++] = rec.getStatement();
- }
-
- // use the database to resolve them into BigdataStatements
- final BigdataStatementIterator it =
- db.asStatementIterator(
- new ChunkedArrayIterator<ISPO>(i, spos, null/* keyOrder */));
-
- /*
- * the BigdataStatementIterator will produce BigdataStatement objects
- * in the same order as the original ISPO array
- */
- for (IChangeRecord rec : unresolved) {
-
- final BigdataStatement stmt = it.next();
-
- resolved.add(new ChangeRecord(stmt, rec.getChangeAction()));
-
- }
-
- return resolved;
-
- }
-
-
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java (from rev 4061, trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/InMemChangeLog.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -0,0 +1,163 @@
+package com.bigdata.rdf.changesets;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import org.apache.log4j.Logger;
+import com.bigdata.rdf.model.BigdataStatement;
+import com.bigdata.rdf.spo.ISPO;
+import com.bigdata.rdf.store.AbstractTripleStore;
+import com.bigdata.rdf.store.BigdataStatementIterator;
+import com.bigdata.striterator.ChunkedArrayIterator;
+
+/**
+ * This is a very simple implementation of a change log. NOTE: This is not
+ * a particularly great implementation. First of all it ends up storing
+ * two copies of the change set. Secondly it needs to be smarter about
+ * concurrency, or maybe we can be smart about it when we do the
+ * implementation on the other side (the SAIL connection can just write
+ * change events to a buffer and then the buffer can be drained by
+ * another thread that doesn't block the actual read/write operations,
+ * although then we need to be careful not to issue the committed()
+ * notification before the buffer is drained).
+ *
+ * @author mike
+ *
+ */
+public class InMemChangeLog implements IChangeLog {
+
+ protected static final Logger log = Logger.getLogger(InMemChangeLog.class);
+
+ /**
+ * Running tally of new changes since the last commit notification.
+ */
+ private final Map<ISPO,IChangeRecord> changeSet =
+ new HashMap<ISPO, IChangeRecord>();
+
+ /**
+ * Keep a record of the change set as of the last commit.
+ */
+ private final Map<ISPO,IChangeRecord> committed =
+ new HashMap<ISPO, IChangeRecord>();
+
+ /**
+ * See {@link IChangeLog#changeEvent(IChangeRecord)}.
+ */
+ public synchronized void changeEvent(final IChangeRecord record) {
+
+ if (log.isInfoEnabled())
+ log.info(record);
+
+ changeSet.put(record.getStatement(), record);
+
+ }
+
+ /**
+ * See {@link IChangeLog#transactionCommited()}.
+ */
+ public synchronized void transactionCommited() {
+
+ if (log.isInfoEnabled())
+ log.info("transaction committed");
+
+ committed.clear();
+
+ committed.putAll(changeSet);
+
+ changeSet.clear();
+
+ }
+
+ /**
+ * See {@link IChangeLog#transactionAborted()}.
+ */
+ public synchronized void transactionAborted() {
+
+ if (log.isInfoEnabled())
+ log.info("transaction aborted");
+
+ changeSet.clear();
+
+ }
+
+ /**
+ * Return the change set as of the last commmit point.
+ *
+ * @return
+ * a collection of {@link IChangeRecord}s as of the last commit
+ * point
+ */
+ public Collection<IChangeRecord> getLastCommit() {
+
+ return committed.values();
+
+ }
+
+ /**
+ * Return the change set as of the last commmit point, using the supplied
+ * database to resolve ISPOs to BigdataStatements.
+ *
+ * @return
+ * a collection of {@link IChangeRecord}s as of the last commit
+ * point
+ */
+ public Collection<IChangeRecord> getLastCommit(final AbstractTripleStore db) {
+
+ return resolve(db, committed.values());
+
+ }
+
+ /**
+ * Use the supplied database to turn a set of ISPO change records into
+ * BigdataStatement change records. BigdataStatements also implement
+ * ISPO, the difference being that BigdataStatements also contain
+ * materialized RDF terms for the 3 (or 4) positions, in addition to just
+ * the internal identifiers (IVs) for those terms.
+ *
+ * @param db
+ * the database containing the lexicon needed to materialize
+ * the BigdataStatement objects
+ * @param unresolved
+ * the ISPO change records that came from IChangeLog notification
+ * events
+ * @return
+ * the fully resolves BigdataStatement change records
+ */
+ private Collection<IChangeRecord> resolve(final AbstractTripleStore db,
+ final Collection<IChangeRecord> unresolved) {
+
+ final Collection<IChangeRecord> resolved =
+ new LinkedList<IChangeRecord>();
+
+ // collect up the ISPOs out of the unresolved change records
+ final ISPO[] spos = new ISPO[unresolved.size()];
+ int i = 0;
+ for (IChangeRecord rec : unresolved) {
+ spos[i++] = rec.getStatement();
+ }
+
+ // use the database to resolve them into BigdataStatements
+ final BigdataStatementIterator it =
+ db.asStatementIterator(
+ new ChunkedArrayIterator<ISPO>(i, spos, null/* keyOrder */));
+
+ /*
+ * the BigdataStatementIterator will produce BigdataStatement objects
+ * in the same order as the original ISPO array
+ */
+ for (IChangeRecord rec : unresolved) {
+
+ final BigdataStatement stmt = it.next();
+
+ resolved.add(new ChangeRecord(stmt, rec.getChangeAction()));
+
+ }
+
+ return resolved;
+
+ }
+
+
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java 2011-01-06 20:13:43 UTC (rev 4061)
+++ branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/changesets/StatementWriter.java 2011-01-06 21:02:44 UTC (rev 4065)
@@ -1,208 +0,0 @@
-package com.bigdata.rdf.changesets;
-
-import java.util.Iterator;
-import java.util.Map;
-import org.apache.log4j.Logger;
-import com.bigdata.rdf.changesets.IChangeRecord.ChangeAction;
-import com.bigdata.rdf.internal.IV;
-import com.bigdata.rdf.model.BigdataBNode;
-import com.bigdata.rdf.spo.ISPO;
-import com.bigdata.rdf.spo.SPO;
-import com.bigdata.rdf.spo.ISPO.ModifiedEnum;
-import com.bigdata.rdf.store.AbstractTripleStore;
-import com.bigdata.relation.accesspath.IElementFilter;
-import com.bigdata.striterator.ChunkedArrayIterator;
-import com.bigdata.striterator.IChunkedOrderedIterator;
-
-public class StatementWriter {
-
- protected static final Logger log = Logger.getLogger(StatementWriter.class);
-
- public static long addStatements(final AbstractTripleStore database,
- final AbstractTripleStore statementStore,
- final boolean copyOnly,
- final IElementFilter<ISPO> filter,
- final IChunkedOrderedIterator<ISPO> itr,
- final IChangeLog changeLog) {
-
- long n = 0;
-
- if (itr.hasNext()) {
-
-// final BigdataStatementIteratorImpl itr2 =
-// new BigdataStatementIteratorImpl(database, bnodes, itr)
-// .start(database.getExecutorService());
-//
-// final BigdataStatement[] stmts =
-// new BigdataStatement[database.getChunkCapacity()];
- final SPO[] stmts = new SPO[database.getChunkCapacity()];
-
- int i = 0;
- while ((i = nextChunk(itr, stmts)) > 0) {
- n += addStatements(database, statementStore, copyOnly, filter,
- stmts, i, changeLog);
- }
-
- }
-
- return n;
-
- }
-
- private static long addStatements(final AbstractTripleStore database,
- final AbstractTripleStore statementStore,
- final boolean copyOnly,
- final IElementFilter<ISPO> filter,
- final ISPO[] stmts,
- final int numStmts,
- final IChangeLog changeLog) {
-
-// final SPO[] tmp = allocateSPOs(stmts, numStmts);
-
- final long n = database.addStatements(statementStore, copyOnly,
- new ChunkedArrayIterator<ISPO>(numStmts, stmts,
- null/* keyOrder */), filter);
-
- // Copy the state of the isModified() flag and notify changeLog
- for (int i = 0; i < numStmts; i++) {
-
- if (stmts[i].isModified()) {
-
-// stmts[i].setModified(true);
-
- if (changeLog != null) {
-
- switch(stmts[i].getModified()) {
- case INSERTED:
- changeLog.cha...
[truncated message content] |
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X