Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[3672] branches/QUADS_QUERY_BRANCH
|
From: <tho...@us...> - 2010-09-28 23:39:51
|
Revision: 3672
http://bigdata.svn.sourceforge.net/bigdata/?rev=3672&view=rev
Author: thompsonbry
Date: 2010-09-28 23:39:42 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Reconciled IFilterConstructor with IFilter.
Added BOpTupleFilter, which parallels TupleFilter and in fact uses the same underlying iterator to protect against reuse of the Tuple instance by the B+Tree iterators.
Fixed a problem in Filterator where eager initialization (in the constructor) caused the next element to be fetched before the filter stack was fully initialized by FilterBase.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/IRangeQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadCommittedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ReadOnlyIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/UnisolatedReadWriteIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/TupleUpdater.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/view/FusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/AccessPathFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/IElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/CacheOnceMetadataIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/CachingMetadataIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/DataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/IDataService.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/NoCacheMetadataIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/AbstractScaleOutClientIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/ClientIndexView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/DataServiceTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/PartitionedTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/service/ndx/RawDataServiceTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/sparse/SparseRowStore.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/striterator/ChunkedStriterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestChunkedIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/TestIterators.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestPrefixFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/filter/TestTupleFilters.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/btree/view/TestFusedView.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/accesspath/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/resources/AbstractResourceManagerTestCase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/service/TestRangeQuery.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/FilterBase.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Striterator.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/WrappedTupleIterator.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestSameVariableConstraint.java
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/FilterConstructor.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/filter/IFilterConstructor.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/ElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/relation/accesspath/TestSameVariableConstraint.java
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/BigdataFileSystem.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -26,7 +26,6 @@
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
import com.bigdata.btree.IndexMetadata;
-import com.bigdata.btree.filter.FilterConstructor;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.btree.proc.ISimpleIndexProcedure;
@@ -52,7 +51,6 @@
import com.bigdata.sparse.LogicalRowSplitHandler;
import com.bigdata.sparse.Schema;
import com.bigdata.sparse.SparseRowStore;
-import com.bigdata.sparse.TPS.TPV;
import cutthecrap.utils.striterators.Resolver;
import cutthecrap.utils.striterators.Striterator;
@@ -375,7 +373,7 @@
/**
* The index in which the file metadata is stored (the index must exist).
*/
- public SparseRowStore getFilleMetadataIndex() {
+ public SparseRowStore getFileMetadataIndex() {
if (fileMetadataIndex == null) {
@@ -553,7 +551,7 @@
metadata.put(FileMetadataSchema.VERSION, AutoIncIntegerCounter.INSTANCE);
// write the metadata (atomic operation).
- final ITPS tps = getFilleMetadataIndex().write(metadataSchema, metadata,
+ final ITPS tps = getFileMetadataIndex().write(metadataSchema, metadata,
AUTO_TIMESTAMP_UNIQUE, null/* filter */, null/*precondition*/);
final int version = (Integer) tps.get(FileMetadataSchema.VERSION).getValue();
@@ -646,7 +644,7 @@
*/
public ITPS readMetadata(final String id, final long timestamp) {
- return getFilleMetadataIndex()
+ return getFileMetadataIndex()
.read(metadataSchema, id, timestamp/* fromTime */,
timestamp + 1/* toTime */, null/* filter */);
@@ -677,7 +675,7 @@
// remove the version identifier if any - we do not want this modified!
metadata.remove(FileMetadataSchema.VERSION);
- return getFilleMetadataIndex().write(metadataSchema, metadata,
+ return getFileMetadataIndex().write(metadataSchema, metadata,
AUTO_TIMESTAMP_UNIQUE, null/* filter */,null/*precondition*/).asMap();
}
@@ -752,7 +750,7 @@
// delete marker.
metadata.put(FileMetadataSchema.VERSION, null);
- getFilleMetadataIndex().write(metadataSchema, metadata, AUTO_TIMESTAMP_UNIQUE,
+ getFileMetadataIndex().write(metadataSchema, metadata, AUTO_TIMESTAMP_UNIQUE,
null/* filter */, null/*precondition*/);
}
@@ -905,7 +903,7 @@
public Iterator<? extends DocumentHeader> getDocumentHeaders(String fromId,
String toId) {
- return new Striterator(getFilleMetadataIndex().rangeIterator(metadataSchema,
+ return new Striterator(getFileMetadataIndex().rangeIterator(metadataSchema,
fromId, toId)).addFilter(new Resolver() {
private static final long serialVersionUID = 1L;
@@ -956,14 +954,13 @@
* range by replacing its VERSION column value with a null value
* (and updating the timestamp in the key).
*/
- getFilleMetadataIndex().getIndex().rangeIterator(
+ getFileMetadataIndex().getIndex().rangeIterator(
fromKey,
toKey,
0/* capacity */,
IRangeQuery.CURSOR,
- new FilterConstructor<TPV>()
- .addFilter(new FileVersionDeleter(
- IRowStoreConstants.AUTO_TIMESTAMP_UNIQUE)));
+ new FileVersionDeleter(
+ IRowStoreConstants.AUTO_TIMESTAMP_UNIQUE));
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/FileVersionDeleter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -30,7 +30,7 @@
private final long timestamp;
/**
- * The timestamp choosen on the server (not serialized).
+ * The timestamp chosen on the server (not serialized).
*/
transient private long choosenTimestamp;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bfs/RepositoryDocumentImpl.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -144,7 +144,7 @@
*/
public RepositoryDocumentImpl(final BigdataFileSystem repo, final String id) {
- this(repo, id, repo.getFilleMetadataIndex().read(
+ this(repo, id, repo.getFileMetadataIndex().read(
BigdataFileSystem.metadataSchema, id,
IRowStoreConstants.MIN_TIMESTAMP,
IRowStoreConstants.CURRENT_ROW, null/* filter */));
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BOpBase.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -89,13 +89,19 @@
* arguments or annotations would have to explicitly overrides
* {@link #clone()} in order to set those fields based on the arguments on
* the cloned {@link BOpBase} class.
+ * <p>
+ * Note: This must be at least "effectively" final per the effectively
+ * immutable contract for {@link BOp}s.
*/
- protected final BOp[] args;
+ private final BOp[] args;
/**
* The operator annotations.
+ * <p>
+ * Note: This must be at least "effectively" final per the effectively
+ * immutable contract for {@link BOp}s.
*/
- protected final Map<String,Object> annotations;
+ private final Map<String,Object> annotations;
/**
* Check the operator argument.
@@ -134,6 +140,13 @@
}
}
+// /**
+// * Deserialization constructor (required).
+// */
+// public BOpBase() {
+//
+// }
+
/**
* Deep copy constructor (required).
* <p>
@@ -218,6 +231,29 @@
}
+ /**
+ * Set the value of an operand.
+ * <p>
+ * Note: This protected to facilitate copy-on-write patterns. It is not
+ * public to prevent arbitrary changes to operators outside of methods which
+ * clone the operator and return the modified version. This is part of the
+ * effectively immutable contract for {@link BOp}s.
+ *
+ * @param index
+ * The index.
+ * @param op
+ * The operand.
+ *
+ * @return The old value.
+ *
+ * @todo thread safety and visibility....
+ */
+ final protected void set(final int index, final BOp op) {
+
+ this.args[index] = op;
+
+ }
+
public int arity() {
return args.length;
@@ -341,6 +377,30 @@
}
+ /**
+ * Set an annotation.
+ * <p>
+ * Note: This protected to facilitate copy-on-write patterns. It is not
+ * public to prevent arbitrary changes to operators outside of methods which
+ * clone the operator and return the modified version. This is part of the
+ * effectively immutable contract for {@link BOp}s.
+ *
+ * @param name
+ * The name.
+ * @param value
+ * The value.
+ *
+ * @return The old value.
+ *
+ * @todo thread safety and visibility for concurrent access to and
+ * modifications of the annotations map.
+ */
+ protected void setProperty(final String name, final Object value) {
+
+ annotations.put(name,value);
+
+ }
+
public int getId() {
return (Integer) getRequiredProperty(Annotations.BOP_ID);
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -31,12 +31,13 @@
import java.io.Serializable;
import com.bigdata.bop.ap.filter.BOpFilterBase;
+import com.bigdata.bop.ap.filter.BOpTupleFilter;
import com.bigdata.bop.ap.filter.DistinctFilter;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IRangeQuery;
import com.bigdata.btree.ITuple;
import com.bigdata.btree.ITupleIterator;
-import com.bigdata.btree.filter.IFilterConstructor;
+import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
@@ -48,6 +49,9 @@
import com.bigdata.relation.rule.eval.pipeline.JoinMasterTask;
import com.bigdata.striterator.IKeyOrder;
+import cutthecrap.utils.striterators.FilterBase;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* An immutable constraint on the elements visited using an {@link IAccessPath}.
* The slots in the predicate corresponding to variables are named and those
@@ -114,22 +118,34 @@
* change the type of the visited objects will result in a runtime
* exception.
* <p>
+ * Note: This filter must be "aware" of the reuse of tuples within tuple
+ * iterators. See {@link BOpTupleFilter} and {@link TupleFilter} for
+ * starting points.
+ * <p>
* You can chain {@link BOpFilterBase} filters by nesting them inside of
- * one another.
+ * one another. You can chain {@link FilterBase} filters together as
+ * well.
*
* @see #ACCESS_PATH_FILTER
*
- * @see IRangeQuery#rangeIterator(byte[], byte[], int, int,
- * IFilterConstructor)
+ * @see IRangeQuery#rangeIterator(byte[], byte[], int, int, IFilter)
*/
String INDEX_LOCAL_FILTER = "indexLocalFilter";
/**
* An optional {@link BOpFilterBase} to be applied to the elements of
- * the relation as they are materialized from the index. Unlike
- * {@link #INDEX_LOCAL_FILTER}, this an {@link #ACCESS_PATH_FILTER} is never
- * sent to a remote index for evaluation. This makes it possible to
- * impose {@link DistinctFilter} across a {@link #REMOTE_ACCESS_PATH}.
+ * the relation as they are materialized from the index. {@link ITuple}s
+ * are automatically resolved into relation "elements" before this
+ * filter is applied.
+ * <p>
+ * Unlike {@link #INDEX_LOCAL_FILTER}, this an
+ * {@link #ACCESS_PATH_FILTER} is never sent to a remote index for
+ * evaluation. This makes it possible to impose {@link DistinctFilter}
+ * across a {@link #REMOTE_ACCESS_PATH}.
+ * <p>
+ * You can chain {@link BOpFilterBase} filters by nesting them inside of
+ * one another. You can chain {@link FilterBase} filters together as
+ * well.
*/
String ACCESS_PATH_FILTER = "accessPathFilter";
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -108,7 +108,7 @@
public int limit() {
- return (Integer) annotations.get(Annotations.LIMIT);
+ return (Integer) getRequiredProperty(Annotations.LIMIT);
}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -139,7 +139,7 @@
public String getOnlyRelationName() {
- final String[] relationName = (String[]) annotations.get(Annotations.RELATION_NAME);
+ final String[] relationName = (String[]) getRequiredProperty(Annotations.RELATION_NAME);
if (relationName.length != 1)
throw new IllegalStateException();
@@ -150,7 +150,7 @@
public String getRelationName(final int index) {
- final String[] relationName = (String[]) annotations.get(Annotations.RELATION_NAME);
+ final String[] relationName = (String[]) getRequiredProperty(Annotations.RELATION_NAME);
return relationName[index];
@@ -160,7 +160,7 @@
public int getRelationCount() {
- final String[] relationName = (String[]) annotations.get(Annotations.RELATION_NAME);
+ final String[] relationName = (String[]) getRequiredProperty(Annotations.RELATION_NAME);
return relationName.length;
@@ -171,7 +171,7 @@
// throw new UnsupportedOperationException();
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.RELATION_NAME, relationName);
+ tmp.setProperty(Annotations.RELATION_NAME, relationName);
return tmp;
@@ -200,7 +200,7 @@
final public boolean isOptional() {
- return (Boolean) annotations.get(Annotations.OPTIONAL);
+ return (Boolean) getProperty(Annotations.OPTIONAL,Boolean.FALSE);
}
@@ -215,14 +215,14 @@
@SuppressWarnings("unchecked")
final public IElementFilter<E> getConstraint() {
- return (IElementFilter<E>) annotations.get(Annotations.CONSTRAINT);
+ return (IElementFilter<E>) getProperty(Annotations.CONSTRAINT);
}
@SuppressWarnings("unchecked")
final public ISolutionExpander<E> getSolutionExpander() {
- return (ISolutionExpander<E>) annotations.get(Annotations.EXPANDER);
+ return (ISolutionExpander<E>) getProperty(Annotations.EXPANDER);
}
@@ -304,16 +304,7 @@
continue;
}
- // bound from the binding set.
-// try {
-// final Field f = tmp.getClass().getField("args");
-// f.setAccessible(true);
-// final BOp[] targs = (BOp[]) f.get(tmp);
-// targs[i] = val.clone();
-// } catch (Exception ex) {
-// throw new RuntimeException(ex);
-// }
- tmp.args[i] = val.clone();
+ tmp.set(i, val.clone());
}
@@ -325,30 +316,12 @@
return get(index).get(bindingSet);
-// if (bindingSet == null)
-// throw new IllegalArgumentException();
-//
-// final IVariableOrConstant<?> t = get(index);
-//
-// final IConstant<?> c;
-// if (t.isVar()) {
-//
-// c = bindingSet.get((IVariable<?>) t);
-//
-// } else {
-//
-// c = (IConstant<?>) t;
-//
-// }
-//
-// return c == null ? null : c.get();
-
}
@SuppressWarnings("unchecked")
public IKeyOrder<E> getKeyOrder() {
- return (IKeyOrder<E>) annotations.get(Annotations.KEY_ORDER);
+ return (IKeyOrder<E>) getProperty(Annotations.KEY_ORDER);
}
@@ -356,7 +329,7 @@
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.KEY_ORDER, keyOrder);
+ tmp.setProperty(Annotations.KEY_ORDER, keyOrder);
return tmp;
@@ -373,7 +346,7 @@
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.PARTITION_ID, partitionId);
+ tmp.setProperty(Annotations.PARTITION_ID, partitionId);
return tmp;
@@ -383,7 +356,7 @@
final Predicate<E> tmp = this.clone();
- tmp.annotations.put(Annotations.BOP_ID, bopId);
+ tmp.setProperty(Annotations.BOP_ID, bopId);
return tmp;
@@ -420,6 +393,7 @@
sb.append(")");
+ final Map<String,Object> annotations = annotations();
if (!annotations.isEmpty()) {
sb.append("[");
boolean first = true;
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -31,12 +31,16 @@
import java.util.Map;
import com.bigdata.bop.BOp;
+import com.bigdata.btree.ITupleIterator;
import cutthecrap.utils.striterators.Filter;
import cutthecrap.utils.striterators.Filterator;
/**
* Used to filter for objects which satisfy some criteria.
+ * <p>
+ * <strong>WARNING : DO NOT USE THIS CLASS ON {@link ITupleIterator}s - use
+ * {@link BOpTupleFilter} instead.</strong>
*
* @author <a href="mailto:tho...@us...">Bryan Thompson</a>
* @version $Id$
@@ -48,7 +52,16 @@
*/
private static final long serialVersionUID = 1L;
+// /**
+// * Deserialization.
+// */
+// public BOpFilter() {
+// super();
+// }
+
/**
+ * Deep copy.
+ *
* @param op
*/
public BOpFilter(BOpFilter op) {
@@ -56,6 +69,8 @@
}
/**
+ * Shallow copy.
+ *
* @param args
* @param annotations
*/
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -62,6 +62,7 @@
}
/**
+ * Deep copy.
* @param op
*/
public BOpFilterBase(BOpFilterBase op) {
@@ -69,6 +70,7 @@
}
/**
+ * Shallow copy.
* @param args
* @param annotations
*/
@@ -82,7 +84,7 @@
src = filterOnce(src, context);
// wrap source with each additional filter from the filter chain.
- for (BOp arg : args) {
+ for (BOp arg : args()) {
src = ((BOpFilterBase) arg).filter(src, context);
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,80 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop.ap.filter;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+
+/**
+ * A filter which does nothing. This may be used to stack existing
+ * {@link BOpFilterBase} operators.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class BOpFilterNOP extends BOpFilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+// /**
+// * Deserialization.
+// */
+// public BOpFilterNOP() {
+// super();
+// }
+
+ /**
+ * Deep copy.
+ *
+ * @param op
+ */
+ public BOpFilterNOP(BOpFilterNOP op) {
+ super(op);
+ }
+
+ /**
+ * Shallow copy.
+ *
+ * @param args
+ * @param annotations
+ */
+ public BOpFilterNOP(BOp[] args, Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ @Override
+ final protected Iterator filterOnce(Iterator src, Object context) {
+ return src;
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterNOP.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,131 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop.ap.filter;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+import com.bigdata.btree.AbstractBTree;
+import com.bigdata.btree.BTree;
+import com.bigdata.btree.IRangeQuery;
+import com.bigdata.btree.ITuple;
+import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.Tuple;
+import com.bigdata.btree.filter.TupleFilter;
+import com.bigdata.btree.filter.TupleFilter.TupleFilterator;
+
+/**
+ * <p>
+ * Filter supporting {@link ITupleIterator}s.
+ * </p>
+ * <p>
+ * <strong>Warning: Unlike {@link BOpFilter}, this class correctly uses a second
+ * {@link Tuple} instance to perform filtering.<strong> This is necessary since
+ * the {@link Tuple} instance for the base {@link ITupleIterator}
+ * implementations for the {@link AbstractBTree} is reused by next() on each
+ * call and the {@link TupleFilter} uses one-step lookahead. Failure to use a
+ * second {@link Tuple} instance will result in <em>overwrite</em> of the
+ * current {@link Tuple} with data from the lookahead {@link Tuple}.
+ * </p>
+ * <p>
+ * Note: You must specify {@link IRangeQuery#KEYS} and/or
+ * {@link IRangeQuery#VALS} in order to filter on the keys and/or values
+ * associated with the visited tuples.
+ * </p>
+ * <p>
+ * Note: YOu must specify {@link IRangeQuery#CURSOR} to enabled
+ * {@link Iterator#remove()} for a <em>local</em> {@link BTree}
+ * </p>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class BOpTupleFilter<E> extends BOpFilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+// /**
+// * Deserialization.
+// */
+// public BOpFilter() {
+// super();
+// }
+
+ /**
+ * Deep copy.
+ *
+ * @param op
+ */
+ public BOpTupleFilter(BOpTupleFilter op) {
+ super(op);
+ }
+
+ /**
+ * Shallow copy.
+ *
+ * @param args
+ * @param annotations
+ */
+ public BOpTupleFilter(BOp[] args, Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ @Override
+ final protected Iterator filterOnce(Iterator src, final Object context) {
+
+ return new TupleFilterator((ITupleIterator<E>) src, context,
+ new FilterImpl());
+
+ }
+
+ /**
+ * Return <code>true</code> iff the object should be accepted.
+ *
+ * @param obj
+ * The object.
+ */
+ protected abstract boolean isValid(ITuple<E> obj);
+
+ private class FilterImpl extends TupleFilter<E> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected boolean isValid(ITuple<E> tuple) {
+
+ return BOpTupleFilter.this.isValid(tuple);
+
+ }
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpTupleFilter.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java (from rev 3554, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/relation/accesspath/SameVariableConstraint.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraint.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,374 @@
+package com.bigdata.bop.ap.filter;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.CognitiveWeb.extser.LongPacker;
+
+import com.bigdata.bop.Constant;
+import com.bigdata.bop.IConstant;
+import com.bigdata.bop.IElement;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.IVariable;
+import com.bigdata.bop.IVariableOrConstant;
+import com.bigdata.relation.accesspath.IAccessPath;
+
+/**
+ * Filter imposes the "same variable" constraint on the elements visited by an
+ * {@link IAccessPath}. The filter is required IFF a {@link IVariable} appears
+ * in more than one position for the {@link IPredicate} associated with the
+ * {@link IAccessPath}. For example, in <code>spo(?g, p1, o1, ?g)</code>, the
+ * variable <i>g</i> shows up at both index ZERO (0) and index THREE (3).
+ * <p>
+ * An instance of the filter is created by passing in an {@link IPredicate}. The
+ * filter creates an array of variables which appear more than once and the
+ * index at which those variables appear in the predicate. The filter then does
+ * the minimum amount of work and just compares the values found in the
+ * different slots in which each variable appears for only those variables which
+ * appear more than once in the {@link IPredicate}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ * @param <E>
+ * The generic type of the elements that will be tested by the
+ * filter.
+ */
+public class SameVariableConstraint<E> implements Externalizable {
+
+ /**
+ * The predicate template.
+ */
+ private IPredicate<E> p;
+
+ /**
+ * An array containing one or more records, each of which has the form
+ *
+ * <pre>
+ * [n,index[0], index[1], ... index[n-1]]
+ * </pre>
+ *
+ * , where <i>n</i> is the number of occurrences of some variable and
+ * <i>index[i]</i>, for <i>i</i> in <code>[0,n-1]</code>, gives each index
+ * in the predicate at which that variable appears.
+ * <p>
+ * For example, given the predicate
+ *
+ * <pre>
+ * spo(?g,p1,o1,?g)
+ * </pre>
+ *
+ * the array would be coded as one record
+ *
+ * <pre>
+ * [2, 0, 3]
+ * </pre>
+ *
+ * Given the predicate
+ *
+ * <pre>
+ * foo(?g,?h,?h,?g,4,?h)
+ * </pre>
+ *
+ * the array would be coded as two records
+ *
+ * <pre>
+ * [[2, 0, 3], [3, 1, 2, 5]]
+ * </pre>
+ *
+ * where the records are indicated by the nested square brackets.
+ */
+ protected int[] indices;
+
+ /**
+ * De-serialization constructor.
+ */
+ public SameVariableConstraint() {
+
+ }
+
+ public SameVariableConstraint(final IPredicate<E> p, final int[] indices) {
+
+ if (p == null)
+ throw new IllegalArgumentException();
+
+ if (indices == null)
+ throw new IllegalArgumentException();
+
+ this.p = p;
+
+ this.indices = indices;
+
+ }
+
+// public boolean canAccept(final Object o) {
+// return true;
+// }
+
+ public boolean accept(final E e) {
+
+ int i = 0;
+ while (i < indices.length) {
+
+ // the #of slots at which that variable appears.
+ final int nslots = indices[i];
+
+ assert nslots >= 2 : "i=" + i + ", nslots=" + nslots + ", indices="
+ + Arrays.toString(indices);
+
+ assert i + nslots < indices.length;
+
+ i++;
+
+ final int firstIndex = indices[i];
+
+ @SuppressWarnings("unchecked")
+ final IConstant<?> c0 = new Constant(((IElement) e).get(firstIndex));
+// final IConstant<?> c0 = p.get(e, firstIndex);
+
+ for (int j = 0; j < nslots; j++, i++) {
+
+ if (j == firstIndex) {
+
+ // no need to compare with self.
+ continue;
+
+ }
+
+ final int thisIndex = indices[i];
+
+ @SuppressWarnings("unchecked")
+ final IConstant<?> c1 = new Constant(((IElement) e)
+ .get(thisIndex));
+// final IConstant<?> c1 = p.get(e, thisIndex);
+
+ // same reference (including null).
+ if (c0 == c1)
+ continue;
+
+ if (c0 != null && c1 == null) {
+
+ return false;
+
+ }
+
+ if (c1 == null && c0 != null) {
+
+ return false;
+
+ }
+
+ if (!c0.equals(c1)) {
+
+ // different constant.
+ return false;
+
+ }
+
+ }
+
+ }
+
+ return true;
+
+ }
+
+ public String toString() {
+
+ return super.toString() + "{p=" + p + ", indices="
+ + Arrays.toString(indices) + "}";
+
+ }
+
+ /**
+ * The initial version.
+ */
+ private static final transient byte VERSION0 = 0;
+
+ /**
+ * The current version.
+ */
+ private static final transient byte VERSION = VERSION0;
+
+ @SuppressWarnings("unchecked")
+ public void readExternal(ObjectInput in) throws IOException,
+ ClassNotFoundException {
+
+ final short version = in.readByte();
+
+ switch (version) {
+ case VERSION0:
+ break;
+ default:
+ throw new UnsupportedOperationException("Unknown version: "
+ + version);
+ }
+
+ p = (IPredicate<E>) in.readObject();
+
+ final int len = (int) LongPacker.unpackLong(in);
+
+ indices = new int[len];
+
+ for (int i = 0; i < len; i++) {
+
+ indices[i] = (int) LongPacker.unpackLong(in);
+
+ }
+
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+
+ out.writeByte(VERSION);
+
+ out.writeObject(p);
+
+ LongPacker.packLong(out, indices.length);
+
+ for (int i = 0; i < indices.length; i++) {
+
+ LongPacker.packLong(out, indices[i]);
+
+ }
+
+ }
+
+ /**
+ * The filter is only created and populated for variables which appear more
+ * than once in the predicate. If there are no variables which appear more
+ * than once, then the filter IS NOT created.
+ *
+ * @param p
+ * The predicate.
+ *
+ * @param <E>
+ * The generic type of the elements to be tested by the filter.
+ *
+ * @return The filter -or- <code>null</code> iff no variables appear more
+ * than once in the {@link IPredicate}.
+ */
+ public static <E> SameVariableConstraint<E> newInstance(
+ final IPredicate<E> p) {
+
+ // #of slots in this predicate.
+ final int arity = p.arity();
+
+ // map exists iff variables are used more than once.
+ Map<IVariableOrConstant<?>, Integer> vars = null;
+
+ // #of occurrences across all variables which are used more than once.
+ int noccurs = 0;
+
+ // #of variables which are reused within the predicate.
+ int nreused = 0;
+
+ {
+
+ for (int i = 0; i < arity; i++) {
+
+ final IVariableOrConstant<?> t = p.get(i);
+
+ if (t != null && t.isVar()) {
+
+ // this slot is a variable.
+
+ if (vars == null) {
+
+ vars = new LinkedHashMap<IVariableOrConstant<?>, Integer>();
+
+ }
+
+ Integer cnt = vars.get(t);
+
+ if (cnt == null) {
+
+ vars.put(t, cnt = Integer.valueOf(1));
+
+ } else {
+
+ final int tmp = cnt.intValue() + 1;
+
+ vars.put(t, cnt = Integer.valueOf(tmp));
+
+ // note: add 2 for the 1st dup and 1 for each
+ // additional dup thereafter.
+ noccurs += (tmp == 2 ? 2 : 1);
+
+ if (tmp == 2) {
+
+ // this variable is reused.
+ nreused++;
+
+ }
+
+ }
+
+ }
+
+ }
+
+ if (nreused == 0) {
+
+ // there are no duplicate variables.
+ return null;
+
+ }
+
+ }
+
+ assert vars != null;
+
+ // allocate the array for the reused variable record(s).
+ final int[] indices = new int[nreused + noccurs];
+
+ // populate that array.
+ {
+
+ int i = 0;
+
+ final Iterator<Map.Entry<IVariableOrConstant<?>, Integer>> itr = vars
+ .entrySet().iterator();
+
+ while (itr.hasNext()) {
+
+ final Map.Entry<IVariableOrConstant<?>, Integer> e = itr.next();
+
+ final int nused = e.getValue().intValue();
+
+ if (nused < 2)
+ continue;
+
+ final IVariable<?> var = (IVariable<?>)e.getKey();
+
+ indices[i++] = nused;
+
+ for (int j = 0; j < arity; j++) {
+
+ final IVariableOrConstant<?> t = p.get(j);
+
+ if (t == var) {
+
+ // variable appears at this index.
+ indices[i++] = j;
+
+ }
+
+ }
+
+ }
+
+ assert i == indices.length;
+
+ }
+
+ return new SameVariableConstraint<E>(p, indices);
+
+ } // factory method.
+
+}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -0,0 +1,105 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop.ap.filter;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import com.bigdata.bop.BOp;
+
+import cutthecrap.utils.striterators.Filter;
+import cutthecrap.utils.striterators.Filterator;
+
+/**
+ * Operator imposing a {@link SameVariableConstraint} filter.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public class SameVariableConstraintFilter extends BOpFilterBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BOpFilterBase.Annotations {
+
+ /**
+ * The constraint to be imposed (required).
+ */
+ String FILTER = SameVariableConstraintFilter.class.getName()
+ + ".filter";
+
+ }
+
+ /**
+ * @param op
+ */
+ public SameVariableConstraintFilter(SameVariableConstraintFilter op) {
+ super(op);
+ }
+
+ /**
+ * @param args
+ * @param annotations
+ */
+ public SameVariableConstraintFilter(BOp[] args,
+ Map<String, Object> annotations) {
+ super(args, annotations);
+ }
+
+ @Override
+ final protected Iterator filterOnce(Iterator src, final Object context) {
+
+ final SameVariableConstraint filter = (SameVariableConstraint) getRequiredProperty(Annotations.FILTER);
+
+ return new Filterator(src, context, new FilterImpl(filter));
+
+ }
+
+ static private class FilterImpl extends Filter {
+
+ private static final long serialVersionUID = 1L;
+
+ private final SameVariableConstraint filter;
+
+ public FilterImpl(final SameVariableConstraint filter) {
+ this.filter = filter;
+ }
+
+ @Override
+ protected boolean isValid(Object obj) {
+
+ return filter.accept(obj);
+
+ }
+
+ }
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/SameVariableConstraintFilter.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/notes.txt 2010-09-28 23:39:42 UTC (rev 3672)
@@ -492,25 +492,8 @@
* <li></li>
* </ul>
+------------------------------------------------------------
-Martyn,
-
-Rationalizing CTC striterators, bigdata striterators, and access paths:
-
-Dropped the ctc_utils jar and added the source files to the classpath.
-
-I've added an IPropertySet interface:
-
-Object getProperty(String name).
-
-There are currently several varieties of IFilter. Each one knows how to create its corresponding implementation class. This exactly parallels the bops, where a concrete bop such as MemorySortOp, knows how to create the corresponding runtime class, MemorySortOp.SortTask. I've added a filterOnce() method to provide the semantics of wrapping an iterator with just that IFilter. The existing filter() method now wraps the chain, starting with _this_ IFilter and then wrapping anything which was added to the chain. The chain is empty if nothing was added. I've modified the various IFilter implementations to conform with this.
-
-I've modified all of the implementation classes (Appenderator, etc.) to accept the new context argument regardless of whether they use it. I have also modified them to always have the reference to the IFilter object so the IPropertySet will be accessible.
-
-Modified Uniqueness filter to use a linked hash map.
-
-Modified Sorter uses Arrays.sort() rather than TreeSort.
-
Some problems:
- Should Appender simply pass the other iterator as m_state to the FilterBase?
@@ -527,21 +510,43 @@
Write unit tests to sanity check these changes.
-Write BOp versions of Append, Expand, etc. These will extend BOpBase.
+done. Replace (I)FilterConstructor with IFilter. FilterConstructor
+implemented clone. I am not sure that I can recall why. Presumably
+because a cloned would have a distinct filter chain. We can not
+easily override clone() on FilterBase because of all the derived
+versions. We could have a deep copy constructor pattern, which is
+what I did for the BOpBase class and then use that for clone.
-The following places directly invoke filterOnce(src,ctx). The first two should be modified to pass in the correct context object. That might require raising a context object into an IRangeIterator parameter.
+Write unit tests at the IPredicate/AccessPath level to use stackable
+filters (for the LOCAL and REMOTE access paths). Write unit tests for
+correct range counts with and without local/remote filters. Write
+unit tests for caching of those range counts.
-AbstractBTree#2933
-FusedView#1281
-TestRemoverator#86
+- The stacking of the BOpFilters may be counter intuitive or just
+ wrong.
-Replace (I)FilterConstructor with IFilter. FilterConstructor implemented clone. I am not sure that I can recall why. Presumably because a cloned would have a distinct filter chain. We can not easily override clone() on FilterBase because of all the derived versions. We could have a deep copy constructor pattern, which is what I did for the BOpBase class and then use that for clone.
+- TestPredicateAccessPath#test_localFilter() does not report the
+ correct tuples. This is weirdness with reuse of the same backing
+ Tuple object. See TupleFilter and BOpFilter. The filter that is
+ being applied in the unit test does not protect against the tuple
+ reuse. It would be best to automatically wrap such asserts. E.g.,
+ BOpTupleFilter. It would also be good to catch runtime type
+ problems with this. E.g., if the source iterator is ITupleFilter
+ then it is an error to wrap with BOpFilter. You must use
+ BOpTupleFilter instead.
-Replace the IPredicate "CONSTRAINT" with "REMOTE_FILTERS" (an IFilter which is evaluated at the BTree) and "LOCAL_FILTERS" (an IFilter which is evaluated against the range iterator once it is materialized on the local node). If the index is local, then the same concepts apply but we do not have to serialize the REMOTE_FILTERS. For the moment, the AccessPath automatically resolves ITuples to relation elements and adds the SameVariableConstraint if necessary. We might shift the burden of responsibility for this to the query planner in the future.
+- Write BOp versions for TupleResolver, ...
-Write unit tests at the IPredicate/AccessPath level to use stackable filters.
+- All of the Filterators need to use deferred prefetch. Prefetch
+ during the constructor causes problems when we are stacking filters
+ using FilterBase.
-Fixed the build.xml script to accomodate the class path changes.
+IPredicate#getConstraint() must go. This will require touching the
+backchainer classes. The (SPO|Magic)Predicate contructors should also
+be touched up since they will otherwise DROP the local and remote
+filters.
-Thanks,
-Bryan
+Reconcile IElementFilter and implementations.
+
+Reconcile the com.bigdata.striterators package. Can we get rid of it?
+Incrementally?
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractBTree.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -54,9 +54,9 @@
import com.bigdata.btree.data.IAbstractNodeData;
import com.bigdata.btree.data.ILeafData;
import com.bigdata.btree.data.INodeData;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.filter.Reverserator;
import com.bigdata.btree.filter.TupleRemover;
+import com.bigdata.btree.filter.WrappedTupleIterator;
import com.bigdata.btree.keys.IKeyBuilder;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor;
@@ -92,6 +92,8 @@
import com.bigdata.util.concurrent.Computable;
import com.bigdata.util.concurrent.Memoizer;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* <p>
* Base class for mutable and immutable B+-Tree implementations.
@@ -2699,7 +2701,7 @@
final public ITupleIterator rangeIterator(Object fromKey, Object toKey,
final int capacity,//
final int flags,//
- final IFilterConstructor filter//
+ final IFilter filter//
) {
fromKey = fromKey == null ? null : metadata.getTupleSerializer()
@@ -2728,7 +2730,7 @@
* iterator.
* <p>
* Note:
- * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilterConstructor)}
+ * {@link FusedView#rangeIterator(byte[], byte[], int, int, IFilter)}
* is also responsible for constructing an {@link ITupleIterator} in a
* manner similar to this method. If you are updating the logic here, then
* check the logic in that method as well!
@@ -2741,7 +2743,7 @@
final byte[] toKey,//
final int capacityIsIgnored,//
final int flags,//
- final IFilterConstructor filter//
+ final IFilter filter//
) {
// btreeCounters.nrangeIterator.incrementAndGet();
@@ -2911,8 +2913,9 @@
* REMOVEALL (those are the assumptions for the flags).
*/
- src = filter.newInstance(src);
-
+ src = new WrappedTupleIterator(filter
+ .filter(src, null/* context */));
+
}
if ((flags & REMOVEALL) != 0) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/AbstractChunkedTupleIterator.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -34,7 +34,6 @@
import org.apache.log4j.Logger;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.keys.SuccessorUtil;
import com.bigdata.io.ByteArrayBuffer;
import com.bigdata.io.DataInputBuffer;
@@ -44,6 +43,8 @@
import com.bigdata.journal.TimestampUtility;
import com.bigdata.rawstore.IBlock;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* A chunked iterator that proceeds a {@link ResultSet} at a time. This
* introduces the concept of a {@link #continuationQuery()} so that the iterator
@@ -99,7 +100,7 @@
/**
* Optional filter.
*/
- protected final IFilterConstructor filter;
+ protected final IFilter filter;
/**
* The #of range query operations executed.
@@ -253,7 +254,7 @@
public AbstractChunkedTupleIterator(final byte[] fromKey,
final byte[] toKey, int capacity, final int flags,
- final IFilterConstructor filter) {
+ final IFilter filter) {
if (capacity < 0) {
@@ -286,7 +287,7 @@
* @return
*/
abstract protected ResultSet getResultSet(long timestamp,byte[] fromKey, byte[] toKey,
- int capacity, int flags, IFilterConstructor filter);
+ int capacity, int flags, IFilter filter);
/**
* Issues the original range query.
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/BigdataMap.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -37,7 +37,6 @@
import java.util.Set;
import java.util.SortedMap;
-import com.bigdata.btree.filter.FilterConstructor;
import com.bigdata.btree.filter.TupleFilter;
import com.bigdata.btree.keys.KeyBuilder;
import com.bigdata.journal.ConcurrencyManager;
@@ -381,7 +380,7 @@
final ITupleIterator itr = ndx.rangeIterator(fromKey, toKey,
0/* capacity */, IRangeQuery.VALS/* flags */,
- new FilterConstructor().addFilter(new TupleFilter() {
+ new TupleFilter() {
private static final long serialVersionUID = 1L;
@@ -391,7 +390,7 @@
return BytesUtil.bytesEqual(tuple.getValue(), val);
}
- }));
+ });
while(itr.hasNext()) {
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/ChunkedLocalRangeIterator.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -32,12 +32,13 @@
import java.nio.ByteBuffer;
import java.util.Iterator;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.filter.ITupleFilter;
import com.bigdata.io.ByteBufferInputStream;
import com.bigdata.rawstore.IBlock;
import com.bigdata.rawstore.IRawStore;
+import cutthecrap.utils.striterators.IFilter;
+
/**
* Chunked range iterator running against a local index or index view.
* <p>
@@ -70,7 +71,7 @@
* @param filter
*/
public ChunkedLocalRangeIterator(IIndex ndx, byte[] fromKey, byte[] toKey,
- int capacity, int flags, IFilterConstructor filter) {
+ int capacity, int flags, IFilter filter) {
super(fromKey, toKey, capacity, flags, filter);
@@ -92,7 +93,7 @@
@Override
protected ResultSet getResultSet(final long timestamp,
final byte[] fromKey, final byte[] toKey, final int capacity,
- final int flags, final IFilterConstructor filter) {
+ final int flags, final IFilter filter) {
/*
* Note: This turns off the REMOVEALL flag unless CURSOR was also
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java 2010-09-28 20:12:26 UTC (rev 3671)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/btree/DelegateIndex.java 2010-09-28 23:39:42 UTC (rev 3672)
@@ -28,7 +28,6 @@
package com.bigdata.btree;
-import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.btree.proc.AbstractKeyArrayIndexProcedureConstructor;
import com.bigdata.btree.proc.IKeyRangeIndexProcedure;
import com.bigdata.btree.proc.IResultHandler;
@@ -36,6 +35,8 @@
import com.bigdata.counters.ICounterSet;
import com.bigdata.mdi.IResourceMetadata;
+import cutthecrap.utils.striterators.IFilter;
+
/**
...
[truncated message content] |
