Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[3665] branches/QUADS_QUERY_BRANCH
|
From: <tho...@us...> - 2010-09-28 16:53:33
|
Revision: 3665
http://bigdata.svn.sourceforge.net/bigdata/?rev=3665&view=rev
Author: thompsonbry
Date: 2010-09-28 16:53:22 +0000 (Tue, 28 Sep 2010)
Log Message:
-----------
Reorganized the pipeline operators, the index sampling operators, and the CTC style striterator operators.
Deprecated IPredicate#CONSTRAINTS. It will be replaced by INDEX_LOCAL_FILTERS and ACCESS_PATH_FILTERS. The former will always run locally at the index against the ITupleIterator. The latter will always run on the AccessPath side against the materialized elements. For example, the default graph "distinct spo" constraint will be imposed as a DistinctFilter wrapping a StripContextFilter. In scale-out, this will use a remote access path as necessary in order to have a total view of the sharded data. My next step is to implement that distinction.
Modified Paths:
--------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/Predicate.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/ConditionalRoutingOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/CopyBindingSetOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/Tee.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/bset/Union.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryClient.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/IQueryDecl.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryDecl.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/QueryEngineTestAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/Rule2BOpUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/engine/RunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/eval/JoinGraph.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/fed/FederatedRunningQuery.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/join/PipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/mutation/InsertOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/DistinctBindingSetOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SliceOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/solutions/SortOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestPredicateAccessPath.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/PipelineDelayOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestPipelineUtility.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/engine/TestRunState.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestNIOChunkMessage.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/fed/TestThickChunkMessage.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/join/TestPipelineJoin.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ndx/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-jini/src/test/com/bigdata/bop/fed/jini/TestJiniFederatedQueryEngine.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/rdf/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-sails/src/java/com/bigdata/rdf/sail/BigdataEvaluationStrategyImpl.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filter.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/Filterator.java
branches/QUADS_QUERY_BRANCH/ctc-striterators/src/java/cutthecrap/utils/striterators/UniquenessFilter.java
Added Paths:
-----------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/SampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpFilterBase.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/BOpResolver.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/DistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/filter/package.html
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestSampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestSampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestDistinctFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/filter/TestResolver.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/filter/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/filter/StripContextFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/DataSetJoin.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/filter/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/filter/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/filter/TestStripContextFilter.java
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/test/com/bigdata/bop/rdf/join/
Removed Paths:
-------------
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/AbstractSampleIndex.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/SampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/SampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/rdf/join/
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ap/TestDistinctElementFilter.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ndx/TestSampleLocalBTree.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/ndx/TestSampleLocalShard.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/rdf/TestAll.java
branches/QUADS_QUERY_BRANCH/bigdata/src/test/com/bigdata/bop/rdf/join/
branches/QUADS_QUERY_BRANCH/bigdata-rdf/src/java/com/bigdata/bop/rdf/join/DataSetJoin.java
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java (from rev 3630, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractAccessPathOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,116 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 18, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+
+import com.bigdata.relation.accesspath.IAccessPath;
+import com.bigdata.striterator.IChunkedOrderedIterator;
+
+/**
+ * Interface for evaluating operations producing chunks of elements (tuples
+ * materialized from some index of a relation).
+ *
+ * @see IAccessPath
+ * @see IChunkedOrderedIterator
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class AbstractAccessPathOp<E> extends BOpBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BOp.Annotations, BufferAnnotations {
+
+ }
+
+ /**
+ * Required shallow copy constructor.
+ *
+ * @param args
+ * @param annotations
+ */
+ public AbstractAccessPathOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ super(args, annotations);
+
+ }
+
+ /**
+ * Required deep copy constructor.
+ *
+ * @param op
+ */
+ public AbstractAccessPathOp(
+ final AbstractAccessPathOp<E> op) {
+ super(op);
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_CAPACITY
+ */
+ protected int getChunkCapacity() {
+
+ return getProperty(Annotations.CHUNK_CAPACITY,
+ Annotations.DEFAULT_CHUNK_CAPACITY);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_OF_CHUNKS_CAPACITY
+ */
+ protected int getChunkOfChunksCapacity() {
+
+ return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
+ Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+
+ }
+
+// protected int getFullyBufferedReadThreshold() {
+//
+// return getProperty(Annotations.FULLY_BUFFERED_READ_THRESHOLD,
+// Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
+//
+// }
+
+ /**
+ * @see BufferAnnotations#CHUNK_TIMEOUT
+ */
+ protected long getChunkTimeout() {
+
+ return getProperty(Annotations.CHUNK_TIMEOUT,
+ Annotations.DEFAULT_CHUNK_TIMEOUT);
+
+ }
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/AbstractChunkedOrderedIteratorOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,112 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Aug 18, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class AbstractChunkedOrderedIteratorOp<E> extends BOpBase
- implements ChunkedOrderedIteratorOp<E> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends ChunkedOrderedIteratorOp.Annotations {
-
- }
-
-// /**
-// * @param args
-// */
-// protected AbstractChunkedOrderedIteratorOp(BOp[] args) {
-// super(args);
-// }
-
- /**
- * Required shallow copy constructor.
- *
- * @param args
- * @param annotations
- */
- protected AbstractChunkedOrderedIteratorOp(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- }
-
- /**
- * Required deep copy constructor.
- *
- * @param op
- */
- public AbstractChunkedOrderedIteratorOp(
- final AbstractChunkedOrderedIteratorOp<E> op) {
- super(op);
- }
-
- protected int getChunkCapacity() {
-
- return getProperty(Annotations.CHUNK_CAPACITY,
- Annotations.DEFAULT_CHUNK_CAPACITY);
-
- }
-
- protected int getChunkOfChunksCapacity() {
-
- return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
- Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
-
- }
-
-// protected int getFullyBufferedReadThreshold() {
-//
-// return getProperty(Annotations.FULLY_BUFFERED_READ_THRESHOLD,
-// Annotations.DEFAULT_FULLY_BUFFERED_READ_THRESHOLD);
-//
-// }
-
- protected long getChunkTimeout() {
-
- return getProperty(Annotations.CHUNK_TIMEOUT,
- Annotations.DEFAULT_CHUNK_TIMEOUT);
-
- }
-
- /**
- * The {@link TimeUnit}s in which the {@link #chunkTimeout} is measured.
- */
- protected static transient final TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
-
-}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,88 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Sep 2, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.Map;
-
-/**
- * Abstract base class for pipeline operators where the data moving along the
- * pipeline is chunks of {@link IBindingSet}s.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class BindingSetPipelineOp extends
- PipelineOp<IBindingSet> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends PipelineOp.Annotations {
-
- /**
- * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
- * the ancestor in the operator tree which serves as the default sink
- * for binding sets (default is the parent).
- */
- String SINK_REF = BindingSetPipelineOp.class.getName() + ".sinkRef";
-
- /**
- * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
- * the ancestor in the operator tree which serves as the alternative
- * sink for binding sets (default is no alternative sink).
- */
- String ALT_SINK_REF = BindingSetPipelineOp.class.getName()
- + ".altSinkRef";
-
- }
-
- /**
- * Required deep copy constructor.
- *
- * @param op
- */
- protected BindingSetPipelineOp(PipelineOp<IBindingSet> op) {
- super(op);
- }
-
- /**
- * Shallow copy constructor.
- *
- * @param args
- * @param annotations
- */
- protected BindingSetPipelineOp(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- }
-
-}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BufferAnnotations.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -27,6 +27,8 @@
package com.bigdata.bop;
+import java.util.concurrent.TimeUnit;
+
import com.bigdata.relation.accesspath.BlockingBuffer;
import com.bigdata.relation.accesspath.IBuffer;
@@ -81,4 +83,9 @@
*/
int DEFAULT_CHUNK_TIMEOUT = 20;
+ /**
+ * The {@link TimeUnit}s in which the {@link #CHUNK_TIMEOUT} is measured.
+ */
+ TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
+
}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ChunkedOrderedIteratorOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,25 +0,0 @@
-package com.bigdata.bop;
-
-import com.bigdata.relation.accesspath.IAccessPath;
-import com.bigdata.striterator.IChunkedOrderedIterator;
-
-/**
- * Interface for evaluating operations producing chunks of elements (tuples
- * materialized from some index of a relation).
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- *
- * @see IAccessPath
- * @see IChunkedOrderedIterator
- */
-public interface ChunkedOrderedIteratorOp<E> extends BOp {
-
- /**
- * Well known annotations.
- */
- public interface Annotations extends BOp.Annotations, BufferAnnotations {
-
-
- }
-
-}
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,51 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Annotations for an operator using an internal concurrent hash map.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public interface ConcurrentHashMapAnnotations extends HashMapAnnotations {
+
+ /**
+ * The concurrency level of the {@link ConcurrentHashMap} used to impose the
+ * distinct constraint.
+ *
+ * @see #DEFAULT_CONCURRENCY_LEVEL
+ */
+ String CONCURRENCY_LEVEL = ConcurrentHashMapAnnotations.class.getName()
+ + ".concurrencyLevel";
+
+ int DEFAULT_CONCURRENCY_LEVEL = 16;
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ConcurrentHashMapAnnotations.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Added: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,61 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 28, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Annotations for an operator using an internal hash map.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+public interface HashMapAnnotations {
+
+ /**
+ * The initial capacity of the {@link ConcurrentHashMap} used to impose the
+ * distinct constraint.
+ *
+ * @see #DEFAULT_INITIAL_CAPACITY
+ */
+ String INITIAL_CAPACITY = HashMapAnnotations.class.getName()
+ + ".initialCapacity";
+
+ int DEFAULT_INITIAL_CAPACITY = 16;
+
+ /**
+ * The load factor of the {@link ConcurrentHashMap} used to impose the
+ * distinct constraint.
+ *
+ * @see #DEFAULT_LOAD_FACTOR
+ */
+ String LOAD_FACTOR = HashMapAnnotations.class.getName() + ".loadFactor";
+
+ float DEFAULT_LOAD_FACTOR = .75f;
+
+}
Property changes on: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/HashMapAnnotations.java
___________________________________________________________________
Added: svn:keywords
+ Id Date Revision Author HeadURL
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,89 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Sep 3, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.FutureTask;
-
-import com.bigdata.bop.engine.BOpStats;
-import com.bigdata.relation.accesspath.IBlockingBuffer;
-
-/**
- * An pipeline operator reads from a source and writes on a sink.
- *
- * @param <E>
- * The generic type of the objects processed by the operator.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-public interface IPipelineOp<E> extends BOp {
-
- /**
- * Return a new object which can be used to collect statistics on the
- * operator evaluation (this may be overridden to return a more specific
- * class depending on the operator).
- */
- BOpStats newStats();
-
- /**
- * Instantiate a buffer suitable as a sink for this operator. The buffer
- * will be provisioned based on the operator annotations.
- * <p>
- * Note: if the operation swallows binding sets from the pipeline (such as
- * operators which write on the database) then the operator MAY return an
- * immutable empty buffer.
- *
- * @param stats
- * The statistics on this object will automatically be updated as
- * elements and chunks are output onto the returned buffer.
- *
- * @return The buffer.
- */
- IBlockingBuffer<E[]> newBuffer(BOpStats stats);
-
- /**
- * Return a {@link FutureTask} which computes the operator against the
- * evaluation context. The caller is responsible for executing the
- * {@link FutureTask} (this gives them the ability to hook the completion of
- * the computation).
- *
- * @param context
- * The evaluation context.
- *
- * @return The {@link FutureTask} which will compute the operator's
- * evaluation.
- *
- * @todo Modify to return a {@link Callable}s for now since we must run each
- * task in its own thread until Java7 gives us fork/join pools and
- * asynchronous file I/O. For the fork/join model we will probably
- * return the ForkJoinTask.
- */
- FutureTask<Void> eval(BOpContext<E> context);
-
-}
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IPredicate.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -30,8 +30,13 @@
import java.io.Serializable;
+import com.bigdata.bop.ap.filter.BOpFilterBase;
+import com.bigdata.bop.ap.filter.DistinctFilter;
import com.bigdata.bop.join.PipelineJoin;
import com.bigdata.btree.IRangeQuery;
+import com.bigdata.btree.ITuple;
+import com.bigdata.btree.ITupleIterator;
+import com.bigdata.btree.filter.IFilterConstructor;
import com.bigdata.mdi.PartitionLocator;
import com.bigdata.relation.IRelation;
import com.bigdata.relation.accesspath.AccessPath;
@@ -88,10 +93,47 @@
/**
* Constraints on the elements read from the relation.
+ *
+ * @deprecated This is being replaced by two classes of filters. One
+ * which is always evaluated local to the index and one
+ * which is evaluated in the JVM in which the access path is
+ * evaluated once the {@link ITuple}s have been resolved to
+ * elements of the relation.
*/
String CONSTRAINT = "constraint";
/**
+ * An optional {@link BOpFilterBase} that will be evaluated local to the
+ * to the index. When the index is remote, the filter will be sent to
+ * the node on which the index resides and evaluated there. This makes
+ * it possible to efficiently filter out tuples which are not of
+ * interest for a given access path.
+ * <p>
+ * Note: The filter MUST NOT change the type of data visited by the
+ * iterator - it must remain an {@link ITupleIterator}. An attempt to
+ * change the type of the visited objects will result in a runtime
+ * exception.
+ * <p>
+ * You can chain {@link BOpFilterBase} filters by nesting them inside of
+ * one another.
+ *
+ * @see #ACCESS_PATH_FILTER
+ *
+ * @see IRangeQuery#rangeIterator(byte[], byte[], int, int,
+ * IFilterConstructor)
+ */
+ String INDEX_LOCAL_FILTER = "indexLocalFilter";
+
+ /**
+ * An optional {@link BOpFilterBase} to be applied to the elements of
+ * the relation as they are materialized from the index. Unlike
+ * {@link #INDEX_LOCAL_FILTER}, this an {@link #ACCESS_PATH_FILTER} is never
+ * sent to a remote index for evaluation. This makes it possible to
+ * impose {@link DistinctFilter} across a {@link #REMOTE_ACCESS_PATH}.
+ */
+ String ACCESS_PATH_FILTER = "accessPathFilter";
+
+ /**
* Expander pattern.
*/
String EXPANDER = "expander";
@@ -144,7 +186,7 @@
*
* @see #DEFAULT_FULLY_BUFFERED_READ_THRESHOLD
*/
- String FULLY_BUFFERED_READ_THRESHOLD = PipelineOp.class.getName()
+ String FULLY_BUFFERED_READ_THRESHOLD = IPredicate.class.getName()
+ ".fullyBufferedReadThreshold";
/**
@@ -168,7 +210,7 @@
*
* @see #DEFAULT_FLAGS
*/
- String FLAGS = PipelineOp.class.getName() + ".flags";
+ String FLAGS = IPredicate.class.getName() + ".flags";
/**
* The default flags will visit the keys and values of the non-deleted
@@ -278,13 +320,17 @@
* which is the only method declared by {@link ISolutionExpander}.
*/
public ISolutionExpander<E> getSolutionExpander();
-
+
/**
* An optional constraint on the visitable elements.
*
* @see Annotations#CONSTRAINT
*
- * @todo rename as get(Element)Filter().
+ * @deprecated This is being replaced by two classes of filters. One which
+ * is always evaluated local to the index and one which is
+ * evaluated in the JVM in which the access path is evaluated
+ * once the {@link ITuple}s have been resolved to elements of
+ * the relation.
*/
public IElementFilter<E> getConstraint();
Modified: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/IShardwisePipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -28,7 +28,7 @@
package com.bigdata.bop;
/**
- * An interface for {@link BindingSetPipelineOp}s which are mapped across
+ * An interface for {@link PipelineOp}s which are mapped across
* shards.
*
* @param <E>
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,230 +0,0 @@
-/**
-
-Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
-
-Contact:
- SYSTAP, LLC
- 4501 Tower Road
- Greensboro, NC 27410
- lic...@bi...
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-*/
-/*
- * Created on Aug 18, 2010
- */
-
-package com.bigdata.bop;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import com.bigdata.bop.engine.BOpStats;
-import com.bigdata.bop.engine.QueryEngine;
-import com.bigdata.relation.accesspath.BlockingBuffer;
-import com.bigdata.relation.accesspath.IBlockingBuffer;
-
-/**
- * An pipeline operator reads from a source and writes on a sink. This is an
- * abstract base class for pipelined operators regardless of the type of data
- * moving along the pipeline.
- *
- * @param <E>
- * The generic type of the objects processed by the operator.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id$
- */
-abstract public class PipelineOp<E> extends BOpBase implements IPipelineOp<E> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * Well known annotations pertaining to the binding set pipeline.
- */
- public interface Annotations extends BOp.Annotations, BufferAnnotations {
-
- }
-
- /**
- * Deep copy constructor.
- *
- * @param op
- */
- protected PipelineOp(final PipelineOp<E> op) {
-
- super(op);
-
- }
-
- /**
- * Shallow copy constructor.
- *
- * @param args
- * @param annotations
- */
- protected PipelineOp(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- }
-
- public int getChunkCapacity() {
-
- return getProperty(Annotations.CHUNK_CAPACITY,
- Annotations.DEFAULT_CHUNK_CAPACITY);
-
- }
-
- public int getChunkOfChunksCapacity() {
-
- return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
- Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
-
- }
-
- public long getChunkTimeout() {
-
- return getProperty(Annotations.CHUNK_TIMEOUT,
- Annotations.DEFAULT_CHUNK_TIMEOUT);
-
- }
-
- /**
- * The {@link TimeUnit}s in which the {@link #chunkTimeout} is measured.
- */
- protected static transient final TimeUnit chunkTimeoutUnit = TimeUnit.MILLISECONDS;
-
- /**
- * Return the {@link PipelineType} of the operator (default
- * {@link PipelineType#Vectored}).
- */
- public PipelineType getPipelineType() {
-
- return PipelineType.Vectored;
-
- }
-
- /**
- * Return <code>true</code> iff {@link #newStats()} must be shared across
- * all invocations of {@link #eval(BOpContext)} for this operator for a
- * given query (default <code>false</code>).
- * <p>
- * Note: {@link BOp#getEvaluationContext()} MUST be overridden to return
- * {@link BOpEvaluationContext#CONTROLLER} if this method is overridden to
- * return <code>true</code>.
- * <p>
- * When <code>true</code>, the {@link QueryEngine} will impose the necessary
- * constraints when the operator is evaluated.
- */
- public boolean isSharedState() {
-
- return false;
-
- }
-
- public BOpStats newStats() {
-
- return new BOpStats();
-
- }
-
- public IBlockingBuffer<E[]> newBuffer(final BOpStats stats) {
-
- if (stats == null)
- throw new IllegalArgumentException();
-
- return new BlockingBufferWithStats<E[]>(getChunkOfChunksCapacity(),
- getChunkCapacity(), getChunkTimeout(), chunkTimeoutUnit, stats);
-
- }
-
- private static class BlockingBufferWithStats<E> extends BlockingBuffer<E> {
-
- private final BOpStats stats;
-
- /**
- * @param chunkOfChunksCapacity
- * @param chunkCapacity
- * @param chunkTimeout
- * @param chunkTimeoutUnit
- * @param stats
- */
- public BlockingBufferWithStats(int chunkOfChunksCapacity,
- int chunkCapacity, long chunkTimeout,
- TimeUnit chunkTimeoutUnit, final BOpStats stats) {
-
- super(chunkOfChunksCapacity, chunkCapacity, chunkTimeout,
- chunkTimeoutUnit);
-
- this.stats = stats;
-
- }
-
- /**
- * Overridden to track {@link BOpStats#unitsOut} and
- * {@link BOpStats#chunksOut}.
- * <p>
- * Note: {@link BOpStats#chunksOut} will report the #of chunks added to
- * this buffer. However, the buffer MAY combine chunks either on add()
- * or when drained by the iterator so the actual #of chunks read back
- * from the iterator MAY differ.
- * <p>
- * {@inheritDoc}
- */
- @Override
- public boolean add(final E e, final long timeout, final TimeUnit unit)
- throws InterruptedException {
-
- final boolean ret = super.add(e, timeout, unit);
-
- if (e.getClass().getComponentType() != null) {
-
- stats.unitsOut.add(((Object[]) e).length);
-
- } else {
-
- stats.unitsOut.increment();
-
- }
-
- stats.chunksOut.increment();
-
- return ret;
-
- }
-
- /**
- * You can uncomment a line in this method to see who is closing the
- * buffer.
- * <p>
- * {@inheritDoc}
- */
- @Override
- public void close() {
-
-// if (isOpen())
-// log.error(toString(), new RuntimeException("STACK TRACE"));
-
- super.close();
-
- }
-
- }
-
-}
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java (from rev 3614, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/BindingSetPipelineOp.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/PipelineOp.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,282 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Sep 2, 2010
+ */
+
+package com.bigdata.bop;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+
+import com.bigdata.bop.ap.E;
+import com.bigdata.bop.engine.BOpStats;
+import com.bigdata.bop.engine.QueryEngine;
+import com.bigdata.relation.accesspath.BlockingBuffer;
+import com.bigdata.relation.accesspath.IBlockingBuffer;
+
+/**
+ * Abstract base class for pipeline operators where the data moving along the
+ * pipeline is chunks of {@link IBindingSet}s.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ */
+abstract public class PipelineOp extends BOpBase {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public interface Annotations extends BOp.Annotations, BufferAnnotations {
+
+ /**
+ * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
+ * the ancestor in the operator tree which serves as the default sink
+ * for binding sets (default is the parent).
+ */
+ String SINK_REF = PipelineOp.class.getName() + ".sinkRef";
+
+ /**
+ * The value of the annotation is the {@link BOp.Annotations#BOP_ID} of
+ * the ancestor in the operator tree which serves as the alternative
+ * sink for binding sets (default is no alternative sink).
+ */
+ String ALT_SINK_REF = PipelineOp.class.getName()
+ + ".altSinkRef";
+
+ }
+
+ /**
+ * Required deep copy constructor.
+ *
+ * @param op
+ */
+ protected PipelineOp(final PipelineOp op) {
+ super(op);
+ }
+
+ /**
+ * Shallow copy constructor.
+ *
+ * @param args
+ * @param annotations
+ */
+ protected PipelineOp(final BOp[] args,
+ final Map<String, Object> annotations) {
+
+ super(args, annotations);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_CAPACITY
+ */
+ public int getChunkCapacity() {
+
+ return getProperty(Annotations.CHUNK_CAPACITY,
+ Annotations.DEFAULT_CHUNK_CAPACITY);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_OF_CHUNKS_CAPACITY
+ */
+ public int getChunkOfChunksCapacity() {
+
+ return getProperty(Annotations.CHUNK_OF_CHUNKS_CAPACITY,
+ Annotations.DEFAULT_CHUNK_OF_CHUNKS_CAPACITY);
+
+ }
+
+ /**
+ * @see BufferAnnotations#CHUNK_TIMEOUT
+ */
+ public long getChunkTimeout() {
+
+ return getProperty(Annotations.CHUNK_TIMEOUT,
+ Annotations.DEFAULT_CHUNK_TIMEOUT);
+
+ }
+
+ /**
+ * Return the {@link PipelineType} of the operator (default
+ * {@link PipelineType#Vectored}).
+ */
+ public PipelineType getPipelineType() {
+
+ return PipelineType.Vectored;
+
+ }
+
+ /**
+ * Return <code>true</code> iff {@link #newStats()} must be shared across
+ * all invocations of {@link #eval(BOpContext)} for this operator for a
+ * given query (default <code>false</code>).
+ * <p>
+ * Note: {@link BOp#getEvaluationContext()} MUST be overridden to return
+ * {@link BOpEvaluationContext#CONTROLLER} if this method is overridden to
+ * return <code>true</code>.
+ * <p>
+ * When <code>true</code>, the {@link QueryEngine} will impose the necessary
+ * constraints when the operator is evaluated.
+ */
+ public boolean isSharedState() {
+
+ return false;
+
+ }
+
+ /**
+ * Return a new object which can be used to collect statistics on the
+ * operator evaluation (this may be overridden to return a more specific
+ * class depending on the operator).
+ */
+ public BOpStats newStats() {
+
+ return new BOpStats();
+
+ }
+
+ /**
+ * Instantiate a buffer suitable as a sink for this operator. The buffer
+ * will be provisioned based on the operator annotations.
+ * <p>
+ * Note: if the operation swallows binding sets from the pipeline (such as
+ * operators which write on the database) then the operator MAY return an
+ * immutable empty buffer.
+ *
+ * @param stats
+ * The statistics on this object will automatically be updated as
+ * elements and chunks are output onto the returned buffer.
+ *
+ * @return The buffer.
+ */
+ public IBlockingBuffer<IBindingSet[]> newBuffer(final BOpStats stats) {
+
+ if (stats == null)
+ throw new IllegalArgumentException();
+
+ return new BlockingBufferWithStats<IBindingSet[]>(
+ getChunkOfChunksCapacity(), getChunkCapacity(),
+ getChunkTimeout(), Annotations.chunkTimeoutUnit, stats);
+
+ }
+
+ /**
+ * Return a {@link FutureTask} which computes the operator against the
+ * evaluation context. The caller is responsible for executing the
+ * {@link FutureTask} (this gives them the ability to hook the completion of
+ * the computation).
+ *
+ * @param context
+ * The evaluation context.
+ *
+ * @return The {@link FutureTask} which will compute the operator's
+ * evaluation.
+ *
+ * @todo Modify to return a {@link Callable}s for now since we must run each
+ * task in its own thread until Java7 gives us fork/join pools and
+ * asynchronous file I/O. For the fork/join model we will probably
+ * return the ForkJoinTask.
+ */
+ abstract public FutureTask<Void> eval(BOpContext<IBindingSet> context);
+
+ private static class BlockingBufferWithStats<E> extends BlockingBuffer<E> {
+
+ private final BOpStats stats;
+
+ /**
+ * @param chunkOfChunksCapacity
+ * @param chunkCapacity
+ * @param chunkTimeout
+ * @param chunkTimeoutUnit
+ * @param stats
+ */
+ public BlockingBufferWithStats(int chunkOfChunksCapacity,
+ int chunkCapacity, long chunkTimeout,
+ TimeUnit chunkTimeoutUnit, final BOpStats stats) {
+
+ super(chunkOfChunksCapacity, chunkCapacity, chunkTimeout,
+ chunkTimeoutUnit);
+
+ this.stats = stats;
+
+ }
+
+ /**
+ * Overridden to track {@link BOpStats#unitsOut} and
+ * {@link BOpStats#chunksOut}.
+ * <p>
+ * Note: {@link BOpStats#chunksOut} will report the #of chunks added to
+ * this buffer. However, the buffer MAY combine chunks either on add()
+ * or when drained by the iterator so the actual #of chunks read back
+ * from the iterator MAY differ.
+ * <p>
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean add(final E e, final long timeout, final TimeUnit unit)
+ throws InterruptedException {
+
+ final boolean ret = super.add(e, timeout, unit);
+
+ if (e.getClass().getComponentType() != null) {
+
+ stats.unitsOut.add(((Object[]) e).length);
+
+ } else {
+
+ stats.unitsOut.increment();
+
+ }
+
+ stats.chunksOut.increment();
+
+ return ret;
+
+ }
+
+ /**
+ * You can uncomment a line in this method to see who is closing the
+ * buffer.
+ * <p>
+ * {@inheritDoc}
+ */
+ @Override
+ public void close() {
+
+// if (isOpen())
+// log.error(toString(), new RuntimeException("STACK TRACE"));
+
+ super.close();
+
+ }
+
+ }
+
+}
Copied: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java (from rev 3630, branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ndx/AbstractSampleIndex.java)
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java (rev 0)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/AbstractSampleIndex.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -0,0 +1,125 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2010. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*/
+/*
+ * Created on Aug 16, 2010
+ */
+
+package com.bigdata.bop.ap;
+
+import com.bigdata.bop.AbstractAccessPathOp;
+import com.bigdata.bop.BOp;
+import com.bigdata.bop.IBindingSet;
+import com.bigdata.bop.IPredicate;
+import com.bigdata.bop.NV;
+import com.bigdata.btree.IIndex;
+import com.bigdata.relation.accesspath.IAccessPath;
+
+/**
+ * Abstract base class for sampling operator for an {@link IIndex}.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id$
+ * @param <E>
+ * The generic type of the elements materialized from that index.
+ *
+ * @todo Implement sample operator. E.g., sampleRange(fromKey,toKey,limit). This
+ * could be on {@link IIndex} or on {@link IAccessPath}. For a shard view,
+ * it must proportionally select from among the ordered components of the
+ * view. For a hash table it would be sample(limit) since range based
+ * operations are not efficient.
+ * <p>
+ * This should accept an index, not a predicate (for RDF we determine the
+ * index an analysis of the bound and unbound arguments on the predicate
+ * and always have a good index, but this is not true in the general
+ * case). When the index is remote, it should be executed at the remote
+ * index.
+ *
+ * @todo This needs to operation on element chunks, not {@link IBindingSet}
+ * chunks. It also may not require pipelining.
+ */
+abstract public class AbstractSampleIndex<E> extends AbstractAccessPathOp<E> {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Known annotations.
+ */
+ public interface Annotations extends BOp.Annotations {
+ /**
+ * The sample limit.
+ */
+ String LIMIT = "limit";
+ }
+
+ protected AbstractSampleIndex(final IPredicate<E> pred, final int limit) {
+
+ super(new BOp[] { pred }, NV.asMap(new NV[] {//
+ new NV(Annotations.LIMIT, Integer.valueOf(limit)) //
+ }));
+
+ if (pred == null)
+ throw new IllegalArgumentException();
+
+ if (limit <= 0)
+ throw new IllegalArgumentException();
+
+ switch (getEvaluationContext()) {
+ case HASHED:
+ case SHARDED:
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ Annotations.EVALUATION_CONTEXT + "="
+ + getEvaluationContext());
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public IPredicate<E> pred() {
+
+ return (IPredicate<E>) get(0);
+
+ }
+
+ public int limit() {
+
+ return (Integer) annotations.get(Annotations.LIMIT);
+
+ }
+
+// /**
+// * This is a shard wise operator.
+// */
+// @Override
+// public BOpEvaluationContext getEvaluationContext() {
+//
+// return BOpEvaluationContext.SHARDED;
+//
+// }
+
+}
Deleted: branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java
===================================================================
--- branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java 2010-09-28 16:10:27 UTC (rev 3664)
+++ branches/QUADS_QUERY_BRANCH/bigdata/src/java/com/bigdata/bop/ap/DistinctElementFilter.java 2010-09-28 16:53:22 UTC (rev 3665)
@@ -1,320 +0,0 @@
-package com.bigdata.bop.ap;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.FutureTask;
-
-import com.bigdata.bop.BOp;
-import com.bigdata.bop.BOpBase;
-import com.bigdata.bop.BOpContext;
-import com.bigdata.bop.IBindingSet;
-import com.bigdata.rdf.spo.DistinctSPOIterator;
-import com.bigdata.rdf.spo.SPO;
-import com.bigdata.relation.accesspath.IElementFilter;
-import com.bigdata.striterator.DistinctFilter;
-import com.bigdata.striterator.IChunkConverter;
-import com.bigdata.striterator.MergeFilter;
-
-/**
- * A DISTINCT operator based for elements in a relation. The operator is based
- * on a hash table. New elements are constructed for each original element in
- * which only the distinct fields are preserved. If the new element is distinct
- * then it is passed by the filter.
- * <p>
- * The filter is capable of changing the type of the accepted elements.
- *
- * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
- * @version $Id: DistinctElementFilter.java 3466 2010-08-27 14:28:04Z
- * thompsonbry $
- *
- * @param <E>
- * The generic type of the source elements for the filter.
- * @param <F>
- * The generic type of the elements passed by the filter.
- *
- * @todo support changing the generic type as part of the filter. this is
- * similar to the {@link IChunkConverter}.
- *
- * @todo Reconcile with {@link IChunkConverter}, {@link DistinctFilter} (handles
- * solutions) and {@link MergeFilter} (handles comparables),
- * {@link DistinctSPOIterator}, etc.
- */
-public class DistinctElementFilter<E> extends BOpBase implements
- IElementFilter<E> {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public interface Annotations extends BOp.Annotations {
-
- /**
- * The initial capacity of the {@link ConcurrentHashMap} used to impose
- * the distinct constraint.
- *
- * @see #DEFAULT_INITIAL_CAPACITY
- */
- String INITIAL_CAPACITY = DistinctElementFilter.class.getName()
- + ".initialCapacity";
-
- int DEFAULT_INITIAL_CAPACITY = 16;
-
- /**
- * The load factor of the {@link ConcurrentHashMap} used to impose the
- * distinct constraint.
- *
- * @see #DEFAULT_LOAD_FACTOR
- */
- String LOAD_FACTOR = DistinctElementFilter.class.getName()
- + ".loadFactor";
-
- float DEFAULT_LOAD_FACTOR = .75f;
-
- /**
- * The concurrency level of the {@link ConcurrentHashMap} used to impose
- * the distinct constraint.
- *
- * @see #DEFAULT_CONCURRENCY_LEVEL
- */
- String CONCURRENCY_LEVEL = DistinctElementFilter.class.getName()
- + ".concurrencyLevel";
-
- int DEFAULT_CONCURRENCY_LEVEL = 16;
-
- /**
- * The set of fields whose values must be distinct.
- *
- * @todo abstract base class to allow easy override for specific element
- * types such as {@link SPO}.
- */
- String FIELDS = DistinctElementFilter.class.getName() + ".fields";
-
- /**
- * An optional constraint on the runtime type of the elements which are
- * acceptable to this filter.
- *
- * @see IElementFilter#canAccept(Object)
- *
- * @todo I am not convinced that we need this. It parallels something
- * which was introduced into the {@link IElementFilter} interface,
- * but I suspect that we do not need that either.
- */
- String CLASS_CONSTRAINT = DistinctElementFilter.class.getName()
- + ".classConstraint";
-
- }
-
- /**
- * Required deep copy constructor.
- */
- public DistinctElementFilter(final DistinctElementFilter<E> op) {
- super(op);
- }
-
- /**
- * Required shallow copy constructor.
- */
- public DistinctElementFilter(final BOp[] args,
- final Map<String, Object> annotations) {
-
- super(args, annotations);
-
- final int[] fields = getFields();
-
- if (fields == null)
- throw new IllegalArgumentException();
-
- if (fields.length == 0)
- throw new IllegalArgumentException();
-
- }
-
- /**
- * @see Annotations#INITIAL_CAPACITY
- */
- public int getInitialCapacity() {
-
- return getProperty(Annotations.INITIAL_CAPACITY,
- Annotations.DEFAULT_INITIAL_CAPACITY);
-
- }
-
- /**
- * @see Annotations#LOAD_FACTOR
- */
- public float getLoadFactor() {
-
- return getProperty(Annotations.LOAD_FACTOR,
- Annotations.DEFAULT_LOAD_FACTOR);
-
- }
-
- /**
- * @see Annotations#CONCURRENCY_LEVEL
- */
- public int getConcurrencyLevel() {
-
- return getProperty(Annotations.CONCURRENCY_LEVEL,
- Annotations.DEFAULT_CONCURRENCY_LEVEL);
-
- }
-
- public int[] getFields() {
-
- return (int[]) getProperty(Annotations.FIELDS);
-
- }
-
- public FutureTask<Void> eval(final BOpContext<IBindingSet> context) {
-
-// return new FutureTask<Void>(new DistinctTask<E>(this, context));
- throw new UnsupportedOperationException();
-
- }
-
- public boolean accept(E e) {
- // TODO Auto-generated method stub
- return false;
- }
-
- public boolean canAccept(Object o) {
- // @todo by annotation giving an optional type constraint.
- return true;
- }
-
-// /**
-// * Task executing on the node.
-// */
-// static private class DistinctTask<E> implements Callable<Void> {
-//
-// private final BOpConte...
[truncated message content] |
