From: <mr...@us...> - 2010-08-31 17:22:41
|
Revision: 3473 http://bigdata.svn.sourceforge.net/bigdata/?rev=3473&view=rev Author: mroycsi Date: 2010-08-31 17:22:33 +0000 (Tue, 31 Aug 2010) Log Message: ----------- Commit CSI changes to csi branch Modified Paths: -------------- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/Banner.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/concurrent/Latch.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/config/LogUtil.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java branches/BIGDATA_RELEASE_0_83_2_CSI/build.properties branches/BIGDATA_RELEASE_0_83_2_CSI/build.xml Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/Banner.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/Banner.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/Banner.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -48,7 +48,7 @@ public class Banner { private static boolean didBanner; - + private static String banner; /** * Environment variables understood by the {@link Banner} class. * @@ -67,18 +67,24 @@ * properties. */ String LOG4J_MBEANS_DISABLE = "com.bigdata.jmx.log4j.disable"; + + /** + * This may be used to disable system errors about log4j + * configuration missing + */ + String LOG4J_QUIET = "com.bigdata.log4j.quiet"; } synchronized static public void banner() { if(!didBanner) { - + final boolean quiet = Boolean.getBoolean(Options.QUIET); if (!quiet) { - System.out.println(banner); + System.out.println(getBanner()); } @@ -130,7 +136,28 @@ } } - + private static final String getBanner(){ + if(banner==null){ + banner=// + "\nBIGDATA(R)"+// + "\n"+// + "\n Flexible"+// + "\n Reliable"+// + "\n Affordable"+// + "\n Web-Scale Computing for the Enterprise"+// + "\n"+// + "\nCopyright SYSTAP, LLC 2006-2010. All rights reserved."+// + "\n"+// + "\n"+AbstractStatisticsCollector.fullyQualifiedHostName+// + "\n"+new Date()+// + "\n"+SystemUtil.operatingSystem() + "/" + SystemUtil.osVersion() + + " " + SystemUtil.architecture() + // + "\n"+SystemUtil.cpuInfo() + " #CPU="+SystemUtil.numProcessors() +// + "\n" + ; + } + return banner; + } /** * Outputs the banner and exits. * @@ -139,26 +166,9 @@ */ public static void main(final String[] args) { - System.out.println(banner); + System.out.println(getBanner()); } - private static final String banner =// - "\nBIGDATA(R)"+// - "\n"+// - "\n Flexible"+// - "\n Reliable"+// - "\n Affordable"+// - "\n Web-Scale Computing for the Enterprise"+// - "\n"+// - "\nCopyright SYSTAP, LLC 2006-2010. All rights reserved."+// - "\n"+// - "\n"+AbstractStatisticsCollector.fullyQualifiedHostName+// - "\n"+new Date()+// - "\n"+SystemUtil.operatingSystem() + "/" + SystemUtil.osVersion() - + " " + SystemUtil.architecture() + // - "\n"+SystemUtil.cpuInfo() + " #CPU="+SystemUtil.numProcessors() +// - "\n" - ; } Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/btree/filter/TupleFilter.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -50,7 +50,7 @@ abstract public class TupleFilter<E> implements ITupleFilter<E> { protected static transient final Logger log = Logger.getLogger(TupleFilter.class); - + protected final transient boolean INFO=log.isInfoEnabled(); /** * Optional state specified by the ctor. */ @@ -173,7 +173,7 @@ // one step lookahead. nextValue = getNext(); - if(log.isInfoEnabled()) { + if(INFO) { log.info("returning: "+returnValue); @@ -231,7 +231,7 @@ final byte[] key = returnValue.getKey(); - if(log.isInfoEnabled()) { + if(INFO) { log.info("key=" + BytesUtil.toString(key)); @@ -270,7 +270,7 @@ if (!isValid(next)) { - if(log.isInfoEnabled()) { + if(INFO) { log.info("rejected : "+next); @@ -280,7 +280,7 @@ } - if(log.isInfoEnabled()) { + if(INFO) { log.info("will visit: "+next); @@ -290,7 +290,7 @@ } - if(log.isInfoEnabled()) { + if(INFO) { log.info("Source is exhausted."); Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/relation/accesspath/AbstractAccessPath.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -86,9 +86,9 @@ static final protected Logger log = Logger.getLogger(IAccessPath.class); -// final static protected boolean log.isInfoEnabled() = log.isInfoEnabled(); + final protected boolean INFO = log.isInfoEnabled(); // -// final static protected boolean log.isDebugEnabled() = log.isDebugEnabled(); + final protected boolean DEBUG = log.isDebugEnabled(); /** Access to the index, resource locator, executor service, etc. */ protected final IIndexManager indexManager; @@ -509,7 +509,7 @@ didInit = true; - if(log.isDebugEnabled()) { + if(DEBUG) { if (fromKey != null && toKey != null) { @@ -577,7 +577,7 @@ } - if(log.isDebugEnabled()) { + if(DEBUG) { log.debug(toString()); @@ -685,14 +685,14 @@ * The access path has already been proven to be empty. */ - if (log.isDebugEnabled()) + if (DEBUG) log.debug("Proven empty by historical range count"); return new EmptyChunkedIterator<R>(keyOrder); } - if (log.isDebugEnabled()) + if (DEBUG) log.debug("offset=" + offset + ", limit=" + limit + ", capacity=" + capacity + ", accessPath=" + this); @@ -703,7 +703,7 @@ if(isFullyBoundForKey) { - if (log.isDebugEnabled()) + if (DEBUG) log.debug("Predicate is fully bound for the key."); /* @@ -808,7 +808,7 @@ final long rangeCountRemaining = rangeCount(false/* exact */) - offset; - if (log.isDebugEnabled()) + if (DEBUG) log.debug("offset=" + offset + ", limit=" + limit + ", rangeCountRemaining=" + rangeCountRemaining + ", fullyBufferedReadThreashold=" @@ -821,7 +821,7 @@ * iterator would not visit anything. */ - if (log.isDebugEnabled()) + if (DEBUG) log.debug("No elements based on range count."); return new EmptyChunkedIterator<R>(keyOrder); @@ -942,7 +942,7 @@ assert limit < MAX_FULLY_BUFFERED_READ_LIMIT : "limit=" + limit + ", max=" + MAX_FULLY_BUFFERED_READ_LIMIT; - if (log.isDebugEnabled()) { + if (DEBUG) { log.debug("offset=" + offset + ", limit=" + limit); @@ -980,7 +980,7 @@ } - if(log.isDebugEnabled()) { + if(DEBUG) { log.debug("Fully buffered: read=" + nread + ", used=" + nused + ", offset=" + offset + ", limit=" + limit); @@ -1018,7 +1018,7 @@ if (src == null) throw new IllegalArgumentException(); - if (log.isDebugEnabled()) + if (DEBUG) log.debug(""); /* @@ -1055,7 +1055,7 @@ // static protected final boolean log.isInfoEnabled() = log.isInfoEnabled(); // -// static protected final boolean log.isDebugEnabled() = log.isDebugEnabled(); + static protected final boolean DEBUG = log.isDebugEnabled(); private final AbstractAccessPath<R> accessPath; @@ -1109,7 +1109,6 @@ long nchunks = 0; long nelements = 0; - try { while (src.hasNext()) { @@ -1119,7 +1118,7 @@ nchunks++; nelements+=chunk.length; - if(log.isDebugEnabled()) + if(DEBUG) log.debug("#chunks=" + nchunks + ", chunkSize=" + chunk.length + ", nelements=" + nelements); @@ -1211,7 +1210,7 @@ } - if (log.isDebugEnabled()) { + if (DEBUG) { log.debug("exact=" + exact + ", filter=" + (filter != null) + ", n=" + n + " : " + toString()); @@ -1255,7 +1254,7 @@ assertInitialized(); - if (log.isDebugEnabled()) { + if (DEBUG) { log.debug(this + " : capacity=" + capacity + ", flags=" + flags + ", filter=" + filter); @@ -1277,7 +1276,7 @@ assertInitialized(); - if (log.isDebugEnabled()) { + if (DEBUG) { log.debug(this.toString()); Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/concurrent/Latch.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/concurrent/Latch.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/concurrent/Latch.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -56,9 +56,10 @@ public class Latch { protected transient static final Logger log = Logger.getLogger(Latch.class); + protected transient final boolean DEBUG = log.isDebugEnabled(); + protected transient final boolean INFO = log.isInfoEnabled(); + private final AtomicLong counter = new AtomicLong(); - private final AtomicLong counter = new AtomicLong(); - private final ReentrantLock lock; private final Condition cond; @@ -138,7 +139,7 @@ if (c <= 0) throw new IllegalStateException(toString()); - if (log.isDebugEnabled()) + if (DEBUG) log.debug(toString()); return c; @@ -169,7 +170,7 @@ final long c = this.counter.addAndGet(delta); - if (log.isDebugEnabled()) + if (DEBUG) log.debug(toString()); if (c == 0) { @@ -219,7 +220,7 @@ final long c = this.counter.decrementAndGet(); - if (log.isDebugEnabled()) + if (DEBUG) log.debug(toString()); if (c == 0) { @@ -264,7 +265,7 @@ //// lock.lockInterruptibly(); // try { - if (log.isInfoEnabled()) + if (INFO) log.info("signalAll()"); // release anyone awaiting our signal. Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/config/LogUtil.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/config/LogUtil.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata/src/java/com/bigdata/util/config/LogUtil.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -32,6 +32,8 @@ import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.xml.DOMConfigurator; +import com.bigdata.Banner; + /** * Utility class that provides a set of static convenience methods related * to the initialization and configuration of the logging mechanism(s) @@ -61,17 +63,19 @@ if (log4jDefaultConfig != null ) { PropertyConfigurator.configureAndWatch(log4jDefaultConfig); } else { - System.out.println - ("ERROR: could not initialize Log4J logging utility"); - System.out.println - (" set system property " - +"'-Dlog4j.configuration=" - +"bigdata/src/resources/logging/log4j.properties" - +"\n and/or \n" - +" set system property " - +"'-Dlog4j.primary.configuration=" - +"<installDir>/" - +"bigdata/src/resources/logging/log4j.properties'"); + if(!Boolean.getBoolean(Banner.Options.LOG4J_QUIET)){ + System.out.println + ("ERROR: could not initialize Log4J logging utility"); + System.out.println + (" set system property " + +"'-Dlog4j.configuration=" + +"bigdata/src/resources/logging/log4j.properties" + +"\n and/or \n" + +" set system property " + +"'-Dlog4j.primary.configuration=" + +"<installDir>/" + +"bigdata/src/resources/logging/log4j.properties'"); + } } } } Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/TermId.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -49,6 +49,8 @@ /** The term identifier. */ private final long termId; + + private transient int hashCode=0; /** * Constructor for a term identifier when you are decoding and already have @@ -160,7 +162,10 @@ * @see Long#hashCode() */ public int hashCode() { - return (int) (termId ^ (termId >>> 32)); + if(hashCode==0){ + hashCode=(int) (termId ^ (termId >>> 32)); + } + return hashCode; } public int byteLength() { Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/internal/XSDDecimalIV.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -64,7 +64,7 @@ public V asValue(final BigdataValueFactory f, final ILexiconConfiguration config) { // @todo factory should cache the XSD URIs. - final V v = (V) f.createLiteral(value.toString(),// + final V v = (V) f.createLiteral(value.toPlainString(),// f.createURI(DTE.XSDDecimal.getDatatype())); v.setIV(this); return v; @@ -105,9 +105,13 @@ return value.shortValue(); } + /** + * Use toPlainString to avoid expression with exponent value that + * would imply xsd:double rather than xsd:decimal + */ @Override public String stringValue() { - return value.toString(); + return value.toPlainString(); } @Override Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/lexicon/LexiconRelation.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -1873,7 +1873,7 @@ * IllegalStateException if the value somehow was assigned * the wrong term identifier (paranoia test). */ - assert value.getIV() == tid : "expecting tid=" + tid + assert value.getIV().equals(tid) : "expecting tid=" + tid + ", but found " + value.getIV(); assert (value).getValueFactory() == valueFactory; Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/DistinctSPOIterator.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -10,6 +10,7 @@ import com.bigdata.rawstore.Bytes; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.store.IRawTripleStore; +import com.bigdata.relation.accesspath.BlockingBuffer; import com.bigdata.striterator.ICloseableIterator; /** @@ -21,7 +22,6 @@ * @see SPORelation#distinctSPOIterator(ICloseableIterator) */ public class DistinctSPOIterator implements ICloseableIterator<ISPO> { - /** * The backing relation, which is only used to obtain the {@link BTree} * instance in {@link #overflowToBTree(Set)}. @@ -35,13 +35,13 @@ /** * Hash set is allocated when the first {@link ISPO} is visited and is used - * until the {@link #MAX_HASH_SET_CAPACITY} is reached, at which point the + * until the {@link #maxHashXSetCapacity} is reached, at which point the * {@link #btreeSet} is allocated. */ private Set<ISPO> hashSet; /** - * B+Tree is used once the {@link #MAX_HASH_SET_CAPACITY} is reached. The + * B+Tree is used once the {@link #maxHashXSetCapacity} is reached. The * B+Tree is slowed than the {@link #hashSet}, but can spill onto the disk * and is appropriate for very large distinct sets. */ @@ -58,6 +58,8 @@ * ahead. */ private ISPO next = null; + + private ISPO first = null; /** * <code>true</code> iff the iterator has been proven to be exhausted. @@ -83,13 +85,8 @@ /** * After this many entries we create the {@link #btreeSet} which can spill * out onto the disk. - * - * @todo configuration parameter (via the constructor). Low memory JVMs - * might want to use a smaller threshold, but the hash set is much - * faster (10x or better). Large memory JVMs might want to use an even - * larger threshold. */ - static final int MAX_HASH_SET_CAPACITY = 100000; + final int maxHashXSetCapacity; /** * @@ -97,7 +94,8 @@ * The source iterator. */ public DistinctSPOIterator(final SPORelation spoRelation, - final ICloseableIterator<ISPO> src) { + final ICloseableIterator<ISPO> src, + final int maxHashSetCapacity) { if (spoRelation == null) throw new IllegalArgumentException(); @@ -109,6 +107,7 @@ this.src = src; + this.maxHashXSetCapacity=maxHashSetCapacity; } public void close() { @@ -160,26 +159,9 @@ if (next != null) return true; - if (hashSet == null) { + if (btreeSet == null && ndistinct >= maxHashXSetCapacity) { /* - * Allocate hash set. - * - * Note: using a linked hash set for faster iterator if we have - * to convert to a B+Tree. - * - * Note: the initial capacity is the default since most access - * paths have low cardinality. - * - * @todo if the caller knows the range count (upper bound) then - * we could plan the hash set capacity more accurately. - */ - - hashSet = new LinkedHashSet<ISPO>(); - - } else if (btreeSet == null && ndistinct >= MAX_HASH_SET_CAPACITY) { - - /* * Open B+Tree. We will not put anything new into the hashSet, * but we will continue to test first against the hashSet and * then against the B+Tree. New distinct ISPOs are inserted into @@ -228,14 +210,16 @@ ISPO tmp = src.next(); nscanned++; // strip off the context (and statement type). - tmp = new SPO(tmp.s(), tmp.p(), tmp.o(), (IV) null/* c */); + //tmp = new SPO(tmp.s(), tmp.p(), tmp.o(), (IV) null/* c */); if (btreeSet == null) { - // Insert into the hash set. + if (hashSet == null&&first!=null) { + hashSet = new LinkedHashSet<ISPO>(); + hashSet.add(first); + } + if (hashSet!=null&&!hashSet.add(tmp)) { - if (!hashSet.add(tmp)) { - // duplicate, keep scanning. continue; @@ -268,7 +252,9 @@ // found a new distinct spo. next = tmp; - + if(first==null){ + first=next; + } ndistinct++; } // while(...) Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPO.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -551,18 +551,16 @@ if (hashCode == 0) { - final int s = this.s.hashCode(); + final int _s = this.s.hashCode(); - final int p = this.p.hashCode(); + final int _p = this.p.hashCode(); - final int o = this.o.hashCode(); + final int _o = this.o.hashCode(); // Note: historical behavior was (s,p,o) based hash. - hashCode = 961 * ((int) (s ^ (s >>> 32))) + 31 - * ((int) (p ^ (p >>> 32))) + ((int) (o ^ (o >>> 32))); - + hashCode = 961 * _s + 31 * _p + _o; } - + return hashCode; } Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOKeyOrder.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -99,14 +99,15 @@ public static final transient int _CSPO = 6; public static final transient int _PCSO = 7; public static final transient int _SOPC = 8; + public static final transient int _SCOP = 9; public static final transient int FIRST_TRIPLE_INDEX = _SPO; public static final transient int LAST_TRIPLE_INDEX = _POS; public static final transient int FIRST_QUAD_INDEX = _SPOC; - public static final transient int LAST_QUAD_INDEX = _SOPC; + public static final transient int LAST_QUAD_INDEX = _SCOP; - public static final transient int MAX_INDEX_COUNT = 9; + public static final transient int MAX_INDEX_COUNT = 10; /* * The three perfect natural orders for triples. @@ -128,6 +129,7 @@ public static final transient SPOKeyOrder CSPO = new SPOKeyOrder(_CSPO); public static final transient SPOKeyOrder PCSO = new SPOKeyOrder(_PCSO); public static final transient SPOKeyOrder SOPC = new SPOKeyOrder(_SOPC); + public static final transient SPOKeyOrder SCOP = new SPOKeyOrder(_SCOP); /** * The name for each of the natural key orders. @@ -143,7 +145,8 @@ "OCSP",// "CSPO",// "PCSO",// - "SOPC" // + "SOPC",// + "SCOP" // }; static final transient SPOKeyOrder[] values = new SPOKeyOrder[] { @@ -158,7 +161,7 @@ CSPO, PCSO, SOPC, - + SCOP, }; static final transient GeneralComparator[] comparators = new GeneralComparator[] { @@ -172,7 +175,8 @@ new GeneralComparator(_OCSP),// new GeneralComparator(_CSPO),// new GeneralComparator(_PCSO),// - new GeneralComparator(_SOPC) }; + new GeneralComparator(_SOPC),// + new GeneralComparator(_SCOP) }; /* * Constants corresponding to the columns of the SPO(C) relation. @@ -197,6 +201,7 @@ new int[] { C, S, P, O }, // new int[] { P, C, S, O }, // new int[] { S, O, P, C }, // + new int[] { S, C, O, P }, // }; /** @@ -286,6 +291,7 @@ case _CSPO: case _PCSO: case _SOPC: + case _SCOP: return 4; default: throw new AssertionError(); @@ -345,6 +351,7 @@ case _CSPO: case _PCSO: case _SOPC: + case _SCOP: return comparators[index]; default: throw new AssertionError(); @@ -633,6 +640,13 @@ c = _3; break; + case SPOKeyOrder._SCOP: + s = _0; + c = _1; + o = _2; + p = _3; + break; + default: throw new UnsupportedOperationException(); @@ -823,7 +837,7 @@ * The quad store indices. */ static private final transient SPOKeyOrder[] quadStoreIndices = { SPOC, - POCS, OCSP, CSPO, PCSO, SOPC }; + POCS, OCSP, CSPO, PCSO, SOPC,SCOP }; /** * Return an iterator which visits the triple store indices ({@link #SPO}, Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOPredicate.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -455,6 +455,10 @@ if(newConstraint == null) throw new IllegalArgumentException(); + if(this.constraint!=null&&this.constraint==newConstraint){ + return this; + } + final IElementFilter<ISPO> tmp = this.constraint == null ? newConstraint : new WrappedSPOFilter(newConstraint, this.constraint); @@ -478,7 +482,7 @@ * @author <a href="mailto:tho...@us...">Bryan Thompson</a> * @version $Id$ */ - private static class WrappedSPOFilter extends SPOFilter { + protected static class WrappedSPOFilter extends SPOFilter { /** * Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPORelation.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -128,6 +128,8 @@ protected static final transient Logger log = Logger .getLogger(SPORelation.class); + protected final transient boolean DEBUG = log.isDebugEnabled(); + private final Set<String> indexNames; private final int keyArity; @@ -217,6 +219,8 @@ } + final protected int maxHashSetCapacity; + public SPORelation(final IIndexManager indexManager, final String namespace, final Long timestamp, final Properties properties) { @@ -332,6 +336,11 @@ // // } + this.maxHashSetCapacity=Integer.parseInt( + getProperty( + AbstractTripleStore.Options.DISTINCT_ITERATOR_MAX_HASH_SET_CAPACITY, + AbstractTripleStore.Options.DEFAULT_DISTINCT_ITERATOR_MAX_HASH_SET_CAPACITY + )); } /** @@ -582,7 +591,7 @@ if (!src.hasNext()) return new EmptyChunkedIterator<ISPO>(SPOKeyOrder.SPO); - return new DistinctSPOIterator(this, src); + return new DistinctSPOIterator(this, src,maxHashSetCapacity); } @@ -1037,7 +1046,7 @@ final SPOAccessPath accessPath = getAccessPath(keyOrder, predicate); - if (log.isDebugEnabled()) + if (DEBUG) log.debug(accessPath.toString()); // System.err.println("new access path: pred="+predicate); @@ -1866,6 +1875,12 @@ filter, sortTime, insertTime, mutationCount, false/* reportMutation */)); + tasks.add(new SPOIndexWriter(this, a, numStmts, + true/* clone */, SPOKeyOrder.SCOP, + SPOKeyOrder.SCOP.isPrimaryIndex(), + filter, sortTime, + insertTime, mutationCount, false/* reportMutation */)); + } } @@ -2030,7 +2045,13 @@ SPOKeyOrder.SOPC, SPOKeyOrder.SOPC.isPrimaryIndex(), true/* clone */, sortTime, writeTime, mutationCount, false/* reportMutation */)); + + tasks.add(new SPOIndexRemover(this, stmts, numStmts, + SPOKeyOrder.SCOP, SPOKeyOrder.SCOP.isPrimaryIndex(), + true/* clone */, sortTime, writeTime, + mutationCount, false/* reportMutation */)); + } } Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/spo/SPOStarJoin.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -20,7 +20,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ + */ package com.bigdata.rdf.spo; import java.io.Serializable; @@ -31,6 +31,7 @@ import java.util.Set; import com.bigdata.rdf.internal.IV; import com.bigdata.rdf.internal.IVUtility; +import com.bigdata.rdf.spo.SPOPredicate.WrappedSPOFilter; import com.bigdata.relation.accesspath.IElementFilter; import com.bigdata.relation.rule.Constant; import com.bigdata.relation.rule.IBindingSet; @@ -41,59 +42,49 @@ import com.bigdata.relation.rule.Var; /** - * Implementation of a star join for SPOs. See {@link IStarJoin}. + * Implementation of a star join for SPOs. See {@link IStarJoin}. * * @author <a href="mailto:mrp...@us...">Mike Personick</a> */ -public class SPOStarJoin extends SPOPredicate - implements IStarJoin<ISPO>, Serializable { +public class SPOStarJoin extends SPOPredicate implements IStarJoin<ISPO>, Serializable { /** - * generated serial version UID + * generated serial version UID */ - private static final long serialVersionUID = 981603459301801862L; - + private static final long serialVersionUID = 981603459301801862L; + /** * The star constraints for this star join. */ protected final Collection<IStarConstraint<ISPO>> starConstraints; /** - * Construct an SPO star join from a normal SPO predicate. The star join - * will have a triple pattern of (S,?,?) instead of the (S,P,O) from the - * original SPO predicate. This way all SPOs for the common subject are - * considered. SPO star constraints must be added later to make this - * star join selective. - * + * Construct an SPO star join from a normal SPO predicate. The star join will have a triple pattern of (S,?,?) instead of the (S,P,O) from the original SPO + * predicate. This way all SPOs for the common subject are considered. SPO star constraints must be added later to make this star join selective. + * * @param pred - * the normal SPO predicate from which to pull the S + * the normal SPO predicate from which to pull the S */ public SPOStarJoin(final SPOPredicate pred) { - - this(pred.relationName, pred.partitionId, pred.s(), Var.var(), - Var.var(), pred.c(), pred.isOptional(), - pred.getConstraint(), pred.getSolutionExpander()); - + + this(pred.relationName, pred.partitionId, pred.s(), Var.var(), Var.var(), pred.c(), pred.isOptional(), pred.getConstraint(), pred.getSolutionExpander()); + } - + /** * Create an SPO star join over the given relation for the given subject. - * + * * @param relationName - * the name of the SPO relation to use + * the name of the SPO relation to use * @param s - * the subject of this star join + * the subject of this star join */ - public SPOStarJoin(final String relationName, - final IVariableOrConstant<IV> s) { + public SPOStarJoin(final String relationName, final IVariableOrConstant<IV> s) { - this(new String[] { relationName }, -1/* partitionId */, s, - Var.var(), Var.var(), - null/* c */, false/* optional */, null/* constraint */, - null/* expander */); + this(new String[] { relationName }, -1/* partitionId */, s, Var.var(), Var.var(), null/* c */, false/* optional */, null/* constraint */, null/* expander */); } - + /** * Fully specified ctor. * @@ -102,7 +93,7 @@ * @param s * @param p * @param o - * @param c + * @param c * MAY be <code>null</code>. * @param optional * @param constraint @@ -119,154 +110,181 @@ final boolean optional, // final IElementFilter<ISPO> constraint,// final ISolutionExpander<ISPO> expander// - ) { - - super(relationName, partitionId, s, p, o, c, optional, constraint, - expander); - + ) { + + super(relationName, partitionId, s, p, o, c, optional, constraint, expander); + this.starConstraints = new LinkedList<IStarConstraint<ISPO>>(); - + } - + /** * Add an SPO star constraint to this star join. */ public void addStarConstraint(IStarConstraint<ISPO> constraint) { - + starConstraints.add(constraint); - + } - + /** * Return an iterator over the SPO star constraints for this star join. */ public Iterator<IStarConstraint<ISPO>> getStarConstraints() { - + return starConstraints.iterator(); - + } - + /** * Return the number of star constraints for this star join. */ public int getNumStarConstraints() { - + return starConstraints.size(); - + } - + /** * Return an iterator over the constraint variables for this star join. */ public Iterator<IVariable> getConstraintVariables() { - + final Set<IVariable> vars = new HashSet<IVariable>(); for (IStarConstraint constraint : starConstraints) { - + if (((SPOStarConstraint) constraint).p.isVar()) { vars.add((IVariable) ((SPOStarConstraint) constraint).p); } - + if (((SPOStarConstraint) constraint).o.isVar()) { vars.add((IVariable) ((SPOStarConstraint) constraint).o); } - + } - + return vars.iterator(); - + } - + /** - * Return an as-bound version of this star join and its star contraints - * using the supplied binding set. + * Return an as-bound version of this star join and its star contraints using the supplied binding set. */ @Override public SPOPredicate asBound(IBindingSet bindingSet) { - + SPOPredicate pred = super.asBound(bindingSet); - - SPOStarJoin starJoin = new SPOStarJoin(pred.relationName, - pred.partitionId, pred.s, pred.p, pred.o, pred.c, pred.optional, - pred.constraint, pred.expander); - + + SPOStarJoin starJoin = new SPOStarJoin(pred.relationName, pred.partitionId, pred.s, pred.p, pred.o, pred.c, pred.optional, pred.constraint, pred.expander); + for (IStarConstraint starConstraint : starConstraints) { - + starJoin.addStarConstraint(starConstraint.asBound(bindingSet)); - + } - + return starJoin; - + } - + + public SPOPredicate reBound(final IVariableOrConstant<IV> s, final IVariableOrConstant<IV> p, final IVariableOrConstant<IV> o, final IVariableOrConstant<IV> c) { + + SPOPredicate pred = super.reBound(s, p, o, c); + SPOStarJoin starJoin = new SPOStarJoin(pred.relationName, pred.partitionId, pred.s, pred.p, pred.o, pred.c, pred.optional, pred.constraint, pred.expander); + + for (IStarConstraint starConstraint : starConstraints) { + + starJoin.addStarConstraint(((SPOStarConstraint) starConstraint).reBound(s, p, o, c)); + + } + + return starJoin; + + } + + @Override + public SPOPredicate setConstraint(IElementFilter<ISPO> newConstraint) { + if(newConstraint == null) + throw new IllegalArgumentException(); + + final IElementFilter<ISPO> tmp = this.constraint == null ? newConstraint + : new WrappedSPOFilter(newConstraint, this.constraint); + + SPOStarJoin starJoin = new SPOStarJoin(relationName, partitionId, s, p, o, c, optional, tmp, expander); + + for (IStarConstraint starConstraint : starConstraints) { + + starJoin.addStarConstraint(starConstraint); + + } + + return starJoin; + + } + protected StringBuilder toStringBuilder(final IBindingSet bindingSet) { - + StringBuilder sb = super.toStringBuilder(bindingSet); - + if (starConstraints.size() > 0) { - + sb.append("star["); - + for (IStarConstraint sc : starConstraints) { - + sb.append(sc); - + sb.append(","); - + } - - sb.setCharAt(sb.length()-1, ']'); - + + sb.setCharAt(sb.length() - 1, ']'); + } - + return sb; - + } - + /** - * Implementation of a star constraint for SPOs. Constraint will specify - * a P and O (variable or constant) and whether the constraint is optional - * or non-optional. + * Implementation of a star constraint for SPOs. Constraint will specify a P and O (variable or constant) and whether the constraint is optional or + * non-optional. */ - public static class SPOStarConstraint implements IStarConstraint<ISPO>, - Serializable { - + public static class SPOStarConstraint implements IStarConstraint<ISPO>, Serializable { + /** * generated serial version UID */ - private static final long serialVersionUID = 997244773880938817L; + private static final long serialVersionUID = 997244773880938817L; /** * Variable or constant P for the constraint. */ protected final IVariableOrConstant<IV> p; - + /** * Variable or constant O for the constraint. */ protected final IVariableOrConstant<IV> o; - + /** * Is the constraint optional or non-optional. */ - protected final boolean optional; - + protected final boolean optional; + /** - * Construct a non-optional SPO star constraint using the supplied P and - * O. + * Construct a non-optional SPO star constraint using the supplied P and O. * * @param p * @param o */ - public SPOStarConstraint(final IVariableOrConstant<IV> p, - final IVariableOrConstant<IV> o) { - + public SPOStarConstraint(final IVariableOrConstant<IV> p, final IVariableOrConstant<IV> o) { + this(p, o, false /* optional */); - + } - + /** * Fully specified ctor. * @@ -274,83 +292,76 @@ * @param o * @param optional */ - public SPOStarConstraint(final IVariableOrConstant<IV> p, - final IVariableOrConstant<IV> o, final boolean optional) { - + public SPOStarConstraint(final IVariableOrConstant<IV> p, final IVariableOrConstant<IV> o, final boolean optional) { + this.p = p; - + this.o = o; - + this.optional = optional; - + } - + final public IVariableOrConstant<IV> p() { - + return p; - + } final public IVariableOrConstant<IV> o() { - + return o; - + } - + final public boolean isOptional() { - + return optional; - + } - + final public int getNumVars() { - + return (p.isVar() ? 1 : 0) + (o.isVar() ? 1 : 0); } - + /** - * Tests the P and O of the supplied SPO against the constraint. Return - * true for a match. + * Tests the P and O of the supplied SPO against the constraint. Return true for a match. */ final public boolean isMatch(ISPO spo) { - - return ((p.isVar() || IVUtility.equals(p.get(), spo.p())) && - (o.isVar() || IVUtility.equals(o.get(), spo.o()))); - + + return ((p.isVar() || IVUtility.equals(p.get(), spo.p())) && (o.isVar() || IVUtility.equals(o.get(), spo.o()))); + } - + /** - * Use the supplied SPO to create variable bindings for supplied - * binding set. + * Use the supplied SPO to create variable bindings for supplied binding set. */ final public void bind(IBindingSet bs, ISPO spo) { - + if (p.isVar()) { - - bs.set((IVariable) p, - new Constant<IV>(spo.p())); - + + bs.set((IVariable) p, new Constant<IV>(spo.p())); + } - + if (o.isVar()) { - - bs.set((IVariable) o, - new Constant<IV>(spo.o())); - + + bs.set((IVariable) o, new Constant<IV>(spo.o())); + } - + } - + /** - * Return an as-bound version of this SPO star constraint for the - * supplied binding set. + * Return an as-bound version of this SPO star constraint for the supplied binding set. */ public IStarConstraint<ISPO> asBound(IBindingSet bindingSet) { - + final IVariableOrConstant<IV> p; { - if (this.p.isVar() && bindingSet.isBound((IVariable)this.p)) { + if (this.p.isVar() && bindingSet.isBound((IVariable) this.p)) { p = bindingSet.get((IVariable) this.p); @@ -360,7 +371,7 @@ } } - + final IVariableOrConstant<IV> o; { if (this.o.isVar() && bindingSet.isBound((IVariable) this.o)) { @@ -373,55 +384,63 @@ } } - + return new SPOStarConstraint(p, o, optional); - + } - + + /** + * Return an as-bound version of this SPO star constraint for the supplied binding set. + */ + public IStarConstraint<ISPO> reBound(final IVariableOrConstant<IV> s, final IVariableOrConstant<IV> p, final IVariableOrConstant<IV> o, final IVariableOrConstant<IV> c) { + if (!p.isVar() || !o.isVar()) { + return new SPOStarConstraint(!p.isVar() ? p : this.p, !o.isVar() ? o : this.o, optional); + } else { + return this; + } + + } + public String toString() { - + return toString(null); - + } - + public String toString(final IBindingSet bindingSet) { final StringBuilder sb = new StringBuilder(); sb.append("("); - sb.append(p.isConstant() || bindingSet == null - || !bindingSet.isBound((IVariable) p) ? p.toString() - : bindingSet.get((IVariable) p)); + sb.append(p.isConstant() || bindingSet == null || !bindingSet.isBound((IVariable) p) ? p.toString() : bindingSet.get((IVariable) p)); sb.append(", "); - sb.append(o.isConstant() || bindingSet == null - || !bindingSet.isBound((IVariable) o) ? o.toString() - : bindingSet.get((IVariable) o)); + sb.append(o.isConstant() || bindingSet == null || !bindingSet.isBound((IVariable) o) ? o.toString() : bindingSet.get((IVariable) o)); sb.append(")"); if (optional) { - + /* * Something special, so do all this stuff. */ - + sb.append("["); - - if(optional) { + + if (optional) { sb.append("optional"); } sb.append("]"); - + } - + return sb.toString(); } } - + } Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/bigdata-rdf/src/java/com/bigdata/rdf/store/AbstractTripleStore.java 2010-08-31 17:22:33 UTC (rev 3473) @@ -104,6 +104,7 @@ import com.bigdata.rdf.rules.RuleContextEnum; import com.bigdata.rdf.spo.BulkCompleteConverter; import com.bigdata.rdf.spo.BulkFilterConverter; +import com.bigdata.rdf.spo.DistinctSPOIterator; import com.bigdata.rdf.spo.ExplicitSPOFilter; import com.bigdata.rdf.spo.ISPO; import com.bigdata.rdf.spo.JustificationWriter; @@ -895,6 +896,21 @@ String DEFAULT_EXTENSION_FACTORY_CLASS = DefaultExtensionFactory.class .getName(); + + + /** + * After this many entries in a DistinctSPOIterator, + * a {@link #btreeSet} is created which can spill + * out onto the disk. + */ + String DISTINCT_ITERATOR_MAX_HASH_SET_CAPACITY = DistinctSPOIterator.class.getName() + + ".maxHashSetCapacity"; + + /** + * Default for {@link #DISTINCT_ITERATOR_MAX_HASH_SET_CAPACITY} + */ + String DEFAULT_DISTINCT_ITERATOR_MAX_HASH_SET_CAPACITY = "100000"; + } protected Class determineAxiomClass() { Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/build.properties =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/build.properties 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/build.properties 2010-08-31 17:22:33 UTC (rev 3473) @@ -37,7 +37,7 @@ # The build version (note: 0.82b -> 0.82.0) build.ver=0.83.2 -build.ver.osgi=0.83 +build.ver.osgi=0.83.2 # Set true to do a snapshot build. This changes the value of ${version} to # include the date. Modified: branches/BIGDATA_RELEASE_0_83_2_CSI/build.xml =================================================================== --- branches/BIGDATA_RELEASE_0_83_2_CSI/build.xml 2010-08-31 16:21:47 UTC (rev 3472) +++ branches/BIGDATA_RELEASE_0_83_2_CSI/build.xml 2010-08-31 17:22:33 UTC (rev 3473) @@ -157,7 +157,7 @@ <fileset dir="bigdata-sails/src/java" /> </jar> <bnd - output="${build.dir}/bundles/com.bigata-${osgi.version}.jar" + output="${build.dir}/bundles/com.bigata_${osgi.version}.jar" classpath="${build.dir}/classes" eclipse="false" failok="false" This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |