[Bigdata-commit] SF.net SVN: bigdata:[3394] trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/ AsynchronousStatementBufferFactory.java

[<mrp...@us...>]

Revision: 3394
          http://bigdata.svn.sourceforge.net/bigdata/?rev=3394&view=rev
Author:   mrpersonick
Date:     2010-08-02 21:37:44 +0000 (Mon, 02 Aug 2010)

Log Message:
-----------
keeping inline values out of the lexicon indices

Modified Paths:
--------------
    trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java

Modified: trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java
===================================================================
--- trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java	2010-08-02 21:36:53 UTC (rev 3393)
+++ trunk/bigdata-rdf/src/java/com/bigdata/rdf/rio/AsynchronousStatementBufferFactory.java	2010-08-02 21:37:44 UTC (rev 3394)
@@ -2803,11 +2803,28 @@
      */
     @SuppressWarnings("unchecked")
     static <V extends BigdataValue> IChunkedIterator<V> newT2IdIterator(
-            final Iterator<V> itr, final int chunkSize) {
+            final LexiconRelation r, final Iterator<V> itr, 
+            final int chunkSize) {
 
-        return new ChunkedWrappedIterator(new Striterator(itr), chunkSize,
-                BigdataValue.class);
+        return new ChunkedWrappedIterator(new Striterator(itr)
+                .addFilter(new Filter() {
 
+                    private static final long serialVersionUID = 1L;
+        
+                    /*
+                     * Filter hides inline terms since we don't want to write
+                     * them on the forward index.
+                     */
+                    @Override
+                    protected boolean isValid(Object obj) {
+        
+                        // will set IV on BigdataValues as a side-effect
+                        return r.getInlineIV((Value) obj) != null;
+        
+                    }
+        
+                }), chunkSize, BigdataValue.class);
+
     }
 
     /**
@@ -3064,6 +3081,13 @@
 
                         }
 
+                        if (v.getIV().isInline()) {
+                            
+                            // Do not write inline terms on the reverse index.
+                            continue;
+                            
+                        }
+                        
                         final byte[] key = v.getIV().encode(tmp.reset())
                                 .getKey();
 
@@ -3767,8 +3791,9 @@
             try {
 
                 final Callable<Void> task = new AsyncTerm2IdIndexWriteTask(
-                        tidsLatch, lexiconRelation, newT2IdIterator(values
-                                .values().iterator(), producerChunkSize),
+                        tidsLatch, lexiconRelation, newT2IdIterator(
+                                lexiconRelation, values.values().iterator(), 
+                                producerChunkSize),
                         buffer_t2id);
 
                 // queue chunks onto the write buffer.


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.