Thread: [Exist-commits] SF.net SVN: exist:[18393] branches/rangeindex

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 454-5900

Revision: 18393
          http://sourceforge.net/p/exist/code/18393
Author:   wolfgang_m
Date:     2013-05-02 20:15:12 +0000 (Thu, 02 May 2013)
Log Message:
-----------
[feature] 
* Updating lucene index to version 4.2.1. Major architectural changes in lucene require some redesigns: nodeId and docId are now stored as docvalue fields, which are much faster than normal fields.

* Enable extension indexes to hook into eXist's query optimizer and rewrite expressions by implementing interface QueryRewriter.

Modified Paths:
--------------
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndex.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndexWorker.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneMatchListener.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneUtil.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/PlainTextHighlighter.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/XMLToQuery.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/xquery/modules/lucene/GetField.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/xquery/modules/lucene/Query.java
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/xquery/modules/lucene/QueryField.java
    branches/rangeindex/extensions/indexes/lucene/test/src/org/exist/indexing/lucene/LuceneIndexTest.java
    branches/rangeindex/extensions/indexes/ngram/src/org/exist/indexing/ngram/NGramIndexWorker.java
    branches/rangeindex/extensions/indexes/sort/src/org/exist/indexing/sort/SortIndexWorker.java
    branches/rangeindex/src/org/exist/collections/CollectionConfigurationManager.java
    branches/rangeindex/src/org/exist/fulltext/FTIndexWorker.java
    branches/rangeindex/src/org/exist/indexing/AbstractIndex.java
    branches/rangeindex/src/org/exist/indexing/IndexController.java
    branches/rangeindex/src/org/exist/indexing/IndexWorker.java
    branches/rangeindex/src/org/exist/storage/NodePath.java
    branches/rangeindex/src/org/exist/storage/statistics/IndexStatisticsWorker.java
    branches/rangeindex/src/org/exist/storage/structural/NativeStructuralIndex.java
    branches/rangeindex/src/org/exist/storage/structural/NativeStructuralIndexWorker.java
    branches/rangeindex/src/org/exist/xquery/Optimizer.java

Added Paths:
-----------
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-common-4.2.1.jar
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-4.2.1.jar
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-4.2.1.jar
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-queryparser-4.2.1.jar
    branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/BinaryTokenStream.java
    branches/rangeindex/src/org/exist/xquery/QueryRewriter.java

Removed Paths:
-------------
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-3.6.1.jar
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-3.6.1.jar
    branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-3.6.1.jar

Added: branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-common-4.2.1.jar
===================================================================
(Binary files differ)

Index: branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-common-4.2.1.jar
===================================================================

--- branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-common-4.2.1.jar	2013-05-02 19:34:41 UTC (rev 18392)
+++ branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-common-4.2.1.jar	2013-05-02 20:15:12 UTC (rev 18393)

Property changes on: branches/rangeindex/extensions/indexes/lucene/lib/lucene-analyzers-common-4.2.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-4.2.1.jar
===================================================================
(Binary files differ)

Index: branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-4.2.1.jar
===================================================================
--- branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-4.2.1.jar	2013-05-02 19:34:41 UTC (rev 18392)
+++ branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-4.2.1.jar	2013-05-02 20:15:12 UTC (rev 18393)

Property changes on: branches/rangeindex/extensions/indexes/lucene/lib/lucene-core-4.2.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-4.2.1.jar
===================================================================
(Binary files differ)

Index: branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-4.2.1.jar
===================================================================
--- branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-4.2.1.jar	2013-05-02 19:34:41 UTC (rev 18392)
+++ branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-4.2.1.jar	2013-05-02 20:15:12 UTC (rev 18393)

Property changes on: branches/rangeindex/extensions/indexes/lucene/lib/lucene-queries-4.2.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: branches/rangeindex/extensions/indexes/lucene/lib/lucene-queryparser-4.2.1.jar
===================================================================
(Binary files differ)

Index: branches/rangeindex/extensions/indexes/lucene/lib/lucene-queryparser-4.2.1.jar
===================================================================
--- branches/rangeindex/extensions/indexes/lucene/lib/lucene-queryparser-4.2.1.jar	2013-05-02 19:34:41 UTC (rev 18392)
+++ branches/rangeindex/extensions/indexes/lucene/lib/lucene-queryparser-4.2.1.jar	2013-05-02 20:15:12 UTC (rev 18393)

Property changes on: branches/rangeindex/extensions/indexes/lucene/lib/lucene-queryparser-4.2.1.jar
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/octet-stream
\ No newline at end of property
Added: branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/BinaryTokenStream.java
===================================================================
--- branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/BinaryTokenStream.java	                        (rev 0)
+++ branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/BinaryTokenStream.java	2013-05-02 20:15:12 UTC (rev 18393)
@@ -0,0 +1,67 @@
+package org.exist.indexing.lucene;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A binary tokenstream that lets you index a single
+ * binary token (BytesRef value).
+ *
+ */
+public final class BinaryTokenStream extends TokenStream {
+    private final ByteTermAttribute bytesAtt = addAttribute(ByteTermAttribute.class);
+    private boolean available = true;
+
+    public BinaryTokenStream(BytesRef bytes) {
+        bytesAtt.setBytesRef(bytes);
+    }
+
+    @Override
+    public boolean incrementToken() {
+        if (available) {
+            available = false;
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public void reset() {
+        available = true;
+    }
+
+    public interface ByteTermAttribute extends TermToBytesRefAttribute {
+        public void setBytesRef(BytesRef bytes);
+    }
+
+    public static class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute,TermToBytesRefAttribute {
+        private BytesRef bytes;
+
+        @Override
+        public int fillBytesRef() {
+            return bytes.hashCode();
+        }
+
+        @Override
+        public BytesRef getBytesRef() {
+            return bytes;
+        }
+
+        @Override
+        public void setBytesRef(BytesRef bytes) {
+            this.bytes = bytes;
+        }
+
+        @Override
+        public void clear() {}
+
+        @Override
+        public void copyTo(AttributeImpl target) {
+            ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
+            other.bytes = bytes;
+        }
+    }
+}
+

Modified: branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndex.java
===================================================================
--- branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndex.java	2013-05-02 19:34:41 UTC (rev 18392)
+++ branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndex.java	2013-05-02 20:15:12 UTC (rev 18393)
@@ -3,8 +3,7 @@
 import org.apache.log4j.Logger;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.*;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
@@ -25,13 +24,11 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.util.Version;
 
 public class LuceneIndex extends AbstractIndex implements RawBackupSupport {
     
-    public final static Version LUCENE_VERSION_IN_USE = Version.LUCENE_34;
+    public final static Version LUCENE_VERSION_IN_USE = Version.LUCENE_42;
 
     private static final Logger LOG = Logger.getLogger(LuceneIndexWorker.class);
 
@@ -42,7 +39,7 @@
     protected Directory directory;
     protected Analyzer defaultAnalyzer;
 
-    protected double bufferSize = IndexWriter.DEFAULT_RAM_BUFFER_SIZE_MB;
+    protected double bufferSize = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
 
     protected IndexWriter cachedWriter = null;
     protected int writerUseCount = 0;
@@ -59,6 +56,10 @@
         //Nothing special to do
     }
 
+    public String getDirName() {
+        return DIR_NAME;
+    }
+
     @Override
     public void configure(BrokerPool pool, String dataDir, Element config) throws DatabaseConfigurationException {
         super.configure(pool, dataDir, config);
@@ -90,7 +91,7 @@
 
     @Override
     public void open() throws DatabaseConfigurationException {
-        File dir = new File(getDataDir(), DIR_NAME);
+        File dir = new File(getDataDir(), getDirName());
         if (LOG.isDebugEnabled())
             LOG.debug("Opening Lucene index directory: " + dir.getAbsolutePath());
         if (dir.exists()) {
@@ -161,11 +162,11 @@
     
     protected boolean needsCommit = false;
 
-    protected IndexWriter getWriter() throws IOException {
+    public IndexWriter getWriter() throws IOException {
         return getWriter(false);
     }
 
-    protected synchronized IndexWriter getWriter(boolean exclusive) throws IOException {
+    public synchronized IndexWriter getWriter(boolean exclusive) throws IOException {
         while (writingReaderUseCount > 0) {
             try {
                 wait();
@@ -200,7 +201,7 @@
         return cachedWriter;
     }
 
-    protected synchronized void releaseWriter(IndexWriter writer) {
+    public synchronized void releaseWriter(IndexWriter writer) {
         if (writer == null)
             return;
         if (writer != cachedWriter)
@@ -233,18 +234,18 @@
         }
     }
     
-    protected synchronized IndexReader getReader() throws IOException {
+    public synchronized IndexReader getReader() throws IOException {
     	commit();
         if (cachedReader != null) {
             readerUseCount++;
         } else {
-            cachedReader = IndexReader.open(directory);
+            cachedReader = DirectoryReader.open(directory);
             readerUseCount = 1;
         }
         return cachedReader;
     }
 
-    protected synchronized void releaseReader(IndexReader reader) {
+    public synchronized void releaseReader(IndexReader reader) {
         if (reader == null)
             return;
         if (reader != cachedReader)
@@ -273,7 +274,7 @@
         if (cachedWritingReader != null) {
             writingReaderUseCount++;
         } else {
-            cachedWritingReader = IndexReader.open(directory, false);
+            cachedWritingReader = DirectoryReader.open(directory);
             writingReaderUseCount = 1;
         }
         notifyAll();
@@ -317,15 +318,15 @@
         try {
         	cachedReader.close();
         	cachedReader = null;
-        	if (cachedSearcher != null)
-        		cachedSearcher.close();
+//        	if (cachedSearcher != null)
+//        		cachedSearcher.close();
         	cachedSearcher = null;
         } catch (IOException e) {
             LOG.warn("Exception while refreshing lucene index: " + e.getMessage(), e);
         }
     }
 
-    protected synchronized IndexSearcher getSearcher() throws IOException {
+    public synchronized IndexSearcher getSearcher() throws IOException {
     	commit();
         if (cachedSearcher != null) {
             searcherUseCount++;
@@ -337,7 +338,7 @@
         return cachedSearcher;
     }
 
-    protected synchronized void releaseSearcher(IndexSearcher searcher) {
+    public synchronized void releaseSearcher(IndexSearcher searcher) {
         if (searcher == null)
             return;
         if (searcher != cachedSearcher)
@@ -349,7 +350,7 @@
 	@Override
 	public void backupToArchive(RawDataBackup backup) throws IOException {
 		for (String name : directory.listAll()) {
-			String path = DIR_NAME + "/" + name;
+			String path = getDirName() + "/" + name;
 			OutputStream os = backup.newEntry(path);
 			InputStream is = new FileInputStream(new File(getDataDir(), path));
 	        byte[] buf = new byte[4096];

Modified: branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndexWorker.java
===================================================================
--- branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndexWorker.java	2013-05-02 19:34:41 UTC (rev 18392)
+++ branches/rangeindex/extensions/indexes/lucene/src/org/exist/indexing/lucene/LuceneIndexWorker.java	2013-05-02 20:15:12 UTC (rev 18393)
@@ -21,77 +21,18 @@
  */
 package org.exist.indexing.lucene;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.Stack;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
 import org.apache.log4j.Logger;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.NumericField;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.TermFreqVector;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Collector;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilteredQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.util.BitVector;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.util.OpenBitSet;
-import org.apache.lucene.util.ReaderUtil;
-import org.apache.lucene.util.Version;
-
+import org.apache.lucene.document.*;
+import org.apache.lucene.index.*;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.*;
 import org.exist.collections.Collection;
-import org.exist.dom.AttrImpl;
-import org.exist.dom.CharacterDataImpl;
-import org.exist.dom.DocumentImpl;
-import org.exist.dom.DocumentSet;
-import org.exist.dom.ElementImpl;
-import org.exist.dom.Match;
-import org.exist.dom.NewArrayNodeSet;
-import org.exist.dom.NodeProxy;
-import org.exist.dom.NodeSet;
-import org.exist.dom.QName;
-import org.exist.dom.StoredNode;
-import org.exist.dom.SymbolTable;
-import org.exist.indexing.AbstractStreamListener;
-import org.exist.indexing.IndexController;
-import org.exist.indexing.IndexWorker;
-import org.exist.indexing.MatchListener;
-import org.exist.indexing.OrderedValuesIndex;
-import org.exist.indexing.QNamedKeysIndex;
-import org.exist.indexing.StreamListener;
+import org.exist.dom.*;
+import org.exist.indexing.*;
 import org.exist.indexing.lucene.PlainTextHighlighter.Offset;
 import org.exist.indexing.lucene.PlainTextIndexConfig.PlainTextDoc;
 import org.exist.indexing.lucene.PlainTextIndexConfig.PlainTextField;
@@ -99,10 +40,7 @@
 import org.exist.memtree.NodeImpl;
 import org.exist.numbering.NodeId;
 import org.exist.security.PermissionDeniedException;
-import org.exist.storage.DBBroker;
-import org.exist.storage.ElementValue;
-import org.exist.storage.IndexSpec;
-import org.exist.storage.NodePath;
+import org.exist.storage.*;
 import org.exist.storage.txn.Txn;
 import org.exist.util.ByteConversion;
 import org.exist.util.DatabaseConfigurationException;
@@ -110,12 +48,14 @@
 import org.exist.xquery.*;
 import org.exist.xquery.value.IntegerValue;
 import org.exist.xquery.value.NodeValue;
-
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.helpers.AttributesImpl;
 
+import java.io.IOException;
+import java.util.*;
+
 /**
  *   Class for handling all Lucene operations.
  *
@@ -129,11 +69,18 @@
     public static final String OPTION_LEADING_WILDCARD = "leading-wildcard";
     public static final String OPTION_FILTER_REWRITE = "filter-rewrite";
     public static final String DEFAULT_OPERATOR_OR = "or";
-    
+
+    public static final org.apache.lucene.document.FieldType TYPE_NODE_ID = new org.apache.lucene.document.FieldType();
+    static {
+        TYPE_NODE_ID.setIndexed(true);
+        TYPE_NODE_ID.setStored(false);
+        TYPE_NODE_ID.setOmitNorms(true);
+        TYPE_NODE_ID.setStoreTermVectors(false);
+        TYPE_NODE_ID.setTokenized(true);
+    }
+
     static final Logger LOG = Logger.getLogger(LuceneIndexWorker.class);
     
-    private static final FieldSelector NODE_FIELD_SELECTOR = new NodeFieldSelector();
-    
     private LuceneIndex index;
     
     @SuppressWarnings("unused")
@@ -160,10 +107,11 @@
     
     private Analyzer analyzer;
 
-    public static final String FIELD_NODE_ID = "nodeId";
     public static final String FIELD_DOC_ID = "docId";
     public static final String FIELD_DOC_URI = "docUri";
 
+    private final byte[] buf = new byte[1024];
+
     public LuceneIndexWorker(LuceneIndex parent, DBBroker broker) {
         this.index = parent;
         this.broker = broker;
@@ -178,6 +126,11 @@
         return index.getIndexName();
     }
 
+    @Override
+    public QueryRewriter getQueryRewriter(XQueryContext context) {
+        return null;
+    }
+
     public Object configure(IndexController controller, NodeList configNodes, Map<String, String> namespaces) throws DatabaseConfigurationException {
         this.controller = controller;
         LOG.debug("Configuring lucene index...");
@@ -308,7 +261,9 @@
     	IndexWriter writer = null;
         try {
             writer = index.getWriter();
-            Term dt = new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(docId));
+            BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
+            NumericUtils.intToPrefixCoded(docId, 0, bytes);
+            Term dt = new Term(FIELD_DOC_ID, bytes);
             writer.deleteDocuments(dt);
         } catch (IOException e) {
             LOG.warn("Error while removing lucene index: " + e.getMessage(), e);
@@ -341,7 +296,9 @@
             writer = index.getWriter();
             for (Iterator<DocumentImpl> i = collection.iterator(broker); i.hasNext(); ) {
                 DocumentImpl doc = i.next();
-                Term dt = new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(doc.getDocId()));
+                BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
+                NumericUtils.intToPrefixCoded(doc.getDocId(), 0, bytes);
+                Term dt = new Term(FIELD_DOC_ID, bytes);
                 writer.deleteDocuments(dt);
             }
         } catch (IOException e) {
@@ -364,47 +321,42 @@
     protected void removeNodes() {
     	if (nodesToRemove == null)
             return;
-        IndexReader reader = null;
+        IndexWriter writer = null;
         try {
-            reader = index.getWritingReader();
-            Term dt = new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(currentDoc.getDocId()));
-            TermDocs docsEnum = reader.termDocs(dt);
-            while (docsEnum.next()) {
-                Document doc = reader.document(docsEnum.doc());
-                NodeId nodeId = readNodeId(doc);
-                if (nodesToRemove.contains(nodeId)) {
-                    reader.deleteDocument(docsEnum.doc());
-                }
+            writer = index.getWriter();
+
+            BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
+            NumericUtils.intToPrefixCoded(currentDoc.getDocId(), 0, bytes);
+            Term dt = new Term(FIELD_DOC_ID, bytes);
+            TermQuery tq = new TermQuery(dt);
+            for (NodeId nodeId : nodesToRemove) {
+                // store the node id
+                int nodeIdLen = nodeId.size();
+                byte[] data = new byte[nodeIdLen + 2];
+                ByteConversion.shortToByte((short) nodeId.units(), data, 0);
+                nodeId.serialize(data, 2);
+
+                Term it = new Term(LuceneUtil.FIELD_NODE_ID, new BytesRef(data));
+                TermQuery iq = new TermQuery(it);
+                BooleanQuery q = new BooleanQuery();
+                q.add(tq, BooleanClause.Occur.MUST);
+                q.add(iq, BooleanClause.Occur.MUST);
+                writer.deleteDocuments(q);
             }
-            nodesToRemove = null;
         } catch (IOException e) {
             LOG.warn("Error while deleting lucene index entries: " + e.getMessage(), e);
         } finally {
-            index.releaseWritingReader(reader);
+            index.releaseWriter(writer);
         }
     }
 
-    private class NodeIdFilter extends Filter {
-    	
-		@Override
-		public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-			LOG.debug("Reader: " + reader.getClass().getName());
-			OpenBitSet bits = new OpenBitSet(reader.maxDoc());
-        	Term dt = new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(currentDoc.getDocId()));
-			TermDocs docsEnum = reader.termDocs(dt);
-            while (docsEnum.next()) {
-            	LOG.debug("docsEnum: " + docsEnum.doc());
-                Document doc = reader.document(docsEnum.doc());
-                NodeId nodeId = readNodeId(doc);
-                if (nodesToRemove.contains(nodeId)) {
-                	LOG.debug("Removing node " + nodeId);
-                    bits.set(docsEnum.doc());
-                }
-            }
-			return bits;
-		}
+    private NodeId readNodeId(int doc, BinaryDocValues nodeIdValues, BrokerPool pool) {
+        BytesRef ref = new BytesRef(buf);
+        nodeIdValues.get(doc, ref);
+        int units = ByteConversion.byteToShort(ref.bytes, ref.offset);
+        return pool.getNodeFactory().createFromData(units, ref.bytes, ref.offset + 2);
     }
-    
+
     /**
      * Query the index. Returns a node set containing all matching nodes. Each node
      * in the node set has a {@link org.exist.indexing.lucene.LuceneIndexWorker.LuceneMatch}
@@ -433,7 +385,7 @@
         try {
             searcher = index.getSearcher();
             for (QName qname : qnames) {
-                String field = encodeQName(qname);
+                String field = LuceneUtil.encodeQName(qname, index.getBrokerPool().getSymbols());
                 Analyzer analyzer = getAnalyzer(null, qname, context.getBroker(), docs);
                 QueryParser parser = new QueryParser(LuceneIndex.LUCENE_VERSION_IN_USE, field, analyzer);
                 setOptions(options, parser);
@@ -506,7 +458,7 @@
         try {
             searcher = index.getSearcher();
             for (QName qname : qnames) {
-                String field = encodeQName(qname);
+                String field = LuceneUtil.encodeQName(qname, index.getBrokerPool().getSymbols());
                 analyzer = getAnalyzer(null, qname, context.getBroker(), docs);
                 Query query = queryTranslator.parse(field, queryRoot, analyzer, options);
                 if (query != null) {
@@ -543,9 +495,8 @@
     private void searchAndProcess(int contextId, QName qname, DocumentSet docs,
             NodeSet contextSet, NodeSet resultSet, boolean returnAncestor,
             IndexSearcher searcher, Query query, XQueryWatchDog watchDog) throws IOException, TerminatedException {
-        LuceneHitCollector collector = new LuceneHitCollector();
+        LuceneHitCollector collector = new LuceneHitCollector(qname, query, docs, contextSet, resultSet, returnAncestor, contextId, watchDog);
         searcher.search(query, collector);
-        processHits(collector.getDocs(), searcher, contextId, qname, docs, contextSet, resultSet, returnAncestor, query, watchDog);
     }
 
     public NodeSet queryField(XQueryContext context, int contextId, DocumentSet docs, NodeSet contextSet,
@@ -568,61 +519,6 @@
         }
         return resultSet;
     }
-    
-    /**
-     * Process the query results collected from the Lucene index and
-     * map them to the corresponding XML nodes in eXist.
-     */
-    private void processHits(List<ScoreDoc> hits, IndexSearcher searcher, int contextId, QName qname, DocumentSet docs, NodeSet contextSet,
-                             NodeSet resultSet, boolean returnAncestor, Query query, XQueryWatchDog watchDog) throws TerminatedException {
-        for (ScoreDoc scoreDoc : hits) {
-            watchDog.proceed(null);
-            try {
-                Document doc = searcher.doc(scoreDoc.doc, NODE_FIELD_SELECTOR);
-                String fDocId = doc.get(FIELD_DOC_ID);
-                int docId = Integer.parseInt(fDocId);
-                DocumentImpl storedDocument = docs.getDoc(docId);
-                if (storedDocument == null)
-                    continue;
-                NodeId nodeId = readNodeId(doc);
-                NodeProxy storedNode = new NodeProxy(storedDocument, nodeId);
-                if (qname != null)
-                	storedNode.setNodeType(qname.getNameType() == ElementValue.ATTRIBUTE ? Node.ATTRIBUTE_NODE : Node.ELEMENT_NODE);
-                // if a context set is specified, we can directly check if the
-                // matching node is a descendant of one of the nodes
-                // in the context set.
-                if (contextSet != null) {
-                    int sizeHint = contextSet.getSizeHint(storedDocument);
-                    if (returnAncestor) {
-                    	NodeProxy parentNode = contextSet.get(storedNode);
-                        // NodeProxy parentNode = contextSet.parentWithChild(storedNode, false, true, NodeProxy.UNKNOWN_NODE_LEVEL);
-                        if (parentNode != null) {
-                            LuceneMatch match = new LuceneMatch(contextId, nodeId, query);
-                            match.setScore(scoreDoc.score);
-                            parentNode.addMatch(match);
-                            resultSet.add(parentNode, sizeHint);
-                            if (Expression.NO_CONTEXT_ID != contextId) {
-                                parentNode.deepCopyContext(storedNode, contextId);
-                            } else
-                                parentNode.copyContext(storedNode);
-                        }
-                    } else {
-                        LuceneMatch match = new LuceneMatch(contextId, nodeId, query);
-                        match.setScore(scoreDoc.score);
-                        storedNode.addMatch(match);
-                        resultSet.add(storedNode, sizeHint);
-                    }
-                } else {
-                    LuceneMatch match = new LuceneMatch(contextId, nodeId, query);
-                    match.setScore(scoreDoc.score);
-                    storedNode.addMatch(match);
-                    resultSet.add(storedNode);
-                }
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
-    }
 
     /**
      * Add SOLR formatted data to lucene index.
@@ -658,12 +554,16 @@
         	pendingDoc = new Document();
         	
         	// Set DocId
-            NumericField fDocId = new NumericField(FIELD_DOC_ID, Field.Store.YES, true);
-            fDocId.setIntValue(currentDoc.getDocId());             
+            IntDocValuesField fDocId = new IntDocValuesField(FIELD_DOC_ID, currentDoc.getDocId());
+
             pendingDoc.add(fDocId);
-            
+
+            IntField fDocIdIdx = new IntField(FIELD_DOC_ID, currentDoc.getDocId(), Field.Store.NO);
+            pendingDoc.add(fDocIdIdx);
+
             // For binary documents the doc path needs to be stored
-            String uri = currentDoc.getURI().toString();  
+            String uri = currentDoc.getURI().toString();
+
             Field fDocUri = new Field(FIELD_DOC_URI, uri, Field.Store.YES, Field.Index.NOT_ANALYZED);
             pendingDoc.add(fDocUri);
         }
@@ -684,19 +584,16 @@
             String contentFieldName = field.getName();
             
             Analyzer fieldAnalyzer = (fieldType == null) ? null : fieldType.getAnalyzer();
-               
-            // Extract (document) Boost factor
-            if (field.getBoost() > 0) {
-                pendingDoc.setBoost(field.getBoost());
-            } 
             
             // Actual field content ; Store flag can be set in solrField
             Field contentField = new Field(contentFieldName, field.getData().toString(), 
                     store, Field.Index.ANALYZED, Field.TermVector.YES);
-            
-            // Set boost value from SOLR config
-            contentField.setBoost(field.getBoost());
-            
+
+            // Extract (document) Boost factor
+            if (field.getBoost() > 0) {
+                contentField.setBoost(field.getBoost());
+            }
+
             pendingDoc.add(contentField);
         }
     }
@@ -706,9 +603,6 @@
         try {
             writer = index.getWriter();
             
-            // by default, Lucene only indexes the first 10,000 terms in a field
-            writer.setMaxFieldLength(Integer.MAX_VALUE);
-            
             writer.addDocument(pendingDoc);
         } catch (IOException e) {
             LOG.warn("An exception was caught while indexing document: " + e.getMessage(), e);
@@ -727,7 +621,7 @@
      * @param queryText
      * @return search report
      */
-    public NodeImpl search(XQueryContext context, List<String> toBeMatchedURIs, String queryText) throws XPathException {
+    public NodeImpl search(XQueryContext context, final List<String> toBeMatchedURIs, String queryText) throws XPathException {
         
         NodeImpl report = null;
         
@@ -737,81 +631,85 @@
             searcher = index.getSearcher();
             
             // Get analyzer : to be retrieved from configuration
-            Analyzer searchAnalyzer = new StandardAnalyzer(Version.LUCENE_29);
+            final Analyzer searchAnalyzer = new StandardAnalyzer(Version.LUCENE_42);
 
             // Setup query Version, default field, analyzer
-            QueryParser parser = new QueryParser(Version.LUCENE_29, "", searchAnalyzer);
-            Query query = parser.parse(queryText);
+            final QueryParser parser = new QueryParser(Version.LUCENE_42, "", searchAnalyzer);
+            final Query query = parser.parse(queryText);
                        
             // extract all used fields from query
-            String[] fields = LuceneUtil.extractFields(query, searcher.getIndexReader());
+            final String[] fields = LuceneUtil.extractFields(query, searcher.getIndexReader());
 
-            // Setup collector for results
-            LuceneHitCollector collector = new LuceneHitCollector();
-            
-            // Perform actual search
-            searcher.search(query, collector);
+            final PlainTextHighlighter highlighter = new PlainTextHighlighter(query, searcher.getIndexReader());
 
-            // Retrieve all documents that match the query
-            List<ScoreDoc> results = collector.getDocsByScore();
-            
-            // reusable attributes
-            AttributesImpl attribs = null;
-            
-            PlainTextHighlighter highlighter = new PlainTextHighlighter(query, searcher.getIndexReader());
-            
-            MemTreeBuilder builder = new MemTreeBuilder();
+            final MemTreeBuilder builder = new MemTreeBuilder();
             builder.startDocument();
-            
+
             // start root element
-            int nodeNr = builder.startElement("", "results", "results", null);
-            
-            BitVector processed = new BitVector(searcher.maxDoc());
-            // Process result documents
-            for (ScoreDoc scoreDoc : results) {
-                if (processed.get(scoreDoc.doc))
-                	continue;
-                processed.set(scoreDoc.doc);
-                
-                Document doc = searcher.doc(scoreDoc.doc);
-                
-                // Get URI field of document                
-                String fDocUri = doc.get(FIELD_DOC_URI);
-                
-                // Get score
-                float score = scoreDoc.score;
-                
-                // Check if document URI has a full match or if a
-                // document is in a collection
-                if(isDocumentMatch(fDocUri, toBeMatchedURIs)){
-                    
-                    // setup attributes
-                    attribs = new AttributesImpl();
-                    attribs.addAttribute("", "uri", "uri", "CDATA", fDocUri);
-                    attribs.addAttribute("", "score", "score", "CDATA", ""+score);
+            final int nodeNr = builder.startElement("", "results", "results", null);
 
-                    // write element and attributes
-                    builder.startElement("", "search", "search", attribs);
-                    for (String field : fields) {
-                    	String[] fieldContent = doc.getValues(field);
-                    	attribs.clear();
-                    	attribs.addAttribute("", "name", "name", "CDATA", field);
-                    	for (String content : fieldContent) {
-                    		List<Offset> offsets = highlighter.getOffsets(content, searchAnalyzer);
-                    		if (offsets != null) {
-                    			builder.startElement("", "field", "field", attribs);
-                    			highlighter.highlight(content, offsets, builder);
-                    			builder.endElement();
-                    		}
-                    	}
+            // Perform actual search
+            searcher.search(query, new Collector() {
+                private Scorer scorer;
+                private AtomicReader reader;
+
+                @Override
+                public void setScorer(Scorer scorer) throws IOException {
+                    this.scorer = scorer;
+                }
+
+                @Override
+                public void collect(int docNum) throws IOException {
+                    Document doc = reader.document(docNum);
+
+                    // Get URI field of document
+                    String fDocUri = doc.get(FIELD_DOC_URI);
+
+                    // Get score
+                    float score = scorer.score();
+
+                    // Check if document URI has a full match or if a
+                    // document is in a collection
+                    if(isDocumentMatch(fDocUri, toBeMatchedURIs)){
+
+                        // setup attributes
+                        AttributesImpl attribs = new AttributesImpl();
+                        attribs.addAttribute("", "uri", "uri", "CDATA", fDocUri);
+                        attribs.addAttribute("", "score", "score", "CDATA", ""+score);
+
+                        // write element and attributes
+                        builder.startElement("", "search", "search", attribs);
+                        for (String field : fields) {
+                            String[] fieldContent = doc.getValues(field);
+                            attribs.clear();
+                            attribs.addAttribute("", "name", "name", "CDATA", field);
+                            for (String content : fieldContent) {
+                                List<Offset> offsets = highlighter.getOffsets(content, searchAnalyzer);
+                                if (offsets != null) {
+                                    builder.startElement("", "field", "field", attribs);
+                                    highlighter.highlight(content, offsets, builder);
+                                    builder.endElement();
+                                }
+                            }
+                        }
+                        builder.endElement();
+
+                        // clean attributes
+                        attribs.clear();
                     }
-                    builder.endElement();
+                }
 
-                    // clean attributes
-                    attribs.clear();
-                }           
-            }
-            
+                @Override
+                public void setNextReader(AtomicReaderContext atomicReaderContext) throws IOException {
+                    this.reader = atomicReaderContext.reader();
+                }
+
+                @Override
+                public boolean acceptsDocsOutOfOrder() {
+                    return true;
+                }
+            });
+
             // finish root element
             builder.endElement();
             
@@ -834,34 +732,50 @@
         return report;
     }
     
-    public String getFieldContent(XQueryContext context, int docId, String field) throws IOException {
-    	Term dt = new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(docId));
+    public String getFieldContent(int docId, String field) throws IOException {
+        BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
+        NumericUtils.intToPrefixCoded(docId, 0, bytes);
+        Term dt = new Term(FIELD_DOC_ID, bytes);
+
     	IndexReader reader = null;
         try {
             reader = index.getReader();
-            TermDocs td = reader.termDocs(dt);
-            if (!td.next())
-            	return null;
-            Document doc = reader.document(td.doc());
-            td.close();
-            return doc.get(field);
-            
+
+            List<AtomicReaderContext> leaves = reader.leaves();
+            for (AtomicReaderContext context : leaves) {
+                DocsEnum docs = context.reader().termDocsEnum(dt);
+                if (docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+                    Document doc = reader.document(docs.docID());
+                    return doc.get(field);
+                }
+            }
         } finally {
             index.releaseReader(reader);
         }
+        return null;
     }
     
     public boolean hasIndex(int docId) throws IOException {
-    	Term dt = new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(docId));
-    	IndexReader reader = null;
+        BytesRef bytes = new BytesRef(NumericUtils.BUF_SIZE_INT);
+        NumericUtils.intToPrefixCoded(docId, 0, bytes);
+        Term dt = new Term(FIELD_DOC_ID, bytes);
+
+        IndexReader reader = null;
         try {
             reader = index.getReader();
-            TermDocs td = reader.termDocs(dt);
-            boolean found = td.next();
-            td.close();
+
+            boolean found = false;
+            List<AtomicReaderContext> leaves = reader.leaves();
+            for (AtomicReaderContext context : leaves) {
+                DocsEnum docs = context.reader().termDocsEnum(dt);
+                if (docs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+                    found = true;
+                    break;
+                }
+            }
             return found;
         } finally {
-        	index.releaseReader(reader);
+            index.releaseReader(reader);
         }
     }
     
@@ -893,56 +807,45 @@
         return false;
     }
 
-    private static class LuceneHitCollector extends Collector {
+    private class LuceneHitCollector extends Collector {
 
-        private List<ScoreDoc> docs = new ArrayList<ScoreDoc>();
-        private int docBase;
         private Scorer scorer;
 
-        private LuceneHitCollector() {
-            //Nothing special to do
-        }
+        private AtomicReader reader;
+        private NumericDocValues docIdValues;
+        private BinaryDocValues nodeIdValues;
+        private final byte[] buf = new byte[1024];
+        private final QName qname;
+        private final DocumentSet docs;
+        private final NodeSet contextSet;
+        private final NodeSet resultSet;
+        private final boolean returnAncestor;
+        private final int contextId;
+        private final Query query;
+        private final XQueryWatchDog watchdog;
 
-        public List<ScoreDoc> getDocs() {
-            Collections.sort(docs, new Comparator<ScoreDoc>() {
-
-                public int compare(ScoreDoc scoreDoc, ScoreDoc scoreDoc1) {
-                    if (scoreDoc.doc == scoreDoc1.doc)
-                        return 0;
-                    else if (scoreDoc.doc < scoreDoc1.doc)
-                        return -1;
-                    return 1;
-                }
-            });
-            return docs;
+        private LuceneHitCollector(QName qname, Query query, DocumentSet docs, NodeSet contextSet, NodeSet resultSet, boolean returnAncestor,
+                                   int contextId, XQueryWatchDog watchDog) {
+            this.qname = qname;
+            this.docs = docs;
+            this.contextSet = contextSet;
+            this.resultSet = resultSet;
+            this.returnAncestor = returnAncestor;
+            this.contextId = contextId;
+            this.query = query;
+            this.watchdog = watchDog;
         }
-        
-        /**
-         * Get matching lucene documents by descending score
-         * @return
-         */
-        public List<ScoreDoc> getDocsByScore() {
-        	Collections.sort(docs, new Comparator<ScoreDoc>() {
 
-                public int compare(ScoreDoc scoreDoc, ScoreDoc scoreDoc1) {
-                    if (scoreDoc.score == scoreDoc1.score)
-                    	return 0;
-                    else if (scoreDoc.score < scoreDoc1.score)
-                    	return 1;
-                    return -1;
-                }
-            });
-            return docs;
-        }
-
         @Override
         public void setScorer(Scorer scorer) throws IOException {
             this.scorer = scorer;
         }
 
         @Override
-        public void setNextReader(IndexReader indexReader, int docBase) throws IOException {
-            this.docBase = docBase;
+        public void setNextReader(AtomicReaderContext atomicReaderContext) throws IOException {
+            this.reader = atomicReaderContext.reader();
+            this.docIdValues = this.reader.getNumericDocValues(FIELD_DOC_ID);
+            this.nodeIdValues = this.reader.getBinaryDocValues(LuceneUtil.FIELD_NODE_ID);
         }
 
         @Override
@@ -954,20 +857,55 @@
         public void collect(int doc) {
             try {
                 float score = scorer.score();
-                docs.add(new ScoreDoc(doc + docBase, score));
+                int docId = (int) this.docIdValues.get(doc);
+                DocumentImpl storedDocument = docs.getDoc(docId);
+                if (storedDocument == null)
+                    return;
+                BytesRef ref = new BytesRef(buf);
+                this.nodeIdValues.get(doc, ref);
+                int units = ByteConversion.byteToShort(ref.bytes, ref.offset);
+                NodeId nodeId = index.getBrokerPool().getNodeFactory().createFromData(units, ref.bytes, ref.offset + 2);
+                //LOG.info("doc: " + docId + "; node: " + nodeId.toString() + "; units: " + units);
+
+                NodeProxy storedNode = new NodeProxy(storedDocument, nodeId);
+                if (qname != null)
+                    storedNode.setNodeType(qname.getNameType() == ElementValue.ATTRIBUTE ? Node.ATTRIBUTE_NODE : Node.ELEMENT_NODE);
+                // if a context set is specified, we can directly check if the
+                // matching node is a descendant of one of the nodes
+                // in the context set.
+                if (contextSet != null) {
+                    int sizeHint = contextSet.getSizeHint(storedDocument);
+                    if (returnAncestor) {
+                        NodeProxy parentNode = contextSet.get(storedNode);
+                        // NodeProxy parentNode = contextSet.parentWithChild(storedNode, false, true, NodeProxy.UNKNOWN_NODE_LEVEL);
+                        if (parentNode != null) {
+                            LuceneMatch match = new LuceneMatch(contextId, nodeId, query);
+                            match.setScore(score);
+                            parentNode.addMatch(match);
+                            resultSet.add(parentNode, sizeHint);
+                            if (Expression.NO_CONTEXT_ID != contextId) {
+                                parentNode.deepCopyContext(storedNode, contextId);
+                            } else
+                                parentNode.copyContext(storedNode);
+                        }
+                    } else {
+                        LuceneMatch match = new LuceneMatch(contextId, nodeId, query);
+                        match.setScore(score);
+                        storedNode.addMatch(match);
+                        resultSet.add(storedNode, sizeHint);
+                    }
+                } else {
+                    LuceneMatch match = new LuceneMatch(contextId, nodeId, query);
+                    match.setScore(score);
+                    storedNode.addMatch(match);
+                    resultSet.add(storedNode);
+                }
             } catch (IOException e) {
                 e.printStackTrace();
             }
         }
     }
 
-    private NodeId readNodeId(Document doc) {
-        byte[] temp = doc.getBinaryValue(FIELD_NODE_ID);
-        int units = ByteConversion.byteToShort(temp, 0);
-        return index.getBrokerPool().getNodeFactory()
-                .createFromData(units, temp, 2);
-    }
-
     /**
      * Check index configurations for all collection in the given DocumentSet and return
      * a list of QNames, which have indexes defined on them.
@@ -992,9 +930,9 @@
         IndexReader reader = null;
         try {
             reader = index.getReader();
-            for (FieldInfo info: ReaderUtil.getMergedFieldInfos(reader)) {
-            	if (!FIELD_DOC_ID.equals(info.name)) {
-                    QName name = decodeQName(info.name);
+            for (FieldInfo info: MultiFields.getMergedFieldInfos(reader)) {
+                if (!FIELD_DOC_ID.equals(info.name)) {
+                    QName name = LuceneUtil.decodeQName(info.name, index.getBrokerPool().getSymbols());
                     if (name != null && (qname == null || matchQName(qname, name)))
                         indexes.add(name);
                 }
@@ -1058,9 +996,7 @@
             IntegerValue vmax = (IntegerValue) hints.get(VALUE_COUNT);
             max = vmax == null ? Long.MAX_VALUE : vmax.getValue();
         }
-        if (nodes == null || max < Long.MAX_VALUE)
-            return scanIndexByQName(qnames, docs, nodes, start, end, max);
-        return scanIndexByNodes(qnames, docs, nodes, start, end, max);
+        return scanIndexByQName(qnames, docs, nodes, start, end, max);
     }
 
     private Occurrences[] scanIndexByQName(List<QName> qnames, DocumentSet docs, NodeSet nodes, String start, String end, long max) {
@@ -1069,59 +1005,61 @@
         try {
             reader = index.getReader();
             for (QName qname : qnames) {
-                String field = encodeQName(qname);
-                TermEnum terms;
-                if (start == null)
-                    terms = reader.terms(new Term(field, ""));
-                else
-                    terms = reader.terms(new Term(field, start));
-                if (terms == null)
-                    continue;
-                Term term;
-                TermDocs termDocs = reader.termDocs();
-                do {
-                    term = terms.term();
-                    if (term != null && term.field().equals(field)) {
+                String field = LuceneUtil.encodeQName(qname, index.getBrokerPool().getSymbols());
+                List<AtomicReaderContext> leaves = reader.leaves();
+                for (AtomicReaderContext context : leaves) {
+                    NumericDocValues docIdValues = context.reader().getNumericDocValues(FIELD_DOC_ID);
+                    BinaryDocValues nodeIdValues = context.reader().getBinaryDocValues(LuceneUtil.FIELD_NODE_ID);
+                    Bits liveDocs = context.reader().getLiveDocs();
+                    Terms terms = context.reader().terms(field);
+                    if (terms == null)
+                        continue;
+                    TermsEnum termsIter = terms.iterator(null);
+                    if (termsIter.next() == null) {
+                        continue;
+                    }
+                    do {
+                        if (map.size() >= max) {
+                            break;
+                        }
+                        BytesRef ref = termsIter.term();
+                        String term = ref.utf8ToString();
                         boolean include = true;
                         if (end != null) {
-                            if (term.text().compareTo(start) > 0)
+                            if (term.compareTo(end) > 0)
                                 include = false;
-                        } else if (start != null && !term.text().startsWith(start))
+                        } else if (start != null && !term.startsWith(start))
                             include = false;
                         if (include) {
-                            termDocs.seek(term);
-                            while (termDocs.next()) {
-                                if (reader.isDeleted(termDocs.doc()))
+                            DocsEnum docsEnum = termsIter.docs(null, null);
+                            while (docsEnum.nextDoc() != DocsEnum.NO_MORE_DOCS) {
+                                if (liveDocs != null && !liveDocs.get(docsEnum.docID())) {
                                     continue;
-                                Document doc = reader.document(termDocs.doc());
-                                String fDocId = doc.get(FIELD_DOC_ID);
-                                int docId = Integer.parseInt(fDocId);
+                                }
+                                int docId = (int) docIdValues.get(docsEnum.docID());
                                 DocumentImpl storedDocument = docs.getDoc(docId);
                                 if (storedDocument == null)
                                     continue;
                                 NodeId nodeId = null;
                                 if (nodes != null) {
-                                    // load document to check if the current node is in the passed context set, if any
-                                    nodeId = readNodeId(doc);
+                                    BytesRef nodeIdRef = new BytesRef(buf);
+                                    nodeIdValues.get(docsEnum.docID(), nodeIdRef);
+                                    int units = ByteConversion.byteToShort(nodeIdRef.bytes, nodeIdRef.offset);
+                                    nodeId = index.getBrokerPool().getNodeFactory().createFromData(units, nodeIdRef.bytes, nodeIdRef.offset + 2);
                                 }
                                 if (nodeId == null || nodes.get(storedDocument, nodeId) != null) {
-                                    Occurrences oc = map.get(term.text());
+                                    Occurrences oc = map.get(term);
                                     if (oc == null) {
-                                        oc = new Occurrences(term.text());
-                                        map.put(term.text(), oc);
+                                        oc = new Occurrences(term);
+                                        map.put(term, oc);
                                     }
                                     oc.addDocument(storedDocument);
-                                    oc.addOccurrences(termDocs.freq());
+                                    oc.addOccurrences(docsEnum.freq());
                                 }
                             }
-                            // termDocs.close();
                         }
-                    }
-                    if (map.size() >= max)
-                        break;
-                } while (terms.next());
-                termDocs.close();
-                terms.close();
+                    } while(termsIter.next() != null);
+                }
             }
         } catch (IOException e) {
             LOG.warn("Error while scanning lucene index entries: " + e.getMessage(), e);
@@ -1132,108 +1070,6 @@
         return map.values().toArray(occur);
     }
 
-    private Occurrences[] scanIndexByNodes(List<QName> qnames, DocumentSet docs, NodeSet nodes, String start, String end, long max) {
-        TreeMap<String, Occurrences> map = new TreeMap<String, Occurrences>();
-
-        FieldSelector selector = new FieldSelector() {
-            private static final long serialVersionUID = 3270211696620175721L;
-            public FieldSelectorResult accept(String field) {
-                if (field.equals(FIELD_NODE_ID))
-                    return FieldSelectorResult.LOAD_AND_BREAK;
-                return FieldSelectorResult.NO_LOAD;
-            }
-        };
-        IndexSearcher searcher = null;
-        try {
-            searcher = index.getSearcher();
-            IndexReader reader = searcher.getIndexReader();
-            for (Iterator<DocumentImpl> i = docs.getDocumentIterator(); i.hasNext(); ) {
-                DocumentImpl doc = i.next();
-                Query query = new TermQuery(new Term(FIELD_DOC_ID, NumericUtils.intToPrefixCoded(doc.getDocId())));
-                DocumentCollector collector = new DocumentCollector(searcher.maxDoc());
-                searcher.search(query, collector);
-
-                DocIdSetIterator iter = collector.docs.iterator();
-                int next;
-                while ((next = iter.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-                    NodeId nodeId = null;
-                    if (nodes != null) {
-                        // load document to check if the current node is in the passed context set, if any
-                        Document luceneDoc = searcher.doc(next, selector);
-                        nodeId = readNodeId(luceneDoc);
-                    }
-                    if (nodeId == null || nodes.get(doc, nodeId) != null) {
-                        for (QName qname : qnames) {
-                            String field = encodeQName(qname);
-                            TermFreqVector tfv = reader.getTermFreqVector(next, field);
-                            if (tfv != null) {
-                                String[] terms = tfv.getTerms();
-                                int[] freq = tfv.getTermFrequencies();
-                                for (int j = 0; j < terms.length; j++) {
-                                    boolean include = true;
-                                    if (end != null) {
-                                        if (terms[j].compareTo(start) > 0)
-                                            include = false;
-                                    } else if (start != null && !terms[j].startsWith(start))
-                                        include = false;
-                                    if (include) {
-                                        Occurrences oc = map.get(terms[j]);
-                                        if (oc == null) {
-                                            oc = new Occurrences(terms[j]);
-                                            map.put(terms[j], oc);
-                                        }
-                                        oc.addDocument(doc);
-                                        oc.addOccurrences(freq[j]);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (IOException e) {
-            LOG.warn("Error while scanning lucene index entries: " + e.getMessage(), e);
-        } finally {
-            index.releaseSearcher(searcher);
-        }
-        return occurrencesToArray(map);
-    }
-
-    private Occurrences[] occurrencesToArray(TreeMap<String, Occurrences> map) {
-        Occurrences[] occur = new Occurrences[map.size()];
-        return map.values().toArray(occur);
-    }
-
-    private static class DocumentCollector extends Collector {
-
-        OpenBitSet docs;
-        int base = 0;
-
-        private DocumentCollector(int size) {
-            docs = new OpenBitSet(size);
-        }
-
-        @Override
-        public void setScorer(Scorer scorer) throws IOException {
-            //What to do there ?
-        }
-
-        @Override
-        public void collect(int doc) throws IOException {
-            docs.set(base + doc);
-        }
-
-        @Override
-        public void setNextReader(IndexReader indexReader, int base) throws IOException {
-            this.base = base;
-        }
-
-        @Override
-        public boolean acceptsDocsOutOfOrder() {
-            return true;
-        }
-    }
-
     /**
      * Adds the passed character sequence to the lucene index. We
      * create one lucene document per XML node, using 2 fields to identify
@@ -1280,23 +1116,30 @@
         IndexWriter writer = null;
         try {
             writer = index.getWriter();
-            // by default, Lucene only indexes the first 10,000 terms in a field
-            writer.setMaxFieldLength(Integer.MAX_VALUE);
-            NumericField fDocId = new NumericField(FIELD_DOC_ID, Field.Store.YES, true);
-            Field fNodeId = new Field(FIELD_NODE_ID, new byte [] { 0 }, Field.Store.YES);
+            // docId and nodeId are stored as doc value
+            IntDocValuesField fDocId = new IntDocValuesField(FIELD_DOC_ID, 0);
+            BinaryDocValuesField fNodeId = new BinaryDocValuesField(LuceneUtil.FIELD_NODE_ID, new BytesRef(8));
+           ...
 
[truncated message content]

Thread: [Exist-commits] SF.net SVN: exist:[18393] branches/rangeindex

eXist-db is a feature rich Open Source native XML database

exist-commits