[ jEdit-commits ] SF.net SVN: jedit:[15334] plugins/LucenePlugin/trunk/src/lucene

[<sh...@us...>]

Revision: 15334
          http://jedit.svn.sourceforge.net/jedit/?rev=15334&view=rev
Author:   shlomy
Date:     2009-05-23 13:27:02 +0000 (Sat, 23 May 2009)

Log Message:
-----------
Created a new analyzer for source code, using that instead of the standard analyzer.
Later, the indexing dialog will allow selection of the analyzer to use.

Also added a field for the indexing timestamp, in order to make it easier to update the index. After updating the index, we can look for documents having an older timestamp - such documents need to be removed.

Modified Paths:
--------------
    plugins/LucenePlugin/trunk/src/lucene/LucenePlugin.java

Added Paths:
-----------
    plugins/LucenePlugin/trunk/src/lucene/SourceCodeAnalyzer.java

Modified: plugins/LucenePlugin/trunk/src/lucene/LucenePlugin.java
===================================================================
--- plugins/LucenePlugin/trunk/src/lucene/LucenePlugin.java	2009-05-22 20:35:59 UTC (rev 15333)
+++ plugins/LucenePlugin/trunk/src/lucene/LucenePlugin.java	2009-05-23 13:27:02 UTC (rev 15334)
@@ -11,7 +11,6 @@
 import javax.swing.JOptionPane;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
@@ -30,7 +29,7 @@
 
 public class LucenePlugin extends EBPlugin {
 	
-	static private Analyzer analyzer = new StandardAnalyzer();
+	static private Analyzer analyzer = new SourceCodeAnalyzer();
 
 	private static boolean validDir(String path)
 	{
@@ -135,6 +134,7 @@
 		Document doc = new Document();
 		doc.add(new Field("file", f.getPath(), Field.Store.YES, Field.Index.NO));
 		doc.add(new Field("modified", String.valueOf(f.lastModified()), Field.Store.YES, Field.Index.NO));
+		doc.add(new Field("indexed", String.valueOf(System.currentTimeMillis()), Field.Store.YES, Field.Index.NO));
 		return doc;
 	}
 
@@ -163,7 +163,6 @@
     			"No matches were found for \"" + queryString + "\"");
     		return null;
     	}
-    	Searcher metaSearcher = new IndexSearcher(metaDir());
     	Vector<FileLine> results = new Vector<FileLine>();
     	for (int i = 0; i < collector.docs.size(); i++)
     	{

Added: plugins/LucenePlugin/trunk/src/lucene/SourceCodeAnalyzer.java
===================================================================
--- plugins/LucenePlugin/trunk/src/lucene/SourceCodeAnalyzer.java	                        (rev 0)
+++ plugins/LucenePlugin/trunk/src/lucene/SourceCodeAnalyzer.java	2009-05-23 13:27:02 UTC (rev 15334)
@@ -0,0 +1,38 @@
+package lucene;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharTokenizer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+
+public class SourceCodeAnalyzer extends Analyzer
+{
+	public TokenStream tokenStream(String fieldName, Reader reader) {
+		return new SourceCodeTokenizer(reader);
+	}
+
+	public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+		Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
+		if (tokenizer == null) {
+			tokenizer = new SourceCodeTokenizer(reader);
+			setPreviousTokenStream(tokenizer);
+		} else
+			tokenizer.reset(reader);
+		return tokenizer;
+	}
+
+	static private class SourceCodeTokenizer extends CharTokenizer
+	{
+		public SourceCodeTokenizer(Reader input) {
+			super(input);
+		}
+
+		@Override
+		protected boolean isTokenChar(char c) {
+			return Character.isJavaIdentifierPart(c);
+		}
+	}
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.