Thread: [Bigdata-commit] SF.net SVN: bigdata:[8226] branches/BIGDATA_RELEASE_1_3_0 (Page 5)

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Revision: 8226
          http://sourceforge.net/p/bigdata/code/8226
Author:   jeremy_carroll
Date:     2014-05-08 01:49:33 +0000 (Thu, 08 May 2014)
Log Message:
-----------
Tests for the AnalyzerFactory's. The tests are for their shared behavior.

Modified Paths:
--------------
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/search/ConfigurableAnalyzerFactory.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractSearchTest.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestAll.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java

Added Paths:
-----------
    branches/BIGDATA_RELEASE_1_3_0/.settings/org.eclipse.core.resources.prefs
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractAnalyzerFactoryTest.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestConfigurableAsDefaultAnalyzerFactory.java
    branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestDefaultAnalyzerFactory.java

Added: branches/BIGDATA_RELEASE_1_3_0/.settings/org.eclipse.core.resources.prefs
===================================================================

--- branches/BIGDATA_RELEASE_1_3_0/.settings/org.eclipse.core.resources.prefs	                        (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/.settings/org.eclipse.core.resources.prefs	2014-05-08 01:49:33 UTC (rev 8226)
@@ -0,0 +1,2 @@
+eclipse.preferences.version=1
+encoding//bigdata/src/test/com/bigdata/search/AbstractAnalyzerFactoryTest.java=UTF-8

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/search/ConfigurableAnalyzerFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/search/ConfigurableAnalyzerFactory.java	2014-05-08 01:49:13 UTC (rev 8225)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/java/com/bigdata/search/ConfigurableAnalyzerFactory.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -547,7 +547,7 @@
 
             	// RussianAnalyzer is missing any way to access stop words.
             	if (RussianAnalyzer.class.equals(cls) && useDefaultStopWords()) {
-            		return new AnalyzerPair(languageRange, new RussianAnalyzer(Version.LUCENE_CURRENT, Collections.EMPTY_SET), new RussianAnalyzer(Version.LUCENE_CURRENT));
+            		return new AnalyzerPair(languageRange, new RussianAnalyzer(Version.LUCENE_CURRENT), new RussianAnalyzer(Version.LUCENE_CURRENT, Collections.EMPTY_SET));
             	}
             	return new VersionSetAnalyzerPair(this, cls);
             }
@@ -612,7 +612,8 @@
      */
     private static final int MAX_LANG_CACHE_SIZE = 500;
     		
-    private final String defaultLanguage;
+    private String defaultLanguage;
+    private final FullTextIndex<?> fullTextIndex;
     
     
     public ConfigurableAnalyzerFactory(final FullTextIndex<?> fullTextIndex) {
@@ -621,9 +622,9 @@
         if (fullTextIndex == null)
             throw new IllegalArgumentException();
         
-        defaultLanguage = getDefaultLanguage(fullTextIndex);
+        this.fullTextIndex = fullTextIndex;
         
-        final Properties properties = initProperties(fullTextIndex);
+        final Properties properties = initProperties();
         
         final Map<String, ConfigOptionsToAnalyzer> analyzers = new HashMap<String, ConfigOptionsToAnalyzer>();
         
@@ -686,6 +687,12 @@
 			
 		}
 	}
+	private String getDefaultLanguage() {
+		if (defaultLanguage == null) {
+            defaultLanguage = getDefaultLanguage(fullTextIndex);
+		}
+		return defaultLanguage;
+	}
 
 	private static boolean hasConstructor(Class<? extends Analyzer> cls, Class<?> ... parameterTypes) {
 		return getConstructor(cls, parameterTypes) != null;
@@ -731,7 +738,7 @@
 		
 	}
 
-	protected Properties initProperties(final FullTextIndex<?> fullTextIndex) {
+	protected Properties initProperties() {
 		final Properties parentProperties = fullTextIndex.getProperties();
         Properties myProps;
         if (Boolean.getBoolean(parentProperties.getProperty(Options.INCLUDE_DEFAULTS, Options.DEFAULT_INCLUDE_DEFAULTS))) {
@@ -773,7 +780,8 @@
 	public Analyzer getAnalyzer(String languageCode, boolean filterStopwords) {
 		
 		if (languageCode == null || languageCode.equals("")) {
-			languageCode = defaultLanguage;
+			
+			languageCode = getDefaultLanguage();
 		}
 		
 		AnalyzerPair pair = langTag2AnalyzerPair.get(languageCode);

Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractAnalyzerFactoryTest.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractAnalyzerFactoryTest.java	                        (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractAnalyzerFactoryTest.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -0,0 +1,174 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2014.  All rights reserved.
+
+Contact:
+     SYSTAP, LLC
+     4501 Tower Road
+     Greensboro, NC 27410
+     lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+/*
+ * Created on May 7, 2014
+ */
+package com.bigdata.search;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public abstract class AbstractAnalyzerFactoryTest extends AbstractSearchTest {
+
+    public AbstractAnalyzerFactoryTest() {
+	}
+    
+    public AbstractAnalyzerFactoryTest(String arg0) {
+    	super(arg0);
+	}
+    
+    public void setUp() throws Exception {
+    	super.setUp();
+	    init(getExtraProperties());
+    }
+    abstract String[] getExtraProperties();
+    
+	private Analyzer getAnalyzer(String lang, boolean filterStopWords) {
+		return getNdx().getAnalyzer(lang, filterStopWords);
+	}
+	
+	private void comparisonTest(String lang, 
+			boolean stopWordsSignificant, 
+			String text, 
+			String spaceSeparated)  throws IOException {
+		compareTokenStream(getAnalyzer(lang, stopWordsSignificant), text,
+				spaceSeparated.split(" "));
+		}
+	private void compareTokenStream(Analyzer a, String text, String expected[]) throws IOException {
+		TokenStream s = a.tokenStream(null, new StringReader(text));
+		int ix = 0;
+        while (s.incrementToken()) {
+            final TermAttribute term = s.getAttribute(TermAttribute.class);
+            final String word = term.term();
+            assertTrue(ix < expected.length);
+        	assertEquals(word, expected[ix++]);
+        }
+        assertEquals(ix, expected.length);
+	}
+	
+
+    public void testEnglishFilterStopWords() throws IOException {
+    	for (String lang: new String[]{ "eng", null, "" }) {
+    	    comparisonTest(lang,
+    			true,
+    			"The test to end all tests! Forever.",
+    			"test end all tests forever"
+    			);
+    	}
+    }
+    public void testEnglishNoFilter() throws IOException {
+    	for (String lang: new String[]{ "eng", null, "" }) {
+    	    comparisonTest(lang,
+    			false,
+    			"The test to end all tests! Forever.",
+    			"the test to end all tests forever"
+    			);
+    	}
+    }
+    
+    // Note we careful use a three letter language code for german.
+    // 'de' is more standard, but the DefaultAnalyzerFactory does not
+    // implement 'de' correctly.
+    public void testGermanFilterStopWords() throws IOException {
+    	comparisonTest("ger",
+    			true,
+    			"Hanoi - Im Streit um die Vorherrschaft im Südchinesischen Meer ist es zu einer " +
+    			"erneuten Auseinandersetzung gekommen:",
+    			"hanoi strei um vorherrschaf sudchinesisch meer zu erneu auseinandersetzung gekomm"
+    			);
+    	
+    }
+
+    // Note we careful use a three letter language code for Russian.
+    // 'ru' is more standard, but the DefaultAnalyzerFactory does not
+    // implement 'ru' correctly.
+    public void testRussianFilterStopWords() throws IOException {
+    	comparisonTest("rus",
+    			true,
+				// I hope this is not offensive text.
+			"Они ответственны полностью и за ту, и за другую трагедию. " +
+		    "Мы уже получили данные от сочувствующих нам офицеров СБУ.",
+    			"ответствен полност ту друг трагед получ дан сочувств нам офицер сбу"
+    			);
+    	
+    }
+    public void testGermanNoStopWords() throws IOException {
+    	comparisonTest("ger",
+    			false,
+    			"Hanoi - Im Streit um die Vorherrschaft im Südchinesischen Meer ist es zu einer " +
+    			"erneuten Auseinandersetzung gekommen:",
+    			"hanoi im strei um die vorherrschaf im sudchinesisch meer ist es zu ein erneu auseinandersetzung gekomm"
+    			);
+    	
+    }
+    public void testRussianNoStopWords() throws IOException {
+    	comparisonTest("rus",
+    			false,
+				// I hope this is not offensive text.
+    			"Они ответственны полностью и за ту, и за другую трагедию. " +
+    		    "Мы уже получили данные от сочувствующих нам офицеров СБУ.",
+    			"он ответствен полност и за ту и за друг трагед мы уж получ дан от сочувств нам офицер сбу"
+    			);
+    	
+    }
+    public void testJapanese() throws IOException {
+    	for (boolean filterStopWords: new Boolean[]{true, false}) {
+    	comparisonTest("jpn",
+      filterStopWords,
+		// I hope this is not offensive text.
+	"高林純示 生態学研究センター教授らの研究グループと松井健二 山口大学医学系研究科（農学系）教授らの研究グループは、",
+    "高林 林純 純示 生態 態学 学研 研究 究セ セン ンタ ター ー教 教授 授ら らの の研 研究 究グ グル ルー " +
+	"ープ プと と松 松井 井健 健二 山口 口大 大学 学医 医学 学系 系研 " +
+    "研究 究科 農学 学系 教授 授ら らの の研 研究 究グ グル ルー ープ プは");
+    	}
+    }
+    public void testConfiguredLanguages() {
+    	checkConfig("BrazilianAnalyzer", "por", "pt");
+        checkConfig("ChineseAnalyzer", "zho", "chi", "zh");
+        checkConfig("CJKAnalyzer", "jpn", "ja", "kor", "ko");
+        checkConfig("CzechAnalyzer", "ces", "cze", "cs");
+        checkConfig("DutchAnalyzer", "dut", "nld", "nl");
+        checkConfig("GermanAnalyzer", "deu", "ger", "de");
+        checkConfig("GreekAnalyzer", "gre", "ell", "el");
+        checkConfig("RussianAnalyzer", "rus", "ru");
+        checkConfig("ThaiAnalyzer", "th", "tha");
+        checkConfig("StandardAnalyzer", "en", "eng", "", null);
+    }
+
+	private void checkConfig(String classname, String ...langs) {
+		for (String lang:langs) {
+			// The DefaultAnalyzerFactory only works for language tags of length exactly three.
+//			if (lang != null && lang.length()==3)
+			{
+			assertEquals(classname, getAnalyzer(lang,true).getClass().getSimpleName());
+			assertEquals(classname, getAnalyzer(lang+"-x-foobar",true).getClass().getSimpleName());
+			}
+		}
+		
+	}
+}

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractSearchTest.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractSearchTest.java	2014-05-08 01:49:13 UTC (rev 8225)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/AbstractSearchTest.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -1,3 +1,29 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2014.  All rights reserved.
+
+Contact:
+     SYSTAP, LLC
+     4501 Tower Road
+     Greensboro, NC 27410
+     lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+/*
+ * Created on May 7, 2014
+ */
 package com.bigdata.search;
 
 import java.util.Properties;
@@ -2,3 +28,2 @@
 
-import com.bigdata.btree.IndexMetadata;
 import com.bigdata.journal.IIndexManager;
@@ -11,7 +36,6 @@
     private String namespace;  
     private IIndexManager indexManager;
     private FullTextIndex<Long> ndx;
-    private IndexMetadata indexMetadata;
     private Properties properties;
     
     public AbstractSearchTest() {
@@ -22,19 +46,29 @@
 	}
 
 	void init(String ...propertyValuePairs) {
-        namespace = getName(); 
-        properties = getProperties();
+        namespace = getClass().getName()+"#"+getName(); 
+        indexManager = getStore();
+        properties = (Properties) getProperties().clone();
+        ndx = createFullTextIndex(namespace, properties, propertyValuePairs);
+    }
+	
+	private FullTextIndex<Long> createFullTextIndex(String namespace, Properties properties, String ...propertyValuePairs) {
         for (int i=0; i<propertyValuePairs.length; ) {
         	properties.setProperty(propertyValuePairs[i++], propertyValuePairs[i++]);
         }
-        indexManager = getStore();
-        ndx = new FullTextIndex<Long>(indexManager, namespace, ITx.UNISOLATED, properties);
+        FullTextIndex<Long> ndx = new FullTextIndex<Long>(indexManager, namespace, ITx.UNISOLATED, properties);
         ndx.create();
-        indexMetadata = ndx.getIndex().getIndexMetadata();
-    }
+        return ndx;
+	}
+
+	FullTextIndex<Long> createFullTextIndex(String namespace, String ...propertyValuePairs) {
+        return createFullTextIndex(namespace, getProperties(), propertyValuePairs);
+	}
 	
 	public void tearDown() throws Exception {
-        indexManager.destroy();
+		if (indexManager != null) {
+           indexManager.destroy();
+		}
 		super.tearDown();
 	}
 
@@ -54,15 +88,8 @@
 		return ndx;
 	}
 
-	IndexMetadata getIndexMetadata() {
-		return indexMetadata;
-	}
-
-
 	Properties getSearchProperties() {
 		return properties;
 	}
 
-
-
 }

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestAll.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestAll.java	2014-05-08 01:49:13 UTC (rev 8225)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestAll.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -104,6 +104,14 @@
         
         // test verifies search index is restart safe.
         suite.addTestSuite(TestSearchRestartSafe.class);
+        
+        // Check behavior of DefaultAnalyzerFactory, see also trac 915
+        suite.addTestSuite(TestDefaultAnalyzerFactory.class);
+        
+        // Check default behavior of ConfigurableAnalyzerFactory
+        // which is intended to be the same as the intended
+        // behavior of DefaultAnalyzerFactory
+        suite.addTestSuite(TestConfigurableAsDefaultAnalyzerFactory.class);
 
         return suite;
     }

Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestConfigurableAsDefaultAnalyzerFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestConfigurableAsDefaultAnalyzerFactory.java	                        (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestConfigurableAsDefaultAnalyzerFactory.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -0,0 +1,43 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2014.  All rights reserved.
+
+Contact:
+     SYSTAP, LLC
+     4501 Tower Road
+     Greensboro, NC 27410
+     lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+/*
+ * Created on May 7, 2014
+ */
+package com.bigdata.search;
+
+public class TestConfigurableAsDefaultAnalyzerFactory extends AbstractAnalyzerFactoryTest {
+
+	public TestConfigurableAsDefaultAnalyzerFactory() {
+	}
+
+	public TestConfigurableAsDefaultAnalyzerFactory(String arg0) {
+		super(arg0);
+	}
+
+	@Override
+	String[] getExtraProperties() {
+		return new String[]{FullTextIndex.Options.ANALYZER_FACTORY_CLASS, ConfigurableAnalyzerFactory.class.getName()};
+	}
+
+}

Added: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestDefaultAnalyzerFactory.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestDefaultAnalyzerFactory.java	                        (rev 0)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestDefaultAnalyzerFactory.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -0,0 +1,43 @@
+/**
+
+Copyright (C) SYSTAP, LLC 2006-2014.  All rights reserved.
+
+Contact:
+     SYSTAP, LLC
+     4501 Tower Road
+     Greensboro, NC 27410
+     lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+/*
+ * Created on May 7, 2014
+ */
+package com.bigdata.search;
+
+public class TestDefaultAnalyzerFactory extends AbstractAnalyzerFactoryTest {
+
+	public TestDefaultAnalyzerFactory() {
+	}
+
+	public TestDefaultAnalyzerFactory(String arg0) {
+		super(arg0);
+	}
+
+	@Override
+	String[] getExtraProperties() {
+		return new String[0];
+	}
+
+}

Modified: branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java
===================================================================
--- branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java	2014-05-08 01:49:13 UTC (rev 8225)
+++ branches/BIGDATA_RELEASE_1_3_0/bigdata/src/test/com/bigdata/search/TestKeyBuilder.java	2014-05-08 01:49:33 UTC (rev 8226)
@@ -93,6 +93,10 @@
         return keyBuilder;
         
     }
+
+	IndexMetadata getIndexMetadata() {
+		return getNdx().getIndex().getIndexMetadata();
+	}
     private IKeyBuilder keyBuilder;
     
     /**

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





Thread: [Bigdata-commit] SF.net SVN: bigdata:[8226] branches/BIGDATA_RELEASE_1_3_0 (Page 5)

Fast, scalable, robust graph database platform

bigdata-commit