[Joafip-svn] SF.net SVN: joafip:[2974] trunk/joafip-4test/src/main/java
Brought to you by:
luc_peuvrier
Menu
▾
▴
[Joafip-svn] SF.net SVN: joafip:[2974] trunk/joafip-4test/src/main/java
|
From: <luc...@us...> - 2011-11-21 07:12:01
|
Revision: 2974
http://joafip.svn.sourceforge.net/joafip/?rev=2974&view=rev
Author: luc_peuvrier
Date: 2011-11-21 07:11:53 +0000 (Mon, 21 Nov 2011)
Log Message:
-----------
comment update. refactoring. test for auto save added.
Modified Paths:
--------------
trunk/joafip-4test/src/main/java/eatmemory/MainEatMemory.java
trunk/joafip-4test/src/main/java/jrat/MainJRat.java
Added Paths:
-----------
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainMemRBTree.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainRBTree.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/RBTree.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/AbstractSpellCheck.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/ISpellCheckDAO.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainCheck.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainReadWords.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/SpellCheckBase.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/SpellCheckException.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/SpellCheckJoafipDAO.java
Removed Paths:
-------------
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/AbstractSpellCheck.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/ISpellCheckDAO.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainCheck.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainReadWords.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckBase.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckException.java
trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckJoafipDAO.java
Modified: trunk/joafip-4test/src/main/java/eatmemory/MainEatMemory.java
===================================================================
--- trunk/joafip-4test/src/main/java/eatmemory/MainEatMemory.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/eatmemory/MainEatMemory.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,3 +1,25 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package eatmemory;
import java.util.Deque;
@@ -3,4 +25,9 @@
import java.util.LinkedList;
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
public final class MainEatMemory {
Modified: trunk/joafip-4test/src/main/java/jrat/MainJRat.java
===================================================================
--- trunk/joafip-4test/src/main/java/jrat/MainJRat.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/jrat/MainJRat.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,3 +1,25 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package jrat;
import org.shiftone.jrat.cli.Cli;
@@ -2,2 +24,7 @@
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
public final class MainJRat {
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/AbstractSpellCheck.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/AbstractSpellCheck.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/AbstractSpellCheck.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,187 +0,0 @@
-package net.sf.joafip.autosave;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
-
-@SuppressWarnings("PMD")
-public abstract class AbstractSpellCheck {
-
- protected static final int MAX_LABELS = 10;
-
- protected AbstractSpellCheck() {
- super();
- }
-
- protected List<String> splitValue(final String value) {
- final List<String> resultList = new LinkedList<String>();
- List<Integer> list = new ArrayList<Integer>(value.length());
- int index = 0;
- while (index < value.length()) {
- final int codePoint = value.codePointAt(index);
- if (!Character.isLetter(codePoint)) {
- if (!list.isEmpty()) {
- resultList.add(codePointToString(list));
- list = new ArrayList<Integer>(value.length());
- }
- index += Character.charCount(codePoint);
- } else {
- list.add(Character.toLowerCase(codePoint));
- index += Character.charCount(codePoint);
- }
- }
- if (!list.isEmpty()) {
- resultList.add(codePointToString(list));
- }
- return resultList;
- }
-
- protected void updateKeyMap(final ISpellCheckDAO spellCheckDAO,
- final NavigableSet<Integer> codePointSet) {
- final int length = codePointSet.size();
- reduce(spellCheckDAO, codePointSet, 0, length / 2);
- }
-
- private void reduce(final ISpellCheckDAO spellCheckDAO,
- final NavigableSet<Integer> keyCodePointSet,
- final int positionOffset, final int minimumSize) {
- if (keyCodePointSet.size() > minimumSize) {
- final String key = codePointToString(keyCodePointSet);
- final ArrayList<Integer> keyCodePointList = new ArrayList<Integer>(
- keyCodePointSet);
- for (int position = positionOffset; position < keyCodePointSet
- .size(); position++) {
- final NavigableSet<Integer> reducedSet = new TreeSet<Integer>();
- reducedSet.addAll(keyCodePointList.subList(0, position));
- reducedSet.addAll(keyCodePointList.subList(position + 1,
- keyCodePointList.size()));
- final String reduceKey = codePointToString(reducedSet);
- Set<String> set = spellCheckDAO.getKeySet(reduceKey);
- if (set == null) {
- set = spellCheckDAO.newSetOfString();
- spellCheckDAO.addKeySet(reduceKey, set);
- }
- set.add(key);
- reduce(spellCheckDAO, reducedSet, position, minimumSize);
- }
- }
- }
-
- protected Set<String> computeReduceKeys(final String key,
- final int minimumSize, final int maxSize) {
- final NavigableSet<Integer> keyInSet = computeKeyInSet(key);
- return computeReduceKeys(keyInSet, minimumSize, maxSize);
- }
-
- protected Set<String> computeReduceKeys(NavigableSet<Integer> keyInSet,
- final int minimumSize, final int maxSize) {
- if (minimumSize <= 0) {
- throw new IllegalArgumentException("bad minimum size "
- + minimumSize);
- }
- if (maxSize <= 0) {
- throw new IllegalArgumentException("bad maximum size "
- + minimumSize);
- }
- if (minimumSize > maxSize) {
- throw new IllegalArgumentException("bad minimum " + minimumSize
- + " greater than maximum " + maxSize);
- }
- // final NavigableSet<Integer> mainKeyInSet = computeKeyInSet(value);
- final Set<String> stringKeySet = new TreeSet<String>();
- stringKeySet.add(codePointToString(keyInSet));
- if (keyInSet.size() > minimumSize) {
- final List<Set<Integer>> list = reduce(keyInSet, 0, minimumSize);
- for (Set<Integer> set : list) {
- if (set.size() <= maxSize) {
- stringKeySet.add(codePointToString(set));
- }
- }
- }
- return stringKeySet;
- }
-
- private List<Set<Integer>> reduce(final NavigableSet<Integer> set,
- final int positionOffset, final int minimumSize) {
- final List<Set<Integer>> listOfSet = new LinkedList<Set<Integer>>();
- if (set.size() > 1) {
- final ArrayList<Integer> setInList = new ArrayList<Integer>(set);
- for (int position = positionOffset; position < set.size(); position++) {
- final NavigableSet<Integer> reducedSet = new TreeSet<Integer>();
- reducedSet.addAll(setInList.subList(0, position));
- reducedSet.addAll(setInList.subList(position + 1,
- setInList.size()));
- listOfSet.add(reducedSet);
- if (reducedSet.size() > minimumSize) {
- listOfSet.addAll(reduce(reducedSet, position, minimumSize));
- }
- }
- }
- return listOfSet;
- }
-
- @SuppressWarnings("unused")
- private String computeKey(final String value) {
- final Set<Integer> codePointSet = computeKeyInSet(value);
- return codePointToString(codePointSet);
- }
-
- protected String codePointToString(final Collection<Integer> codePointSet) {
- final StringBuilder stringBuilder = new StringBuilder();
- for (int codePoint : codePointSet) {
- stringBuilder.appendCodePoint(codePoint);
- }
- return stringBuilder.toString();
- }
-
- protected NavigableSet<Integer> computeKeyInSet(final String value) {
- final NavigableSet<Integer> codePointSet = new TreeSet<Integer>();
- int index = 0;
- while (index < value.length()) {
- final int codePoint = value.codePointAt(index);
- index += Character.charCount(codePoint);
- codePointSet.add(codePoint);
- }
- return codePointSet;
- }
-
- protected int computeAverrage(final String word) {
- int index = 0;
- int sum = 0;
- while (index < word.length()) {
- final int codePoint = word.codePointAt(index);
- index += Character.charCount(codePoint);
- sum += codePoint;
- }
- return sum * 1000 / index;
- }
-
- public static int computeLevenshteinDistance(CharSequence str1,
- CharSequence str2) {
- int[][] distance = new int[str1.length() + 1][str2.length() + 1];
-
- for (int i = 0; i <= str1.length(); i++)
- distance[i][0] = i;
- for (int j = 0; j <= str2.length(); j++)
- distance[0][j] = j;
-
- for (int i = 1; i <= str1.length(); i++)
- for (int j = 1; j <= str2.length(); j++)
- distance[i][j] = minimum(
- distance[i - 1][j] + 1,
- distance[i][j - 1] + 1,
- distance[i - 1][j - 1]
- + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0
- : 1));
-
- return distance[str1.length()][str2.length()];
- }
-
- private static int minimum(int a, int b, int c) {
- return Math.min(Math.min(a, b), c);
- }
-}
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/ISpellCheckDAO.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/ISpellCheckDAO.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/ISpellCheckDAO.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,20 +0,0 @@
-package net.sf.joafip.autosave;
-
-import java.util.Set;
-
-public interface ISpellCheckDAO {
-
- Set<String> getWordSet(String key);
-
- void addWordSet(String key, Set<String> set);
-
- Set<String> getKeySet(String key);
-
- void addKeySet(String key, Set<String> set);
-
- Set<String> newSetOfString();
-
- void commit() throws SpellCheckException;
-
- void close(boolean save) throws SpellCheckException;
-}
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainCheck.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainCheck.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainCheck.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,116 +0,0 @@
-package net.sf.joafip.autosave;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-
-import net.sf.joafip.NotStorableClass;
-import net.sf.joafip.entity.EnumFilePersistenceCloseAction;
-import net.sf.joafip.service.FilePersistenceBuilder;
-import net.sf.joafip.service.FilePersistenceClassNotFoundException;
-import net.sf.joafip.service.FilePersistenceDataCorruptedException;
-import net.sf.joafip.service.FilePersistenceException;
-import net.sf.joafip.service.FilePersistenceInvalidClassException;
-import net.sf.joafip.service.FilePersistenceNotSerializableException;
-import net.sf.joafip.service.FilePersistenceTooBigForSerializationException;
-import net.sf.joafip.service.IDataAccessSession;
-import net.sf.joafip.service.IFilePersistence;
-
-@SuppressWarnings("PMD")
-@NotStorableClass
-public class MainCheck extends AbstractSpellCheck {
-
- private static final String SPELL_CHECK_BASE = "spellCheckBase";
-
- private IFilePersistence filePersistence;
-
- private IDataAccessSession session;
-
- public static void main(String[] args) {
- (new MainCheck()).run();
- }
-
- private void run() {
- try {
- openFilePersistence();
- final InputStream in = getClass().getClassLoader()
- .getResourceAsStream("word_list.txt");
- ;
- final BufferedReader reader = new BufferedReader(
- new InputStreamReader(in));
-
- int wordValueCount = 0;
- int labelCount = 0;
- String line;
- final long startTime = System.currentTimeMillis();
- boolean done = false;
- while (!done && labelCount < MAX_LABELS
- && (line = reader.readLine()) != null) {
- labelCount++;
- for (String wordValue : splitValue(line)) {
- final NavigableSet<Integer> keyCodePointSet = computeKeyInSet(wordValue);
- final String key = codePointToString(keyCodePointSet);
- session.open();
- final SpellCheckBase spellCheckBase = (SpellCheckBase) session
- .getObject(SPELL_CHECK_BASE);
- final Map<String, Set<String>> wordsByKeyMap = spellCheckBase
- .getWordsByKeyMap();
- final Set<String> set = wordsByKeyMap.get(key);
- if (set == null) {
- System.out.println("ERROR no set for key " + key
- + " from word " + wordValue);
- done = true;
- } else if (!set.contains(wordValue)) {
- System.out.println("ERROR missing word " + wordValue
- + " in set ");
- done = true;
- }
- wordValueCount++;
- System.out.println(wordValue + " " + wordValueCount);
- session.close(EnumFilePersistenceCloseAction.DO_NOT_SAVE);
- }
- }
- System.out.println("labelCount " + labelCount);
- reader.close();
-
- closeFilePersistence();
-
- System.out.println("elapsed "
- + (System.currentTimeMillis() - startTime) + " mS");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- private void openFilePersistence() throws FilePersistenceException,
- FilePersistenceInvalidClassException,
- FilePersistenceNotSerializableException,
- FilePersistenceClassNotFoundException,
- FilePersistenceDataCorruptedException {
- final FilePersistenceBuilder builder = new FilePersistenceBuilder();
- // builder.setPathName("runtime/spellcheckbase");
- builder.setPathName("Z:/");
- builder.setRemoveFiles(false);
- builder.setFileCache(16 * 1024/* pageSize */, 8 * 1024/* maxPage */);
- builder.setGarbageManagement(false);
- builder.setCrashSafeMode(false);
- filePersistence = builder.build();
- session = filePersistence.createDataAccessSession();
- }
-
- private void closeFilePersistence() throws FilePersistenceException,
- FilePersistenceInvalidClassException,
- FilePersistenceNotSerializableException,
- FilePersistenceClassNotFoundException,
- FilePersistenceDataCorruptedException,
- FilePersistenceTooBigForSerializationException {
- if (session.isOpened()) {
- session.close(EnumFilePersistenceCloseAction.DO_NOT_SAVE);
- }
- filePersistence.close();
- }
-
-}
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainReadWords.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainReadWords.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainReadWords.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,67 +0,0 @@
-package net.sf.joafip.autosave;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.NavigableSet;
-import java.util.Set;
-
-import net.sf.joafip.NotStorableClass;
-
-/**
- * @author luc peuvrier
- *
- */
-@SuppressWarnings("PMD")
-@NotStorableClass
-public class MainReadWords extends AbstractSpellCheck {
-
- public static void main(String[] args) {
- (new MainReadWords()).run();
- }
-
- private void run() {
- try {
- final ISpellCheckDAO spellCheckDAO = new SpellCheckJoafipDAO(true);
-
- final InputStream in = getClass().getClassLoader()
- .getResourceAsStream("word_list.txt");
-
- final BufferedReader reader = new BufferedReader(
- new InputStreamReader(in));
-
- int wordValueCount = 0;
- int labelCount = 0;
- String line;
- final long startTime = System.currentTimeMillis();
- while (labelCount < MAX_LABELS
- && (line = reader.readLine()) != null) {
- labelCount++;
- for (String wordValue : splitValue(line)) {
- final NavigableSet<Integer> keyCodePointSet = computeKeyInSet(wordValue);
- final String key = codePointToString(keyCodePointSet);
- Set<String> set = spellCheckDAO.getWordSet(key);
- if (set == null) {
- set = spellCheckDAO.newSetOfString();
- spellCheckDAO.addWordSet(key, set);
- // updateKeyMap(spellCheckDAO, keyCodePointSet);
- }
- if (set.add(wordValue)) {
- wordValueCount++;
- System.out.println(wordValue + " " + wordValueCount);
- }
- spellCheckDAO.commit();
- }
- }
- System.out.println("labelCount " + labelCount);
- reader.close();
-
- spellCheckDAO.close(true);
-
- System.out.println("elapsed "
- + (System.currentTimeMillis() - startTime) + " mS");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-}
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckBase.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckBase.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckBase.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,46 +0,0 @@
-package net.sf.joafip.autosave;
-
-import java.util.Map;
-import java.util.Set;
-
-import net.sf.joafip.StorableClass;
-import net.sf.joafip.java.util.PTreeMap;
-import net.sf.joafip.store.service.proxy.IInstanceFactory;
-
-@SuppressWarnings("PMD")
-@StorableClass
-public class SpellCheckBase {
-
- Map<String, Set<String>> wordsByKeyMap;// = new TreeMap<String,
- // Set<String>>();
-
- Map<String, Set<String>> keysByKeyMap;// = new TreeMap<String,
- // Set<String>>();
-
- public static SpellCheckBase newInstance(
- final IInstanceFactory instanceFactory) {
- return (SpellCheckBase) instanceFactory.newInstance(
- SpellCheckBase.class, new Class[] { IInstanceFactory.class },
- new Object[] { instanceFactory });
- }
-
- @SuppressWarnings("unchecked")
- public SpellCheckBase(final IInstanceFactory instanceFactory) {
- super();
- if (instanceFactory == null) {
- wordsByKeyMap = new PTreeMap<String, Set<String>>();
- keysByKeyMap = new PTreeMap<String, Set<String>>();
- } else {
- wordsByKeyMap = PTreeMap.newInstance(instanceFactory);
- keysByKeyMap = PTreeMap.newInstance(instanceFactory);
- }
- }
-
- public Map<String, Set<String>> getWordsByKeyMap() {
- return wordsByKeyMap;
- }
-
- public Map<String, Set<String>> getKeysByKeyMap() {
- return keysByKeyMap;
- }
-}
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckException.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckException.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckException.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,25 +0,0 @@
-package net.sf.joafip.autosave;
-
-public class SpellCheckException extends Exception {
-
- /**
- *
- */
- private static final long serialVersionUID = -3735427947716224960L;
-
- public SpellCheckException() {
- super();
- }
-
- public SpellCheckException(final String message, final Throwable cause) {
- super(message, cause);
- }
-
- public SpellCheckException(final String message) {
- super(message);
- }
-
- public SpellCheckException(final Throwable cause) {
- super(cause);
- }
-}
Deleted: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckJoafipDAO.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckJoafipDAO.java 2011-11-21 07:08:50 UTC (rev 2973)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckJoafipDAO.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -1,282 +0,0 @@
-package net.sf.joafip.autosave;
-
-import java.util.Map;
-import java.util.Set;
-
-import net.sf.joafip.NotStorableClass;
-import net.sf.joafip.entity.EnumFilePersistenceCloseAction;
-import net.sf.joafip.java.util.PTreeSet;
-import net.sf.joafip.service.FilePersistenceBuilder;
-import net.sf.joafip.service.FilePersistenceClassNotFoundException;
-import net.sf.joafip.service.FilePersistenceDataCorruptedException;
-import net.sf.joafip.service.FilePersistenceException;
-import net.sf.joafip.service.FilePersistenceInvalidClassException;
-import net.sf.joafip.service.FilePersistenceNotSerializableException;
-import net.sf.joafip.service.FilePersistenceTooBigForSerializationException;
-import net.sf.joafip.service.IAutoSaveEventListener;
-import net.sf.joafip.service.IDataAccessSession;
-import net.sf.joafip.service.IExclusiveDataAccessSession;
-import net.sf.joafip.service.IFilePersistence;
-import net.sf.joafip.store.service.proxy.IInstanceFactory;
-
-@SuppressWarnings("PMD")
-@NotStorableClass
-public class SpellCheckJoafipDAO implements IAutoSaveEventListener,
- ISpellCheckDAO {
-
- private static final String STORAGE_DIR_PATH = "Z:/";
-
- private static final boolean EX_SESSION = true;
-
- private static final boolean AUTOSAVE_ENABLED = false;
-
- private static final boolean COMMIT_SAVE = false;
-
- private static final String SPELL_CHECK_BASE = "spellCheckBase";
-
- private IFilePersistence filePersistence;
-
- private IExclusiveDataAccessSession exSession;
-
- private IDataAccessSession session;
-
- private transient IInstanceFactory instanceFactory;
-
- private SpellCheckBase spellCheckBase;
-
- private int maxVisited = Integer.MIN_VALUE;
-
- private int minVisited = Integer.MAX_VALUE;
-
- public SpellCheckJoafipDAO(final boolean removeFile)
- throws FilePersistenceException,
- FilePersistenceInvalidClassException,
- FilePersistenceNotSerializableException,
- FilePersistenceClassNotFoundException,
- FilePersistenceDataCorruptedException,
- FilePersistenceTooBigForSerializationException {
- super();
- final FilePersistenceBuilder builder = new FilePersistenceBuilder();
- builder.setPathName(STORAGE_DIR_PATH);
- builder.setRemoveFiles(removeFile);
- builder.setFileCache(16 * 1024/* pageSize */, 8 * 1024/* maxPage */);
- builder.setGarbageManagement(false);
- builder.setCrashSafeMode(false);
- filePersistence = builder.build();
- if (EX_SESSION) {
- if (AUTOSAVE_ENABLED) {
- // FIXMELUC ___TEST auto save=0
- filePersistence.autoSaveSetup(1000);
- filePersistence.setAutoSaveEnabled(AUTOSAVE_ENABLED);
- filePersistence.setAutoSaveEventListener(this);
- // FIXMELUC ___TEST maintain in memory
- // filePersistence.maintainInMemorySetup(100);
- // filePersistence.setMaintainedInMemoryEnabled(true);
- }
- exSession = filePersistence.createExclusiveDataAccessSession();
- instanceFactory = exSession.getInstanceFactory();
- exSession.open();
- if (removeFile) {
- if (AUTOSAVE_ENABLED) {
- spellCheckBase = SpellCheckBase
- .newInstance(instanceFactory);
- } else {
- spellCheckBase = new SpellCheckBase(null);
- }
- exSession.setObject(SPELL_CHECK_BASE, spellCheckBase);
- if (AUTOSAVE_ENABLED) {
- exSession.save();
- } else {
- if (COMMIT_SAVE) {
- exSession.save();
- } else {
- exSession.close();
- exSession.open();
- spellCheckBase = (SpellCheckBase) exSession
- .getObject(SPELL_CHECK_BASE);
- }
- }
- } else {
- spellCheckBase = (SpellCheckBase) exSession
- .getObject(SPELL_CHECK_BASE);
- if (spellCheckBase == null) {
- throw new IllegalStateException("no data");
- }
- }
- } else {
- session = filePersistence.createDataAccessSession();
- session.open();
- if (removeFile) {
- spellCheckBase = new SpellCheckBase(null);
- session.setObject(SPELL_CHECK_BASE, spellCheckBase);
- session.close(EnumFilePersistenceCloseAction.SAVE);
- session.open();
- spellCheckBase = (SpellCheckBase) session
- .getObject(SPELL_CHECK_BASE);
- } else {
- spellCheckBase = (SpellCheckBase) session
- .getObject(SPELL_CHECK_BASE);
- if (spellCheckBase == null) {
- throw new IllegalStateException("no data");
- }
- }
- }
- }
-
- @Override
- public void commit() throws SpellCheckException {
- try {
- if (EX_SESSION) {
- if (!AUTOSAVE_ENABLED) {
- if (COMMIT_SAVE) {
- exSession.save();
- updateVisitedForSave();
- } else {
- exSession.close();
- updateVisitedForSave();
- exSession.open();
- spellCheckBase = (SpellCheckBase) exSession
- .getObject(SPELL_CHECK_BASE);
- }
- }
- } else {
- session.close(EnumFilePersistenceCloseAction.SAVE);
- updateVisitedForSave();
- session.open();
- spellCheckBase = (SpellCheckBase) session
- .getObject(SPELL_CHECK_BASE);
- }
- } catch (FilePersistenceException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceInvalidClassException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceNotSerializableException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceClassNotFoundException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceDataCorruptedException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceTooBigForSerializationException e) {
- throw new SpellCheckException(e);
- }
- }
-
- private void updateVisitedForSave() throws FilePersistenceException {
- final int visited = filePersistence.getNumberOfVisited();
- if (visited > maxVisited) {
- maxVisited = visited;
- }
- if (visited < minVisited) {
- minVisited = visited;
- }
- // System.out.println("visited "+visited+" "+minVisited+"/"+maxVisited);
- }
-
- @Override
- public void close(final boolean save) throws SpellCheckException {
- try {
- if (EX_SESSION) {
- exSession.close();
- } else {
- session.close(EnumFilePersistenceCloseAction.SAVE);
- }
- System.out.println("nb of data records "
- + filePersistence.getNumberOfDataRecord());
- export();
- filePersistence.close();
- System.out.println("min visited " + minVisited + " max visited "
- + maxVisited);
- } catch (FilePersistenceException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceInvalidClassException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceNotSerializableException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceClassNotFoundException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceDataCorruptedException e) {
- throw new SpellCheckException(e);
- } catch (FilePersistenceTooBigForSerializationException e) {
- throw new SpellCheckException(e);
- }
- }
-
- private void export() throws FilePersistenceException,
- FilePersistenceClassNotFoundException,
- FilePersistenceInvalidClassException,
- FilePersistenceDataCorruptedException,
- FilePersistenceNotSerializableException,
- FilePersistenceTooBigForSerializationException {
- final String directoryName;
- if (EX_SESSION) {
- if (AUTOSAVE_ENABLED) {
- directoryName = "runtime/export/autosave";
- } else {
- directoryName = "runtime/export/exsession";
- }
- } else {
- directoryName = "runtime/export/session";
- }
- filePersistence.xmlExport(directoryName, STORAGE_DIR_PATH + "/tmp",
- false/* exportPersistedClassByteCode */);
- final int nbExported = filePersistence.getNumberOfObjectExported();
- System.out.println("number of object exported=" + nbExported);
- }
-
- @Override
- public Set<String> getWordSet(final String key) {
- final Map<String, Set<String>> wordsByKeyMap = spellCheckBase
- .getWordsByKeyMap();
- return wordsByKeyMap.get(key);
- }
-
- @Override
- public void addWordSet(final String key, final Set<String> set) {
- final Map<String, Set<String>> wordsByKeyMap = spellCheckBase
- .getWordsByKeyMap();
- wordsByKeyMap.put(key, set);
- }
-
- @Override
- public Set<String> getKeySet(final String key) {
- final Map<String, Set<String>> keysByKeyMap = spellCheckBase
- .getKeysByKeyMap();
- return keysByKeyMap.get(key);
- }
-
- @Override
- public void addKeySet(final String key, final Set<String> set) {
- final Map<String, Set<String>> keysByKeyMap = spellCheckBase
- .getKeysByKeyMap();
- keysByKeyMap.put(key, set);
- }
-
- @SuppressWarnings({ "unchecked", "unused" })
- public Set<String> newSetOfString() {
- Set<String> set;
- if (EX_SESSION && AUTOSAVE_ENABLED) {
- set = PTreeSet.newInstance(instanceFactory);
- } else {
- set = new PTreeSet<String>();
- }
- return set;
- }
-
- @Override
- public boolean doSave() {
- assert EX_SESSION && AUTOSAVE_ENABLED;
- return true;
- }
-
- @Override
- public void saveDone(final int numberOfObjectState,
- final int numberOfWeakreference, final long msDuration) {
- assert EX_SESSION && AUTOSAVE_ENABLED;
- try {
- updateVisitedForSave();
- } catch (FilePersistenceException e) {
- e.printStackTrace();
- }
- System.out.println("SAVE");
- }
-}
Added: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainMemRBTree.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainMemRBTree.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainMemRBTree.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.rbtree;
+
+import net.sf.joafip.redblacktree.service.RBTException;
+import net.sf.joafip.service.FilePersistenceClassNotFoundException;
+import net.sf.joafip.service.FilePersistenceDataCorruptedException;
+import net.sf.joafip.service.FilePersistenceException;
+import net.sf.joafip.service.FilePersistenceInvalidClassException;
+import net.sf.joafip.service.FilePersistenceNotSerializableException;
+import net.sf.joafip.service.FilePersistenceTooBigForSerializationException;
+
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
+public final class MainMemRBTree {
+
+ private MainMemRBTree() {
+ super();
+ }
+
+ public static void main(final String[] args) {
+ try {
+ final MainMemRBTree main = new MainMemRBTree();
+ main.run();
+ } catch (Exception exception) {
+ exception.printStackTrace();// NOPMD
+ }
+ }
+
+ private void run() throws FilePersistenceException,
+ FilePersistenceInvalidClassException,
+ FilePersistenceNotSerializableException,
+ FilePersistenceClassNotFoundException,
+ FilePersistenceDataCorruptedException,
+ FilePersistenceTooBigForSerializationException, RBTException {
+
+ final RBTree tree = RBTree.newInstance(null);
+
+ for (int count = 0; count < 1000000; count++) {
+ tree.append("" + count);// NOPMD
+ if (count % 1000 == 0) {
+ System.out.println("c=" + count);// NOPMD
+ }
+ if (count > 8) {
+ tree.remove("" + (count - 8));// NOPMD
+ }
+ }
+ }
+
+}
Added: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainRBTree.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainRBTree.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/MainRBTree.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.rbtree;
+
+import net.sf.joafip.redblacktree.service.RBTException;
+import net.sf.joafip.service.FilePersistenceBuilder;
+import net.sf.joafip.service.FilePersistenceClassNotFoundException;
+import net.sf.joafip.service.FilePersistenceDataCorruptedException;
+import net.sf.joafip.service.FilePersistenceException;
+import net.sf.joafip.service.FilePersistenceInvalidClassException;
+import net.sf.joafip.service.FilePersistenceNotSerializableException;
+import net.sf.joafip.service.FilePersistenceTooBigForSerializationException;
+import net.sf.joafip.service.IAutoSaveEventListener;
+import net.sf.joafip.service.IExclusiveDataAccessSession;
+import net.sf.joafip.service.IFilePersistence;
+import net.sf.joafip.store.service.proxy.IInstanceFactory;
+
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
+public final class MainRBTree implements IAutoSaveEventListener {
+
+ private MainRBTree() {
+ super();
+ }
+
+ public static void main(final String[] args) {
+ try {
+ final MainRBTree main = new MainRBTree();
+ main.run();
+ } catch (Exception exception) {
+ exception.printStackTrace();// NOPMD
+ }
+ }
+
+ private void run() throws FilePersistenceException,
+ FilePersistenceInvalidClassException,
+ FilePersistenceNotSerializableException,
+ FilePersistenceClassNotFoundException,
+ FilePersistenceDataCorruptedException,
+ FilePersistenceTooBigForSerializationException, RBTException {
+ final IFilePersistence filePersistence = createFilePersistence();
+ // filePersistence.setAutoSaveEventListener(this);
+ final IExclusiveDataAccessSession session = filePersistence
+ .createExclusiveDataAccessSession();
+ final IInstanceFactory instanceFactory = session.getInstanceFactory();
+
+ session.open();
+
+ final RBTree tree = RBTree.newInstance(instanceFactory);
+
+ for (int count = 0; count < 1000000; count++) {
+ tree.append("" + count);// NOPMD
+ if (count % 1000 == 0) {
+ System.out.println("c=" + count);// NOPMD
+ }
+ if (count > 8) {
+ tree.remove("" + (count - 8));// NOPMD
+ }
+ }
+
+ session.close();
+
+ filePersistence.close();
+ }
+
+ private IFilePersistence createFilePersistence()
+ throws FilePersistenceException,
+ FilePersistenceInvalidClassException,
+ FilePersistenceNotSerializableException,
+ FilePersistenceClassNotFoundException,
+ FilePersistenceDataCorruptedException {
+ final FilePersistenceBuilder builder = new FilePersistenceBuilder();
+ builder.setPathName("runtime");
+ // builder.setFileCache(setup.getPageSize(), setup.getMaxPage());
+ builder.setProxyMode(true);
+ builder.setRemoveFiles(true);
+ builder.setCrashSafeMode(false);
+ builder.setGarbageManagement(false);
+ builder.setAutoSaveEnabled(true);
+ builder.setMaxInMemoryThreshold(0);
+ // builder.setMaintenedInMemory(true);
+ // builder.setMaintenedInMemoryQuota(500);
+ builder.setFileCache(1000, 10 * 1024);
+ return builder.build();
+ }
+
+ @Override
+ public boolean doSave() {
+ return true;
+ }
+
+ @SuppressWarnings("PMD")
+ @Override
+ public void saveDone(final int numberOfObjectState,
+ final int numberOfWeakreference, final long msDuration) {
+ System.out.print("n=");
+ System.out.print(numberOfObjectState);
+ System.out.print(" w=");
+ System.out.print(numberOfWeakreference);
+ System.out.print(" d=");
+ System.out.print(msDuration);
+ System.out.println(" mS");
+ }
+
+}
Added: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/RBTree.java
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/RBTree.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/rbtree/RBTree.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.rbtree;
+
+import net.sf.joafip.redblacktree.entity.IRBTNode;
+import net.sf.joafip.redblacktree.entity.RBTSentinel;
+import net.sf.joafip.redblacktree.impl.memory.entity.RBTNode;
+import net.sf.joafip.redblacktree.service.IRBTNodeManager;
+import net.sf.joafip.redblacktree.service.RBTException;
+import net.sf.joafip.redblacktree.service.RedBlackTree;
+import net.sf.joafip.store.service.proxy.IInstanceFactory;
+
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
+public class RBTree implements IRBTNodeManager<String> {
+
+ private final IInstanceFactory instanceFactory;
+
+ private final RedBlackTree<String> tree;
+
+ private IRBTNode<String> rootNode;
+
+ @SuppressWarnings("unchecked")
+ public RBTree(final IInstanceFactory instanceFactory) {
+ super();
+ this.instanceFactory = instanceFactory;
+ tree = RedBlackTree.newInstance(instanceFactory, this,
+ true/* manageNodeIndex */, false/* uniqueValue */);
+ }
+
+ public static RBTree newInstance(final IInstanceFactory instanceFactory) {
+ final RBTree tree;
+ if (instanceFactory == null) {
+ tree = new RBTree(instanceFactory);
+ } else {
+ tree = (RBTree) instanceFactory.newInstance(RBTree.class,
+ new Class<?>[] { IInstanceFactory.class },
+ new Object[] { instanceFactory });
+ }
+ return tree;
+ }
+
+ @Override
+ public IRBTNode<String> getRootNode() throws RBTException {
+ return rootNode;
+ }
+
+ @Override
+ public void setRootNode(final IRBTNode<String> rootNode)
+ throws RBTException {
+ this.rootNode = rootNode;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IRBTNode<String> newSentinel() {
+ return RBTSentinel.newInstance(instanceFactory);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void append(final String value) throws RBTException {
+ final IRBTNode<String> nodeToAppend = RBTNode.newInstance(
+ instanceFactory, value);
+ tree.append(nodeToAppend);
+ }
+
+ public void remove(final String value) throws RBTException {
+ tree.delete(value);
+ }
+}
Copied: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/AbstractSpellCheck.java (from rev 2972, trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/AbstractSpellCheck.java)
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/AbstractSpellCheck.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/AbstractSpellCheck.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,214 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.spellcheck;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
+@SuppressWarnings("PMD")
+public abstract class AbstractSpellCheck {
+
+ protected static final int MAX_LABELS = 10;
+
+ protected AbstractSpellCheck() {
+ super();
+ }
+
+ protected List<String> splitValue(final String value) {
+ final List<String> resultList = new LinkedList<String>();
+ List<Integer> list = new ArrayList<Integer>(value.length());
+ int index = 0;
+ while (index < value.length()) {
+ final int codePoint = value.codePointAt(index);
+ if (!Character.isLetter(codePoint)) {
+ if (!list.isEmpty()) {
+ resultList.add(codePointToString(list));
+ list = new ArrayList<Integer>(value.length());
+ }
+ index += Character.charCount(codePoint);
+ } else {
+ list.add(Character.toLowerCase(codePoint));
+ index += Character.charCount(codePoint);
+ }
+ }
+ if (!list.isEmpty()) {
+ resultList.add(codePointToString(list));
+ }
+ return resultList;
+ }
+
+ protected void updateKeyMap(final ISpellCheckDAO spellCheckDAO,
+ final NavigableSet<Integer> codePointSet) {
+ final int length = codePointSet.size();
+ reduce(spellCheckDAO, codePointSet, 0, length / 2);
+ }
+
+ private void reduce(final ISpellCheckDAO spellCheckDAO,
+ final NavigableSet<Integer> keyCodePointSet,
+ final int positionOffset, final int minimumSize) {
+ if (keyCodePointSet.size() > minimumSize) {
+ final String key = codePointToString(keyCodePointSet);
+ final ArrayList<Integer> keyCodePointList = new ArrayList<Integer>(
+ keyCodePointSet);
+ for (int position = positionOffset; position < keyCodePointSet
+ .size(); position++) {
+ final NavigableSet<Integer> reducedSet = new TreeSet<Integer>();
+ reducedSet.addAll(keyCodePointList.subList(0, position));
+ reducedSet.addAll(keyCodePointList.subList(position + 1,
+ keyCodePointList.size()));
+ final String reduceKey = codePointToString(reducedSet);
+ Set<String> set = spellCheckDAO.getKeySet(reduceKey);
+ if (set == null) {
+ set = spellCheckDAO.newSetOfString();
+ spellCheckDAO.addKeySet(reduceKey, set);
+ }
+ set.add(key);
+ reduce(spellCheckDAO, reducedSet, position, minimumSize);
+ }
+ }
+ }
+
+ protected Set<String> computeReduceKeys(final String key,
+ final int minimumSize, final int maxSize) {
+ final NavigableSet<Integer> keyInSet = computeKeyInSet(key);
+ return computeReduceKeys(keyInSet, minimumSize, maxSize);
+ }
+
+ protected Set<String> computeReduceKeys(NavigableSet<Integer> keyInSet,
+ final int minimumSize, final int maxSize) {
+ if (minimumSize <= 0) {
+ throw new IllegalArgumentException("bad minimum size "
+ + minimumSize);
+ }
+ if (maxSize <= 0) {
+ throw new IllegalArgumentException("bad maximum size "
+ + minimumSize);
+ }
+ if (minimumSize > maxSize) {
+ throw new IllegalArgumentException("bad minimum " + minimumSize
+ + " greater than maximum " + maxSize);
+ }
+ // final NavigableSet<Integer> mainKeyInSet = computeKeyInSet(value);
+ final Set<String> stringKeySet = new TreeSet<String>();
+ stringKeySet.add(codePointToString(keyInSet));
+ if (keyInSet.size() > minimumSize) {
+ final List<Set<Integer>> list = reduce(keyInSet, 0, minimumSize);
+ for (Set<Integer> set : list) {
+ if (set.size() <= maxSize) {
+ stringKeySet.add(codePointToString(set));
+ }
+ }
+ }
+ return stringKeySet;
+ }
+
+ private List<Set<Integer>> reduce(final NavigableSet<Integer> set,
+ final int positionOffset, final int minimumSize) {
+ final List<Set<Integer>> listOfSet = new LinkedList<Set<Integer>>();
+ if (set.size() > 1) {
+ final ArrayList<Integer> setInList = new ArrayList<Integer>(set);
+ for (int position = positionOffset; position < set.size(); position++) {
+ final NavigableSet<Integer> reducedSet = new TreeSet<Integer>();
+ reducedSet.addAll(setInList.subList(0, position));
+ reducedSet.addAll(setInList.subList(position + 1,
+ setInList.size()));
+ listOfSet.add(reducedSet);
+ if (reducedSet.size() > minimumSize) {
+ listOfSet.addAll(reduce(reducedSet, position, minimumSize));
+ }
+ }
+ }
+ return listOfSet;
+ }
+
+ @SuppressWarnings("unused")
+ private String computeKey(final String value) {
+ final Set<Integer> codePointSet = computeKeyInSet(value);
+ return codePointToString(codePointSet);
+ }
+
+ protected String codePointToString(final Collection<Integer> codePointSet) {
+ final StringBuilder stringBuilder = new StringBuilder();
+ for (int codePoint : codePointSet) {
+ stringBuilder.appendCodePoint(codePoint);
+ }
+ return stringBuilder.toString();
+ }
+
+ protected NavigableSet<Integer> computeKeyInSet(final String value) {
+ final NavigableSet<Integer> codePointSet = new TreeSet<Integer>();
+ int index = 0;
+ while (index < value.length()) {
+ final int codePoint = value.codePointAt(index);
+ index += Character.charCount(codePoint);
+ codePointSet.add(codePoint);
+ }
+ return codePointSet;
+ }
+
+ protected int computeAverrage(final String word) {
+ int index = 0;
+ int sum = 0;
+ while (index < word.length()) {
+ final int codePoint = word.codePointAt(index);
+ index += Character.charCount(codePoint);
+ sum += codePoint;
+ }
+ return sum * 1000 / index;
+ }
+
+ public static int computeLevenshteinDistance(CharSequence str1,
+ CharSequence str2) {
+ int[][] distance = new int[str1.length() + 1][str2.length() + 1];
+
+ for (int i = 0; i <= str1.length(); i++)
+ distance[i][0] = i;
+ for (int j = 0; j <= str2.length(); j++)
+ distance[0][j] = j;
+
+ for (int i = 1; i <= str1.length(); i++)
+ for (int j = 1; j <= str2.length(); j++)
+ distance[i][j] = minimum(
+ distance[i - 1][j] + 1,
+ distance[i][j - 1] + 1,
+ distance[i - 1][j - 1]
+ + ((str1.charAt(i - 1) == str2.charAt(j - 1)) ? 0
+ : 1));
+
+ return distance[str1.length()][str2.length()];
+ }
+
+ private static int minimum(int a, int b, int c) {
+ return Math.min(Math.min(a, b), c);
+ }
+}
Copied: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/ISpellCheckDAO.java (from rev 2972, trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/ISpellCheckDAO.java)
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/ISpellCheckDAO.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/ISpellCheckDAO.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.spellcheck;
+
+import java.util.Set;
+
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
+public interface ISpellCheckDAO {
+
+ Set<String> getWordSet(String key);
+
+ void addWordSet(String key, Set<String> set);
+
+ Set<String> getKeySet(String key);
+
+ void addKeySet(String key, Set<String> set);
+
+ Set<String> newSetOfString();
+
+ void commit() throws SpellCheckException;
+
+ void close(boolean save) throws SpellCheckException;
+}
Copied: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainCheck.java (from rev 2972, trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainCheck.java)
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainCheck.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainCheck.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.spellcheck;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+
+import net.sf.joafip.NotStorableClass;
+import net.sf.joafip.entity.EnumFilePersistenceCloseAction;
+import net.sf.joafip.service.FilePersistenceBuilder;
+import net.sf.joafip.service.FilePersistenceClassNotFoundException;
+import net.sf.joafip.service.FilePersistenceDataCorruptedException;
+import net.sf.joafip.service.FilePersistenceException;
+import net.sf.joafip.service.FilePersistenceInvalidClassException;
+import net.sf.joafip.service.FilePersistenceNotSerializableException;
+import net.sf.joafip.service.FilePersistenceTooBigForSerializationException;
+import net.sf.joafip.service.IDataAccessSession;
+import net.sf.joafip.service.IFilePersistence;
+
+/**
+ *
+ * @author luc peuvrier
+ *
+ */
+@SuppressWarnings("PMD")
+@NotStorableClass
+public class MainCheck extends AbstractSpellCheck {
+
+ private static final String SPELL_CHECK_BASE = "spellCheckBase";
+
+ private IFilePersistence filePersistence;
+
+ private IDataAccessSession session;
+
+ public static void main(String[] args) {
+ (new MainCheck()).run();
+ }
+
+ private void run() {
+ try {
+ openFilePersistence();
+ final InputStream in = getClass().getClassLoader()
+ .getResourceAsStream("word_list.txt");
+ ;
+ final BufferedReader reader = new BufferedReader(
+ new InputStreamReader(in));
+
+ int wordValueCount = 0;
+ int labelCount = 0;
+ String line;
+ final long startTime = System.currentTimeMillis();
+ boolean done = false;
+ while (!done && labelCount < MAX_LABELS
+ && (line = reader.readLine()) != null) {
+ labelCount++;
+ for (String wordValue : splitValue(line)) {
+ final NavigableSet<Integer> keyCodePointSet = computeKeyInSet(wordValue);
+ final String key = codePointToString(keyCodePointSet);
+ session.open();
+ final SpellCheckBase spellCheckBase = (SpellCheckBase) session
+ .getObject(SPELL_CHECK_BASE);
+ final Map<String, Set<String>> wordsByKeyMap = spellCheckBase
+ .getWordsByKeyMap();
+ final Set<String> set = wordsByKeyMap.get(key);
+ if (set == null) {
+ System.out.println("ERROR no set for key " + key
+ + " from word " + wordValue);
+ done = true;
+ } else if (!set.contains(wordValue)) {
+ System.out.println("ERROR missing word " + wordValue
+ + " in set ");
+ done = true;
+ }
+ wordValueCount++;
+ System.out.println(wordValue + " " + wordValueCount);
+ session.close(EnumFilePersistenceCloseAction.DO_NOT_SAVE);
+ }
+ }
+ System.out.println("labelCount " + labelCount);
+ reader.close();
+
+ closeFilePersistence();
+
+ System.out.println("elapsed "
+ + (System.currentTimeMillis() - startTime) + " mS");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private void openFilePersistence() throws FilePersistenceException,
+ FilePersistenceInvalidClassException,
+ FilePersistenceNotSerializableException,
+ FilePersistenceClassNotFoundException,
+ FilePersistenceDataCorruptedException {
+ final FilePersistenceBuilder builder = new FilePersistenceBuilder();
+ // builder.setPathName("runtime/spellcheckbase");
+ builder.setPathName("Z:/");
+ builder.setRemoveFiles(false);
+ builder.setFileCache(16 * 1024/* pageSize */, 8 * 1024/* maxPage */);
+ builder.setGarbageManagement(false);
+ builder.setCrashSafeMode(false);
+ filePersistence = builder.build();
+ session = filePersistence.createDataAccessSession();
+ }
+
+ private void closeFilePersistence() throws FilePersistenceException,
+ FilePersistenceInvalidClassException,
+ FilePersistenceNotSerializableException,
+ FilePersistenceClassNotFoundException,
+ FilePersistenceDataCorruptedException,
+ FilePersistenceTooBigForSerializationException {
+ if (session.isOpened()) {
+ session.close(EnumFilePersistenceCloseAction.DO_NOT_SAVE);
+ }
+ filePersistence.close();
+ }
+
+}
Copied: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainReadWords.java (from rev 2972, trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/MainReadWords.java)
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainReadWords.java (rev 0)
+++ trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/MainReadWords.java 2011-11-21 07:11:53 UTC (rev 2974)
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2011 Luc Peuvrier
+ *
+ * This file is a part of JOAFIP.
+ *
+ * JOAFIP is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License.
+ *
+ * Licensed under the GNU LESSER GENERAL PUBLIC LICENSE
+ * Licensed under the LGPL License, Version 3, 29 June 2007 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.gnu.org/licenses/lgpl.html
+ *
+ * JOAFIP is distributed in the hope that it will be useful, but
+ * unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.sf.joafip.autosave.spellcheck;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.NavigableSet;
+import java.util.Set;
+
+import net.sf.joafip.NotStorableClass;
+
+/**
+ * @author luc peuvrier
+ *
+ */
+@SuppressWarnings("PMD")
+@NotStorableClass
+public class MainReadWords extends AbstractSpellCheck {
+
+ public static void main(String[] args) {
+ (new MainReadWords()).run();
+ }
+
+ private void run() {
+ try {
+ final ISpellCheckDAO spellCheckDAO = new SpellCheckJoafipDAO(true);
+
+ final InputStream in = getClass().getClassLoader()
+ .getResourceAsStream("word_list.txt");
+
+ final BufferedReader reader = new BufferedReader(
+ new InputStreamReader(in));
+
+ int wordValueCount = 0;
+ int labelCount = 0;
+ String line;
+ final long startTime = System.currentTimeMillis();
+ while (labelCount < MAX_LABELS
+ && (line = reader.readLine()) != null) {
+ labelCount++;
+ for (String wordValue : splitValue(line)) {
+ final NavigableSet<Integer> keyCodePointSet = computeKeyInSet(wordValue);
+ final String key = codePointToString(keyCodePointSet);
+ Set<String> set = spellCheckDAO.getWordSet(key);
+ if (set == null) {
+ set = spellCheckDAO.newSetOfString();
+ spellCheckDAO.addWordSet(key, set);
+ // updateKeyMap(spellCheckDAO, keyCodePointSet);
+ }
+ if (set.add(wordValue)) {
+ wordValueCount++;
+ System.out.println(wordValue + " " + wordValueCount);
+ }
+ spellCheckDAO.commit();
+ }
+ }
+ System.out.println("labelCount " + labelCount);
+ reader.close();
+
+ spellCheckDAO.close(true);
+
+ System.out.println("elapsed "
+ + (System.currentTimeMillis() - startTime) + " mS");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
Copied: trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/SpellCheckBase.java (from rev 2972, trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/SpellCheckBase.java)
===================================================================
--- trunk/joafip-4test/src/main/java/net/sf/joafip/autosave/spellcheck/SpellCheckBase.java ...
[truncated message content] |
