foray-commit Mailing List for FOray (Page 78)

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Revision: 11945
          http://sourceforge.net/p/foray/code/11945
Author:   victormote
Date:     2021-10-26 23:47:05 +0000 (Tue, 26 Oct 2021)
Log Message:
-----------
Improvements to spell-checker & English dictionary.

Modified Paths:
--------------
    trunk/foray/foray-hyphen/src/main/data/word-lists/eng-word-list-moby.txt
    trunk/foray/foray-hyphen/src/main/java/org/foray/hyphen/SpellChecker.java

Modified: trunk/foray/foray-hyphen/src/main/data/word-lists/eng-word-list-moby.txt
===================================================================

--- trunk/foray/foray-hyphen/src/main/data/word-lists/eng-word-list-moby.txt	2021-10-26 19:17:20 UTC (rev 11944)
+++ trunk/foray/foray-hyphen/src/main/data/word-lists/eng-word-list-moby.txt	2021-10-26 23:47:05 UTC (rev 11945)
@@ -37,6 +37,7 @@
 # readable as the bullet symbol, and is unlikely to be used for any other purpose.
 #
 #
+a
 a cap"pel"la
 a for"ti"o"ri
 a go"go
@@ -3556,6 +3557,7 @@
 Al"ys
 Al"y"at"tes
 Al"y"son
+am
 ambs"ace
 ames"ace
 Ames"bur"y
@@ -7839,6 +7841,7 @@
 ar"y"te"noi"dal
 ar"y"te"no"ep"i"glot"tic
 Ar"za"chel
+as
 ascetic
 asdic
 ash
@@ -8437,6 +8440,7 @@
 as"ymp"tot"i"cal"ly
 as"yn"det"ic
 as"yn"det"i"cal"ly
+at
 at-home
 Atch"e"son
 Atch"i"son
@@ -10980,6 +10984,7 @@
 A"my"clas
 a"my"o"to"ni"a
 A"mé"dée
+an
 a"na
 a"nab"a"sine
 a"nab"a"sis
@@ -14172,6 +14177,7 @@
 bdl
 BDS
 bds
+be
 Bea
 BEA
 Beach
@@ -21226,6 +21232,7 @@
 BVM
 bwa"na
 BWG
+by
 by-bid"der
 by-bid"ding
 by-e"lec"tion
@@ -43924,6 +43931,7 @@
 Dnies"ter
 Dnie"per
 Dnie"ster
+do
 do o"ver
 do with"out
 do-all
@@ -45037,6 +45045,7 @@
 DPP
 dpt
 DPW
+Dr
 drab
 drab"ber
 drab"best
@@ -62736,6 +62745,7 @@
 GNP
 gns
 gnu
+go
 go a"bout
 go a"gainst
 go a"head
@@ -67629,6 +67639,7 @@
 hcf
 hdbk
 hdqrs
+he
 he'd
 he'll
 he's
@@ -73530,6 +73541,7 @@
 IEE
 ier"pe"tra"ble
 Ie"per
+if
 IFC
 IFS
 if"fy
@@ -74697,6 +74709,7 @@
 im"pu"tres"ci"ble
 Im"re
 Im"roz
+in
 in ab"sen"ti"a
 in ae"ter"num
 in cam"er"a
@@ -80119,6 +80132,7 @@
 I"rus
 I"rène
 I"r"kli"on
+is
 I"s
 I"saac
 I"saacs
@@ -80384,6 +80398,7 @@
 i"sta"na
 I"sus
 I"sère
+it
 I"tal"ian
 I"tal"ian East Af"ri"ca
 I"tal"ian sixth
@@ -93241,6 +93256,7 @@
 Mc"Teague
 MDS
 mdse
+me
 me-too
 me-too"ism
 Mead
@@ -98470,6 +98486,7 @@
 MRA
 MRC
 mri"dang
+Mr
 Mrs
 Ms-Th
 MSc
@@ -99350,6 +99367,7 @@
 Mwan"za
 mwa"li"mu
 Mwe"ru
+my
 mycol
 Myc"e"ri"nus
 myd"ri"at"ic
@@ -101800,6 +101818,7 @@
 Nnam"di
 NNE
 NNW
+no
 no-ac"count
 no-ball
 no-claim bo"nus
@@ -109323,6 +109342,7 @@
 op"u"lent
 op"u"lent"ly
 op"u"len"cy
+or
 orb
 orc
 orch
@@ -111393,6 +111413,7 @@
 o"dyl"ic
 O"dys"seus
 O"dys"se"us
+of
 o"fay
 O"gal"lal"a
 O"ga"sa"wa"ra Gun"to
@@ -111538,6 +111559,7 @@
 o"mo"pha"gi"a
 o"mo"pho"ri"on
 O"mu"ta
+on
 O"na
 o"nan"ism
 o"nan"ist
@@ -149487,6 +149509,7 @@
 sny
 snye
 sny"ing
+so
 so-so
 soak
 soak"age
@@ -163046,6 +163069,7 @@
 tme"sis
 tng
 TNT
+to
 to-be
 to-do
 to-mor"row
@@ -178731,6 +178755,7 @@
 un"zone
 un"zoned
 un"zon"ing
+up
 up-and-com"ing
 up-and-o"ver
 up-and-un"der
@@ -179420,6 +179445,7 @@
 U"ru"guay"an
 U"ru"gua"ia"na
 u"ru"shi"ol
+us
 u"shab"ti
 u"shab"tis
 u"shab"ti"u
@@ -183011,6 +183037,7 @@
 WBC
 WCC
 WCTU
+we
 we'd
 we'll
 we're
@@ -186281,6 +186308,7 @@
 Ya"va"r
 Ya"wa"ta
 ya"za"ta
+ye
 ye'se
 yea
 yeah

Modified: trunk/foray/foray-hyphen/src/main/java/org/foray/hyphen/SpellChecker.java
===================================================================
--- trunk/foray/foray-hyphen/src/main/java/org/foray/hyphen/SpellChecker.java	2021-10-26 19:17:20 UTC (rev 11944)
+++ trunk/foray/foray-hyphen/src/main/java/org/foray/hyphen/SpellChecker.java	2021-10-26 23:47:05 UTC (rev 11945)
@@ -64,9 +64,12 @@
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParserFactory;
@@ -127,6 +130,21 @@
     /** Command-line return status constant indicating that there was an error parsing the input file. */
     public static final byte STATUS_PARSING_ERROR = 3;
 
+    /** The list of patterns for which matching words should NOT be considered as misspelled. */
+    private static final List<Pattern> VALID_WORD_PATTERNS = new ArrayList<Pattern>();
+    static {
+        /* The word is all Arabic digits. */
+        VALID_WORD_PATTERNS.add(Pattern.compile("^[0-9]+$"));
+        /* The word is all uppercase Roman numeral characters. */
+        VALID_WORD_PATTERNS.add(Pattern.compile("^[IVXLCDM]+$"));
+        /* The word is all lowercase Roman numeral characters. */
+        VALID_WORD_PATTERNS.add(Pattern.compile("^[ivxlcdm]+$"));
+        /* The word is currency. */
+        VALID_WORD_PATTERNS.add(Pattern.compile("^[$\xA3][0-9]+[0-9,\\.]*$"));
+        /* The word is a single capital letter, such as a person's initial. */
+        VALID_WORD_PATTERNS.add(Pattern.compile("^[A-Z]$"));
+    }
+
     /** The input source to be pretty-printed. */
     private InputSource input;
 
@@ -162,6 +180,9 @@
     /** The list of dictionaries that are currently active, i.e. that match the current orthography. */
 //    private List<Dictionary> currentDictionaries;
 
+    /** The counter for "Not found" words. */
+    private int notFoundCounter = 0;
+
     /**
      * Constructor.
      * @param input The input source encapsulating the document to be spell-checked.
@@ -272,6 +293,7 @@
 
     @Override
     public void endDocument() throws SAXException {
+        this.output.println("Count of \"Not found\" items: " + this.notFoundCounter);
     }
 
 
@@ -385,6 +407,7 @@
                             } else {
                                 if (flagThisWord(dictionary, word)) {
                                     this.output.println("Not found: " + word);
+                                    this.notFoundCounter ++;
                                 }
                             }
                         }
@@ -424,10 +447,51 @@
      * word for any other reason.
      */
     private boolean flagThisWord(final SegmentDictionary dictionary, final CharSequence word) {
-        final Word dictWord = dictionary.getWord(word);
+        /* TODO: This is all eng-us-lat specific. Need to think about how plurals and other almost-the-same cases should
+         * be handled. */
+
+        /* Is the word itself directly in the dictionary. */
+        Word dictWord = dictionary.getWord(word);
         if (dictWord != null) {
             return false;
         }
+
+        /* Check against patterns that are considered to be valid words, but that are not in the dictionary. */
+        for (int index = 0; index < VALID_WORD_PATTERNS.size(); index ++) {
+            final Pattern pattern = VALID_WORD_PATTERNS.get(index);
+            final Matcher matcher = pattern.matcher(word);
+            if (matcher.matches()) {
+                return false;
+            }
+        }
+
+        /* Starting here, we will change the word content in various ways, looking for a match. The changes are
+         * cumulative. */
+        final StringBuilder builder = new StringBuilder(word);
+
+        /* If the first character is capitalized, convert to lowercase & check again. */
+        if (Character.isUpperCase(builder.charAt(0))) {
+            builder.setCharAt(0, Character.toLowerCase(builder.charAt(0)));
+            dictWord = dictionary.getWord(builder);
+        }
+        if (dictWord != null) {
+            return false;
+        }
+
+        /* If the last character is a lowercase "s" or "'s" (apostrophe "s"), this may be a plural noun, a
+         * present-tense verb, or a possessive. Remove them and try again. */
+        if (builder.charAt(builder.length() - 1) == 's') {
+            builder.deleteCharAt(builder.length() - 1);
+            if (builder.length() > 0
+                    && builder.charAt(builder.length() - 1) == '\x92') {
+                builder.deleteCharAt(builder.length() - 1);
+            }
+            dictWord = dictionary.getWord(builder);
+        }
+        if (dictWord != null) {
+            return false;
+        }
+
         return true;
     }
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





2006	Jan	Feb	Mar (139)	Apr (98)	May (250)	Jun (394)	Jul (84)	Aug (13)	Sep (420)	Oct (186)	Nov (1)	Dec (3)
2007	Jan (108)	Feb (202)	Mar (291)	Apr (247)	May (374)	Jun (227)	Jul (231)	Aug (60)	Sep (31)	Oct (45)	Nov (18)	Dec
2008	Jan (38)	Feb (71)	Mar (142)	Apr	May (59)	Jun (6)	Jul (10)	Aug	Sep	Oct	Nov	Dec
2009	Jan (12)	Feb (4)	Mar (88)	Apr (121)	May (17)	Jun (30)	Jul	Aug (5)	Sep	Oct (1)	Nov	Dec
2010	Jan (11)	Feb (76)	Mar (11)	Apr	May (11)	Jun	Jul	Aug (44)	Sep (14)	Oct (7)	Nov	Dec
2011	Jan	Feb	Mar	Apr	May (9)	Jun	Jul	Aug	Sep	Oct (10)	Nov	Dec
2012	Jan	Feb	Mar	Apr	May	Jun (3)	Jul (4)	Aug	Sep	Oct	Nov	Dec
2016	Jan	Feb	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec (168)
2017	Jan (77)	Feb (11)	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec
2018	Jan	Feb	Mar (1)	Apr (6)	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec
2019	Jan	Feb (88)	Mar (118)	Apr (1)	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec
2020	Jan	Feb	Mar	Apr	May (6)	Jun	Jul	Aug	Sep	Oct	Nov	Dec (141)
2021	Jan (170)	Feb (20)	Mar	Apr	May	Jun	Jul (1)	Aug	Sep	Oct (62)	Nov (189)	Dec (162)
2022	Jan (201)	Feb (118)	Mar (8)	Apr	May (2)	Jun (47)	Jul (19)	Aug (14)	Sep (3)	Oct	Nov (28)	Dec (235)
2023	Jan (112)	Feb (23)	Mar (2)	Apr (2)	May	Jun (1)	Jul	Aug (70)	Sep (92)	Oct (20)	Nov (1)	Dec (1)
2024	Jan	Feb	Mar (1)	Apr (1)	May (14)	Jun (11)	Jul (1)	Aug	Sep	Oct	Nov	Dec
2025	Jan (10)	Feb (29)	Mar	Apr (162)	May (245)	Jun (83)	Jul	Aug (1)	Sep	Oct	Nov (4)	Dec

foray-commit Mailing List for FOray (Page 78)

Modular XSL-FO Implementation for Java.

foray-commit — FOray repository commit log messages