|
From: <vic...@us...> - 2026-07-30 14:13:16
|
Revision: 14951
http://sourceforge.net/p/foray/code/14951
Author: victormote
Date: 2026-07-30 14:13:14 +0000 (Thu, 30 Jul 2026)
Log Message:
-----------
Move token-based word recognition logic from SpellChecker to Orthography4a, for reuse.
Modified Paths:
--------------
trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Lexer4a.java
trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java
trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/SpellChecker.java
Modified: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Lexer4a.java
===================================================================
--- trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Lexer4a.java 2026-07-30 13:27:21 UTC (rev 14950)
+++ trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Lexer4a.java 2026-07-30 14:13:14 UTC (rev 14951)
@@ -470,7 +470,7 @@
}
@Override
- public Token previous() {
+ public Token4a previous() {
if (! this.isLocked) {
throw new IllegalStateException("This lexer is not locked.");
}
@@ -482,6 +482,17 @@
return returnToken;
}
+ /**
+ * Does exactly what {@link #next()} does, but without incrementing the next token.
+ * This allows client code to see the next token without advancing to it.
+ * @return The next token.
+ */
+ public Token4a peekPrevious() {
+ final Token4a token = previous();
+ this.iteratorIndex ++;
+ return token;
+ }
+
@Override
public void remove() {
throw new UnsupportedOperationException("Method \"remove\" is not supported.");
Modified: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java
===================================================================
--- trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java 2026-07-30 13:27:21 UTC (rev 14950)
+++ trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java 2026-07-30 14:13:14 UTC (rev 14951)
@@ -230,6 +230,36 @@
return null;
}
+ /**
+ * Recognizes a word from a lexer token and its lexer.
+ * @param token The token whose content should be converted to a word, if possible.
+ * @param lexer The lexer that produced {@code token}, in case context is needed.
+ * @return The word from this orthography that matches {@code token}, or null if none exists.
+ */
+ public Word4a recognizeWord(final Lexer4a.Token4a token, final Lexer4a lexer) {
+ if (token.getTokenType() != Lexer.TokenType.WORD) {
+ throw new IllegalArgumentException("Token must be a word token.");
+ }
+ final CharSequence text = token.getText();
+ Word4a recognizedWord = recognizeWord(text, 0, text.length(), null);
+ if (recognizedWord == null) {
+ final Lexer.Token savedToken = token.getImmutableCopy();
+ if (lexer.hasNext()) {
+ final Lexer4a.Token4a nextToken = lexer.peekNext();
+ if (nextToken.getTokenType() == Lexer.TokenType.AMBIGUOUS_TRAILING_PUNCTUATION) {
+ final String testWord = savedToken.getText().toString() + nextToken.getText().toString();
+ recognizedWord = recognizeWord(testWord, 0, testWord.length(), null);
+ if (recognizedWord != null) {
+ return recognizedWord;
+ }
+ }
+ }
+ } else {
+ return recognizedWord;
+ }
+ return null;
+ }
+
@Override
public Word4a hyphenateUnrecognizedWord(final CharSequence word, final int offset, final int length) {
/* The character sequence containing the characters in the word that we are looking for. */
Modified: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/SpellChecker.java
===================================================================
--- trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/SpellChecker.java 2026-07-30 13:27:21 UTC (rev 14950)
+++ trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/SpellChecker.java 2026-07-30 14:13:14 UTC (rev 14951)
@@ -434,7 +434,7 @@
orthography = this.server.getOrthography(lastWritingSystem);
}
while (lexer.hasNext()) {
- final Lexer.Token token = lexer.next();
+ final Lexer4a.Token4a token = lexer.next();
if (token.getTokenType() == TokenType.WORD) {
if (! lastWritingSystem.equals(token.getWritingSystem())) {
orthography = this.server.getOrthography(token.getWritingSystem());
@@ -452,27 +452,14 @@
* @param token The word token to be checked.
* @param element The element containing the content being tokenized.
*/
- private void checkWord(final Orthography4a orthography, final Lexer.Token token, final TextElement element) {
- CharSequence text = StringUtils.EMPTY_STRING;
+ private void checkWord(final Orthography4a orthography, final Lexer4a.Token4a token, final TextElement element) {
if (orthography == null) {
/* Treat as an error. */
- text = "(no config) " + token.getText();
+ final String message = "(no config) " + token.getText();
+ this.output.println(message);
} else {
- text = token.getText();
- Word4a recognizedWord = orthography.recognizeWord(text, 0, text.length(), null);
- if (recognizedWord == null) {
- final Lexer.Token savedToken = token.getImmutableCopy();
- if (this.lexer.hasNext()) {
- final Lexer.Token nextToken = lexer.peekNext();
- if (nextToken.getTokenType() == Lexer.TokenType.AMBIGUOUS_TRAILING_PUNCTUATION) {
- final String testWord = savedToken.getText().toString() + nextToken.getText().toString();
- recognizedWord = orthography.recognizeWord(testWord, 0, testWord.length(), null);
- if (recognizedWord != null) {
- return;
- }
- }
- }
- } else {
+ final Word4a word = orthography.recognizeWord(token, this.lexer);
+ if (word != null) {
return;
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
