foray-commit Mailing List for FOray (Page 47)

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Revision: 12726
          http://sourceforge.net/p/foray/code/12726
Author:   victormote
Date:     2022-08-30 18:39:31 +0000 (Tue, 30 Aug 2022)
Log Message:
-----------
Rough-in parsing and storage of explicit tokens.

Modified Paths:
--------------
    trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java
    trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/OrthographyParser.java

Added Paths:
-----------
    trunk/foray/foray-orthography/src/main/java/org/foray/orthography/ExplicitTokens.java

Added: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/ExplicitTokens.java
===================================================================

--- trunk/foray/foray-orthography/src/main/java/org/foray/orthography/ExplicitTokens.java	                        (rev 0)
+++ trunk/foray/foray-orthography/src/main/java/org/foray/orthography/ExplicitTokens.java	2022-08-30 18:39:31 UTC (rev 12726)
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2022 The FOray Project.
+ *      http://www.foray.org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This work is in part derived from the following work(s), used with the
+ * permission of the licensor:
+ *      Apache FOP, licensed by the Apache Software Foundation
+ *
+ */
+
+/*
+ * $LastChangedRevision$
+ * $LastChangedDate$
+ * $LastChangedBy$
+ */
+
+package org.foray.orthography;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A collection of strings that should be treated as explicit tokens by a tokenizer/lexer.
+ */
+public class ExplicitTokens {
+
+    /** The initial size of the token collection. */
+    private static final int INITIAL_MAP_CAPACITY = 50;
+
+    /** The collection of explicit tokens. */
+    private Set<String> tokens = new HashSet<String>(INITIAL_MAP_CAPACITY);
+    /* TODO: Convert this to a TernaryTree. */
+
+    /** The size, in chars, of the largest member of {@link #tokens}. */
+    private int maxTokenSize;
+
+    /**
+     * Adds an explicit token to this orthography.
+     * @param token The explicit token to be added.
+     */
+    public void addToken(final String token) {
+        if (token.length() > this.maxTokenSize) {
+            this.maxTokenSize = token.length();
+        }
+        this.tokens.add(token);
+    }
+
+    /**
+     * Searches a character sequence starting at a given index to see if the indexed character is the beginning of an
+     * explicit token, returning that token if it is.
+     * @param sequence The character sequence (usually a {@link String}) being searched.
+     * @param start The index to the first character in {@code sequence} to be tested.
+     * @return The matching token, if there is one, or null if not.
+     */
+    public String findToken(final CharSequence sequence, final int start) {
+        int index = start + 1;
+        while (index < sequence.length()
+                && index - start < this.maxTokenSize) {
+            /* TODO: This String creation horrible and should only be used for proof of concept. */
+            final String testString = sequence.subSequence(start, index).toString();
+            if (this.tokens.contains(testString)) {
+                return testString;
+            }
+            index ++;
+        }
+        return null;
+    }
+
+}


Property changes on: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/ExplicitTokens.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Rev
\ No newline at end of property
Modified: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java
===================================================================
--- trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java	2022-08-30 13:06:58 UTC (rev 12725)
+++ trunk/foray/foray-orthography/src/main/java/org/foray/orthography/Orthography4a.java	2022-08-30 18:39:31 UTC (rev 12726)
@@ -81,6 +81,9 @@
     /** Regex pattern used to break compound words into their components. */
     private Pattern compoundWordBreaker = Pattern.compile(Character.toString(compoundWordMarker));
 
+    /** The explicit tokens for this orthography. */
+    private ExplicitTokens explicitTokens;
+
     /**
      * Constructor.
      * @param server The parent hyphenation server.
@@ -480,4 +483,26 @@
         return false;
     }
 
+    /**
+     * Sets the explicit tokens for this orthography.
+     * @param tokens The new explicit tokens for this orthography.
+     */
+    public void setExplicitTokens(final ExplicitTokens tokens) {
+        this.explicitTokens = tokens;
+    }
+
+    /**
+     * Searches a character sequence starting at a given index to see if the indexed character is the beginning of an
+     * explicit token, returning that token if it is.
+     * @param sequence The character sequence (usually a {@link String}) being searched.
+     * @param start The index to the first character in {@code sequence} to be tested.
+     * @return The matching token, if there is one, or null if not.
+     */
+    public String findToken(final CharSequence sequence, final int start) {
+        if (this.explicitTokens == null) {
+            return null;
+        }
+        return this.explicitTokens.findToken(sequence, start);
+    }
+
 }

Modified: trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/OrthographyParser.java
===================================================================
--- trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/OrthographyParser.java	2022-08-30 13:06:58 UTC (rev 12725)
+++ trunk/foray/foray-orthography/src/main/java/org/foray/orthography/util/OrthographyParser.java	2022-08-30 18:39:31 UTC (rev 12726)
@@ -39,6 +39,7 @@
 import org.foray.orthography.DerivativePattern;
 import org.foray.orthography.DerivativeRule;
 import org.foray.orthography.DictionaryResource;
+import org.foray.orthography.ExplicitTokens;
 import org.foray.orthography.HyphenationPatternsResource;
 import org.foray.orthography.Orthography4a;
 import org.foray.orthography.OrthographyServer4a;
@@ -79,6 +80,9 @@
     /** Stateful variable. */
     private DictionaryResource currentDictionaryResource;
 
+    /** The current ExplicitTokens instance being parsed. */
+    private ExplicitTokens currentExplicitTokens;
+
     /** Stateful variable. */
     private DictionaryResource.WordListElement currentWordListElement;
 
@@ -123,7 +127,10 @@
 
 //    /** The map of match rule lists, keyed by id. */
 //    private Map<String, List<Pattern>> matchRuleLists = new HashMap<String, List<Pattern>>();
-//
+
+    /** The map of parsed {@link ExplicitTokens} instances, keyed by id. */
+    private Map<String, ExplicitTokens> explicitTokensMap = new HashMap<String, ExplicitTokens>();
+
     /** The map of derivative factory lists, keyed by id. */
     private Map<String, List<WordWrapperFactory<?>>> derivativeLists =
             new HashMap<String, List<WordWrapperFactory<?>>>();
@@ -415,9 +422,19 @@
         case "convertible-to-possessive": return;
         case "possessive": return;
         case "extensible": return;
-        case "explicit-token-list": return;
+        case "explicit-token-list": {
+            final String idString = attributes.getValue("id");
+            this.currentExplicitTokens = new ExplicitTokens();
+            this.explicitTokensMap.put(idString, currentExplicitTokens);
+            return;
+        }
         case "explicit-token": return;
-        case "explicit-tokens": return;
+        case "explicit-tokens": {
+            final String reference = attributes.getValue("reference");
+            final ExplicitTokens tokens = this.explicitTokensMap.get(reference);
+            this.currentOrthographyConfig.setExplicitTokens(tokens);
+            return;
+        }
         default: {
             /* Make sure user knows about unknown tag. */
             errorMessage("Unknown tag in orthography configuration: {}", localName);
@@ -602,9 +619,14 @@
         case "orthography": {
             return;
         }
+        case "explicit-token-list": {
+            this.currentExplicitTokens = null;
+            return;
+        }
         case "explicit-token": {
-            /* TODO: Complete this. For now, clear the text. */
-            getAndClearText();
+            final String token = getAndClearText();
+            this.currentExplicitTokens.addToken(token);
+            return;
         }
         }
     }

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





2006	Jan	Feb	Mar (139)	Apr (98)	May (250)	Jun (394)	Jul (84)	Aug (13)	Sep (420)	Oct (186)	Nov (1)	Dec (3)
2007	Jan (108)	Feb (202)	Mar (291)	Apr (247)	May (374)	Jun (227)	Jul (231)	Aug (60)	Sep (31)	Oct (45)	Nov (18)	Dec
2008	Jan (38)	Feb (71)	Mar (142)	Apr	May (59)	Jun (6)	Jul (10)	Aug	Sep	Oct	Nov	Dec
2009	Jan (12)	Feb (4)	Mar (88)	Apr (121)	May (17)	Jun (30)	Jul	Aug (5)	Sep	Oct (1)	Nov	Dec
2010	Jan (11)	Feb (76)	Mar (11)	Apr	May (11)	Jun	Jul	Aug (44)	Sep (14)	Oct (7)	Nov	Dec
2011	Jan	Feb	Mar	Apr	May (9)	Jun	Jul	Aug	Sep	Oct (10)	Nov	Dec
2012	Jan	Feb	Mar	Apr	May	Jun (3)	Jul (4)	Aug	Sep	Oct	Nov	Dec
2016	Jan	Feb	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec (168)
2017	Jan (77)	Feb (11)	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec
2018	Jan	Feb	Mar (1)	Apr (6)	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec
2019	Jan	Feb (88)	Mar (118)	Apr (1)	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec
2020	Jan	Feb	Mar	Apr	May (6)	Jun	Jul	Aug	Sep	Oct	Nov	Dec (141)
2021	Jan (170)	Feb (20)	Mar	Apr	May	Jun	Jul (1)	Aug	Sep	Oct (62)	Nov (189)	Dec (162)
2022	Jan (201)	Feb (118)	Mar (8)	Apr	May (2)	Jun (47)	Jul (19)	Aug (14)	Sep (3)	Oct	Nov (28)	Dec (235)
2023	Jan (112)	Feb (23)	Mar (2)	Apr (2)	May	Jun (1)	Jul	Aug (70)	Sep (92)	Oct (20)	Nov (1)	Dec (1)
2024	Jan	Feb	Mar (1)	Apr (1)	May (14)	Jun (11)	Jul (1)	Aug	Sep	Oct	Nov	Dec
2025	Jan (10)	Feb (29)	Mar	Apr (162)	May (245)	Jun (83)	Jul	Aug (1)	Sep	Oct	Nov (2)	Dec

foray-commit Mailing List for FOray (Page 47)

Modular XSL-FO Implementation for Java.

foray-commit — FOray repository commit log messages