[DL-Learner SVN] SF.net SVN: dl-learner:[4185] trunk/components-core/src/main/java/org/ dllearner/algorithms/isle/index/TextDocument.java

[<dfl...@us...>]

Revision: 4185
          http://sourceforge.net/p/dl-learner/code/4185
Author:   dfleischhacker
Date:     2013-12-02 15:19:06 +0000 (Mon, 02 Dec 2013)
Log Message:
-----------
Add getTokensStartingAtToken without numerOfToken parameter

Modified Paths:
--------------
    trunk/components-core/src/main/java/org/dllearner/algorithms/isle/index/TextDocument.java

Modified: trunk/components-core/src/main/java/org/dllearner/algorithms/isle/index/TextDocument.java
===================================================================
--- trunk/components-core/src/main/java/org/dllearner/algorithms/isle/index/TextDocument.java	2013-12-02 14:59:36 UTC (rev 4184)
+++ trunk/components-core/src/main/java/org/dllearner/algorithms/isle/index/TextDocument.java	2013-12-02 15:19:06 UTC (rev 4185)
@@ -99,6 +99,31 @@
         return tokens;
     }
 
+    /**
+     * Returns a list containing all successive tokens from this document starting at the given start
+     * token. If {@code ignorePunctuation} is set, tokens which represent punctuation are added to the result but not
+     * counted for the number of tokens.
+     *
+     * @param start             token to start collecting tokens from the document
+     * @param ignorePunctuation if true, punctuation are not counted towards the number of tokens to return
+     * @return list containing all relevant tokens, depending in the value of ignorePunctuation, the
+     *          list might contain additional non-relevant (punctuation) tokens
+     */
+    public List<Token> getTokensStartingAtToken(Token start, boolean ignorePunctuation) {
+        ArrayList<Token> tokens = new ArrayList<Token>();
+
+        boolean found = false;
+
+        for (int i = 0; i < this.size(); i++) {
+            Token t = this.get(i);
+            if (t == start) {
+                return this.subList(i, this.size());
+            }
+        }
+
+        return tokens;
+    }
+
     private String getStringForLevel(Token t, SurfaceFormLevel l) {
         switch (l) {
             case RAW:

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.