[Jython-checkins] SF.net SVN: jython:[5181] branches/asm/src/org/python/antlr/ PythonTokenSource.java

[<fwi...@us...>]

Revision: 5181
          http://jython.svn.sourceforge.net/jython/?rev=5181&view=rev
Author:   fwierzbicki
Date:     2008-08-15 19:44:57 +0000 (Fri, 15 Aug 2008)

Log Message:
-----------
Refactor: put repeated code into one method.

Modified Paths:
--------------
    branches/asm/src/org/python/antlr/PythonTokenSource.java

Modified: branches/asm/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonTokenSource.java	2008-08-15 18:37:14 UTC (rev 5180)
+++ branches/asm/src/org/python/antlr/PythonTokenSource.java	2008-08-15 19:44:57 UTC (rev 5181)
@@ -168,11 +168,7 @@
         } else if (t.getType() == PythonLexer.NEWLINE) {
             // save NEWLINE in the queue
             //System.out.println("found newline: "+t+" stack is "+stackString());
-            List hiddenTokens = stream.getTokens(lastTokenAddedIndex + 1,t.getTokenIndex() - 1);
-            if (hiddenTokens!=null) {
-                tokens.addAll(hiddenTokens);
-            }
-            lastTokenAddedIndex = t.getTokenIndex();
+            updateLastTokenAddedIndex(t);
             tokens.addElement(t);
             Token newline = t;
 
@@ -180,11 +176,7 @@
             t = stream.LT(1);
             stream.consume();
 
-            hiddenTokens = stream.getTokens(lastTokenAddedIndex + 1,t.getTokenIndex() - 1);
-            if (hiddenTokens!=null) {
-                tokens.addAll(hiddenTokens);
-            }
-            lastTokenAddedIndex = t.getTokenIndex();
+            updateLastTokenAddedIndex(t);
 
             // compute cpos as the char pos of next non-WS token in line
             int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
@@ -230,12 +222,16 @@
     }
     
     private void enqueue(Token t) {
+        updateLastTokenAddedIndex(t);
+        tokens.addElement(t);
+    }
+
+    private void updateLastTokenAddedIndex(Token t) {
         List hiddenTokens = stream.getTokens(lastTokenAddedIndex + 1,t.getTokenIndex() - 1);
         if (hiddenTokens != null) {
             tokens.addAll(hiddenTokens);
         }
         lastTokenAddedIndex = t.getTokenIndex();
-        tokens.addElement(t);
     }
 
     private void handleIndents(int cpos, CommonToken t) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.