[Jython-checkins] SF.net SVN: jython:[5190] branches/asm/src/org/python/antlr/ PythonTokenSource.java

[<fwi...@us...>]

Revision: 5190
          http://jython.svn.sourceforge.net/jython/?rev=5190&view=rev
Author:   fwierzbicki
Date:     2008-08-16 22:46:04 +0000 (Sat, 16 Aug 2008)

Log Message:
-----------
Fix comment/newline parsing problems in interactive (single) parsing.

Modified Paths:
--------------
    branches/asm/src/org/python/antlr/PythonTokenSource.java

Modified: branches/asm/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonTokenSource.java	2008-08-16 19:31:55 UTC (rev 5189)
+++ branches/asm/src/org/python/antlr/PythonTokenSource.java	2008-08-16 22:46:04 UTC (rev 5190)
@@ -146,7 +146,6 @@
     }
 
     private void generateNewline(Token t) {
-        // Imaginary newline before EOF
         CommonToken newline = new CommonToken(PythonLexer.NEWLINE, "\n");
         newline.setLine(t.getLine());
         newline.setCharPositionInLine(t.getCharPositionInLine());
@@ -158,9 +157,11 @@
         stream.consume();
 
         if (t.getType() == Token.EOF) {
-            Token prev = stream.LT(-1);
-            if (!inSingle && (prev == null || prev.getType() != PythonLexer.NEWLINE)) {
-                generateNewline(t);
+            if (!inSingle) {
+                Token prev = stream.LT(-1);
+                if (prev == null || prev.getType() != PythonLexer.NEWLINE) {
+                    generateNewline(t);
+                }
             }
 
             handleDedents(-1, (CommonToken)t);
@@ -168,7 +169,7 @@
         } else if (t.getType() == PythonLexer.NEWLINE) {
             // save NEWLINE in the queue
             //System.out.println("found newline: "+t+" stack is "+stackString());
-            updateLastTokenAddedIndex(t);
+            enqueueHiddens(t);
             tokens.addElement(t);
             Token newline = t;
 
@@ -176,7 +177,7 @@
             t = stream.LT(1);
             stream.consume();
 
-            updateLastTokenAddedIndex(t);
+            enqueueHiddens(t);
 
             // compute cpos as the char pos of next non-WS token in line
             int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
@@ -222,11 +223,24 @@
     }
     
     private void enqueue(Token t) {
-        updateLastTokenAddedIndex(t);
+        enqueueHiddens(t);
         tokens.addElement(t);
     }
 
-    private void updateLastTokenAddedIndex(Token t) {
+    private void enqueueHiddens(Token t) {
+        if (inSingle && t.getType() == Token.EOF) {
+            if (stream.size() > lastTokenAddedIndex + 1) {
+                Token hidden = stream.get(lastTokenAddedIndex + 1);
+                if (hidden.getType() == PythonLexer.COMMENT) {
+                    String text = hidden.getText();
+                    int i = text.indexOf("\n");
+                    while(i != -1) {
+                        generateNewline(hidden);
+                        i = text.indexOf("\n", i + 1);
+                    }
+                }
+            }
+        }
         List hiddenTokens = stream.getTokens(lastTokenAddedIndex + 1,t.getTokenIndex() - 1);
         if (hiddenTokens != null) {
             tokens.addAll(hiddenTokens);


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.