[Jython-checkins] SF.net SVN: jython:[5931] trunk/jython

[<fwi...@us...>]

Revision: 5931
          http://jython.svn.sourceforge.net/jython/?rev=5931&view=rev
Author:   fwierzbicki
Date:     2009-01-14 23:56:22 +0000 (Wed, 14 Jan 2009)

Log Message:
-----------
DEDENT problem with comments in single mode fixed.

Modified Paths:
--------------
    trunk/jython/Lib/test/test_codeop_jy.py
    trunk/jython/src/org/python/antlr/PythonTokenSource.java

Modified: trunk/jython/Lib/test/test_codeop_jy.py
===================================================================
--- trunk/jython/Lib/test/test_codeop_jy.py	2009-01-14 14:09:58 UTC (rev 5930)
+++ trunk/jython/Lib/test/test_codeop_jy.py	2009-01-14 23:56:22 UTC (rev 5931)
@@ -102,6 +102,8 @@
         # Failed for Jython 2.5a2.  See http://bugs.jython.org/issue1116.
         av("@a.b.c\ndef f():\n pass") 
 
+        av("def f():\n pass\n#foo")
+
     def test_incomplete(self):
         ai = self.assertIncomplete
 

Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTokenSource.java	2009-01-14 14:09:58 UTC (rev 5930)
+++ trunk/jython/src/org/python/antlr/PythonTokenSource.java	2009-01-14 23:56:22 UTC (rev 5931)
@@ -195,7 +195,7 @@
             t = stream.LT(1);
             stream.consume();
 
-            enqueueHiddens(t);
+            List<Token> commentedNewlines = enqueueHiddens(t);
 
             // compute cpos as the char pos of next non-WS token in line
             int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
@@ -230,6 +230,9 @@
                 for(int i=1;i<newlines.length();i++) {
                     generateNewline(newline);
                 }
+                for (Token c : commentedNewlines) {
+                    generateNewline(c);
+                }
             }
 
             if (t.getType() != PythonLexer.LEADING_WS) { // discard WS
@@ -246,7 +249,8 @@
         tokens.addElement(t);
     }
 
-    private void enqueueHiddens(Token t) {
+    private List<Token> enqueueHiddens(Token t) {
+        List<Token> newlines = new ArrayList<Token>();
         if (inSingle && t.getType() == Token.EOF) {
             if (stream.size() > lastTokenAddedIndex + 1) {
                 Token hidden = stream.get(lastTokenAddedIndex + 1);
@@ -254,7 +258,7 @@
                     String text = hidden.getText();
                     int i = text.indexOf("\n");
                     while(i != -1) {
-                        generateNewline(hidden);
+                        newlines.add(hidden);
                         i = text.indexOf("\n", i + 1);
                     }
                 }
@@ -265,6 +269,7 @@
             tokens.addAll(hiddenTokens);
         }
         lastTokenAddedIndex = t.getTokenIndex();
+        return newlines;
     }
 
     private void handleIndents(int cpos, CommonToken t) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.