jython-checkins

[Jython-checkins] SF.net SVN: jython:[5289] trunk/jython/src/org/python/antlr/ PythonTokenSource.java

[<fwi...@us...>]

Revision: 5289
          http://jython.svn.sourceforge.net/jython/?rev=5289&view=rev
Author:   fwierzbicki
Date:     2008-09-05 03:36:04 +0000 (Fri, 05 Sep 2008)

Log Message:
-----------
Adjustment to offset fix.  Overthought the offsets for simple
dedents.

Modified Paths:
--------------
    trunk/jython/src/org/python/antlr/PythonTokenSource.java

Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-04 16:57:30 UTC (rev 5288)
+++ trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-05 03:36:04 UTC (rev 5289)
@@ -208,11 +208,7 @@
             }
             else if (cpos < lastIndent) { // they dedented
                 Token prev = stream.LT(-1);
-                if (prev != null) {
-                    handleDedents(cpos, (CommonToken)prev);
-                } else {
-                    handleDedents(cpos, (CommonToken)t);
-                }
+                handleDedents(cpos, (CommonToken)newline);
             }
 
             if (t.getType() == Token.EOF && inSingle) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Jython-checkins] SF.net SVN: jython:[5304] trunk/jython/src/org/python/antlr/ PythonTokenSource.java

[<fwi...@us...>]

Revision: 5304
          http://jython.svn.sourceforge.net/jython/?rev=5304&view=rev
Author:   fwierzbicki
Date:     2008-09-09 18:42:57 +0000 (Tue, 09 Sep 2008)

Log Message:
-----------
Fix some offset problems in DEDENT, INDENT, and EOF.

Modified Paths:
--------------
    trunk/jython/src/org/python/antlr/PythonTokenSource.java

Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-08 17:48:28 UTC (rev 5303)
+++ trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-09 18:42:57 UTC (rev 5304)
@@ -154,6 +154,7 @@
 
     private void handleEOF(CommonToken eof, CommonToken prev) {
         if (prev != null) {
+            eof.setStartIndex(prev.getStopIndex());
             eof.setStopIndex(prev.getStopIndex());
         }
     }
@@ -258,9 +259,11 @@
     private void handleIndents(int cpos, CommonToken t) {
         push(cpos);
         //System.out.println("push("+cpos+"): "+stackString());
-        Token indent = new CommonToken(PythonParser.INDENT,"");
+        CommonToken indent = new CommonToken(PythonParser.INDENT,"");
         indent.setCharPositionInLine(t.getCharPositionInLine());
         indent.setLine(t.getLine());
+        indent.setStartIndex(t.getStartIndex() - 1);
+        indent.setStopIndex(t.getStartIndex() - 1);
         tokens.addElement(indent);
     }
 
@@ -274,9 +277,8 @@
             dedent.setCharPositionInLine(t.getCharPositionInLine());
             dedent.setLine(t.getLine());
 
-            //XXX: this will get messed up by comments.
-            dedent.setStartIndex(t.getStartIndex());
-            dedent.setStopIndex(t.getStopIndex());
+            dedent.setStartIndex(t.getStartIndex() - 1);
+            dedent.setStopIndex(t.getStartIndex() - 1);
 
             tokens.addElement(dedent);
         }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.