[Jython-checkins] SF.net SVN: jython: [4460] trunk/jython/grammar/Python.g

[<fwi...@us...>]

Revision: 4460
          http://jython.svn.sourceforge.net/jython/?rev=4460&view=rev
Author:   fwierzbicki
Date:     2008-05-27 09:56:34 -0700 (Tue, 27 May 2008)

Log Message:
-----------
Fail fast on lexer errors.  Also correct lexing of
imaginary numbers (can start with zero, are never
octal or hex).

Modified Paths:
--------------
    trunk/jython/grammar/Python.g

Modified: trunk/jython/grammar/Python.g
===================================================================
--- trunk/jython/grammar/Python.g	2008-05-27 15:56:25 UTC (rev 4459)
+++ trunk/jython/grammar/Python.g	2008-05-27 16:56:34 UTC (rev 4460)
@@ -187,6 +187,7 @@
 @header { 
 package org.python.antlr;
 
+import org.python.antlr.ParseException;
 import org.python.antlr.PythonTree;
 } 
 
@@ -258,6 +259,33 @@
  */
 int implicitLineJoiningLevel = 0;
 int startPos=-1;
+
+    public Token nextToken() {
+		while (true) {
+			token = null;
+			channel = Token.DEFAULT_CHANNEL;
+			tokenStartCharIndex = input.index();
+			tokenStartCharPositionInLine = input.getCharPositionInLine();
+			tokenStartLine = input.getLine();
+			text = null;
+			if ( input.LA(1)==CharStream.EOF ) {
+                return Token.EOF_TOKEN;
+            }
+            try {
+                mTokens();
+				if ( token==null ) {
+					emit();
+				}
+				else if ( token==Token.SKIP_TOKEN ) {
+					continue;
+				}
+				return token;
+			}
+            catch (RecognitionException re) {
+                throw new ParseException(getErrorMessage(re, this.getTokenNames()));
+            }
+        }
+    }
 }
 
 //single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
@@ -985,12 +1013,12 @@
 INT :   // Hex
         '0' ('x' | 'X') ( '0' .. '9' | 'a' .. 'f' | 'A' .. 'F' )+
     |   // Octal
-        '0' DIGITS*
+        '0'  ( '0' .. '7' )*
     |   '1'..'9' DIGITS*
     ;
 
 COMPLEX
-    :   INT ('j'|'J')
+    :   DIGITS+ ('j'|'J')
     |   FLOAT ('j'|'J')
     ;
 


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.