[Jython-checkins] SF.net SVN: jython: [4704] branches/asm

[<fwi...@us...>]

Revision: 4704
          http://jython.svn.sourceforge.net/jython/?rev=4704&view=rev
Author:   fwierzbicki
Date:     2008-06-20 10:32:49 -0700 (Fri, 20 Jun 2008)

Log Message:
-----------
Cleared up some bad behavior in Python.g (was reporting and recovering
from missing tokens).

Modified Paths:
--------------
    branches/asm/grammar/Python.g
    branches/asm/grammar/PythonPartial.g
    branches/asm/src/org/python/antlr/InteractiveParser.java
    branches/asm/src/org/python/antlr/PythonPartialTokenSource.java

Modified: branches/asm/grammar/Python.g
===================================================================
--- branches/asm/grammar/Python.g	2008-06-20 17:08:28 UTC (rev 4703)
+++ branches/asm/grammar/Python.g	2008-06-20 17:32:49 UTC (rev 4704)
@@ -394,6 +394,14 @@
         throw e;
     }
 
+	protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow)
+		throws RecognitionException
+	{
+        mismatch(input, ttype, follow);
+        return null;
+    }
+
+
     /**
      * A list holding the error message(s) encountered during parse.
      */

Modified: branches/asm/grammar/PythonPartial.g
===================================================================
--- branches/asm/grammar/PythonPartial.g	2008-06-20 17:08:28 UTC (rev 4703)
+++ branches/asm/grammar/PythonPartial.g	2008-06-20 17:32:49 UTC (rev 4704)
@@ -75,7 +75,7 @@
 } 
 
 @members {
-    boolean debugOn = false;
+    boolean debugOn = true;
 
     private void debug(String message) {
         if (debugOn) {
@@ -90,15 +90,17 @@
         throw e;
     }
 
-    /*
 	protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow)
 		throws RecognitionException
 	{
         mismatch(input, ttype, follow);
         return null;
     }
-    */
 
+	public void emitErrorMessage(String msg) {
+		System.err.print("[EMITTING] ");
+    }
+
 	public void reportError(RecognitionException e) {
 		System.err.print("[REPORTING] ");
 		// if we've already reported an error and have not matched a token
@@ -161,6 +163,22 @@
             }
         }
     }
+
+	public void reportError(RecognitionException e) {
+		System.err.print("[LEXER REPORTING] ");
+		// if we've already reported an error and have not matched a token
+		// yet successfully, don't report any errors.
+		if ( state.errorRecovery ) {
+			System.err.print("[SPURIOUS] ");
+			return;
+		}
+		state.syntaxErrors++; // don't count spurious
+		state.errorRecovery = true;
+
+		displayRecognitionError(this.getTokenNames(), e);
+	}
+
+
 }
 
 single_input : NEWLINE
@@ -208,7 +226,7 @@
      | compound_stmt
      ;
 
-simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? NEWLINE
+simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? (NEWLINE|ENDMARK)
             ;
 
 small_stmt : expr_stmt
@@ -440,6 +458,7 @@
      | FLOAT
      | COMPLEX
      | (STRING)+ {debug("matched STRING");} 
+     | STRINGPART
      ;
 
 listmaker : test 
@@ -660,13 +679,18 @@
     :   ('r'|'u'|'ur')?
         (   '\'\'\'' (options {greedy=false;}:TRIAPOS)* '\'\'\''
         |   '"""' (options {greedy=false;}:TRIQUOTE)* '"""'
-        |   '\'\'\'' ~('\'\'\'')*
-        |   '"""' ~('"""')*
         |   '"' (ESC|~('\\'|'\n'|'"'))* '"'
         |   '\'' (ESC|~('\\'|'\n'|'\''))* '\''
         )
     ;
 
+STRINGPART
+    :   ('r'|'u'|'ur')?
+        (   '\'\'\'' ~('\'\'\'')*
+        |   '"""' ~('"""')*
+        )
+    ;
+
 /** the two '"'? cause a warning -- is there a way to avoid that? */
 fragment
 TRIQUOTE

Modified: branches/asm/src/org/python/antlr/InteractiveParser.java
===================================================================
--- branches/asm/src/org/python/antlr/InteractiveParser.java	2008-06-20 17:08:28 UTC (rev 4703)
+++ branches/asm/src/org/python/antlr/InteractiveParser.java	2008-06-20 17:32:49 UTC (rev 4704)
@@ -65,8 +65,9 @@
             PythonWalker walker = new PythonWalker(nodes);
             tree = walker.interactive();
         } catch (RecognitionException e) {
-            //XXX: this can't happen.  Need to strip the throws from antlr
-            //     generated code.
+            //I am only throwing ParseExceptions, but "throws RecognitionException" still gets
+            //into the generated code.
+            System.err.println("FIXME: pretty sure this can't happen -- but needs to be checked");
         }
         return tree;
     }

Modified: branches/asm/src/org/python/antlr/PythonPartialTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonPartialTokenSource.java	2008-06-20 17:08:28 UTC (rev 4703)
+++ branches/asm/src/org/python/antlr/PythonPartialTokenSource.java	2008-06-20 17:32:49 UTC (rev 4704)
@@ -81,6 +81,14 @@
     {
         Token t = stream.LT(1);
         stream.consume();
+        /*
+        if ( t.getType()==PythonPartialLexer.STRINGPART ) {
+            Token nl = new ClassicToken(PythonPartialParser.NEWLINE,"\n");
+            nl.setCharPositionInLine(t.getCharPositionInLine());
+            nl.setLine(t.getLine());
+            tokens.addElement(nl);
+        }
+        */
 
         // if not a NEWLINE, doesn't signal indent/dedent work; just enqueue
         if ( t.getType()!=PythonPartialLexer.NEWLINE ) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.