Thread: [Jython-checkins] SF.net SVN: jython:[5178] branches/asm (Page 3)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Revision: 5178
          http://jython.svn.sourceforge.net/jython/?rev=5178&view=rev
Author:   fwierzbicki
Date:     2008-08-15 11:43:53 +0000 (Fri, 15 Aug 2008)

Log Message:
-----------
Use same lexer and token source for parsing and partial parsing.

Modified Paths:
--------------
    branches/asm/grammar/Python.g
    branches/asm/grammar/PythonPartial.g
    branches/asm/src/org/python/antlr/InteractiveParser.java
    branches/asm/src/org/python/core/ParserFacade.java
    branches/asm/tests/java/org/python/antlr/PythonPartialTester.java

Removed Paths:
-------------
    branches/asm/src/org/python/antlr/PythonPartialTokenSource.java

Modified: branches/asm/grammar/Python.g
===================================================================

--- branches/asm/grammar/Python.g	2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/grammar/Python.g	2008-08-15 11:43:53 UTC (rev 5178)
@@ -236,13 +236,17 @@
  *       4]
  */
 
+//For use in partial parsing.
+public boolean eofWhileNested = false;
+public boolean partial = false;
+
+int implicitLineJoiningLevel = 0;
+int startPos=-1;
+
 //If you want to use another error recovery mechanisms change this
 //and the same one in the parser.
 private ErrorHandler errorHandler;
 
-int implicitLineJoiningLevel = 0;
-int startPos=-1;
-
     public void setErrorHandler(ErrorHandler eh) {
         this.errorHandler = eh;
     }
@@ -261,6 +265,9 @@
             state.tokenStartLine = input.getLine();
             state.text = null;
             if ( input.LA(1)==CharStream.EOF ) {
+                if (implicitLineJoiningLevel > 0) {
+                    eofWhileNested = true;
+                }
                 return Token.EOF_TOKEN;
             }
             try {
@@ -1124,6 +1131,13 @@
         }
     ;
 
+STRINGPART
+    : {partial}?=> ('r'|'u'|'ur'|'R'|'U'|'UR'|'uR'|'Ur')?
+        (   '\'\'\'' ~('\'\'\'')*
+        |   '"""' ~('"""')*
+        )
+    ;
+
 /** the two '"'? cause a warning -- is there a way to avoid that? */
 fragment
 TRIQUOTE

Modified: branches/asm/grammar/PythonPartial.g
===================================================================
--- branches/asm/grammar/PythonPartial.g	2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/grammar/PythonPartial.g	2008-08-15 11:43:53 UTC (rev 5178)
@@ -64,10 +64,8 @@
 
 grammar PythonPartial;
 
-tokens {
-    INDENT;
-    DEDENT;
-    ENDMARKER;
+options {
+    tokenVocab=Python;
 }
 
 @header { 
@@ -188,12 +186,12 @@
 }
 
 single_input : NEWLINE
-             | simple_stmt {debug("matched simple_stmt");}
+             | simple_stmt
              | compound_stmt NEWLINE?
              ;
 
 //eval_input: testlist NEWLINE* ENDMARKER
-eval_input : LEADING_WS? (NEWLINE)* testlist? (NEWLINE)* ENDMARKER
+eval_input : LEADING_WS? (NEWLINE)* testlist? (NEWLINE)* EOF
            ;
 
 decorators: decorator+
@@ -206,7 +204,7 @@
     : NAME (DOT NAME)*
     ;
 
-funcdef : decorators? 'def' NAME parameters COLON suite
+funcdef : decorators? DEF NAME parameters COLON suite
         ;
 
 parameters : LPAREN (varargslist)? RPAREN
@@ -236,7 +234,7 @@
      | compound_stmt
      ;
 
-simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? (NEWLINE|ENDMARKER)
+simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? (NEWLINE|EOF)
             ;
 
 small_stmt : expr_stmt
@@ -284,17 +282,17 @@
           | DOUBLESLASHEQUAL
           ;
 
-print_stmt : 'print' (printlist | RIGHTSHIFT printlist)?
+print_stmt : PRINT (printlist | RIGHTSHIFT printlist)?
            ;
 
 printlist returns [boolean newline]
     : test (options {k=2;}: COMMA test)* (COMMA)?
     ;
 
-del_stmt : 'del' exprlist
+del_stmt : DELETE exprlist
          ;
 
-pass_stmt : 'pass'
+pass_stmt : PASS
           ;
 
 flow_stmt : break_stmt
@@ -304,29 +302,29 @@
           | yield_stmt
           ;
 
-break_stmt : 'break'
+break_stmt : BREAK
            ;
 
-continue_stmt : 'continue'
+continue_stmt : CONTINUE
               ;
 
-return_stmt : 'return' (testlist)?
+return_stmt : RETURN (testlist)?
             ;
 
 yield_stmt : yield_expr
            ;
 
-raise_stmt: 'raise' (test (COMMA test (COMMA test)?)?)?
+raise_stmt: RAISE (test (COMMA test (COMMA test)?)?)?
           ;
 
 import_stmt : import_name
             | import_from
             ;
 
-import_name : 'import' dotted_as_names
+import_name : IMPORT dotted_as_names
             ;
 
-import_from: 'from' (DOT* dotted_name | DOT+) 'import'
+import_from: FROM (DOT* dotted_name | DOT+) IMPORT
               (STAR
               | import_as_names
               | LPAREN import_as_names RPAREN
@@ -336,10 +334,10 @@
 import_as_names : import_as_name (COMMA import_as_name)* (COMMA)?
                 ;
 
-import_as_name : NAME ('as' NAME)?
+import_as_name : NAME (AS NAME)?
                ;
 
-dotted_as_name : dotted_name ('as' NAME)?
+dotted_as_name : dotted_name (AS NAME)?
                ;
 
 dotted_as_names : dotted_as_name (COMMA dotted_as_name)*
@@ -347,13 +345,13 @@
 dotted_name : NAME (DOT NAME)*
             ;
 
-global_stmt : 'global' NAME (COMMA NAME)*
+global_stmt : GLOBAL NAME (COMMA NAME)*
             ;
 
-exec_stmt : 'exec' expr ('in' test (COMMA test)?)?
+exec_stmt : EXEC expr (IN test (COMMA test)?)?
           ;
 
-assert_stmt : 'assert' test (COMMA test)?
+assert_stmt : ASSERT test (COMMA test)?
             ;
 
 compound_stmt : if_stmt
@@ -365,39 +363,44 @@
               | classdef
               ;
 
-if_stmt: 'if' test COLON suite elif_clause*  ('else' COLON suite)?
+if_stmt: IF test COLON suite elif_clause*  (ORELSE COLON suite)?
        ;
 
-elif_clause : 'elif' test COLON suite
+elif_clause : ELIF test COLON suite
             ;
 
-while_stmt : 'while' test COLON suite ('else' COLON suite)?
+while_stmt : WHILE test COLON suite (ORELSE COLON suite)?
            ;
 
-for_stmt : 'for' exprlist 'in' testlist COLON suite ('else' COLON suite)?
+for_stmt : FOR exprlist IN testlist COLON suite (ELSE COLON suite)?
          ;
 
-try_stmt : 'try' COLON suite
-           ( except_clause+ ('else' COLON suite)? ('finally' COLON suite)?
-           | 'finally' COLON suite
+try_stmt : TRY COLON suite
+           ( except_clause+ (ELSE COLON suite)? (FINALLY COLON suite)?
+           | FINALLY COLON suite
            )?
          ;
 
-with_stmt: 'with' test (with_var)? COLON suite
+with_stmt: WITH test (with_var)? COLON suite
          ;
 
-with_var: ('as' | NAME) expr
+with_var: (AS | NAME) expr
         ;
 
-except_clause : 'except' (test (COMMA test)?)? COLON suite
+except_clause : EXCEPT (test (COMMA test)?)? COLON suite
               ;
 
 suite : simple_stmt
-      | NEWLINE ((INDENT (stmt)+ (DEDENT|ENDMARKER))|ENDMARKER)
+      | NEWLINE (EOF
+                |DEDENT EOF
+                |INDENT (stmt)+ (DEDENT
+                                |EOF
+                                )
+                )
       ;
 
 test: or_test {debug("matched test: or_test");} 
-    ( ('if' or_test 'else') => 'if' or_test 'else' test)?
+    ( (IF or_test ELSE) => IF or_test ELSE test)?
     | lambdef
     ;
 
@@ -421,10 +424,10 @@
         | LESSEQUAL
         | ALT_NOTEQUAL
         | NOTEQUAL
-        | 'in'
-        | NOT 'in'
-        | 'is'
-        | 'is' NOT
+        | IN
+        | NOT IN
+        | IS
+        | IS NOT
         ;
 
 expr : xor_expr (VBAR xor_expr)* {debug("matched expr");}
@@ -484,7 +487,7 @@
            
     ;
 
-lambdef: 'lambda' (varargslist)? COLON test
+lambdef: LABMDA (varargslist)? COLON test
        ;
 
 trailer : LPAREN (arglist)? RPAREN
@@ -513,7 +516,7 @@
 dictmaker : test COLON test (options {k=2;}:COMMA test COLON test)* (COMMA)?
           ;
 
-classdef: 'class' NAME (LPAREN testlist? RPAREN)? COLON suite
+classdef: CLASS NAME (LPAREN testlist? RPAREN)? COLON suite
         ;
 
 arglist : argument (COMMA argument)*
@@ -533,270 +536,24 @@
           | list_if
           ;
 
-list_for : 'for' exprlist 'in' testlist (list_iter)?
+list_for : FOR exprlist IN testlist (list_iter)?
          ;
 
-list_if : 'if' test (list_iter)?
+list_if : IF test (list_iter)?
         ;
 
 gen_iter: gen_for
         | gen_if
         ;
 
-gen_for: 'for' exprlist 'in' or_test gen_iter?
+gen_for: FOR exprlist IN or_test gen_iter?
        ;
 
-gen_if: 'if' test gen_iter?
+gen_if: IF test gen_iter?
       ;
 
-yield_expr : 'yield' testlist?
+yield_expr : YIELD testlist?
            ;
+//XXX:
+//testlist1: test (',' test)*
 
-LPAREN    : '(' {implicitLineJoiningLevel++;} ;
-
-RPAREN    : ')' {implicitLineJoiningLevel--;} ;
-
-LBRACK    : '[' {implicitLineJoiningLevel++;} ;
-
-RBRACK    : ']' {implicitLineJoiningLevel--;} ;
-
-COLON     : ':' ;
-
-COMMA    : ',' ;
-
-SEMI    : ';' ;
-
-PLUS    : '+' ;
-
-MINUS    : '-' ;
-
-STAR    : '*' ;
-
-SLASH    : '/' ;
-
-VBAR    : '|' ;
-
-AMPER    : '&' ;
-
-LESS    : '<' ;
-
-GREATER    : '>' ;
-
-ASSIGN    : '=' ;
-
-PERCENT    : '%' ;
-
-BACKQUOTE    : '`' ;
-
-LCURLY    : '{' {implicitLineJoiningLevel++;} ;
-
-RCURLY    : '}' {implicitLineJoiningLevel--;} ;
-
-CIRCUMFLEX    : '^' ;
-
-TILDE    : '~' ;
-
-EQUAL    : '==' ;
-
-NOTEQUAL    : '!=' ;
-
-ALT_NOTEQUAL: '<>' ;
-
-LESSEQUAL    : '<=' ;
-
-LEFTSHIFT    : '<<' ;
-
-GREATEREQUAL    : '>=' ;
-
-RIGHTSHIFT    : '>>' ;
-
-PLUSEQUAL    : '+=' ;
-
-MINUSEQUAL    : '-=' ;
-
-DOUBLESTAR    : '**' ;
-
-STAREQUAL    : '*=' ;
-
-DOUBLESLASH    : '//' ;
-
-SLASHEQUAL    : '/=' ;
-
-VBAREQUAL    : '|=' ;
-
-PERCENTEQUAL    : '%=' ;
-
-AMPEREQUAL    : '&=' ;
-
-CIRCUMFLEXEQUAL    : '^=' ;
-
-LEFTSHIFTEQUAL    : '<<=' ;
-
-RIGHTSHIFTEQUAL    : '>>=' ;
-
-DOUBLESTAREQUAL    : '**=' ;
-
-DOUBLESLASHEQUAL    : '//=' ;
-
-DOT : '.' ;
-
-AT : '@' ;
-
-AND : 'and' ;
-
-OR : 'or' ;
-
-NOT : 'not' ;
-
-FLOAT
-    :   '.' DIGITS (Exponent)?
-    |   DIGITS '.' Exponent
-    |   DIGITS ('.' (DIGITS (Exponent)?)? | Exponent)
-    ;
-
-LONGINT
-    :   INT ('l'|'L')
-    ;
-
-fragment
-Exponent
-    :    ('e' | 'E') ( '+' | '-' )? DIGITS
-    ;
-
-INT :   // Hex
-        '0' ('x' | 'X') ( '0' .. '9' | 'a' .. 'f' | 'A' .. 'F' )+
-    |   // Octal
-        '0' ( '0' .. '7' )*
-    |   '1'..'9' DIGITS*
-    ;
-
-COMPLEX
-    :   DIGITS+ ('j'|'J')
-    |   FLOAT ('j'|'J')
-    ;
-
-fragment
-DIGITS : ( '0' .. '9' )+ ;
-
-NAME:    ( 'a' .. 'z' | 'A' .. 'Z' | '_')
-        ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '0' .. '9' )*
-    ;
-
-/** Match various string types.  Note that greedy=false implies '''
- *  should make us exit loop not continue.
- */
-STRING
-    :   ('r'|'u'|'ur')?
-        (   '\'\'\'' (options {greedy=false;}:TRIAPOS)* '\'\'\''
-        |   '"""' (options {greedy=false;}:TRIQUOTE)* '"""'
-        |   '"' (ESC|~('\\'|'\n'|'"'))* '"'
-        |   '\'' (ESC|~('\\'|'\n'|'\''))* '\''
-        )
-    ;
-
-STRINGPART
-    :   ('r'|'u'|'ur')?
-        (   '\'\'\'' ~('\'\'\'')*
-        |   '"""' ~('"""')*
-        )
-    ;
-
-/** the two '"'? cause a warning -- is there a way to avoid that? */
-fragment
-TRIQUOTE
-    : '"'? '"'? (ESC|~('\\'|'"'))+
-    ;
-
-/** the two '\''? cause a warning -- is there a way to avoid that? */
-fragment
-TRIAPOS
-    : '\''? '\''? (ESC|~('\\'|'\''))+
-    ;
-
-fragment
-ESC
-    :    '\\' .
-    ;
-
-/** Consume a newline and any whitespace at start of next line
- *  unless the next line contains only white space, in that case
- *  emit a newline.
- */
-CONTINUED_LINE
-    :    '\\' ('\r')? '\n' (' '|'\t')*  { $channel=HIDDEN; }
-         ( nl=NEWLINE {emit(new ClassicToken(NEWLINE,nl.getText()));}
-         |
-         )
-    ;
-
-/** Treat a sequence of blank lines as a single blank line.  If
- *  nested within a (..), {..}, or [..], then ignore newlines.
- *  If the first newline starts in column one, they are to be ignored.
- *
- *  Frank Wierzbicki added: Also ignore FORMFEEDS (\u000C).
- */
-NEWLINE
-    :   (('\u000C')?('\r')? '\n' )+
-        {if ( startPos==0 || implicitLineJoiningLevel>0 )
-            $channel=HIDDEN;
-        }
-    ;
-
-WS  :    {startPos>0}?=> (' '|'\t'|'\u000C')+ {$channel=HIDDEN;}
-    ;
-    
-/** Grab everything before a real symbol.  Then if newline, kill it
- *  as this is a blank line.  If whitespace followed by comment, kill it
- *  as it's a comment on a line by itself.
- *
- *  Ignore leading whitespace when nested in [..], (..), {..}.
- */
-LEADING_WS
-@init {
-    int spaces = 0;
-}
-    :   {startPos==0}?=>
-        (   {implicitLineJoiningLevel>0}? ( ' ' | '\t' )+ {$channel=HIDDEN;}
-           |    (     ' '  { spaces++; }
-            |    '\t' { spaces += 8; spaces -= (spaces \% 8); }
-               )+
-            {
-            // make a string of n spaces where n is column number - 1
-            char[] indentation = new char[spaces];
-            for (int i=0; i<spaces; i++) {
-                indentation[i] = ' ';
-            }
-            String s = new String(indentation);
-            emit(new ClassicToken(LEADING_WS,new String(indentation)));
-            }
-            // kill trailing newline if present and then ignore
-            ( ('\r')? '\n' {if (state.token!=null) state.token.setChannel(HIDDEN); else $channel=HIDDEN;})*
-           // {state.token.setChannel(99); }
-        )
-    ;
-
-/** Comments not on line by themselves are turned into newlines.
-
-    b = a # end of line comment
-
-    or
-
-    a = [1, # weird
-         2]
-
-    This rule is invoked directly by nextToken when the comment is in
-    first column or when comment is on end of nonwhitespace line.
-
-    Only match \n here if we didn't start on left edge; let NEWLINE return that.
-    Kill if newlines if we live on a line by ourselves
-    
-    Consume any leading whitespace if it starts on left edge.
- */
-COMMENT
-@init {
-    $channel=HIDDEN;
-}
-    :    {startPos==0}?=> (' '|'\t')* '#' (~'\n')* '\n'+
-    |    {startPos>0}?=> '#' (~'\n')* // let NEWLINE handle \n unless char pos==0 for '#'
-    ;
-

Modified: branches/asm/src/org/python/antlr/InteractiveParser.java
===================================================================
--- branches/asm/src/org/python/antlr/InteractiveParser.java	2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/src/org/python/antlr/InteractiveParser.java	2008-08-15 11:43:53 UTC (rev 5178)
@@ -21,17 +21,6 @@
 
     private BufferedReader bufreader;
 
-    public static class PPLexer extends PythonPartialLexer {
-        public PPLexer(CharStream lexer) {
-            super(lexer);
-        }
-
-        public Token nextToken() {
-            startPos = getCharPositionInLine();
-            return super.nextToken();
-        }
-    }
-
     public InteractiveParser(BufferedReader br, String filename) {
         this.bufreader = br;
         this.filename = filename;

Deleted: branches/asm/src/org/python/antlr/PythonPartialTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonPartialTokenSource.java	2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/src/org/python/antlr/PythonPartialTokenSource.java	2008-08-15 11:43:53 UTC (rev 5178)
@@ -1,220 +0,0 @@
-package org.python.antlr;
-
-/*
- [The "BSD licence"]
- Copyright (c) 2004 Terence Parr and Loring Craymer
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
- 1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
- 3. The name of the author may not be used to endorse or promote products
-    derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-import org.antlr.runtime.*;
-import java.util.*;
-
-/** This file is a copy of PythonTokenSource with some comments removed
- *  so I can play with it to implement interactive mode partial parsing.
- */
-
-public class PythonPartialTokenSource implements TokenSource {
-    public static final int MAX_INDENTS = 100;
-    public static final int FIRST_CHAR_POSITION = 0;
-
-    /** The stack of indent levels (column numbers) */
-    int[] indentStack = new int[MAX_INDENTS];
-    /** stack pointer */
-    int sp=-1; // grow upwards
-
-    /** The queue of tokens */
-    Vector tokens = new Vector();
-
-    /** We pull real tokens from this lexer */
-    CommonTokenStream stream;
-
-    int lastTokenAddedIndex = -1;
-
-    boolean atEnd = false;
-
-    public PythonPartialTokenSource(PythonLexer lexer) {
-    }
-
-    public PythonPartialTokenSource(CommonTokenStream stream) {
-        this.stream = stream;
-        // "state" of indent level is FIRST_CHAR_POSITION
-        push(FIRST_CHAR_POSITION);
-    }
-
-    public Token nextToken() {
-        // if something in queue, just remove and return it
-        if ( tokens.size()>0 ) {
-            Token t = (Token)tokens.firstElement();
-            tokens.removeElementAt(0);
-            //System.out.println(t);
-            return t;
-        }
-
-        insertImaginaryIndentDedentTokens();
-
-        return nextToken();
-    }
-
-    protected void insertImaginaryIndentDedentTokens()
-    {
-        Token t = stream.LT(1);
-        stream.consume();
-        if ( t.getType()==Token.EOF ) {
-            atEnd = true;
-            Token em = new CommonToken(PythonPartialParser.ENDMARKER,"");
-            em.setCharPositionInLine(t.getCharPositionInLine());
-            em.setLine(t.getLine());
-            tokens.addElement(em);
-         }
-
-        // if not a NEWLINE, doesn't signal indent/dedent work; just enqueue
-        if ( t.getType()!=PythonPartialLexer.NEWLINE ) {
-            List hiddenTokens = stream.getTokens(lastTokenAddedIndex+1,t.getTokenIndex()-1);
-            if ( hiddenTokens!=null ) {
-                tokens.addAll(hiddenTokens);
-            }
-            lastTokenAddedIndex = t.getTokenIndex();
-            tokens.addElement(t);
-            return;
-        }
-
-        // save NEWLINE in the queue
-        //System.out.println("found newline: "+t+" stack is "+stackString());
-        List hiddenTokens = stream.getTokens(lastTokenAddedIndex+1,t.getTokenIndex()-1);
-        if ( hiddenTokens!=null ) {
-            tokens.addAll(hiddenTokens);
-        }
-        lastTokenAddedIndex = t.getTokenIndex();
-        tokens.addElement(t);
-
-        // grab first token of next line
-        t = stream.LT(1);
-        stream.consume();
-
-        hiddenTokens = stream.getTokens(lastTokenAddedIndex+1,t.getTokenIndex()-1);
-        if ( hiddenTokens!=null ) {
-            tokens.addAll(hiddenTokens);
-        }
-        lastTokenAddedIndex = t.getTokenIndex();
-
-        // compute cpos as the char pos of next non-WS token in line
-        int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
-        if ( t.getType()==Token.EOF ) {
-            atEnd = true;
-            Token em = new CommonToken(PythonPartialParser.ENDMARKER,"");
-            em.setCharPositionInLine(t.getCharPositionInLine());
-            em.setLine(t.getLine());
-            tokens.addElement(em);
-            
-            cpos = -1; // pretend EOF always happens at left edge
-        }
-        else if ( t.getType()==PythonPartialLexer.LEADING_WS ) {
-            cpos = t.getText().length();
-        }
-
-        //System.out.println("next token is: "+t);
-
-        // compare to last indent level
-        int lastIndent = peek();
-        //System.out.println("cpos, lastIndent = "+cpos+", "+lastIndent);
-        if ( cpos > lastIndent ) { // they indented; track and gen INDENT
-            push(cpos);
-            //System.out.println("push("+cpos+"): "+stackString());
-            Token indent = new CommonToken(PythonPartialParser.INDENT,"");
-            indent.setCharPositionInLine(t.getCharPositionInLine());
-            indent.setLine(t.getLine());
-            tokens.addElement(indent);
-        }
-        else if ( cpos < lastIndent ) { // they dedented
-            // how far back did we dedent?
-            int prevIndex = findPreviousIndent(cpos);
-            //System.out.println("dedented; prevIndex of cpos="+cpos+" is "+prevIndex);
-            // generate DEDENTs for each indent level we backed up over
-            for (int d=sp-1; d>=prevIndex; d--) {
-                Token tok;
-                if (atEnd) {
-                    tok = new CommonToken(PythonPartialParser.ENDMARKER,"");
-                } else {
-                    tok = new CommonToken(PythonPartialParser.DEDENT,"");
-                }
-                tok.setCharPositionInLine(t.getCharPositionInLine());
-                tok.setLine(t.getLine());
-                tokens.addElement(tok);
-            }
-            sp = prevIndex; // pop those off indent level
-        }
-        if ( t.getType()!=PythonPartialLexer.LEADING_WS ) { // discard WS
-            tokens.addElement(t);
-        }
-    }
-
-    //  T O K E N  S T A C K  M E T H O D S
-
-    protected void push(int i) {
-        if (sp>=MAX_INDENTS) {
-            throw new IllegalStateException("stack overflow");
-        }
-        sp++;
-        indentStack[sp] = i;
-    }
-
-    protected int pop() {
-        if (sp<0) {
-            throw new IllegalStateException("stack underflow");
-        }
-        int top = indentStack[sp];
-        sp--;
-        return top;
-    }
-
-    protected int peek() {
-        return indentStack[sp];
-    }
-
-    /** Return the index on stack of previous indent level == i else -1 */
-    protected int findPreviousIndent(int i) {
-        for (int j=sp-1; j>=0; j--) {
-            if ( indentStack[j]==i ) {
-                return j;
-            }
-        }
-        return FIRST_CHAR_POSITION;
-    }
-
-    public String stackString() {
-        StringBuffer buf = new StringBuffer();
-        for (int j=sp; j>=0; j--) {
-            buf.append(" ");
-            buf.append(indentStack[j]);
-        }
-        return buf.toString();
-    }
-
-    //FIXME: needed this for the Antlr 3.1b interface change.
-    public String getSourceName() {
-        return "XXX-need-real-name.py";
-    }
-
-}

Modified: branches/asm/src/org/python/core/ParserFacade.java
===================================================================
--- branches/asm/src/org/python/core/ParserFacade.java	2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/src/org/python/core/ParserFacade.java	2008-08-15 11:43:53 UTC (rev 5178)
@@ -16,6 +16,7 @@
 import org.antlr.runtime.CharStream;
 import org.antlr.runtime.CommonTokenStream;
 
+import org.python.antlr.BaseParser;
 import org.python.antlr.ExpressionParser;
 import org.python.antlr.InteractiveParser;
 import org.python.antlr.ParseException;
@@ -24,9 +25,9 @@
 import org.python.antlr.PythonParser;
 import org.python.antlr.PythonTree;
 import org.python.antlr.PythonTree;
-import org.python.antlr.PythonPartialLexer;
+import org.python.antlr.PythonLexer;
 import org.python.antlr.PythonPartialParser;
-import org.python.antlr.PythonPartialTokenSource;
+import org.python.antlr.PythonTokenSource;
 import org.python.antlr.ast.modType;
 import org.python.core.io.StreamIO;
 import org.python.core.io.TextIOInputStream;
@@ -158,7 +159,7 @@
             }
         } catch (Throwable t) {
             PyException p = fixParseError(bufreader, t, filename);
-            if (validPartialSentence(bufreader, kind)) {
+            if (validPartialSentence(bufreader, kind, filename)) {
                 return null;
             }
             throw p;
@@ -166,15 +167,16 @@
         return node;
     }
 
-    private static boolean validPartialSentence(BufferedReader bufreader, String kind) {
-        PythonPartialLexer lexer = null;
+    private static boolean validPartialSentence(BufferedReader bufreader, String kind, String filename) {
+        PythonLexer lexer = null;
         try {
             bufreader.reset();
             CharStream cs = new NoCloseReaderStream(bufreader);
-            lexer = new InteractiveParser.PPLexer(cs);
+            lexer = new BaseParser.PyLexer(cs);
+            lexer.partial = true;
             CommonTokenStream tokens = new CommonTokenStream(lexer);
             tokens.discardOffChannelTokens(true);
-            PythonPartialTokenSource indentedSource = new PythonPartialTokenSource(tokens);
+            PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename);
             tokens = new CommonTokenStream(indentedSource);
             PythonPartialParser parser = new PythonPartialParser(tokens);
             if (kind.equals("single")) {

Modified: branches/asm/tests/java/org/python/antlr/PythonPartialTester.java
===================================================================
--- branches/asm/tests/java/org/python/antlr/PythonPartialTester.java	2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/tests/java/org/python/antlr/PythonPartialTester.java	2008-08-15 11:43:53 UTC (rev 5178)
@@ -10,26 +10,15 @@
  */
 public class PythonPartialTester {
 
-    public static class PPLexer extends PythonPartialLexer {
-        public PPLexer(CharStream lexer) {
-            super(lexer);
-        }
-
-        public Token nextToken() {
-            startPos = getCharPositionInLine();
-            return super.nextToken();
-        }
-    }
-
 	public void parse(String[] args) throws Exception {
         try {
             PythonTree result = null;
             CharStream input = new ANTLRFileStream(args[0]);
-            PythonPartialLexer lexer = new PPLexer(input);
+            PythonLexer lexer = new BaseParser.PyLexer(input);
             CommonTokenStream tokens = new CommonTokenStream(lexer);
             tokens.discardOffChannelTokens(true);
             //PythonTokenSource indentedSource = new PythonTokenSource(tokens);
-            PythonPartialTokenSource indentedSource = new PythonPartialTokenSource(tokens);
+            PythonTokenSource indentedSource = new PythonTokenSource(tokens, "<test>");
             tokens = new CommonTokenStream(indentedSource);
             PythonPartialParser parser = new PythonPartialParser(tokens);
             parser.single_input();


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Thread: [Jython-checkins] SF.net SVN: jython:[5178] branches/asm (Page 3)

jython-checkins