Menu
▾
▴
[Jython-checkins] SF.net SVN: jython:[5178] branches/asm
|
From: <fwi...@us...> - 2008-08-15 11:43:58
|
Revision: 5178
http://jython.svn.sourceforge.net/jython/?rev=5178&view=rev
Author: fwierzbicki
Date: 2008-08-15 11:43:53 +0000 (Fri, 15 Aug 2008)
Log Message:
-----------
Use same lexer and token source for parsing and partial parsing.
Modified Paths:
--------------
branches/asm/grammar/Python.g
branches/asm/grammar/PythonPartial.g
branches/asm/src/org/python/antlr/InteractiveParser.java
branches/asm/src/org/python/core/ParserFacade.java
branches/asm/tests/java/org/python/antlr/PythonPartialTester.java
Removed Paths:
-------------
branches/asm/src/org/python/antlr/PythonPartialTokenSource.java
Modified: branches/asm/grammar/Python.g
===================================================================
--- branches/asm/grammar/Python.g 2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/grammar/Python.g 2008-08-15 11:43:53 UTC (rev 5178)
@@ -236,13 +236,17 @@
* 4]
*/
+//For use in partial parsing.
+public boolean eofWhileNested = false;
+public boolean partial = false;
+
+int implicitLineJoiningLevel = 0;
+int startPos=-1;
+
//If you want to use another error recovery mechanisms change this
//and the same one in the parser.
private ErrorHandler errorHandler;
-int implicitLineJoiningLevel = 0;
-int startPos=-1;
-
public void setErrorHandler(ErrorHandler eh) {
this.errorHandler = eh;
}
@@ -261,6 +265,9 @@
state.tokenStartLine = input.getLine();
state.text = null;
if ( input.LA(1)==CharStream.EOF ) {
+ if (implicitLineJoiningLevel > 0) {
+ eofWhileNested = true;
+ }
return Token.EOF_TOKEN;
}
try {
@@ -1124,6 +1131,13 @@
}
;
+STRINGPART
+ : {partial}?=> ('r'|'u'|'ur'|'R'|'U'|'UR'|'uR'|'Ur')?
+ ( '\'\'\'' ~('\'\'\'')*
+ | '"""' ~('"""')*
+ )
+ ;
+
/** the two '"'? cause a warning -- is there a way to avoid that? */
fragment
TRIQUOTE
Modified: branches/asm/grammar/PythonPartial.g
===================================================================
--- branches/asm/grammar/PythonPartial.g 2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/grammar/PythonPartial.g 2008-08-15 11:43:53 UTC (rev 5178)
@@ -64,10 +64,8 @@
grammar PythonPartial;
-tokens {
- INDENT;
- DEDENT;
- ENDMARKER;
+options {
+ tokenVocab=Python;
}
@header {
@@ -188,12 +186,12 @@
}
single_input : NEWLINE
- | simple_stmt {debug("matched simple_stmt");}
+ | simple_stmt
| compound_stmt NEWLINE?
;
//eval_input: testlist NEWLINE* ENDMARKER
-eval_input : LEADING_WS? (NEWLINE)* testlist? (NEWLINE)* ENDMARKER
+eval_input : LEADING_WS? (NEWLINE)* testlist? (NEWLINE)* EOF
;
decorators: decorator+
@@ -206,7 +204,7 @@
: NAME (DOT NAME)*
;
-funcdef : decorators? 'def' NAME parameters COLON suite
+funcdef : decorators? DEF NAME parameters COLON suite
;
parameters : LPAREN (varargslist)? RPAREN
@@ -236,7 +234,7 @@
| compound_stmt
;
-simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? (NEWLINE|ENDMARKER)
+simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? (NEWLINE|EOF)
;
small_stmt : expr_stmt
@@ -284,17 +282,17 @@
| DOUBLESLASHEQUAL
;
-print_stmt : 'print' (printlist | RIGHTSHIFT printlist)?
+print_stmt : PRINT (printlist | RIGHTSHIFT printlist)?
;
printlist returns [boolean newline]
: test (options {k=2;}: COMMA test)* (COMMA)?
;
-del_stmt : 'del' exprlist
+del_stmt : DELETE exprlist
;
-pass_stmt : 'pass'
+pass_stmt : PASS
;
flow_stmt : break_stmt
@@ -304,29 +302,29 @@
| yield_stmt
;
-break_stmt : 'break'
+break_stmt : BREAK
;
-continue_stmt : 'continue'
+continue_stmt : CONTINUE
;
-return_stmt : 'return' (testlist)?
+return_stmt : RETURN (testlist)?
;
yield_stmt : yield_expr
;
-raise_stmt: 'raise' (test (COMMA test (COMMA test)?)?)?
+raise_stmt: RAISE (test (COMMA test (COMMA test)?)?)?
;
import_stmt : import_name
| import_from
;
-import_name : 'import' dotted_as_names
+import_name : IMPORT dotted_as_names
;
-import_from: 'from' (DOT* dotted_name | DOT+) 'import'
+import_from: FROM (DOT* dotted_name | DOT+) IMPORT
(STAR
| import_as_names
| LPAREN import_as_names RPAREN
@@ -336,10 +334,10 @@
import_as_names : import_as_name (COMMA import_as_name)* (COMMA)?
;
-import_as_name : NAME ('as' NAME)?
+import_as_name : NAME (AS NAME)?
;
-dotted_as_name : dotted_name ('as' NAME)?
+dotted_as_name : dotted_name (AS NAME)?
;
dotted_as_names : dotted_as_name (COMMA dotted_as_name)*
@@ -347,13 +345,13 @@
dotted_name : NAME (DOT NAME)*
;
-global_stmt : 'global' NAME (COMMA NAME)*
+global_stmt : GLOBAL NAME (COMMA NAME)*
;
-exec_stmt : 'exec' expr ('in' test (COMMA test)?)?
+exec_stmt : EXEC expr (IN test (COMMA test)?)?
;
-assert_stmt : 'assert' test (COMMA test)?
+assert_stmt : ASSERT test (COMMA test)?
;
compound_stmt : if_stmt
@@ -365,39 +363,44 @@
| classdef
;
-if_stmt: 'if' test COLON suite elif_clause* ('else' COLON suite)?
+if_stmt: IF test COLON suite elif_clause* (ORELSE COLON suite)?
;
-elif_clause : 'elif' test COLON suite
+elif_clause : ELIF test COLON suite
;
-while_stmt : 'while' test COLON suite ('else' COLON suite)?
+while_stmt : WHILE test COLON suite (ORELSE COLON suite)?
;
-for_stmt : 'for' exprlist 'in' testlist COLON suite ('else' COLON suite)?
+for_stmt : FOR exprlist IN testlist COLON suite (ELSE COLON suite)?
;
-try_stmt : 'try' COLON suite
- ( except_clause+ ('else' COLON suite)? ('finally' COLON suite)?
- | 'finally' COLON suite
+try_stmt : TRY COLON suite
+ ( except_clause+ (ELSE COLON suite)? (FINALLY COLON suite)?
+ | FINALLY COLON suite
)?
;
-with_stmt: 'with' test (with_var)? COLON suite
+with_stmt: WITH test (with_var)? COLON suite
;
-with_var: ('as' | NAME) expr
+with_var: (AS | NAME) expr
;
-except_clause : 'except' (test (COMMA test)?)? COLON suite
+except_clause : EXCEPT (test (COMMA test)?)? COLON suite
;
suite : simple_stmt
- | NEWLINE ((INDENT (stmt)+ (DEDENT|ENDMARKER))|ENDMARKER)
+ | NEWLINE (EOF
+ |DEDENT EOF
+ |INDENT (stmt)+ (DEDENT
+ |EOF
+ )
+ )
;
test: or_test {debug("matched test: or_test");}
- ( ('if' or_test 'else') => 'if' or_test 'else' test)?
+ ( (IF or_test ELSE) => IF or_test ELSE test)?
| lambdef
;
@@ -421,10 +424,10 @@
| LESSEQUAL
| ALT_NOTEQUAL
| NOTEQUAL
- | 'in'
- | NOT 'in'
- | 'is'
- | 'is' NOT
+ | IN
+ | NOT IN
+ | IS
+ | IS NOT
;
expr : xor_expr (VBAR xor_expr)* {debug("matched expr");}
@@ -484,7 +487,7 @@
;
-lambdef: 'lambda' (varargslist)? COLON test
+lambdef: LABMDA (varargslist)? COLON test
;
trailer : LPAREN (arglist)? RPAREN
@@ -513,7 +516,7 @@
dictmaker : test COLON test (options {k=2;}:COMMA test COLON test)* (COMMA)?
;
-classdef: 'class' NAME (LPAREN testlist? RPAREN)? COLON suite
+classdef: CLASS NAME (LPAREN testlist? RPAREN)? COLON suite
;
arglist : argument (COMMA argument)*
@@ -533,270 +536,24 @@
| list_if
;
-list_for : 'for' exprlist 'in' testlist (list_iter)?
+list_for : FOR exprlist IN testlist (list_iter)?
;
-list_if : 'if' test (list_iter)?
+list_if : IF test (list_iter)?
;
gen_iter: gen_for
| gen_if
;
-gen_for: 'for' exprlist 'in' or_test gen_iter?
+gen_for: FOR exprlist IN or_test gen_iter?
;
-gen_if: 'if' test gen_iter?
+gen_if: IF test gen_iter?
;
-yield_expr : 'yield' testlist?
+yield_expr : YIELD testlist?
;
+//XXX:
+//testlist1: test (',' test)*
-LPAREN : '(' {implicitLineJoiningLevel++;} ;
-
-RPAREN : ')' {implicitLineJoiningLevel--;} ;
-
-LBRACK : '[' {implicitLineJoiningLevel++;} ;
-
-RBRACK : ']' {implicitLineJoiningLevel--;} ;
-
-COLON : ':' ;
-
-COMMA : ',' ;
-
-SEMI : ';' ;
-
-PLUS : '+' ;
-
-MINUS : '-' ;
-
-STAR : '*' ;
-
-SLASH : '/' ;
-
-VBAR : '|' ;
-
-AMPER : '&' ;
-
-LESS : '<' ;
-
-GREATER : '>' ;
-
-ASSIGN : '=' ;
-
-PERCENT : '%' ;
-
-BACKQUOTE : '`' ;
-
-LCURLY : '{' {implicitLineJoiningLevel++;} ;
-
-RCURLY : '}' {implicitLineJoiningLevel--;} ;
-
-CIRCUMFLEX : '^' ;
-
-TILDE : '~' ;
-
-EQUAL : '==' ;
-
-NOTEQUAL : '!=' ;
-
-ALT_NOTEQUAL: '<>' ;
-
-LESSEQUAL : '<=' ;
-
-LEFTSHIFT : '<<' ;
-
-GREATEREQUAL : '>=' ;
-
-RIGHTSHIFT : '>>' ;
-
-PLUSEQUAL : '+=' ;
-
-MINUSEQUAL : '-=' ;
-
-DOUBLESTAR : '**' ;
-
-STAREQUAL : '*=' ;
-
-DOUBLESLASH : '//' ;
-
-SLASHEQUAL : '/=' ;
-
-VBAREQUAL : '|=' ;
-
-PERCENTEQUAL : '%=' ;
-
-AMPEREQUAL : '&=' ;
-
-CIRCUMFLEXEQUAL : '^=' ;
-
-LEFTSHIFTEQUAL : '<<=' ;
-
-RIGHTSHIFTEQUAL : '>>=' ;
-
-DOUBLESTAREQUAL : '**=' ;
-
-DOUBLESLASHEQUAL : '//=' ;
-
-DOT : '.' ;
-
-AT : '@' ;
-
-AND : 'and' ;
-
-OR : 'or' ;
-
-NOT : 'not' ;
-
-FLOAT
- : '.' DIGITS (Exponent)?
- | DIGITS '.' Exponent
- | DIGITS ('.' (DIGITS (Exponent)?)? | Exponent)
- ;
-
-LONGINT
- : INT ('l'|'L')
- ;
-
-fragment
-Exponent
- : ('e' | 'E') ( '+' | '-' )? DIGITS
- ;
-
-INT : // Hex
- '0' ('x' | 'X') ( '0' .. '9' | 'a' .. 'f' | 'A' .. 'F' )+
- | // Octal
- '0' ( '0' .. '7' )*
- | '1'..'9' DIGITS*
- ;
-
-COMPLEX
- : DIGITS+ ('j'|'J')
- | FLOAT ('j'|'J')
- ;
-
-fragment
-DIGITS : ( '0' .. '9' )+ ;
-
-NAME: ( 'a' .. 'z' | 'A' .. 'Z' | '_')
- ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '0' .. '9' )*
- ;
-
-/** Match various string types. Note that greedy=false implies '''
- * should make us exit loop not continue.
- */
-STRING
- : ('r'|'u'|'ur')?
- ( '\'\'\'' (options {greedy=false;}:TRIAPOS)* '\'\'\''
- | '"""' (options {greedy=false;}:TRIQUOTE)* '"""'
- | '"' (ESC|~('\\'|'\n'|'"'))* '"'
- | '\'' (ESC|~('\\'|'\n'|'\''))* '\''
- )
- ;
-
-STRINGPART
- : ('r'|'u'|'ur')?
- ( '\'\'\'' ~('\'\'\'')*
- | '"""' ~('"""')*
- )
- ;
-
-/** the two '"'? cause a warning -- is there a way to avoid that? */
-fragment
-TRIQUOTE
- : '"'? '"'? (ESC|~('\\'|'"'))+
- ;
-
-/** the two '\''? cause a warning -- is there a way to avoid that? */
-fragment
-TRIAPOS
- : '\''? '\''? (ESC|~('\\'|'\''))+
- ;
-
-fragment
-ESC
- : '\\' .
- ;
-
-/** Consume a newline and any whitespace at start of next line
- * unless the next line contains only white space, in that case
- * emit a newline.
- */
-CONTINUED_LINE
- : '\\' ('\r')? '\n' (' '|'\t')* { $channel=HIDDEN; }
- ( nl=NEWLINE {emit(new ClassicToken(NEWLINE,nl.getText()));}
- |
- )
- ;
-
-/** Treat a sequence of blank lines as a single blank line. If
- * nested within a (..), {..}, or [..], then ignore newlines.
- * If the first newline starts in column one, they are to be ignored.
- *
- * Frank Wierzbicki added: Also ignore FORMFEEDS (\u000C).
- */
-NEWLINE
- : (('\u000C')?('\r')? '\n' )+
- {if ( startPos==0 || implicitLineJoiningLevel>0 )
- $channel=HIDDEN;
- }
- ;
-
-WS : {startPos>0}?=> (' '|'\t'|'\u000C')+ {$channel=HIDDEN;}
- ;
-
-/** Grab everything before a real symbol. Then if newline, kill it
- * as this is a blank line. If whitespace followed by comment, kill it
- * as it's a comment on a line by itself.
- *
- * Ignore leading whitespace when nested in [..], (..), {..}.
- */
-LEADING_WS
-@init {
- int spaces = 0;
-}
- : {startPos==0}?=>
- ( {implicitLineJoiningLevel>0}? ( ' ' | '\t' )+ {$channel=HIDDEN;}
- | ( ' ' { spaces++; }
- | '\t' { spaces += 8; spaces -= (spaces \% 8); }
- )+
- {
- // make a string of n spaces where n is column number - 1
- char[] indentation = new char[spaces];
- for (int i=0; i<spaces; i++) {
- indentation[i] = ' ';
- }
- String s = new String(indentation);
- emit(new ClassicToken(LEADING_WS,new String(indentation)));
- }
- // kill trailing newline if present and then ignore
- ( ('\r')? '\n' {if (state.token!=null) state.token.setChannel(HIDDEN); else $channel=HIDDEN;})*
- // {state.token.setChannel(99); }
- )
- ;
-
-/** Comments not on line by themselves are turned into newlines.
-
- b = a # end of line comment
-
- or
-
- a = [1, # weird
- 2]
-
- This rule is invoked directly by nextToken when the comment is in
- first column or when comment is on end of nonwhitespace line.
-
- Only match \n here if we didn't start on left edge; let NEWLINE return that.
- Kill if newlines if we live on a line by ourselves
-
- Consume any leading whitespace if it starts on left edge.
- */
-COMMENT
-@init {
- $channel=HIDDEN;
-}
- : {startPos==0}?=> (' '|'\t')* '#' (~'\n')* '\n'+
- | {startPos>0}?=> '#' (~'\n')* // let NEWLINE handle \n unless char pos==0 for '#'
- ;
-
Modified: branches/asm/src/org/python/antlr/InteractiveParser.java
===================================================================
--- branches/asm/src/org/python/antlr/InteractiveParser.java 2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/src/org/python/antlr/InteractiveParser.java 2008-08-15 11:43:53 UTC (rev 5178)
@@ -21,17 +21,6 @@
private BufferedReader bufreader;
- public static class PPLexer extends PythonPartialLexer {
- public PPLexer(CharStream lexer) {
- super(lexer);
- }
-
- public Token nextToken() {
- startPos = getCharPositionInLine();
- return super.nextToken();
- }
- }
-
public InteractiveParser(BufferedReader br, String filename) {
this.bufreader = br;
this.filename = filename;
Deleted: branches/asm/src/org/python/antlr/PythonPartialTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonPartialTokenSource.java 2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/src/org/python/antlr/PythonPartialTokenSource.java 2008-08-15 11:43:53 UTC (rev 5178)
@@ -1,220 +0,0 @@
-package org.python.antlr;
-
-/*
- [The "BSD licence"]
- Copyright (c) 2004 Terence Parr and Loring Craymer
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
- 1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 3. The name of the author may not be used to endorse or promote products
- derived from this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-import org.antlr.runtime.*;
-import java.util.*;
-
-/** This file is a copy of PythonTokenSource with some comments removed
- * so I can play with it to implement interactive mode partial parsing.
- */
-
-public class PythonPartialTokenSource implements TokenSource {
- public static final int MAX_INDENTS = 100;
- public static final int FIRST_CHAR_POSITION = 0;
-
- /** The stack of indent levels (column numbers) */
- int[] indentStack = new int[MAX_INDENTS];
- /** stack pointer */
- int sp=-1; // grow upwards
-
- /** The queue of tokens */
- Vector tokens = new Vector();
-
- /** We pull real tokens from this lexer */
- CommonTokenStream stream;
-
- int lastTokenAddedIndex = -1;
-
- boolean atEnd = false;
-
- public PythonPartialTokenSource(PythonLexer lexer) {
- }
-
- public PythonPartialTokenSource(CommonTokenStream stream) {
- this.stream = stream;
- // "state" of indent level is FIRST_CHAR_POSITION
- push(FIRST_CHAR_POSITION);
- }
-
- public Token nextToken() {
- // if something in queue, just remove and return it
- if ( tokens.size()>0 ) {
- Token t = (Token)tokens.firstElement();
- tokens.removeElementAt(0);
- //System.out.println(t);
- return t;
- }
-
- insertImaginaryIndentDedentTokens();
-
- return nextToken();
- }
-
- protected void insertImaginaryIndentDedentTokens()
- {
- Token t = stream.LT(1);
- stream.consume();
- if ( t.getType()==Token.EOF ) {
- atEnd = true;
- Token em = new CommonToken(PythonPartialParser.ENDMARKER,"");
- em.setCharPositionInLine(t.getCharPositionInLine());
- em.setLine(t.getLine());
- tokens.addElement(em);
- }
-
- // if not a NEWLINE, doesn't signal indent/dedent work; just enqueue
- if ( t.getType()!=PythonPartialLexer.NEWLINE ) {
- List hiddenTokens = stream.getTokens(lastTokenAddedIndex+1,t.getTokenIndex()-1);
- if ( hiddenTokens!=null ) {
- tokens.addAll(hiddenTokens);
- }
- lastTokenAddedIndex = t.getTokenIndex();
- tokens.addElement(t);
- return;
- }
-
- // save NEWLINE in the queue
- //System.out.println("found newline: "+t+" stack is "+stackString());
- List hiddenTokens = stream.getTokens(lastTokenAddedIndex+1,t.getTokenIndex()-1);
- if ( hiddenTokens!=null ) {
- tokens.addAll(hiddenTokens);
- }
- lastTokenAddedIndex = t.getTokenIndex();
- tokens.addElement(t);
-
- // grab first token of next line
- t = stream.LT(1);
- stream.consume();
-
- hiddenTokens = stream.getTokens(lastTokenAddedIndex+1,t.getTokenIndex()-1);
- if ( hiddenTokens!=null ) {
- tokens.addAll(hiddenTokens);
- }
- lastTokenAddedIndex = t.getTokenIndex();
-
- // compute cpos as the char pos of next non-WS token in line
- int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
- if ( t.getType()==Token.EOF ) {
- atEnd = true;
- Token em = new CommonToken(PythonPartialParser.ENDMARKER,"");
- em.setCharPositionInLine(t.getCharPositionInLine());
- em.setLine(t.getLine());
- tokens.addElement(em);
-
- cpos = -1; // pretend EOF always happens at left edge
- }
- else if ( t.getType()==PythonPartialLexer.LEADING_WS ) {
- cpos = t.getText().length();
- }
-
- //System.out.println("next token is: "+t);
-
- // compare to last indent level
- int lastIndent = peek();
- //System.out.println("cpos, lastIndent = "+cpos+", "+lastIndent);
- if ( cpos > lastIndent ) { // they indented; track and gen INDENT
- push(cpos);
- //System.out.println("push("+cpos+"): "+stackString());
- Token indent = new CommonToken(PythonPartialParser.INDENT,"");
- indent.setCharPositionInLine(t.getCharPositionInLine());
- indent.setLine(t.getLine());
- tokens.addElement(indent);
- }
- else if ( cpos < lastIndent ) { // they dedented
- // how far back did we dedent?
- int prevIndex = findPreviousIndent(cpos);
- //System.out.println("dedented; prevIndex of cpos="+cpos+" is "+prevIndex);
- // generate DEDENTs for each indent level we backed up over
- for (int d=sp-1; d>=prevIndex; d--) {
- Token tok;
- if (atEnd) {
- tok = new CommonToken(PythonPartialParser.ENDMARKER,"");
- } else {
- tok = new CommonToken(PythonPartialParser.DEDENT,"");
- }
- tok.setCharPositionInLine(t.getCharPositionInLine());
- tok.setLine(t.getLine());
- tokens.addElement(tok);
- }
- sp = prevIndex; // pop those off indent level
- }
- if ( t.getType()!=PythonPartialLexer.LEADING_WS ) { // discard WS
- tokens.addElement(t);
- }
- }
-
- // T O K E N S T A C K M E T H O D S
-
- protected void push(int i) {
- if (sp>=MAX_INDENTS) {
- throw new IllegalStateException("stack overflow");
- }
- sp++;
- indentStack[sp] = i;
- }
-
- protected int pop() {
- if (sp<0) {
- throw new IllegalStateException("stack underflow");
- }
- int top = indentStack[sp];
- sp--;
- return top;
- }
-
- protected int peek() {
- return indentStack[sp];
- }
-
- /** Return the index on stack of previous indent level == i else -1 */
- protected int findPreviousIndent(int i) {
- for (int j=sp-1; j>=0; j--) {
- if ( indentStack[j]==i ) {
- return j;
- }
- }
- return FIRST_CHAR_POSITION;
- }
-
- public String stackString() {
- StringBuffer buf = new StringBuffer();
- for (int j=sp; j>=0; j--) {
- buf.append(" ");
- buf.append(indentStack[j]);
- }
- return buf.toString();
- }
-
- //FIXME: needed this for the Antlr 3.1b interface change.
- public String getSourceName() {
- return "XXX-need-real-name.py";
- }
-
-}
Modified: branches/asm/src/org/python/core/ParserFacade.java
===================================================================
--- branches/asm/src/org/python/core/ParserFacade.java 2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/src/org/python/core/ParserFacade.java 2008-08-15 11:43:53 UTC (rev 5178)
@@ -16,6 +16,7 @@
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
+import org.python.antlr.BaseParser;
import org.python.antlr.ExpressionParser;
import org.python.antlr.InteractiveParser;
import org.python.antlr.ParseException;
@@ -24,9 +25,9 @@
import org.python.antlr.PythonParser;
import org.python.antlr.PythonTree;
import org.python.antlr.PythonTree;
-import org.python.antlr.PythonPartialLexer;
+import org.python.antlr.PythonLexer;
import org.python.antlr.PythonPartialParser;
-import org.python.antlr.PythonPartialTokenSource;
+import org.python.antlr.PythonTokenSource;
import org.python.antlr.ast.modType;
import org.python.core.io.StreamIO;
import org.python.core.io.TextIOInputStream;
@@ -158,7 +159,7 @@
}
} catch (Throwable t) {
PyException p = fixParseError(bufreader, t, filename);
- if (validPartialSentence(bufreader, kind)) {
+ if (validPartialSentence(bufreader, kind, filename)) {
return null;
}
throw p;
@@ -166,15 +167,16 @@
return node;
}
- private static boolean validPartialSentence(BufferedReader bufreader, String kind) {
- PythonPartialLexer lexer = null;
+ private static boolean validPartialSentence(BufferedReader bufreader, String kind, String filename) {
+ PythonLexer lexer = null;
try {
bufreader.reset();
CharStream cs = new NoCloseReaderStream(bufreader);
- lexer = new InteractiveParser.PPLexer(cs);
+ lexer = new BaseParser.PyLexer(cs);
+ lexer.partial = true;
CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.discardOffChannelTokens(true);
- PythonPartialTokenSource indentedSource = new PythonPartialTokenSource(tokens);
+ PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename);
tokens = new CommonTokenStream(indentedSource);
PythonPartialParser parser = new PythonPartialParser(tokens);
if (kind.equals("single")) {
Modified: branches/asm/tests/java/org/python/antlr/PythonPartialTester.java
===================================================================
--- branches/asm/tests/java/org/python/antlr/PythonPartialTester.java 2008-08-14 18:48:37 UTC (rev 5177)
+++ branches/asm/tests/java/org/python/antlr/PythonPartialTester.java 2008-08-15 11:43:53 UTC (rev 5178)
@@ -10,26 +10,15 @@
*/
public class PythonPartialTester {
- public static class PPLexer extends PythonPartialLexer {
- public PPLexer(CharStream lexer) {
- super(lexer);
- }
-
- public Token nextToken() {
- startPos = getCharPositionInLine();
- return super.nextToken();
- }
- }
-
public void parse(String[] args) throws Exception {
try {
PythonTree result = null;
CharStream input = new ANTLRFileStream(args[0]);
- PythonPartialLexer lexer = new PPLexer(input);
+ PythonLexer lexer = new BaseParser.PyLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.discardOffChannelTokens(true);
//PythonTokenSource indentedSource = new PythonTokenSource(tokens);
- PythonPartialTokenSource indentedSource = new PythonPartialTokenSource(tokens);
+ PythonTokenSource indentedSource = new PythonTokenSource(tokens, "<test>");
tokens = new CommonTokenStream(indentedSource);
PythonPartialParser parser = new PythonPartialParser(tokens);
parser.single_input();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
