Menu
▾
▴
[Jython-checkins] SF.net SVN: jython:[5073] branches/asm
|
From: <fwi...@us...> - 2008-08-03 22:54:17
|
Revision: 5073
http://jython.svn.sourceforge.net/jython/?rev=5073&view=rev
Author: fwierzbicki
Date: 2008-08-03 22:54:14 +0000 (Sun, 03 Aug 2008)
Log Message:
-----------
CommonToken now appears to cover the cases that ClassicToken was left around
to cover. Removed all ClassicToken.
Modified Paths:
--------------
branches/asm/grammar/Python.g
branches/asm/src/org/python/antlr/PythonPartialTokenSource.java
branches/asm/src/org/python/antlr/PythonTokenSource.java
branches/asm/src/org/python/antlr/PythonTree.java
branches/asm/src/org/python/antlr/PythonTreeAdaptor.java
Removed Paths:
-------------
branches/asm/src/org/python/antlr/ImaginaryToken.java
Modified: branches/asm/grammar/Python.g
===================================================================
--- branches/asm/grammar/Python.g 2008-08-03 21:45:49 UTC (rev 5072)
+++ branches/asm/grammar/Python.g 2008-08-03 22:54:14 UTC (rev 5073)
@@ -1337,7 +1337,7 @@
*/
CONTINUED_LINE
: '\\' ('\r')? '\n' (' '|'\t')* { $channel=HIDDEN; }
- ( nl=NEWLINE {emit(new ClassicToken(NEWLINE,nl.getText()));}
+ ( nl=NEWLINE {emit(new CommonToken(NEWLINE,nl.getText()));}
|
)
;
@@ -1383,7 +1383,7 @@
for (int i=0; i<spaces; i++) {
indentation[i] = ' ';
}
- ClassicToken c = new ClassicToken(LEADING_WS,new String(indentation));
+ CommonToken c = new CommonToken(LEADING_WS,new String(indentation));
c.setLine(input.getLine());
c.setCharPositionInLine(input.getCharPositionInLine());
emit(c);
Deleted: branches/asm/src/org/python/antlr/ImaginaryToken.java
===================================================================
--- branches/asm/src/org/python/antlr/ImaginaryToken.java 2008-08-03 21:45:49 UTC (rev 5072)
+++ branches/asm/src/org/python/antlr/ImaginaryToken.java 2008-08-03 22:54:14 UTC (rev 5073)
@@ -1,29 +0,0 @@
-package org.python.antlr;
-
-import org.antlr.runtime.ClassicToken;
-
-public class ImaginaryToken extends ClassicToken {
-
- protected int start;
- protected int stop;
-
- public ImaginaryToken(int ttype, String text) {
- super(ttype, text);
- }
-
- public int getStartIndex() {
- return start;
- }
-
- public void setStartIndex(int start) {
- this.start = start;
- }
-
- public int getStopIndex() {
- return stop;
- }
-
- public void setStopIndex(int stop) {
- this.stop = stop;
- }
-}
Modified: branches/asm/src/org/python/antlr/PythonPartialTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonPartialTokenSource.java 2008-08-03 21:45:49 UTC (rev 5072)
+++ branches/asm/src/org/python/antlr/PythonPartialTokenSource.java 2008-08-03 22:54:14 UTC (rev 5073)
@@ -83,7 +83,7 @@
stream.consume();
if ( t.getType()==Token.EOF ) {
atEnd = true;
- Token em = new ClassicToken(PythonPartialParser.ENDMARKER,"");
+ Token em = new CommonToken(PythonPartialParser.ENDMARKER,"");
em.setCharPositionInLine(t.getCharPositionInLine());
em.setLine(t.getLine());
tokens.addElement(em);
@@ -123,7 +123,7 @@
int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
if ( t.getType()==Token.EOF ) {
atEnd = true;
- Token em = new ClassicToken(PythonPartialParser.ENDMARKER,"");
+ Token em = new CommonToken(PythonPartialParser.ENDMARKER,"");
em.setCharPositionInLine(t.getCharPositionInLine());
em.setLine(t.getLine());
tokens.addElement(em);
@@ -142,7 +142,7 @@
if ( cpos > lastIndent ) { // they indented; track and gen INDENT
push(cpos);
//System.out.println("push("+cpos+"): "+stackString());
- Token indent = new ClassicToken(PythonPartialParser.INDENT,"");
+ Token indent = new CommonToken(PythonPartialParser.INDENT,"");
indent.setCharPositionInLine(t.getCharPositionInLine());
indent.setLine(t.getLine());
tokens.addElement(indent);
@@ -155,9 +155,9 @@
for (int d=sp-1; d>=prevIndex; d--) {
Token tok;
if (atEnd) {
- tok = new ClassicToken(PythonPartialParser.ENDMARKER,"");
+ tok = new CommonToken(PythonPartialParser.ENDMARKER,"");
} else {
- tok = new ClassicToken(PythonPartialParser.DEDENT,"");
+ tok = new CommonToken(PythonPartialParser.DEDENT,"");
}
tok.setCharPositionInLine(t.getCharPositionInLine());
tok.setLine(t.getLine());
Modified: branches/asm/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonTokenSource.java 2008-08-03 21:45:49 UTC (rev 5072)
+++ branches/asm/src/org/python/antlr/PythonTokenSource.java 2008-08-03 22:54:14 UTC (rev 5073)
@@ -200,7 +200,7 @@
if (cpos > lastIndent) { // they indented; track and gen INDENT
push(cpos);
//System.out.println("push("+cpos+"): "+stackString());
- Token indent = new ImaginaryToken(PythonParser.INDENT,"");
+ Token indent = new CommonToken(PythonParser.INDENT,"");
indent.setCharPositionInLine(t.getCharPositionInLine());
indent.setLine(t.getLine());
tokens.addElement(indent);
@@ -211,7 +211,7 @@
//System.out.println("dedented; prevIndex of cpos="+cpos+" is "+prevIndex);
// generate DEDENTs for each indent level we backed up over
for (int d = sp - 1; d >= prevIndex; d--) {
- ImaginaryToken dedent = new ImaginaryToken(PythonParser.DEDENT,"");
+ CommonToken dedent = new CommonToken(PythonParser.DEDENT,"");
dedent.setCharPositionInLine(t.getCharPositionInLine());
dedent.setLine(t.getLine());
Modified: branches/asm/src/org/python/antlr/PythonTree.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonTree.java 2008-08-03 21:45:49 UTC (rev 5072)
+++ branches/asm/src/org/python/antlr/PythonTree.java 2008-08-03 22:54:14 UTC (rev 5073)
@@ -152,12 +152,7 @@
public int getCharStartIndex() {
if (charStartIndex == -1 && token != null) {
- if (token instanceof CommonToken) {
- return ((CommonToken)token).getStartIndex();
- }
- if (token instanceof ImaginaryToken) {
- return ((ImaginaryToken)token).getStartIndex();
- }
+ return ((CommonToken)token).getStartIndex();
}
return charStartIndex ;
}
@@ -178,12 +173,7 @@
public int getCharStopIndex() {
if (charStopIndex == -1 && token != null) {
- if (token instanceof CommonToken) {
- return ((CommonToken)token).getStopIndex() + 1;
- }
- if (token instanceof ImaginaryToken) {
- return ((ImaginaryToken)token).getStopIndex() + 1;
- }
+ return ((CommonToken)token).getStopIndex() + 1;
}
return charStopIndex;
}
Modified: branches/asm/src/org/python/antlr/PythonTreeAdaptor.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonTreeAdaptor.java 2008-08-03 21:45:49 UTC (rev 5072)
+++ branches/asm/src/org/python/antlr/PythonTreeAdaptor.java 2008-08-03 22:54:14 UTC (rev 5073)
@@ -18,23 +18,11 @@
int stopChar = 0;
if (startToken!=null) {
start = startToken.getTokenIndex();
- //XXX: instanceof is necessary because of the use of ClassicToken
- // for leading whitespace lexing.
- if (startToken instanceof CommonToken) {
- startChar = ((CommonToken)startToken).getStartIndex();
- } else if (startToken instanceof ImaginaryToken) {
- startChar = ((ImaginaryToken)startToken).getStartIndex();
- }
+ startChar = ((CommonToken)startToken).getStartIndex();
}
if (stopToken!=null) {
stop = stopToken.getTokenIndex();
- //XXX: instanceof is necessary because of the use of ClassicToken
- // for leading whitespace lexing.
- if (stopToken instanceof CommonToken) {
- stopChar = ((CommonToken)stopToken).getStopIndex() + 1;
- } else if (stopToken instanceof ImaginaryToken) {
- stopChar = ((ImaginaryToken)stopToken).getStopIndex() + 1;
- }
+ stopChar = ((CommonToken)stopToken).getStopIndex() + 1;
}
PythonTree pt = (PythonTree)t;
pt.setTokenStartIndex(start);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
