From: <fwi...@us...> - 2008-09-05 16:30:39
|
Revision: 5291 http://jython.svn.sourceforge.net/jython/?rev=5291&view=rev Author: fwierzbicki Date: 2008-09-05 16:30:36 +0000 (Fri, 05 Sep 2008) Log Message: ----------- Broke test_traceback.py in my last commit. This fixes it again. Modified Paths: -------------- trunk/jython/grammar/Python.g trunk/jython/src/org/python/antlr/PythonTokenSource.java Modified: trunk/jython/grammar/Python.g =================================================================== --- trunk/jython/grammar/Python.g 2008-09-05 16:21:55 UTC (rev 5290) +++ trunk/jython/grammar/Python.g 2008-09-05 16:30:36 UTC (rev 5291) @@ -286,10 +286,14 @@ @after { if (!stypes.isEmpty()) { //The EOF token messes up the end offsets, so set them manually. + //XXX: this may no longer be true now that PythonTokenSource is + // adjusting EOF offsets -- but needs testing before I remove + // this. PythonTree stop = (PythonTree)stypes.get(stypes.size() -1); mtype.setCharStopIndex(stop.getCharStopIndex()); mtype.setTokenStopIndex(stop.getTokenStopIndex()); } + $file_input.tree = mtype; } : (NEWLINE @@ -1778,6 +1782,8 @@ CommonToken c = new CommonToken(LEADING_WS,new String(indentation)); c.setLine(input.getLine()); c.setCharPositionInLine(input.getCharPositionInLine()); + c.setStartIndex(input.index() - 1); + c.setStopIndex(input.index() - 1); emit(c); // kill trailing newline if present and then ignore if (newlines != 0) { @@ -1793,7 +1799,12 @@ for (int i=0; i<newlines; i++) { nls[i] = '\n'; } - emit(new CommonToken(NEWLINE,new String(nls))); + CommonToken c = new CommonToken(NEWLINE,new String(nls)); + c.setLine(input.getLine()); + c.setCharPositionInLine(input.getCharPositionInLine()); + c.setStartIndex(input.index() - 1); + c.setStopIndex(input.index() - 1); + emit(c); } } ) Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java =================================================================== --- trunk/jython/src/org/python/antlr/PythonTokenSource.java 2008-09-05 16:21:55 UTC (rev 5290) +++ trunk/jython/src/org/python/antlr/PythonTokenSource.java 2008-09-05 16:30:36 UTC (rev 5291) @@ -152,23 +152,25 @@ tokens.addElement(newline); } + private void handleEOF(CommonToken eof, CommonToken prev) { + if (prev != null) { + eof.setStopIndex(prev.getStopIndex()); + } + } + protected void insertImaginaryIndentDedentTokens() { Token t = stream.LT(1); stream.consume(); if (t.getType() == Token.EOF) { Token prev = stream.LT(-1); + handleEOF((CommonToken)t, (CommonToken)prev); if (!inSingle) { if (prev == null || prev.getType() != PythonLexer.NEWLINE) { generateNewline(t); } } - if (prev != null) { - ((CommonToken)t).setStopIndex(((CommonToken)prev).getStopIndex()); - handleDedents(-1, (CommonToken)prev); - } else { - handleDedents(-1, (CommonToken)t); - } + handleDedents(-1, (CommonToken)t); enqueue(t); } else if (t.getType() == PythonLexer.NEWLINE) { // save NEWLINE in the queue @@ -186,6 +188,7 @@ // compute cpos as the char pos of next non-WS token in line int cpos = t.getCharPositionInLine(); // column dictates indent/dedent if (t.getType() == Token.EOF) { + handleEOF((CommonToken)t, (CommonToken)newline); cpos = -1; // pretend EOF always happens at left edge } else if (t.getType() == PythonLexer.LEADING_WS) { @@ -207,8 +210,7 @@ handleIndents(cpos, (CommonToken)t); } else if (cpos < lastIndent) { // they dedented - Token prev = stream.LT(-1); - handleDedents(cpos, (CommonToken)newline); + handleDedents(cpos, (CommonToken)t); } if (t.getType() == Token.EOF && inSingle) { This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |