From: <fwi...@us...> - 2008-09-04 16:57:33
|
Revision: 5288 http://jython.svn.sourceforge.net/jython/?rev=5288&view=rev Author: fwierzbicki Date: 2008-09-04 16:57:30 +0000 (Thu, 04 Sep 2008) Log Message: ----------- Fix for some offset problems -- the biggest one being dedents now take on the offset values of the previous token. Modified Paths: -------------- trunk/jython/grammar/Python.g trunk/jython/src/org/python/antlr/PythonTokenSource.java trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java Modified: trunk/jython/grammar/Python.g =================================================================== --- trunk/jython/grammar/Python.g 2008-09-03 15:24:20 UTC (rev 5287) +++ trunk/jython/grammar/Python.g 2008-09-04 16:57:30 UTC (rev 5288) @@ -284,6 +284,12 @@ List stypes = new ArrayList(); } @after { + if (!stypes.isEmpty()) { + //The EOF token messes up the end offsets, so set them manually. + PythonTree stop = (PythonTree)stypes.get(stypes.size() -1); + mtype.setCharStopIndex(stop.getCharStopIndex()); + mtype.setTokenStopIndex(stop.getTokenStopIndex()); + } $file_input.tree = mtype; } : (NEWLINE Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java =================================================================== --- trunk/jython/src/org/python/antlr/PythonTokenSource.java 2008-09-03 15:24:20 UTC (rev 5287) +++ trunk/jython/src/org/python/antlr/PythonTokenSource.java 2008-09-04 16:57:30 UTC (rev 5288) @@ -157,14 +157,18 @@ stream.consume(); if (t.getType() == Token.EOF) { + Token prev = stream.LT(-1); if (!inSingle) { - Token prev = stream.LT(-1); if (prev == null || prev.getType() != PythonLexer.NEWLINE) { generateNewline(t); } } - - handleDedents(-1, (CommonToken)t); + if (prev != null) { + ((CommonToken)t).setStopIndex(((CommonToken)prev).getStopIndex()); + handleDedents(-1, (CommonToken)prev); + } else { + handleDedents(-1, (CommonToken)t); + } enqueue(t); } else if (t.getType() == PythonLexer.NEWLINE) { // save NEWLINE in the queue @@ -203,7 +207,12 @@ handleIndents(cpos, (CommonToken)t); } else if (cpos < lastIndent) { // they dedented - handleDedents(cpos, (CommonToken)t); + Token prev = stream.LT(-1); + if (prev != null) { + handleDedents(cpos, (CommonToken)prev); + } else { + handleDedents(cpos, (CommonToken)t); + } } if (t.getType() == Token.EOF && inSingle) { Modified: trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java =================================================================== --- trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java 2008-09-03 15:24:20 UTC (rev 5287) +++ trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java 2008-09-04 16:57:30 UTC (rev 5288) @@ -12,6 +12,8 @@ if (t==null) { return; } + //System.out.println("setting boundries on '"+t+"' with start: '" + startToken + "' and '" + + // stopToken + "'"); int start = 0; int stop = 0; int startChar = 0; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |