Menu
▾
▴
[Jython-checkins] SF.net SVN: jython:[5339] trunk/jython
|
From: <fwi...@us...> - 2008-09-18 12:07:38
|
Revision: 5339
http://jython.svn.sourceforge.net/jython/?rev=5339&view=rev
Author: fwierzbicki
Date: 2008-09-18 19:07:35 +0000 (Thu, 18 Sep 2008)
Log Message:
-----------
Fixed another eof/whitespace parsing issue from Django. Turned all of these
problems into a new test: test_eof_jy.py.
Modified Paths:
--------------
trunk/jython/grammar/Python.g
trunk/jython/src/org/python/antlr/PythonTokenSource.java
Added Paths:
-----------
trunk/jython/Lib/test/eof_fodder1.py
trunk/jython/Lib/test/eof_fodder2.py
trunk/jython/Lib/test/eof_fodder3.py
trunk/jython/Lib/test/eof_fodder4.py
trunk/jython/Lib/test/eof_fodder5.py
trunk/jython/Lib/test/eof_fodder6.py
trunk/jython/Lib/test/test_eof_jy.py
Added: trunk/jython/Lib/test/eof_fodder1.py
===================================================================
--- trunk/jython/Lib/test/eof_fodder1.py (rev 0)
+++ trunk/jython/Lib/test/eof_fodder1.py 2008-09-18 19:07:35 UTC (rev 5339)
@@ -0,0 +1,3 @@
+def d():
+ def e():
+ pass
\ No newline at end of file
Added: trunk/jython/Lib/test/eof_fodder2.py
===================================================================
--- trunk/jython/Lib/test/eof_fodder2.py (rev 0)
+++ trunk/jython/Lib/test/eof_fodder2.py 2008-09-18 19:07:35 UTC (rev 5339)
@@ -0,0 +1,4 @@
+def f():
+ def g():
+ pass
+
\ No newline at end of file
Added: trunk/jython/Lib/test/eof_fodder3.py
===================================================================
--- trunk/jython/Lib/test/eof_fodder3.py (rev 0)
+++ trunk/jython/Lib/test/eof_fodder3.py 2008-09-18 19:07:35 UTC (rev 5339)
@@ -0,0 +1,4 @@
+def f():
+ def g():
+ pass
+
Added: trunk/jython/Lib/test/eof_fodder5.py
===================================================================
--- trunk/jython/Lib/test/eof_fodder5.py (rev 0)
+++ trunk/jython/Lib/test/eof_fodder5.py 2008-09-18 19:07:35 UTC (rev 5339)
@@ -0,0 +1,3 @@
+def f():
+ pass
+# just a comment
\ No newline at end of file
Added: trunk/jython/Lib/test/eof_fodder6.py
===================================================================
--- trunk/jython/Lib/test/eof_fodder6.py (rev 0)
+++ trunk/jython/Lib/test/eof_fodder6.py 2008-09-18 19:07:35 UTC (rev 5339)
@@ -0,0 +1,5 @@
+def f():
+ def g():
+ pass
+
+
\ No newline at end of file
Added: trunk/jython/Lib/test/test_eof_jy.py
===================================================================
--- trunk/jython/Lib/test/test_eof_jy.py (rev 0)
+++ trunk/jython/Lib/test/test_eof_jy.py 2008-09-18 19:07:35 UTC (rev 5339)
@@ -0,0 +1,55 @@
+import unittest
+from test import test_support
+
+class TestEof(unittest.TestCase):
+ """
+ Oddities originally found in Django involving whitespace and newlines or
+ lack thereof at the end of files. I can't use __builtin__.compile()
+ because newlines get added and hide these problems, so I have opted to
+ import Python files containing these oddities.
+ """
+
+ def test_indented_no_newline(self):
+ try:
+ import eof_fodder1
+ except ImportError, cause:
+ self.fail(cause)
+
+ def test_trailing_ws_no_newline(self):
+ try:
+ import eof_fodder2
+ except ImportError, cause:
+ self.fail(cause)
+
+ def test_trailing_ws(self):
+ try:
+ import eof_fodder3
+ except ImportError, cause:
+ self.fail(cause)
+
+ def test_empty(self):
+ try:
+ import eof_fodder4
+ except ImportError, cause:
+ self.fail(cause)
+
+ def test_just_a_comment_no_newline(self):
+ try:
+ import eof_fodder5
+ except ImportError, cause:
+ self.fail(cause)
+
+ def test_junky_ws_after_indent(self):
+ try:
+ import eof_fodder6
+ except ImportError, cause:
+ self.fail(cause)
+
+#==============================================================================
+
+def test_main(verbose=None):
+ test_classes = [TestEof]
+ test_support.run_unittest(*test_classes)
+
+if __name__ == "__main__":
+ test_main(verbose=True)
Modified: trunk/jython/grammar/Python.g
===================================================================
--- trunk/jython/grammar/Python.g 2008-09-17 22:37:23 UTC (rev 5338)
+++ trunk/jython/grammar/Python.g 2008-09-18 19:07:35 UTC (rev 5339)
@@ -1785,7 +1785,7 @@
)+
( ('\r')? '\n' {newlines++; }
)* {
- if (input.LA(1) != -1) {
+ if (input.LA(1) != -1 || newlines == 0) {
// make a string of n spaces where n is column number - 1
char[] indentation = new char[spaces];
for (int i=0; i<spaces; i++) {
Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTokenSource.java 2008-09-17 22:37:23 UTC (rev 5338)
+++ trunk/jython/src/org/python/antlr/PythonTokenSource.java 2008-09-18 19:07:35 UTC (rev 5339)
@@ -167,11 +167,18 @@
Token prev = stream.LT(-1);
handleEOF((CommonToken)t, (CommonToken)prev);
if (!inSingle) {
- if (prev == null || prev.getType() != PythonLexer.NEWLINE) {
+ if (prev == null) {
generateNewline(t);
+ } else if (prev.getType() == PythonLexer.LEADING_WS) {
+ handleDedents(-1, (CommonToken)t);
+ generateNewline(t);
+ } else if (prev.getType() != PythonLexer.NEWLINE) {
+ generateNewline(t);
+ handleDedents(-1, (CommonToken)t);
}
+ } else {
+ handleDedents(-1, (CommonToken)t);
}
- handleDedents(-1, (CommonToken)t);
enqueue(t);
} else if (t.getType() == PythonLexer.NEWLINE) {
// save NEWLINE in the queue
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
