[Jython-checkins] SF.net SVN: jython:[5189] branches/asm

[<fwi...@us...>]

Revision: 5189
          http://jython.svn.sourceforge.net/jython/?rev=5189&view=rev
Author:   fwierzbicki
Date:     2008-08-16 19:31:55 +0000 (Sat, 16 Aug 2008)

Log Message:
-----------
Shouldn't be discarding hidden tokens.

Modified Paths:
--------------
    branches/asm/src/org/python/antlr/ExpressionParser.java
    branches/asm/src/org/python/antlr/InteractiveParser.java
    branches/asm/src/org/python/antlr/ModuleParser.java
    branches/asm/src/org/python/core/ParserFacade.java
    branches/asm/tests/java/org/python/antlr/PythonPartialTester.java
    branches/asm/tests/java/org/python/antlr/PythonTreeTester.java

Modified: branches/asm/src/org/python/antlr/ExpressionParser.java
===================================================================
--- branches/asm/src/org/python/antlr/ExpressionParser.java	2008-08-16 13:56:28 UTC (rev 5188)
+++ branches/asm/src/org/python/antlr/ExpressionParser.java	2008-08-16 19:31:55 UTC (rev 5189)
@@ -26,7 +26,6 @@
         PythonLexer lexer = new PyLexer(this.charStream);
         lexer.setErrorHandler(errorHandler);
         CommonTokenStream tokens = new CommonTokenStream(lexer);
-        tokens.discardOffChannelTokens(true);
         PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename);
         tokens = new CommonTokenStream(indentedSource);
         PythonParser parser = new PythonParser(tokens);

Modified: branches/asm/src/org/python/antlr/InteractiveParser.java
===================================================================
--- branches/asm/src/org/python/antlr/InteractiveParser.java	2008-08-16 13:56:28 UTC (rev 5188)
+++ branches/asm/src/org/python/antlr/InteractiveParser.java	2008-08-16 19:31:55 UTC (rev 5189)
@@ -31,7 +31,6 @@
         PythonLexer lexer = new PyLexer(new NoCloseReaderStream(bufreader));
         lexer.setErrorHandler(errorHandler);
         CommonTokenStream tokens = new CommonTokenStream(lexer);
-        tokens.discardOffChannelTokens(true);
         PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename, true);
         tokens = new CommonTokenStream(indentedSource);
         PythonParser parser = new PythonParser(tokens);

Modified: branches/asm/src/org/python/antlr/ModuleParser.java
===================================================================
--- branches/asm/src/org/python/antlr/ModuleParser.java	2008-08-16 13:56:28 UTC (rev 5188)
+++ branches/asm/src/org/python/antlr/ModuleParser.java	2008-08-16 19:31:55 UTC (rev 5189)
@@ -30,7 +30,6 @@
         PythonLexer lexer = new PyLexer(this.charStream);
         lexer.setErrorHandler(errorHandler);
         CommonTokenStream tokens = new CommonTokenStream(lexer);
-        tokens.discardOffChannelTokens(true);
         PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename);
         tokens = new CommonTokenStream(indentedSource);
         PythonParser parser = new PythonParser(tokens);

Modified: branches/asm/src/org/python/core/ParserFacade.java
===================================================================
--- branches/asm/src/org/python/core/ParserFacade.java	2008-08-16 13:56:28 UTC (rev 5188)
+++ branches/asm/src/org/python/core/ParserFacade.java	2008-08-16 19:31:55 UTC (rev 5189)
@@ -170,7 +170,6 @@
             lexer = new BaseParser.PyLexer(cs);
             lexer.partial = true;
             CommonTokenStream tokens = new CommonTokenStream(lexer);
-            tokens.discardOffChannelTokens(true);
             PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename);
             tokens = new CommonTokenStream(indentedSource);
             PythonPartialParser parser = new PythonPartialParser(tokens);

Modified: branches/asm/tests/java/org/python/antlr/PythonPartialTester.java
===================================================================
--- branches/asm/tests/java/org/python/antlr/PythonPartialTester.java	2008-08-16 13:56:28 UTC (rev 5188)
+++ branches/asm/tests/java/org/python/antlr/PythonPartialTester.java	2008-08-16 19:31:55 UTC (rev 5189)
@@ -16,7 +16,6 @@
             CharStream input = new ANTLRFileStream(args[0]);
             PythonLexer lexer = new BaseParser.PyLexer(input);
             CommonTokenStream tokens = new CommonTokenStream(lexer);
-            tokens.discardOffChannelTokens(true);
             //PythonTokenSource indentedSource = new PythonTokenSource(tokens);
             PythonTokenSource indentedSource = new PythonTokenSource(tokens, "<test>");
             tokens = new CommonTokenStream(indentedSource);

Modified: branches/asm/tests/java/org/python/antlr/PythonTreeTester.java
===================================================================
--- branches/asm/tests/java/org/python/antlr/PythonTreeTester.java	2008-08-16 13:56:28 UTC (rev 5188)
+++ branches/asm/tests/java/org/python/antlr/PythonTreeTester.java	2008-08-16 19:31:55 UTC (rev 5189)
@@ -33,7 +33,6 @@
         PythonLexer lexer = new ModuleParser.PyLexer(input);
         lexer.setErrorHandler(eh);
         CommonTokenStream tokens = new CommonTokenStream(lexer);
-        tokens.discardOffChannelTokens(true);
         PythonTokenSource indentedSource = new PythonTokenSource(tokens, args[0]);
         tokens = new CommonTokenStream(indentedSource);
         PythonParser parser = new PythonParser(tokens);


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.