[Pydev-cvs] org.python.pydev.parser/src/org/python/pydev/parser/jython IParserHost.java,NONE,1.1 PythonGrammarTreeConstants.java,NONE,1.1 PythonGrammarConstants.java,NONE,1.1 .cvsignore,NONE,1.1 ParseException.java,NONE,1.1 Visitor.java,NONE,1.1 PythonGrammar.java,NONE,1.1 ParseError.java,NONE,1.1 package.html,NONE,1.1 build.xml,NONE,1.1 ICompilerAPI.java,NONE,1.1 ReaderCharStream.java,NONE,1.1 Node.java,NONE,1.1 python.jjt,NONE,1.1 JJTPythonGrammarState.java,NONE,1.1 Token.java,NONE,1.1 TreeBuilder.java,NONE,1.1 PythonGrammarTokenManager.java,NONE,1.1 SimpleNode.java,NONE,1.1 CharStream.java,NONE,1.1 TokenMgrError.java,NONE,1.1 SpecialStr.java,NONE,1.1

[Fabio Z. <fa...@us...>]

Update of /cvsroot/pydev/org.python.pydev.parser/src/org/python/pydev/parser/jython
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv12282/src/org/python/pydev/parser/jython

Added Files:
	IParserHost.java PythonGrammarTreeConstants.java 
	PythonGrammarConstants.java .cvsignore ParseException.java 
	Visitor.java PythonGrammar.java ParseError.java package.html 
	build.xml ICompilerAPI.java ReaderCharStream.java Node.java 
	python.jjt JJTPythonGrammarState.java Token.java 
	TreeBuilder.java PythonGrammarTokenManager.java 
	SimpleNode.java CharStream.java TokenMgrError.java 
	SpecialStr.java 
Log Message:


--- NEW FILE: TokenMgrError.java ---
/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 0.7pre2 */

// Hacked by baw 12-Mar-1999 because JavaCC does not seem to be generating
// the same class structure as it was before :-(

package org.python.pydev.parser.jython;

public class TokenMgrError extends Error
{
   /*
    * Ordinals for various reasons why an Error of this type can be thrown.
    */

   /**
    * Lexical error occured.
    */
   static public final int LEXICAL_ERROR = 0;

   /**
    * An attempt wass made to create a second instance of a static token manager.
    */
   static final int STATIC_LEXER_ERROR = 1;

   /**
    * Tried to change to an invalid lexical state.
    */
   static final int INVALID_LEXICAL_STATE = 2;

   /**
    * Detected (and bailed out of) an infinite loop in the token manager.
    */
   static final int LOOP_DETECTED = 3;

   /**
    * Indicates the reason why the exception is thrown. It will have
    * one of the above 4 values.
    */
   public int errorCode;

   /**
    * Replaces unprintable characters by their espaced (or unicode escaped)
    * equivalents in the given string
    */
   protected static final String addEscapes(String str) {
      StringBuffer retval = new StringBuffer();
      char ch;
      for (int i = 0; i < str.length(); i++) {
        switch (str.charAt(i))
        {
           case 0 :
              continue;
           case '\b':
              retval.append("\\b");
              continue;
           case '\t':
              retval.append("\\t");
              continue;
           case '\n':
              retval.append("\\n");
              continue;
           case '\f':
              retval.append("\\f");
              continue;
           case '\r':
              retval.append("\\r");
              continue;
           case '\"':
              retval.append("\\\"");
              continue;
           case '\'':
              retval.append("\\\'");
              continue;
           case '\\':
              retval.append("\\\\");
              continue;
           default:
              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
                 String s = "0000" + Integer.toString(ch, 16);
                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
              } else {
                 retval.append(ch);
              }
              continue;
        }
      }
      return retval.toString();
   }

   /**
    * Returns a detailed message for the Error when it is thrown by the
    * token manager to indicate a lexical error.
    * Parameters :
    *    EOFSeen     : indicates if EOF caused the lexicl error
    *    curLexState : lexical state in which this error occured
    *    errorLine   : line number when the error occured
    *    errorColumn : column number when the error occured
    *    errorAfter  : prefix that was seen before this error occured
    *    curchar     : the offending character
    * Note: You can customize the lexical error message by modifying this method.
    */

   // added: 12-Mar-1999 (baw)
   public boolean EOFSeen;
   public int errorLine, errorColumn;
   public String curChar;
    
   // 8-May-2003 (pedronis)
   public int lexState = -1;

   private static final String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
      return("Lexical error at line " +
           errorLine + ", column " +
           errorColumn + ".  Encountered: " +
           (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
           "after : \"" + addEscapes(errorAfter) + "\"");
   }

   /**
    * You can also modify the body of this method to customize your error messages.
    * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
    * of end-users concern, so you can return something like :
    *
    *     "Internal Error : Please file a bug report .... "
    *
    * from this method for such cases in the release version of your parser.
    */
   public String getMessage() {
      return super.getMessage();
   }

   /*
    * Constructors of various flavors follow.
    */

   public TokenMgrError() {
   }

   public TokenMgrError(String message, int reason) {
      super(message);
      errorCode = reason;
   }

    // added: 12-Mar-1999 baw
   public TokenMgrError(String message, int errorLine, int errorColumn) {
      this(message, LEXICAL_ERROR);
           this.EOFSeen = false;
           this.errorLine = errorLine;
           this.errorColumn = errorColumn;
   }

   public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
      this(LexicalError(EOFSeen, lexState, errorLine, errorColumn,
                        errorAfter, curChar), reason);
      // modified: 12-Mar-1999 baw
      this.EOFSeen = EOFSeen;
      this.errorLine = errorLine;
      this.errorColumn = errorColumn;
      this.curChar = addEscapes(String.valueOf(curChar));
      
      // 8-May-2003 (pedronis)
      this.lexState = lexState;
   }
}

--- NEW FILE: PythonGrammarTokenManager.java ---
/* Generated By:JJTree&JavaCC: Do not edit this line. PythonGrammarTokenManager.java */
package org.python.pydev.parser.jython;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;

import org.python.pydev.parser.jython.ast.Call;
import org.python.pydev.parser.jython.ast.Import;
import org.python.pydev.parser.jython.ast.Str;
import org.python.pydev.parser.jython.ast.Tuple;
import org.python.pydev.parser.jython.ast.Yield;
import org.python.pydev.parser.jython.ast.modType;

public class PythonGrammarTokenManager implements PythonGrammarConstants
{
    int indentation[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
    int level = 0;
    int dedents = 0;
    int parens = 0;
[...3871 lines suppressed...]
            image.append(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));
    matchedToken.image = image.toString();
         break;
      case 115 :
        if (image == null)
            image = new StringBuffer();
            image.append(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));
    matchedToken.image = image.toString();
         break;
      case 116 :
        if (image == null)
            image = new StringBuffer();
            image.append(input_stream.GetSuffix(jjimageLen + (lengthOfMatch = jjmatchedPos + 1)));
    matchedToken.image = image.toString();
         break;
      default : 
         break;
   }
}
}

--- NEW FILE: PythonGrammarTreeConstants.java ---
/* Generated By:JJTree: Do not edit this line. E:\eclipse_workspace\org.python.pydev.parser\src\org\python\parser\PythonGrammarTreeConstants.java */

package org.python.pydev.parser.jython;

public interface PythonGrammarTreeConstants
{
  public int JJTSINGLE_INPUT = 0;
  public int JJTFILE_INPUT = 1;
  public int JJTEVAL_INPUT = 2;
  public int JJTFUNCDEF = 3;
  public int JJTDECORATORS = 4;
  public int JJTBEGIN_DECORATOR = 5;
  public int JJTVOID = 6;
  public int JJTEXTRAARGLIST = 7;
  public int JJTEXTRAKEYWORDLIST = 8;
  public int JJTDEFAULTARG = 9;
  public int JJTTUPLE = 10;
  public int JJTAUG_PLUS = 11;
  public int JJTAUG_MINUS = 12;
  public int JJTAUG_MULTIPLY = 13;
  public int JJTAUG_DIVIDE = 14;
  public int JJTAUG_FLOORDIVIDE = 15;
  public int JJTAUG_MODULO = 16;
  public int JJTAUG_AND = 17;
  public int JJTAUG_OR = 18;
  public int JJTAUG_XOR = 19;
  public int JJTAUG_LSHIFT = 20;
  public int JJTAUG_RSHIFT = 21;
  public int JJTAUG_POWER = 22;
  public int JJTEXPR_STMT = 23;
  public int JJTPRINTEXT_STMT = 24;
  public int JJTPRINT_STMT = 25;
  public int JJTDEL_STMT = 26;
  public int JJTBEGIN_DEL_STMT = 27;
  public int JJTPASS_STMT = 28;
  public int JJTBREAK_STMT = 29;
  public int JJTCONTINUE_STMT = 30;
  public int JJTRETURN_STMT = 31;
  public int JJTBEGIN_RETURN_STMT = 32;
  public int JJTYIELD_STMT = 33;
  public int JJTRAISE_STMT = 34;
  public int JJTIMPORT = 35;
  public int JJTIMPORTFROM = 36;
  public int JJTDOTTED_AS_NAME = 37;
  public int JJTDOTTED_NAME = 38;
  public int JJTIMPORT_AS_NAME = 39;
  public int JJTGLOBAL_STMT = 40;
  public int JJTEXEC_STMT = 41;
  public int JJTASSERT_STMT = 42;
  public int JJTIF_STMT = 43;
  public int JJTBEGIN_IF_STMT = 44;
  public int JJTBEGIN_ELIF_STMT = 45;
  public int JJTWHILE_STMT = 46;
  public int JJTBEGIN_WHILE_STMT = 47;
  public int JJTBEGIN_ELSE_STMT = 48;
  public int JJTFOR_STMT = 49;
  public int JJTBEGIN_FOR_STMT = 50;
  public int JJTBEGIN_FOR_ELSE_STMT = 51;
  public int JJTTRY_STMT = 52;
  public int JJTTRYFINALLY_STMT = 53;
  public int JJTBEGIN_TRY_STMT = 54;
  public int JJTBEGIN_TRY_ELSE_STMT = 55;
  public int JJTBEGIN_FINALLY_STMT = 56;
  public int JJTEXCEPT_CLAUSE = 57;
  public int JJTBEGIN_EXCEPT_CLAUSE = 58;
  public int JJTSUITE = 59;
  public int JJTOR_BOOLEAN = 60;
  public int JJTAND_BOOLEAN = 61;
  public int JJTNOT_1OP = 62;
  public int JJTCOMPARISION = 63;
  public int JJTLESS_CMP = 64;
  public int JJTGREATER_CMP = 65;
  public int JJTEQUAL_CMP = 66;
  public int JJTGREATER_EQUAL_CMP = 67;
  public int JJTLESS_EQUAL_CMP = 68;
  public int JJTNOTEQUAL_CMP = 69;
  public int JJTIN_CMP = 70;
  public int JJTNOT_IN_CMP = 71;
  public int JJTIS_NOT_CMP = 72;
  public int JJTIS_CMP = 73;
  public int JJTOR_2OP = 74;
  public int JJTXOR_2OP = 75;
  public int JJTAND_2OP = 76;
  public int JJTLSHIFT_2OP = 77;
  public int JJTRSHIFT_2OP = 78;
  public int JJTADD_2OP = 79;
  public int JJTSUB_2OP = 80;
  public int JJTMUL_2OP = 81;
  public int JJTDIV_2OP = 82;
  public int JJTFLOORDIV_2OP = 83;
  public int JJTMOD_2OP = 84;
  public int JJTPOS_1OP = 85;
  public int JJTNEG_1OP = 86;
  public int JJTINVERT_1OP = 87;
  public int JJTPOW_2OP = 88;
  public int JJTCALL_OP = 89;
  public int JJTINDEX_OP = 90;
  public int JJTDOT_OP = 91;
  public int JJTLIST = 92;
  public int JJTDICTIONARY = 93;
  public int JJTSTR_1OP = 94;
  public int JJTSTRJOIN = 95;
  public int JJTLAMBDEF = 96;
  public int JJTSUBSCRIPTLIST = 97;
  public int JJTELLIPSES = 98;
  public int JJTSLICE = 99;
  public int JJTCOLON = 100;
  public int JJTCOMMA = 101;
  public int JJTLIST_FOR = 102;
  public int JJTCLASSDEF = 103;
  public int JJTEXTRAARGVALUELIST = 104;
  public int JJTEXTRAKEYWORDVALUELIST = 105;
  public int JJTKEYWORD = 106;
  public int JJTNUM = 107;
  public int JJTCOMPLEX = 108;
  public int JJTNAME = 109;
  public int JJTSTRING = 110;
  public int JJTUNICODE = 111;


  public String[] jjtNodeName = {
    "single_input",
    "file_input",
    "eval_input",
    "funcdef",
    "decorators",
    "begin_decorator",
    "void",
    "ExtraArgList",
    "ExtraKeywordList",
    "defaultarg",
    "tuple",
    "aug_plus",
    "aug_minus",
    "aug_multiply",
    "aug_divide",
    "aug_floordivide",
    "aug_modulo",
    "aug_and",
    "aug_or",
    "aug_xor",
    "aug_lshift",
    "aug_rshift",
    "aug_power",
    "expr_stmt",
    "printext_stmt",
    "print_stmt",
    "del_stmt",
    "begin_del_stmt",
    "pass_stmt",
    "break_stmt",
    "continue_stmt",
    "return_stmt",
    "begin_return_stmt",
    "yield_stmt",
    "raise_stmt",
    "Import",
    "ImportFrom",
    "dotted_as_name",
    "dotted_name",
    "import_as_name",
    "global_stmt",
    "exec_stmt",
    "assert_stmt",
    "if_stmt",
    "begin_if_stmt",
    "begin_elif_stmt",
    "while_stmt",
    "begin_while_stmt",
    "begin_else_stmt",
    "for_stmt",
    "begin_for_stmt",
    "begin_for_else_stmt",
    "try_stmt",
    "tryfinally_stmt",
    "begin_try_stmt",
    "begin_try_else_stmt",
    "begin_finally_stmt",
    "except_clause",
    "begin_except_clause",
    "suite",
    "or_boolean",
    "and_boolean",
    "not_1op",
    "comparision",
    "less_cmp",
    "greater_cmp",
    "equal_cmp",
    "greater_equal_cmp",
    "less_equal_cmp",
    "notequal_cmp",
    "in_cmp",
    "not_in_cmp",
    "is_not_cmp",
    "is_cmp",
    "or_2op",
    "xor_2op",
    "and_2op",
    "lshift_2op",
    "rshift_2op",
    "add_2op",
    "sub_2op",
    "mul_2op",
    "div_2op",
    "floordiv_2op",
    "mod_2op",
    "pos_1op",
    "neg_1op",
    "invert_1op",
    "pow_2op",
    "Call_Op",
    "Index_Op",
    "Dot_Op",
    "list",
    "dictionary",
    "str_1op",
    "strjoin",
    "lambdef",
    "subscriptlist",
    "Ellipses",
    "Slice",
    "Colon",
    "Comma",
    "list_for",
    "classdef",
    "ExtraArgValueList",
    "ExtraKeywordValueList",
    "Keyword",
    "Num",
    "Complex",
    "Name",
    "String",
    "Unicode",
  };
}

--- NEW FILE: SpecialStr.java ---
package org.python.pydev.parser.jython;

public class SpecialStr {
	public String str;
	public int beginLine;
	public int beginCol;

	public SpecialStr(String str, int beginLine, int beginCol){
		this.str = str;
		this.beginLine = beginLine;
		this.beginCol = beginCol;
	}
	
	@Override
	public String toString() {
		return str;
	}

	@Override
	public int hashCode() {
		return str.hashCode();
	}
	
	@Override
	public boolean equals(Object obj) {
		if(!(obj instanceof SpecialStr)){
			return false;
		}
		return str.equals(((SpecialStr)obj).str);
	}
}

--- NEW FILE: Visitor.java ---
package org.python.pydev.parser.jython;

import org.python.pydev.parser.jython.ast.*;

public class Visitor extends VisitorBase {

    /**
     * Visit each of the children one by one.
     * @args node The node whose children will be visited.
     */
    public void traverse(SimpleNode node) throws Exception {
        node.traverse(this);
    }


    public void visit(SimpleNode[] nodes) throws Exception {
        for (int i = 0; i < nodes.length; i++) {
            visit(nodes[i]);
        }
    }

    /**
     * Visit the node by calling a visitXXX method.
     */
    public Object visit(SimpleNode node) throws Exception {
        open_level(node);
        Object ret = node.accept(this);
        close_level(node);
        return ret;
    }


    protected Object unhandled_node(SimpleNode node) throws Exception {
        return this;
    }

    protected void open_level(SimpleNode node) throws Exception {
    }

    protected void close_level(SimpleNode node) throws Exception {
    }
}

--- NEW FILE: build.xml ---
<project name="jython-parser" default="parser" basedir="E:\eclipse_workspace\org.python.pydev.parser\src\org\python\parser">
<!--
<project name="jython-parser" default="parser" basedir="D:\eclipse_workspace\org.python.pydev.parser\src\org\python\parser">
-->

    <property name="javaccHome" value="E:\install\Dev.JavaCC\javacc-4.0" />
    <property name="parser.dir" value="${basedir}" />

    <!-- print out some usage hints -->
    <target name="usage" description="prints a short usage">
        <echo>
If this build.xml is called directly, you have to make sure the property javaccHome is set correctly.
This can be done either inside this build.xml (first top property), or on the command line.

This build only handles javacc. If changes are done to the python.asdl file, you have to
re-generate it by hand with "python asdl_java.py Python.asdl" inside of the ast package.
        	
Current setting is: ${javaccHome}
        </echo>
    </target>

    <!-- checks if work to do / can be done -->
    <target name="pre">
        <!-- set the property parser.regen.notreq if PythonGrammer.java is more up to date than python.jjt 
        <uptodate property="parser.regen.notreq" targetfile="${parser.dir}/PythonGrammar.java">
            <srcfiles dir="${parser.dir}" includes="python.jjt" />
        </uptodate>
        -->
        <!-- fail if generation should be done, but javacc.jar is not available -->
        <property name="javacc.jar" value="${javaccHome}/bin/lib/javacc.jar" />
        <available file="${javacc.jar}" property="javacc.jar.present" />
        <fail message="${javacc.jar} not found: correct the javaccHome property">
            <condition>
                <and>
                    <not>
                        <isset property="parser.regen.notreq" />
                    </not>
                    <not>
                        <isset property="javacc.jar.present" />
                    </not>
                </and>
            </condition>
        </fail>
    </target>

    <!-- information if up to date -->
    <target name="uptodate" depends="pre" if="parser.regen.notreq">
        <echo>parser generated files are up to date</echo>
    </target>

    <!-- delete the parser generated files -->
    <target name="clean" depends="uptodate" unless="parser.regen.notreq">
        <!-- do not delete file="${parser.dir}/TokenMgrError.java" -->
        <!-- do not delete file="${parser.dir}/ParseException.java" -->
        <delete file="${parser.dir}/PythonGrammar.java" />
        <delete file="${parser.dir}/PythonGrammarTreeConstants.java" />
        <delete file="${parser.dir}/PythonGrammarTokenManager.java" />
        <delete file="${parser.dir}/PythonGrammarConstants.java" />
        <delete file="${parser.dir}/CharStream.java" />
        <!-- +++ TODO: maybe this could be allowed (checkin): -->
        <!-- <delete file="${parser.dir}/Token.java" /> -->
    </target>

    <!-- invoke jjtree on python.jjt -->
    <target name="tree" depends="clean" unless="parser.regen.notreq">
        <java classname="jjtree" classpath="${javacc.jar}" fork="yes">
            <arg value="-OUTPUT_DIRECTORY=${parser.dir}" />
            <arg file="${parser.dir}/python.jjt" />
        </java>
    </target>

    <!-- invoke javacc on python.jj -->
    <target name="gen" depends="tree" unless="parser.regen.notreq">
        <java classname="javacc" classpath="${javacc.jar}" fork="yes">
            <arg value="-OUTPUT_DIRECTORY=${parser.dir}" />
            <arg file="${parser.dir}/python.jj" />
        </java>
        <delete file="${parser.dir}/python.jj" />
    </target>

    <!-- confirmation -->
    <target name="done" depends="gen" unless="parser.regen.notreq">
        <echo>parser generated to directory ${parser.dir}</echo>
    </target>

    <!-- default target -->
    <target name="parser" depends="done" description="the default target" />

</project>
--- NEW FILE: ParseError.java ---
/* Generated By:JavaCC: Do not edit this line. ParseError.java Version 0.6 */
package org.python.pydev.parser.jython;

public class ParseError extends Exception {
}

--- NEW FILE: IParserHost.java ---
package org.python.pydev.parser.jython;

/**
 * 
 * literal creation callbacks from the parser to the its host
 * 
 **/

public interface IParserHost {

       public Object newLong(String s);

       public Object newLong(java.math.BigInteger i);

       public Object newFloat(double v);
       
       public Object newImaginary(double v);
       
       public Object newInteger(int i);
       
       public String decode_UnicodeEscape(String str, int start, int end,
               String errors, boolean unicode);
}

--- NEW FILE: PythonGrammarConstants.java ---
/* Generated By:JJTree&JavaCC: Do not edit this line. PythonGrammarConstants.java */
package org.python.pydev.parser.jython;

public interface PythonGrammarConstants {

  int EOF = 0;
  int SPACE = 1;
  int CONTINUATION = 4;
  int NEWLINE1 = 5;
  int NEWLINE = 6;
  int NEWLINE2 = 7;
  int CRLF1 = 12;
  int DEDENT = 14;
  int INDENT = 15;
  int TRAILING_COMMENT = 16;
  int SINGLE_LINE_COMMENT = 17;
  int LPAREN = 18;
  int RPAREN = 19;
  int LBRACE = 20;
  int RBRACE = 21;
  int LBRACKET = 22;
  int RBRACKET = 23;
  int SEMICOLON = 24;
  int COMMA = 25;
  int DOT = 26;
  int COLON = 27;
  int PLUS = 28;
  int MINUS = 29;
  int MULTIPLY = 30;
  int DIVIDE = 31;
  int FLOORDIVIDE = 32;
  int POWER = 33;
  int LSHIFT = 34;
  int RSHIFT = 35;
  int MODULO = 36;
  int NOT = 37;
  int XOR = 38;
  int OR = 39;
  int AND = 40;
  int EQUAL = 41;
  int GREATER = 42;
  int LESS = 43;
  int EQEQUAL = 44;
  int EQLESS = 45;
  int EQGREATER = 46;
  int LESSGREATER = 47;
  int NOTEQUAL = 48;
  int PLUSEQ = 49;
  int MINUSEQ = 50;
  int MULTIPLYEQ = 51;
  int DIVIDEEQ = 52;
  int FLOORDIVIDEEQ = 53;
  int MODULOEQ = 54;
  int ANDEQ = 55;
  int OREQ = 56;
  int XOREQ = 57;
  int LSHIFTEQ = 58;
  int RSHIFTEQ = 59;
  int POWEREQ = 60;
  int OR_BOOL = 61;
  int AND_BOOL = 62;
  int NOT_BOOL = 63;
  int IS = 64;
  int IN = 65;
  int LAMBDA = 66;
  int IF = 67;
  int ELSE = 68;
  int ELIF = 69;
  int WHILE = 70;
  int FOR = 71;
  int TRY = 72;
  int EXCEPT = 73;
  int DEF = 74;
  int CLASS = 75;
  int FINALLY = 76;
  int PRINT = 77;
  int PASS = 78;
  int BREAK = 79;
  int CONTINUE = 80;
  int RETURN = 81;
  int YIELD = 82;
  int IMPORT = 83;
  int FROM = 84;
  int DEL = 85;
  int RAISE = 86;
  int GLOBAL = 87;
  int EXEC = 88;
  int ASSERT = 89;
  int AS = 90;
  int AT = 91;
  int NAME = 92;
  int LETTER = 93;
  int DECNUMBER = 94;
  int HEXNUMBER = 95;
  int OCTNUMBER = 96;
  int FLOAT = 97;
  int COMPLEX = 98;
  int EXPONENT = 99;
  int DIGIT = 100;
  int SINGLE_STRING = 109;
  int SINGLE_STRING2 = 110;
  int TRIPLE_STRING = 111;
  int TRIPLE_STRING2 = 112;
  int SINGLE_USTRING = 113;
  int SINGLE_USTRING2 = 114;
  int TRIPLE_USTRING = 115;
  int TRIPLE_USTRING2 = 116;

  int DEFAULT = 0;
  int FORCE_NEWLINE1 = 1;
  int FORCE_NEWLINE2 = 2;
  int MAYBE_FORCE_NEWLINE_IF_EOF = 3;
  int INDENTING = 4;
  int INDENTATION_UNCHANGED = 5;
  int UNREACHABLE = 6;
  int IN_STRING11 = 7;
  int IN_STRING21 = 8;
  int IN_STRING13 = 9;
  int IN_STRING23 = 10;
  int IN_USTRING11 = 11;
  int IN_USTRING21 = 12;
  int IN_USTRING13 = 13;
  int IN_USTRING23 = 14;
  int IN_STRING1NLC = 15;
  int IN_STRING2NLC = 16;
  int IN_USTRING1NLC = 17;
  int IN_USTRING2NLC = 18;

  String[] tokenImage = {
    "<EOF>",
    "\" \"",
    "\"\\t\"",
    "\"\\f\"",
    "<CONTINUATION>",
    "<NEWLINE1>",
    "<NEWLINE>",
    "<NEWLINE2>",
    "\"\"",
    "\"\\t\"",
    "\" \"",
    "\"\\f\"",
    "<CRLF1>",
    "\"\"",
    "\"\"",
    "\"<INDENT>\"",
    "<TRAILING_COMMENT>",
    "<SINGLE_LINE_COMMENT>",
    "\"(\"",
    "\")\"",
    "\"{\"",
    "\"}\"",
    "\"[\"",
    "\"]\"",
    "\";\"",
    "\",\"",
    "\".\"",
    "\":\"",
    "\"+\"",
    "\"-\"",
    "\"*\"",
    "\"/\"",
    "\"//\"",
    "\"**\"",
    "\"<<\"",
    "\">>\"",
    "\"%\"",
    "\"~\"",
    "\"^\"",
    "\"|\"",
    "\"&\"",
    "\"=\"",
    "\">\"",
    "\"<\"",
    "\"==\"",
    "\"<=\"",
    "\">=\"",
    "\"<>\"",
    "\"!=\"",
    "\"+=\"",
    "\"-=\"",
    "\"*=\"",
    "\"/=\"",
    "\"//=\"",
    "\"%=\"",
    "\"&=\"",
    "\"|=\"",
    "\"^=\"",
    "\"<<=\"",
    "\">>=\"",
    "\"**=\"",
    "\"or\"",
    "\"and\"",
    "\"not\"",
    "\"is\"",
    "\"in\"",
    "\"lambda\"",
    "\"if\"",
    "\"else\"",
    "\"elif\"",
    "\"while\"",
    "\"for\"",
    "\"try\"",
    "\"except\"",
    "\"def\"",
    "\"class\"",
    "\"finally\"",
    "\"print\"",
    "\"pass\"",
    "\"break\"",
    "\"continue\"",
    "\"return\"",
    "\"yield\"",
    "\"import\"",
    "\"from\"",
    "\"del\"",
    "\"raise\"",
    "\"global\"",
    "\"exec\"",
    "\"assert\"",
    "\"as\"",
    "\"@\"",
    "<NAME>",
    "<LETTER>",
    "<DECNUMBER>",
    "<HEXNUMBER>",
    "<OCTNUMBER>",
    "<FLOAT>",
    "<COMPLEX>",
    "<EXPONENT>",
    "<DIGIT>",
    "<token of kind 101>",
    "<token of kind 102>",
    "<token of kind 103>",
    "<token of kind 104>",
    "<token of kind 105>",
    "<token of kind 106>",
    "<token of kind 107>",
    "<token of kind 108>",
    "\"\\\'\"",
    "\"\\\"\"",
    "\"\\\'\\\'\\\'\"",
    "\"\\\"\\\"\\\"\"",
    "\"\\\'\"",
    "\"\\\"\"",
    "\"\\\'\\\'\\\'\"",
    "\"\\\"\\\"\\\"\"",
    "\"\\\\\\r\\n\"",
    "<token of kind 118>",
    "\"\\\\\\r\\n\"",
    "<token of kind 120>",
    "\"\\\\\\r\\n\"",
    "<token of kind 122>",
    "\"\\\\\\r\\n\"",
    "<token of kind 124>",
    "\"\"",
    "\"\"",
    "\"\"",
    "\"\"",
    "<token of kind 129>",
    "<token of kind 130>",
    "\"\\r\\n\"",
    "\"\\n\"",
    "\"\\r\"",
    "<token of kind 134>",
    "<token of kind 135>",
    "\"`\"",
  };

}

--- NEW FILE: CharStream.java ---
/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 4.0 */
package org.python.pydev.parser.jython;

/**
 * This interface describes a character stream that maintains line and
 * column number positions of the characters.  It also has the capability
 * to backup the stream to some extent.  An implementation of this
 * interface is used in the TokenManager implementation generated by
 * JavaCCParser.
 *
 * All the methods except backup can be implemented in any fashion. backup
 * needs to be implemented correctly for the correct operation of the lexer.
 * Rest of the methods are all used to get information like line number,
 * column number and the String that constitutes a token and are not used
 * by the lexer. Hence their implementation won't affect the generated lexer's
 * operation.
 */

public interface CharStream {

  /**
   * Returns the next character from the selected input.  The method
   * of selecting the input is the responsibility of the class
   * implementing this interface.  Can throw any java.io.IOException.
   */
  char readChar() throws java.io.IOException;

  /**
   * Returns the column position of the character last read.
   * @deprecated 
   * @see #getEndColumn
   */
  int getColumn();

  /**
   * Returns the line number of the character last read.
   * @deprecated 
   * @see #getEndLine
   */
  int getLine();

  /**
   * Returns the column number of the last character for current token (being
   * matched after the last call to BeginTOken).
   */
  int getEndColumn();

  /**
   * Returns the line number of the last character for current token (being
   * matched after the last call to BeginTOken).
   */
  int getEndLine();

  /**
   * Returns the column number of the first character for current token (being
   * matched after the last call to BeginTOken).
   */
  int getBeginColumn();

  /**
   * Returns the line number of the first character for current token (being
   * matched after the last call to BeginTOken).
   */
  int getBeginLine();

  /**
   * Backs up the input stream by amount steps. Lexer calls this method if it
   * had already read some characters, but could not use them to match a
   * (longer) token. So, they will be used again as the prefix of the next
   * token and it is the implemetation's responsibility to do this right.
   */
  void backup(int amount);

  /**
   * Returns the next character that marks the beginning of the next token.
   * All characters must remain in the buffer between two successive calls
   * to this method to implement backup correctly.
   */
  char BeginToken() throws java.io.IOException;

  /**
   * Returns a string made up of characters from the marked token beginning 
   * to the current buffer position. Implementations have the choice of returning
   * anything that they want to. For example, for efficiency, one might decide
   * to just return null, which is a valid implementation.
   */
  String GetImage();

  /**
   * Returns an array of characters that make up the suffix of length 'len' for
   * the currently matched token. This is used to build up the matched string
   * for use in actions in the case of MORE. A simple and inefficient
   * implementation of this is as follows :
   *
   *   {
   *      String t = GetImage();
   *      return t.substring(t.length() - len, t.length()).toCharArray();
   *   }
   */
  char[] GetSuffix(int len);

  /**
   * The lexer calls this function to indicate that it is done with the stream
   * and hence implementations can free any resources held by this class.
   * Again, the body of this function can be just empty and it will not
   * affect the lexer's operation.
   */
  void Done();

}

--- NEW FILE: python.jjt ---
// -*- java -*-
//good reading for error handling: https://javacc.dev.java.net/doc/errorrecovery.html
options
{
    // only non-defaults are given here.
    NODE_SCOPE_HOOK       = true;   // call jjtree*NodeScope()
    NODE_FACTORY          = true;
    NODE_USES_PARSER      = true;
    STATIC                = false;  // multiple parsers
    COMMON_TOKEN_ACTION   = true;   // CommonTokenAction(Token)
    DEBUG_PARSER          = false;  // must be used to enable tracing 
    DEBUG_TOKEN_MANAGER   = false;  // used to debug the tokens we have generating 

    USER_CHAR_STREAM      = true;
    UNICODE_INPUT         = true;
    ERROR_REPORTING 	  = true;  // this is NOT used to shut down errors regarding javacc org.python.parser.PythonGrammar$LookaheadSuccess
   								   // because the generated information is not complete enough if we do it.
}

[...1421 lines suppressed...]
|   ( t = <TRY> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <EXCEPT> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <DEF> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <CLASS> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <FINALLY> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <PRINT> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <PASS> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <BREAK> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <CONTINUE> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <RETURN> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <YIELD> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <IMPORT> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <FROM> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <DEL> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <RAISE> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <GLOBAL> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <EXEC> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <ASSERT> { jjtThis.setImage(t.image); return t; } {} )#Name
|   ( t = <AS> { jjtThis.setImage(t.image); return t; } {} )#Name
}

--- NEW FILE: SimpleNode.java ---
// Copyright (c) Corporation for National Research Initiatives
package org.python.pydev.parser.jython;

import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.python.pydev.parser.jython.ast.VisitorIF;
import org.python.pydev.parser.jython.ast.commentType;

public class SimpleNode implements Node {
    
    public int beginLine, beginColumn;
    
    /**
     * each node may have a number of associated comments, altought they are not in the visiting structure by default
     * (it appears separately from that in this attribute, so, when doing a traverse in some node, the comments
     * will NOT be added in the visitor by default, as in the grammar it is not associated).
     * 
     * If they are not of commentType, they should be strings, with things such as colons, parenthesis, etc, to help
     * in the process of pretty-printing the ast.
     */
    public List<Object> specialsBefore = new ArrayList<Object>();
    public List<Object> specialsAfter  = new ArrayList<Object>();

    public SimpleNode() { }

    public static Node jjtCreate(PythonGrammar p, int id) {
        return p.jjtree.openNode(id);
    }

    public int getId() {
        return -1;
    }

    public Object getImage() {
        return null;
    }

    public void setImage(Object image) {
    }

    /**
     * @param special The 'special token' added (comment or some literal)
     * @param after defines if it was found before or after the token
     */
    public void addSpecial(Object special, boolean after) {
        if(special != null){
            if(special instanceof Token){
                Token t = (Token) special;
                commentType comment = new commentType(t.image.trim());
                comment.beginColumn = t.beginColumn;
                comment.beginLine = t.beginLine;
                special = comment;
            }
            
            if(after){
                if(special instanceof commentType){
                    specialsAfter.add(special);
                }else{
                    int addAt = countAfter(special);
                    specialsAfter.add(addAt, special);
                }
            }else{
                if(special instanceof commentType){
                    specialsBefore.add(special);
                }else{
                    int addAt = countBefore(special);
                    specialsBefore.add(addAt, special);
                }
            }
//            if(this instanceof DefaultArg){
//                DefaultArg a = (DefaultArg) this;
//                System.out.println("Adding:"+special+" after:"+after+" to:"+a.parameter);
//            }else{
//                System.out.println("Adding:"+special+" after:"+after+" to:"+this+" class:"+special.getClass());
//            }
        }
    }

    private int countStrings() {
        List l = specialsAfter;
        return doCount(l);
    }
    private int countStringsBefore() {
        List l = specialsBefore;
        return doCount(l);
    }
    
    private int countAfter(Object special) {
        int[] lineCol = getLineCol(special);
        if(lineCol == null){
            return countStrings();
        }
        
        return getIndex(lineCol, specialsAfter);
    }
    
    private int countBefore(Object special) {
        int[] lineCol = getLineCol(special);
        if(lineCol == null){
            return countStringsBefore();
        }
        
        return getIndex(lineCol, specialsBefore);
    }

    /**
     * @param lineCol
     * @param l
     * @return
     */
    private int getIndex(int[] lineCol, List l) {
        int i=0;
        for(Object o : l){
            int[] existing = getLineCol(o);
            if(existing == null){
                //do nothing... (will be after it)
                
            }else if(existing[0] > lineCol[0]){
                return i;
                
            } else if(existing[0] == lineCol[0]){
                if(existing[1] > lineCol[1]){
                    return i;
                }
                
            }
            i++;
        }
        return i;
    }

    private int[] getLineCol(Object o) {
        if (o instanceof SpecialStr){
            SpecialStr s = (SpecialStr) o;
            return new int[]{s.beginLine, s.beginCol};
        }
        if (o instanceof commentType){
            commentType c = (commentType) o;
            return new int[]{c.beginLine, c.beginColumn};
        }
        return null;
    }

    /**
     * @param l
     * @return
     */
    private int doCount(List l) {
        int i=0;
        for(Object o : l){
            if (o instanceof String || o instanceof SpecialStr){
                i++;
            }
        }
        return i;
    }

    /* You can override these two methods in subclasses of SimpleNode to
       customize the way the node appears when the tree is dumped.  If
       your output uses more than one line you should override
       toString(String), otherwise overriding toString() is probably all
       you need to do. */

    public String toString() {
        return super.toString() + " at line "+beginLine;
    }
    public String toString(String prefix) { return prefix + toString(); }

    public Object accept(VisitorIF visitor) throws Exception {
        throw new ParseException("Unexpected node: "+this);
    }

    public void traverse(VisitorIF visitor) throws Exception {
        throw new ParseException("Unexpected node: "+this);
    }

    /* Override this method if you want to customize how the node dumps
       ut its children. */

    protected String dumpThis(String s) {
        return s;
    }

    protected String dumpThis(Object o) {
        return String.valueOf(o);
    }

    protected String dumpThis(Object[] s) {
        StringBuffer sb = new StringBuffer();
        if (s == null) {
            sb.append("null");
        } else {
            sb.append("[");
            for (int i = 0; i < s.length; i++) {
                if (i > 0)
                    sb.append(", ");
                sb.append(String.valueOf(s[i]));
            }
            sb.append("]");
        }
        
        return sb.toString();
    }

    protected String dumpThis(int i) {
        return Integer.toString(i);
    }

    protected String dumpThis(int i, String[] names) {
        // XXX Verify bounds.
        return names[i];
    }

    protected String dumpThis(int[] arr, String[] names) {
        StringBuffer sb = new StringBuffer();
        if (arr == null) {
            sb.append("null");
        } else {
            sb.append("[");
            for (int i = 0; i < arr.length; i++) {
                if (i > 0)
                    sb.append(", ");
                // XXX Verify bounds.
                sb.append(names[arr[i]]);
            }
            sb.append("]");
        }
        return sb.toString();
    }

    protected String dumpThis(boolean b) {
        return String.valueOf(b);
    }


    public void pickle(DataOutputStream ostream) throws IOException {
        throw new IOException("Pickling not implemented");
    }

    protected void pickleThis(String s, DataOutputStream ostream)
        throws IOException
    {
        if (s == null) {
            ostream.writeInt(-1);
        } else {
            ostream.writeInt(s.length());
            ostream.writeBytes(s);
        }
    }

    protected void pickleThis(String[] s, DataOutputStream ostream)
        throws IOException
    {
        if (s == null) {
            ostream.writeInt(-1);
        } else {
            ostream.writeInt(s.length);
            for (int i = 0; i < s.length; i++) {
                pickleThis(s[i], ostream);
            }
        }
    }

    protected void pickleThis(SimpleNode o, DataOutputStream ostream)
        throws IOException
    {
        if (o == null) {
            ostream.writeInt(-1);
        } else {
            o.pickle(ostream);
        }
    }

    protected void pickleThis(SimpleNode[] s, DataOutputStream ostream)
        throws IOException
    {
        if (s == null) {
            ostream.writeInt(-1);
        } else {
            ostream.writeInt(s.length);
            for (int i = 0; i < s.length; i++) {
                pickleThis(s[i], ostream);
            }
        }
    }

    protected void pickleThis(int i, DataOutputStream ostream)
        throws IOException
    {
        ostream.writeInt(i);
    }

    protected void pickleThis(int[] arr, DataOutputStream ostream)
        throws IOException
    {
        if (arr == null) {
            ostream.writeInt(-1);
        } else {
            ostream.writeInt(arr.length);
            for (int i = 0; i < arr.length; i++) {
                ostream.writeInt(arr[i]);
            }
        }
    }

    protected void pickleThis(boolean b, DataOutputStream ostream)
        throws IOException
    {
        ostream.writeBoolean(b);
    }

    protected void pickleThis(Object n, DataOutputStream ostream)
        throws IOException
    {
        String s = n.toString();
        ostream.writeInt(s.length());
        ostream.writeBytes(s);
    }
}

--- NEW FILE: TreeBuilder.java ---
package org.python.pydev.parser.jython;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;

import org.python.pydev.parser.jython.ast.Assert;
import org.python.pydev.parser.jython.ast.Assign;
import org.python.pydev.parser.jython.ast.Attribute;
import org.python.pydev.parser.jython.ast.AugAssign;
import org.python.pydev.parser.jython.ast.BinOp;
import org.python.pydev.parser.jython.ast.BoolOp;
import org.python.pydev.parser.jython.ast.Break;
import org.python.pydev.parser.jython.ast.Call;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.Compare;
import org.python.pydev.parser.jython.ast.Comprehension;
import org.python.pydev.parser.jython.ast.Continue;
import org.python.pydev.parser.jython.ast.Delete;
[...1110 lines suppressed...]
                    "augmented assign to tuple not possible", node);
        }
        node.ctx = ctx;
        traverse(node);
        return null;
    }

    public Object visitCall(Call node) throws Exception {
        throw new ParseException("can't assign to function call", node);
    }

    public Object visitListComp(Call node) throws Exception {
        throw new ParseException("can't assign to list comprehension call",
                                 node);
    }

    public Object unhandled_node(SimpleNode node) throws Exception {
        throw new ParseException("can't assign to operator", node);
    }
}

--- NEW FILE: ICompilerAPI.java ---
/*
 * Author: atotic
 * Created: Jul 29, 2003
 */
package org.python.pydev.parser.jython;

/**
 * callbacks from the parser to the compiler
 *
 * org.totic.pydev needed to use org.python.parser outside of the the jython tree
 * and this class abstracts all calls from parser to the compiler
 */

public interface ICompilerAPI {

	public Object newLong(String s);

	public Object newLong(java.math.BigInteger i);

	public Object newFloat(double v);
	
	public Object newImaginary(double v);
	
	public Object newInteger(int i);
	
	public String decode_UnicodeEscape(String str, int start, int end,
		String errors, boolean unicode);
}

--- NEW FILE: .cvsignore ---
python.jj
.AppleDouble
*.class

--- NEW FILE: PythonGrammar.java ---
/* Generated By:JJTree&JavaCC: Do not edit this line. PythonGrammar.java */
package org.python.pydev.parser.jython;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;

import org.python.pydev.parser.jython.ast.Call;
import org.python.pydev.parser.jython.ast.Import;
import org.python.pydev.parser.jython.ast.Str;
import org.python.pydev.parser.jython.ast.Tuple;
import org.python.pydev.parser.jython.ast.Yield;
import org.python.pydev.parser.jython.ast.modType;

public class PythonGrammar/*@bgen(jjtree)*/implements PythonGrammarTreeConstants, PythonGrammarConstants {/*@bgen(jjtree)*/
  protected JJTPythonGrammarState jjtree = new JJTPythonGrammarState();
    public IParserHost hostLiteralMkr;
    private SimpleNode prev;
    private SimpleNode lastIdNode;
    private static boolean DEBUG = false;
[...8143 lines suppressed...]
    jj_rescan = false;
  }

  final private void jj_save(int index, int xla) {
    JJCalls p = jj_2_rtns[index];
    while (p.gen > jj_gen) {
      if (p.next == null) { p = p.next = new JJCalls(); break; }
      p = p.next;
    }
    p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
  }

  static final class JJCalls {
    int gen;
    Token first;
    int arg;
    JJCalls next;
  }

}

--- NEW FILE: package.html ---
<body>
Thanks to jython's team for this implementation of JavaCC python parser.

It uses JavaCC and adslgen.

Fabio:
It has been updated to support:
- list comprehension correctly (in tuples and in method calls, which required only changes to javacc)
- decorators, which required changes to javacc and python.asdl (a decorators type was created, as oposed
to an array of expressions, as was in the prior version in the python tree).

Still to do:
- allow for error recovery

</body>
--- NEW FILE: ParseException.java ---
/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 0.7pre6 */
package org.python.pydev.parser.jython;

/**
 * This exception is thrown when parse errors are encountered.
 * You can explicitly create objects of this exception type by
 * calling the method generateParseException in the generated
 * parser.
 *
 * You can modify this class to customize your error reporting
 * mechanisms so long as you retain the public fields.
 */
public class ParseException extends Exception {

  /**
   * This constructor is used by the method "generateParseException"
   * in the generated parser.  Calling this constructor generates
   * a new object of this type with the fields "currentToken",
   * "expectedTokenSequences", and "tokenImage" set.  The boolean
   * flag "specialConstructor" is also set to true to indicate that
   * this constructor was used to create this object.
   * This constructor calls its super class with the empty string
   * to force the "toString" method of parent class "Throwable" to
   * print the error message in the form:
   *     ParseException: <result of getMessage>
   */
  public ParseException(Token currentTokenVal,
                        int[][] expectedTokenSequencesVal,
                        String[] tokenImageVal
                       )
  {
    super("");
    specialConstructor = true;
    currentToken = currentTokenVal;
    expectedTokenSequences = expectedTokenSequencesVal;
    tokenImage = tokenImageVal;
  }

  /**
   * The following constructors are for use by you for whatever
   * purpose you can think of.  Constructing the exception in this
   * manner makes the exception behave in the normal way - i.e., as
   * documented in the class "Throwable".  The fields "errorToken",
   * "expectedTokenSequences", and "tokenImage" do not contain
   * relevant information.  The JavaCC generated code does not use
   * these constructors.
   */

  public ParseException() {
    super();
    specialConstructor = false;
  }

  public ParseException(String message) {
    super(message);
    specialConstructor = false;
  }

  public ParseException(String message, SimpleNode node) {
      this(message, node.beginLine, node.beginColumn);
  }
  
  public ParseException(String message, Token t) {
      this(message, t.beginLine, t.beginColumn);
  }
  
  public ParseException(String message, int beginLine, int beginColumn) {
    super(message);
    // If a node is passed in, provide just enough info to get current line/column out
    Token t = new Token();
    t.beginLine = beginLine;
    t.beginColumn = beginColumn;
    
    currentToken = new Token();
    currentToken.next = t;
    t = currentToken;
    t.beginLine = beginLine;
    t.beginColumn = beginColumn;
    
    specialConstructor = false;
  }

  /**
   * This variable determines which constructor was used to create
   * this object and thereby affects the semantics of the
   * "getMessage" method (see below).
   */
  protected boolean specialConstructor;

  /**
   * This is the last token that has been consumed successfully.  If
   * this object has been created due to a parse error, the token
   * followng this token will (therefore) be the first error token.
   */
  public Token currentToken;

  /**
   * Each entry in this array is an array of integers.  Each array
   * of integers represents a sequence of tokens (by their ordinal
   * values) that is expected at this point of the parse.
   */
  public int[][] expectedTokenSequences;

  /**
   * This is a reference to the "tokenImage" array of the generated
   * parser within which the parse error occurred.  This array is
   * defined in the generated ...Constants interface.
   */
  public String[] tokenImage;

  /**
   * This method has the standard behavior when this object has been
   * created using the standard constructors.  Otherwise, it uses
   * "currentToken" and "expectedTokenSequences" to generate a parse
   * error message and returns it.  If this object has been created
   * due to a parse error, and you do not catch it (it gets thrown
   * from the parser), then this method is called during the printing
   * of the final stack trace, and hence the correct error message
   * gets displayed.
   */
   public static boolean verboseExceptions = false;
  public String getMessage() {
    if (!specialConstructor) {
      return super.getMessage();
    }
    if (verboseExceptions) {
        String expected = "";
        int maxSize = 0;
        for (int i = 0; i < expectedTokenSequences.length; i++) {
          if (maxSize < expectedTokenSequences[i].length) {
            maxSize = expectedTokenSequences[i].length;
          }
          for (int j = 0; j < expectedTokenSequences[i].length; j++) {
            expected += tokenImage[expectedTokenSequences[i][j]] + " ";
          }
          if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
            expected += "...";
          }
          expected += eol + "    ";
        }
        String retval = "Encountered \"";
        Token tok = currentToken.next;
        for (int i = 0; i < maxSize; i++) {
          if (i != 0) retval += " ";
          if (tok.kind == 0) {
            retval += tokenImage[0];
            break;
          }
          retval += add_escapes(tok.image);
          tok = tok.next; 
        }
        retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn + "." + eol;
        if (expectedTokenSequences.length == 1) {
          retval += "Was expecting:" + eol + "    ";
        } else {
          retval += "Was expecting one of:" + eol + "    ";
        }
        retval += expected;
        return retval;
    } else {
        return "invalid syntax";
    }
  }

  /**
   * The end of line string for this machine.
   */
  protected String eol = System.getProperty("line.separator", "\n");
 
  /**
   * Used to convert raw characters to their escaped version
   * when these raw version cannot be used as part of an ASCII
   * string literal.
   */
  protected String add_escapes(String str) {
      StringBuffer retval = new StringBuffer();
      char ch;
      for (int i = 0; i < str.length(); i++) {
        switch (str.charAt(i))
        {
           case 0 :
              continue;
           case '\b':
              retval.append("\\b");
              continue;
           case '\t':
              retval.append("\\t");
              continue;
           case '\n':
              retval.append("\\n");
              continue;
           case '\f':
              retval.append("\\f");
              continue;
           case '\r':
              retval.append("\\r");
              continue;
           case '\"':
              retval.append("\\\"");
              continue;
           case '\'':
              retval.append("\\\'");
              continue;
           case '\\':
              retval.append("\\\\");
              continue;
           default:
              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
                 String s = "0000" + Integer.toString(ch, 16);
                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
              } else {
                 retval.append(ch);
              }
              continue;
        }
      }
      return retval.toString();
   }

}

--- NEW FILE: Node.java ---
/* Generated By:JJTree: Do not edit this line. Node.java */

package org.python.pydev.parser.jython;

/* All AST nodes must implement this interface.  It provides basic
   machinery for constructing the parent and child relationships
   between nodes. */

public interface Node {

}


--- NEW FILE: ReaderCharStream.java ---
package org.python.pydev.parser.jython;

/**
 * An implementation of interface CharStream, where the data is read from
 * a Reader.
 * This file started life as a copy of ASCII_CharStream.java.
 */

public final class ReaderCharStream implements CharStream
{
  public static final boolean staticFlag = false;
  int bufsize;
  int available;
  int tokenBegin;
  public int bufpos = -1;
  private int bufline[];
  private int bufcolumn[];

  private int column = 0;
  private int line = 1;

  private boolean prevCharIsCR = false;
  private boolean prevCharIsLF = false;

  private java.io.Reader inputStream;

  private char[] buffer;
  private int maxNextCharInd = 0;
  private int inBuf = 0;

  private final void ExpandBuff(boolean wrapAround)
  {
     char[] newbuffer = new char[bufsize + 2048];
     int newbufline[] = new int[bufsize + 2048];
     int newbufcolumn[] = new int[bufsize + 2048];

     try
     {
        if (wrapAround)
        {
           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
           System.arraycopy(buffer, 0, newbuffer,
                                             bufsize - tokenBegin, bufpos);
           buffer = newbuffer;

           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
           System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
           bufline = newbufline;

           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
           System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
           bufcolumn = newbufcolumn;

           maxNextCharInd = (bufpos += (bufsize - tokenBegin));
        }
        else
        {
           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
           buffer = newbuffer;

           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
           bufline = newbufline;

           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
           bufcolumn = newbufcolumn;

           maxNextCharInd = (bufpos -= tokenBegin);
        }
     }
     catch (Throwable t)
     {
        throw new Error(t.getMessage());
     }


     bufsize += 2048;
     available = bufsize;
     tokenBegin = 0;
  }

  private final void FillBuff() throws java.io.IOException
  {
     if (maxNextCharInd == available)
     {
        if (available == bufsize)
        {
           if (tokenBegin > 2048)
           {
              bufpos = maxNextCharInd = 0;
              available = tokenBegin;
           }
           else if (tokenBegin < 0)
              bufpos = maxNextCharInd = 0;
           else
              ExpandBuff(false);
        }
        else if (available > tokenBegin)
           available = bufsize;
        else if ((tokenBegin - available) < 2048)
           ExpandBuff(true);
        else
           available = tokenBegin;
     }

     int i;
     try {
        if ((i = inputStream.read(buffer, maxNextCharInd,
                                    available - maxNextCharInd)) == -1)
        {
           inputStream.close();
           throw new java.io.IOException();
        }
        else
           maxNextCharInd += i;
        return;
     }
     catch(java.io.IOException e) {
        --bufpos;
        backup(0);
        if (tokenBegin == -1)
           tokenBegin = bufpos;
        throw e;
     }
  }

  public final char BeginToken() throws java.io.IOException
  {
     tokenBegin = -1;
     char c = readChar();
     tokenBegin = bufpos;

     return c;
  }

  private final void UpdateLineColumn(char c)
  {
     column++;

     if (prevCharIsLF)
     {
        prevCharIsLF = false;
        line += (column = 1);
     }
     else if (prevCharIsCR)
     {
        prevCharIsCR = false;
        if (c == '\n')
        {
           prevCharIsLF = true;
        }
        else
           line += (column = 1);
     }

     switch (c)
     {
        case '\r' :
           prevCharIsCR = true;
           break;
        case '\n' :
           prevCharIsLF = true;
           break;
//ok, this was commented out because the position would not reflect correctly the positions found in the ast.
//this may have other problems, but they have to be analyzed better to see the problems this may bring
//(files that mix tabs and spaces may suffer, but I could not find out very well the problems -- anyway,
//restricting the analysis to files that have only tabs or only spaces seems reasonable -- shortcuts are available
//so that we can convert a file from one type to another, so, what remains is making some lint analysis to be sure of it).
//        case '\t' :
//           column--;
//           column += (8 - (column & 07));
//           break;
        default :
           break;
     }

     bufline[bufpos] = line;
     bufcolumn[bufpos] = column;
  }

  public final char readChar() throws java.io.IOException
  {
     if (inBuf > 0)
     {
        --inBuf;
        return buffer[(bufpos == bufsize - 1) ? (bufpos = 0) : ++bufpos];
     }

     if (++bufpos >= maxNextCharInd)
        FillBuff();

     char c = buffer[bufpos];

     UpdateLineColumn(c);
     return (c);
  }

  /**
   * @deprecated
   * @see #getEndColumn
   */

  public final int getColumn() {
     return bufcolumn[bufpos];
  }

  /**
   * @deprecated
   * @see #getEndLine
   */

  public final int getLine() {
     return bufline[bufpos];
  }

  public final int getEndColumn() {
     return bufcolumn[bufpos];
  }

  public final int getEndLine() {
     return bufline[bufpos];
  }

  public final int getBeginColumn() {
     return bufcolumn[tokenBegin];
  }

  public final int getBeginLine() {
     return bufline[tokenBegin];
  }

  public final void backup(int amount) {

    inBuf += amount;
    if ((bufpos -= amount) < 0)
       bufpos += bufsize;
  }

  public ReaderCharStream(java.io.Reader dstream)
  {
    inputStream = dstream;
    line = 1;
    column = 0;

    available = bufsize = 4096;
    buffer = new char[bufsize];
    bufline = new int[bufsize];
    bufcolumn = new int[bufsize];
  }

  public final String GetImage()
  {
     if (bufpos >= tokenBegin)
        return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
     else
        return new String(buffer, tokenBegin, bufsize - tokenBegin) +
                              new String(buffer, 0, bufpos + 1);
  }

  public final char[] GetSuffix(int len)
  {
     char[] ret = new char[len];

     if ((bufpos + 1) >= len)
        System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
     else
     {
        System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
                                                          len - bufpos - 1);
        System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
     }

     return ret;
  }

  public void Done()
  {
     buffer = null;
     bufline = null;
     bufcolumn = null;
  }

  /**
   * Method to adjust line and column numbers for the start of a token.<BR>
   */
  public void adjustBeginLineColumn(int newLine, int newCol)
  {
     int start = tokenBegin;
     int len;

     if (bufpos >= tokenBegin)
     {
        len = bufpos - tokenBegin + inBuf + 1;
     }
     else
     {
        len = bufsize - tokenBegin + bufpos + 1 + inBuf;
     }

     int i = 0, j = 0, k = 0;
     int nextColDiff = 0, columnDiff = 0;

     while (i < len &&
            bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
     {
        bufline[j] = newLine;
        nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
        bufcolumn[j] = newCol + columnDiff;
        columnDiff = nextColDiff;
        i++;
     }

     if (i < len)
     {
        bufline[j] = newLine++;
        bufcolumn[j] = newCol + columnDiff;

        while (i++ < len)
        {
           if (bufline[j = start % bufsize] != bufline[++start % bufsize])
              bufline[j] = newLine++;
           else
              bufline[j] = newLine;
        }
     }

     line = bufline[j];
     column = bufcolumn[j];
  }

}

--- NEW FILE: Token.java ---
/* Generated By:JavaCC: Do not edit this line. Token.java Version 0.7pre3 */
package org.python.pydev.parser.jython;

/**
 * Describes the input token stream.
 */

public class Token {

  /**
   * An integer that describes the kind of this token.  This numbering
   * system is determined by JavaCCParser, and a table of these numbers is
   * stored in the file ...Constants.java.
   */
  public int kind;

  /**
   * beginLine and beginColumn describe the position of the first character
   * of this token; endLine and endColumn describe the position of the
   * last character of this token.
   */
  public int beginLine, beginColumn, endLine, endColumn;

  /**
   * The string image of the token.
   */
  public String image;

  /**
   * A reference to the next regular (non-special) token from the input
   * stream.  If this is the last token from the input stream, or if the
   * token manager has not read tokens beyond this one, this field is
   * set to null.  This is true only if this token is also a regular
   * token.  Otherwise, see below for a description of the contents of
   * this field.
   */
  public Token nex...
 
[truncated message content]