From: Jan P. <jp...@us...> - 2005-12-27 02:02:43
|
Update of /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv23331/src/org/epic/core/parser Modified Files: expectStringEnd.g expectSubstExpr.g expectStringSuffix.g expectHereDocEnd.g PerlLexerBase.java sharedTokenTypes.txt perl.g pod.g expectString.g PerlMultiLexer.java Log Message: Improved syntax highlighting, particularly for Perl special variables. Added recognition of more operators. Added a recovery mechanism to skip over remaining chars of a line on error. Index: expectStringSuffix.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/expectStringSuffix.g,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- expectStringSuffix.g 23 Dec 2005 23:15:56 -0000 1.1 +++ expectStringSuffix.g 27 Dec 2005 02:02:33 -0000 1.2 @@ -8,7 +8,7 @@ options { k = 2; - charVocabulary = '\3'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; importVocab = shared; } Index: expectSubstExpr.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/expectSubstExpr.g,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- expectSubstExpr.g 25 Dec 2005 15:54:47 -0000 1.2 +++ expectSubstExpr.g 27 Dec 2005 02:02:33 -0000 1.3 @@ -8,7 +8,7 @@ options { k = 2; - charVocabulary = '\3'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; importVocab = shared; } { Index: expectString.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/expectString.g,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- expectString.g 23 Dec 2005 23:15:56 -0000 1.1 +++ expectString.g 27 Dec 2005 02:02:33 -0000 1.2 @@ -8,7 +8,7 @@ options { k = 2; - charVocabulary = '\3'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; importVocab = shared; } { Index: PerlLexerBase.java =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/PerlLexerBase.java,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- PerlLexerBase.java 25 Dec 2005 14:06:03 -0000 1.2 +++ PerlLexerBase.java 27 Dec 2005 02:02:33 -0000 1.3 @@ -105,7 +105,7 @@ super.setInputState(state); pc = 0; qmarkRegexp = slashRegexp = glob = afterArrow = afterSub = format = - proto = afterColon = false; + afterColon = false; } protected CurlyToken createCurlyToken(int type, String text) Index: perl.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/perl.g,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- perl.g 25 Dec 2005 14:06:03 -0000 1.2 +++ perl.g 27 Dec 2005 02:02:33 -0000 1.3 @@ -8,7 +8,7 @@ options { k = 4; - charVocabulary = '\1'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; importVocab = shared; exportVocab = Perl; } @@ -45,26 +45,81 @@ OPEN_SQUOTE: '\'' { getParent().expectStringEnd('\''); }; OPEN_DQUOTE: '"' { getParent().expectStringEnd('"'); }; -SPECIAL_VAR - : ("*<" | "*/" | "*?" | "*\"" - | "$$" | "$$m" | "$$s" | "$`" | "$\"" | "$'" | "$%" | "$/" | "$," | "$?" - | "$;" | "$[" | "$]") - { glob = false; } +MAYBE_SPECIAL_VAR + : { !proto }? ( + ("**") + => "**" { $setToken(createOperatorToken(PerlTokenTypes.OPER_MULMUL, "**")); } + | ("*=") + => "*=" { $setToken(createOperatorToken(PerlTokenTypes.OPER_MULEQ, "*=")); } + | (VAR) + => VAR { $setType(PerlTokenTypes.VAR); } + | (SPECIAL_VAR) + => SPECIAL_VAR { $setType(PerlTokenTypes.SPECIAL_VAR); glob = false; } + | ('*') + => '*' { $setToken(createOperatorToken(PerlTokenTypes.OPER_MUL, "*")); } + | ("%=") + => "%=" { $setToken(createOperatorToken(PerlTokenTypes.OPER_MODEQ, "%=")); } + | ('%') + => '%' { $setToken(createOperatorToken(PerlTokenTypes.OPER_MOD, "%")); } + | (VAR_START) // incomplete variable + => VAR_START { $setType(PerlTokenTypes.VAR); } + ); + +protected OPER_MODEQ: ; +protected OPER_MULEQ: ; +protected OPER_MULMUL: ; +protected OPER_MUL: ; +protected OPER_MOD: ; + +protected SPECIAL_VAR + // see English.pm for the *? operators + : ( + "*^A" | "*^C" | "*^D" | "*^E" | "*^F" | "*^I" | "*^L" | "*^N" + | "*^O" | "*^P" | "*^R" | "*^S" | "*^T" | "*^V" | "*^W" | "*^X" + | "*/" | "*?" | "*%" | "*@" | "*_" | "*-" | "*+" | "*." | "*|" | "*," + | "*;" | "*~" | "*:" | "*^" | "*<" | "*>" | "*(" | "*)" /* | "*$" TODO, watch out: 5*$x */ + | "*\"" | "*\\" + + | "$$m" | "$$s" + | "$^A" | "$^C" | "$^D" | "$^E" | "$^F" | "$^H" | "$^I" | "$^L" | "$^M" + | "$^N" | "$^O" | "$^P" | "$^R" | "$^S" | "$^T" | "$^V" | "$^W" | "$^X" + | "$/" | "$?" | "$%" | "$@" | "$_" | "$-" | "$+" | "$." | "$|" | "$!" + | "$;" | "$~" | "$$" | "$`" | "$'" | "$<" | "$>" | "$(" | "$)" | "$," + | "$[" | "$]" | "$:" | "$*" | "$#" | "$=" | "$^" | "$&" + | "$\"" | "$\\" + + | "@+" | "@-" | "@_" | "@$" + + | "%!" | "%@" | "%^H" + ); + +protected VAR + : { !proto }? VAR_START (ID | CURLY | '@' | '\uFFFF'!) + { glob = qmarkRegexp = slashRegexp = false; }; + +protected VAR_START + : ('@' | '$' | '%' /* | '*' TODO, but think of $x*5 */) + ('@' | '$' | '%' | '*' | '#' /* | ' ' TODO, but think of $x % $y */)* + ; + +protected CURLY + : '{' + ( CURLY | NEWLINE | ~('}' | '\uFFFF'))* + ('}' | '\uFFFF'!) ; OPER_AND: "&&" { $setToken(createOperatorToken(PerlTokenTypes.OPER_AND, "&&")); }; OPER_OR: "||" { $setToken(createOperatorToken(PerlTokenTypes.OPER_OR, "||")); }; -OPER_MULMUL: "**" { $setToken(createOperatorToken(PerlTokenTypes.OPER_MULMUL, "**")); }; -OPER_LT: "<=" { $setToken(createOperatorToken(PerlTokenTypes.OPER_LT, "<=")); }; -OPER_GT: ">=" { $setToken(createOperatorToken(PerlTokenTypes.OPER_GT, ">=")); }; +OPER_LTEQ: "<=" { $setToken(createOperatorToken(PerlTokenTypes.OPER_LTEQ, "<=")); }; +OPER_GTEQ: ">=" { $setToken(createOperatorToken(PerlTokenTypes.OPER_GTEQ, ">=")); }; PROTO : { proto }? - ('$' | '@' | '%' | '*' | ';' | '\\' | '&')+ + ('$' | '@' | '%' | '*' | ';' | '\\' | '&' | WS)+ { proto = false; } ; -OPER_S: "-s" { $setToken(createOperatorToken(PerlTokenTypes.OPER_S, "**")); }; +OPER_S: "-s" { $setToken(createOperatorToken(PerlTokenTypes.OPER_S, "-s")); }; OPEN_SLASH : '/' @@ -99,7 +154,7 @@ }; SUBST_OR_MATCH_OR_WORD // this disambiguation rule disfavours EXPRs too much :-( - : (("tr" | 's' | 'y') ~('a'..'z' | '0'..'9' | '_' | '}' | '\r' | '\n' | ' ')) + : (("tr" | 's' | 'y') ~('a'..'z' | '0'..'9' | '_' | '}' | '\r' | '\n')) => SUBST_EXPR { $setType(PerlTokenTypes.SUBST_EXPR); } | (("qq" | "qx" | "qw" | "qr" | 'm' | 'q') ~('a'..'z' | '0'..'9' | '_' | '}' | '\r' | '\n' | ' ')) => MATCH_EXPR { $setType(PerlTokenTypes.MATCH_EXPR); } @@ -145,6 +200,10 @@ : "==" { $setToken(createOperatorToken(PerlTokenTypes.OPER_DOUBLEEQ, "==")); }; +OPER_NOTEQ + : "!=" + { $setToken(createOperatorToken(PerlTokenTypes.OPER_NOTEQ, "!=")); }; + OPER_EQMATCH : "=~" { $setToken(createOperatorToken(PerlTokenTypes.OPER_EQMATCH, "=~")); }; @@ -160,6 +219,13 @@ qmarkRegexp = false; }; +OPER_MINUSEQ + : "-=" + { + $setToken(createOperatorToken(PerlTokenTypes.OPER_PLUSPLUS, "-=")); + qmarkRegexp = false; + }; + OPER_PLUSPLUS : "++" { @@ -167,6 +233,13 @@ qmarkRegexp = false; }; +OPER_PLUSEQ + : "+=" + { + $setToken(createOperatorToken(PerlTokenTypes.OPER_PLUSPLUS, "+=")); + qmarkRegexp = false; + }; + OPER_COMMA : ',' { $setToken(createOperatorToken(PerlTokenTypes.OPER_COMMA, ",")); }; @@ -178,10 +251,6 @@ glob = true; }; -OPER_MUL - : '*' - { $setToken(createOperatorToken(PerlTokenTypes.OPER_MUL, "*")); }; - OPER_DIV : '/' { $setToken(createOperatorToken(PerlTokenTypes.OPER_DIV, "/")); }; @@ -194,6 +263,10 @@ : '-' { $setToken(createOperatorToken(PerlTokenTypes.OPER_MINUS, "-")); }; +OPER_DOTDOT + : ".." + { $setToken(createOperatorToken(PerlTokenTypes.OPER_DOTDOT, "..")); }; + OPER_DOT : '.' { $setToken(createOperatorToken(PerlTokenTypes.OPER_DOT, ".")); }; @@ -217,6 +290,21 @@ glob = qmarkRegexp = slashRegexp = false; }; +OPEN_BRACKET + : '[' + { + $setToken(createOperatorToken(PerlTokenTypes.OPEN_BRACKET, "[")); + format = false; + glob = qmarkRegexp = slashRegexp = true; + }; + +CLOSE_BRACKET + : ']' + { + $setToken(createOperatorToken(PerlTokenTypes.CLOSE_BRACKET, "]")); + glob = qmarkRegexp = slashRegexp = false; + }; + // see Pod/Functions.pm:148 FORMAT_STMT : { format }? "=" ~('>') (~(';'))* SEMI @@ -228,16 +316,12 @@ | (VAR) => VAR; -VAR - : { !proto }? VAR_START (ID | CURLY | '@') - { glob = qmarkRegexp = slashRegexp = false; }; - PROC_REF : '&' ID { qmarkRegexp = slashRegexp = false; }; OPER_LSHIFT_OR_HEREDOC - : ("<<" (WS)? ('"' | '\'' | '`' | 'A'..'Z')) + : (OPEN_HEREDOC) //("<<" (WS)? ('"' | '\'' | '`' | 'A'..'Z')) => OPEN_HEREDOC { $setType(PerlTokenTypes.OPEN_HEREDOC); } | ("<<" (WS)? ~('"' | '\'' | '`' | 'A'..'Z')) @@ -251,11 +335,11 @@ : ( ("<<" (WS)? '"') - => "<<" (WS!)? '"'! ("\\\"" | ~('"'))+ + => "<<" (WS!)? '"'! ("\\\"" | ~('"'))* | ("<<" (WS)? "'") - => "<<" (WS!)? "'"! ("\\'" | ~('\''))+ + => "<<" (WS!)? "'"! ("\\'" | ~('\''))* | ("<<" (WS)? '`') - => "<<" (WS!)? '`'! ("\\`" | ~('`'))+ + => "<<" (WS!)? '`'! ("\\`" | ~('`'))* | ("<<" (WS)? WORD_CHAR) => "<<" (WORD_CHAR)+ ) @@ -268,7 +352,33 @@ GLOB : { glob }? - '<' (~('<' | '>' | '\n' | '\r'))* '>'; + '<' (~('<' | '>' | '\n' | '\r'))* '>' + ; + +OPER_RSHIFT + : ">>" + { + $setToken(createOperatorToken(PerlTokenTypes.OPER_RSHIFT, ">>")); + qmarkRegexp = slashRegexp = false; + } + ; + +OPER_GT + : '>' + { + $setToken(createOperatorToken(PerlTokenTypes.OPER_GT, ">")); + qmarkRegexp = slashRegexp = false; + } + ; + +OPER_LT + : { !glob }? + '<' + { + $setToken(createOperatorToken(PerlTokenTypes.OPER_LT, "<")); + qmarkRegexp = slashRegexp = false; + } + ; OPEN_POD : { getColumn() == 1 }? @@ -298,7 +408,7 @@ { if (KEYWORDS1.contains(str)) { - glob = true; + if ("while".equals(str)) glob = true; $setType(PerlTokenTypes.KEYWORD1); } else if (KEYWORDS2.contains(str)) @@ -320,7 +430,8 @@ ; protected ID - : { afterColon = false; } ( + : { afterColon = false; } + ( { // keep going if we have "::", break on ":" // there must be a better way to implement it X-( @@ -339,16 +450,6 @@ OTHER: ~('\uFFFF'); -protected CURLY - : '{' - ( CURLY | NEWLINE | ~('}' | '\uFFFF'))* - ('}' | '\uFFFF'!) - ; - -protected VAR_START - : ('@' | '$' | '%' | '*') ('@' | '$' | '%' | '*' | '#')* - ; - protected KEYWORD1: ; protected KEYWORD2: ; protected KEYWORD_USE: ; Index: pod.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/pod.g,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- pod.g 23 Dec 2005 23:15:56 -0000 1.1 +++ pod.g 27 Dec 2005 02:02:33 -0000 1.2 @@ -8,7 +8,7 @@ options { k = 5; - charVocabulary = '\3'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; importVocab = shared; } Index: expectStringEnd.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/expectStringEnd.g,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- expectStringEnd.g 23 Dec 2005 23:15:56 -0000 1.1 +++ expectStringEnd.g 27 Dec 2005 02:02:33 -0000 1.2 @@ -8,7 +8,7 @@ options { k = 2; - charVocabulary = '\3'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; importVocab = shared; } { Index: PerlMultiLexer.java =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/PerlMultiLexer.java,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- PerlMultiLexer.java 23 Dec 2005 23:15:56 -0000 1.1 +++ PerlMultiLexer.java 27 Dec 2005 02:02:33 -0000 1.2 @@ -90,6 +90,13 @@ return mainLexer.getCurlyLevel(); } + public void recover() + { + LexerSharedInputState inputState = mainLexer.getInputState(); + if (inputState instanceof PerlLexerSharedInputState) + ((PerlLexerSharedInputState) inputState).recover(); + } + public void reset( Reader reader, IDocument doc, @@ -242,5 +249,19 @@ throw new RuntimeException(e); } } + + public void recover() + { + try + { + for (;;) + { + char c = input.LA(1); + if (c == '\r' || c == '\n' || c == '\uFFFF') break; + else { input.consume(); column++; } + } + } + catch (CharStreamException e) { /* can't happen */ } + } } } \ No newline at end of file Index: expectHereDocEnd.g =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/expectHereDocEnd.g,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- expectHereDocEnd.g 23 Dec 2005 23:15:56 -0000 1.1 +++ expectHereDocEnd.g 27 Dec 2005 02:02:33 -0000 1.2 @@ -8,7 +8,7 @@ options { k = 2; - charVocabulary = '\3'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; + charVocabulary = '\0'..'\377' | '\u0080'..'\u00FF' | '\u0100'..'\u017F' | '\u0180'..'\u024F'; } { private boolean endOfHeredoc; Index: sharedTokenTypes.txt =================================================================== RCS file: /cvsroot/e-p-i-c/org.epic.perleditor/src/org/epic/core/parser/sharedTokenTypes.txt,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- sharedTokenTypes.txt 25 Dec 2005 14:06:03 -0000 1.2 +++ sharedTokenTypes.txt 27 Dec 2005 02:02:33 -0000 1.3 @@ -31,17 +31,6 @@ OPEN_BQUOTE=31 OPEN_SQUOTE=32 OPEN_DQUOTE=33 -MUL_LBRACE_SPECIAL_VAR=34 -MUL_SLASH_SPECIAL_VAR=35 -MUL_DQUOTE_SPECIAL_VAR=36 -DOLLAR_DOLLAR_SPECIAL_VAR=37 -DOLLAR_DOLLAR_SPECIAL_VAR_M=38 -DOLLAR_DOLLAR_SPECIAL_VAR_S=39 -DOLLAR_BQUOTE_SPECIAL_VAR=40 -DOLLAR_DQUOTE_SPECIAL_VAR=41 -DOLLAR_SQUOTE_SPECIAL_VAR=42 -DOLLAR_PERCENT_SPECIAL_VAR=43 -DOLLAR_SLASH_SPECIAL_VAR=44 OPER_AND=45 OPER_OR=46 OPER_MULMUL=47 @@ -81,6 +70,7 @@ WORD_CHAR=81 OTHER=82 CURLY=83 +OPER_RSHIFT=84 VAR_START=85 KEYWORD1=86 KEYWORD2=87 @@ -92,12 +82,18 @@ OPER_QMARK=93 OPER_COLON=94 OPER_DOT=95 -MUL_QMARK_SPECIAL_VAR=96 -DOLLAR_COMMA_SPECIAL_VAR=97 -DOLLAR_QMARK_SPECIAL_VAR=98 -OPER_LT=99 -OPER_GT=100 -DOLLAR_SEMI_SPECIAL_VAR=101 +OPER_LT=96 +OPER_GT=97 +OPER_LTEQ=99 +OPER_GTEQ=100 +MAYBE_SPECIAL_VAR=101 KEYWORD_PACKAGE=102 OPER_BSLASH=103 -OPER_OTHER=104 \ No newline at end of file +OPER_OTHER=104 +OPER_DOTDOT=105 +OPER_MOD=106 +OPER_MODEQ=107 +OPER_MULEQ=108 +OPER_PLUSEQ=109 +OPER_MINUSEQ=110 +OPER_NOTEQ=111 \ No newline at end of file |