>>>>> "EL" == Eric M Ludlam <eric@...> writes:
EL>
EL> Hi,
EL> I got the new lexer (semantic-lex) to work in C code. I updated
EL> `semantic-flex' to be a go-between till all modes which need a lexer
EL> are updated to the new lexer, at which time we can delete the old
EL> lexer.
EL>
EL> The recent check-in to semantic-c shows how to use the new lexer API
EL> to replace the old settings of extensions and other lexical flags.
EL>
EL> [...]
Eric,
I've updated the python lexer using semantic-c.el as a
model. The diff below shows the missing quote that I added
for the newline token in semantic.el. Aside from that, I
followed what you did for C and was able to write the new
lexer without any problem very quickly.
I will be taking some time off too for a couple of weeks.
Hopefully I can try to write the proper semantic rules for
the python parser in August.
Here is the diff. I already checked in
wisent-python.el/wisent-python.wy
changes.
Index: semantic-lex.el
===================================================================
RCS file: /cvsroot/cedet/cedet/semantic/semantic-lex.el,v
retrieving revision 1.3
diff -u -r1.3 semantic-lex.el
--- semantic-lex.el 10 Jul 2002 03:51:16 -0000 1.3
+++ semantic-lex.el 11 Jul 2002 07:54:34 -0000
@@ -30,7 +30,8 @@
;;;###autoload
(defvar semantic-lex-analyzer nil
- "The lexical analyzer used for a given buffer.")
+ "The lexical analyzer used for a given buffer.
+See `semantic-lex' for documentation.")
;;; Code:
(defvar semantic-lex-tokens
@@ -395,7 +396,7 @@
(define-lex-simple-regex-analyzer semantic-lex-newline
"Detect and create newline tokens."
- "\\s-*\\(\n\\|\\s>\\)" newline 1)
+ "\\s-*\\(\n\\|\\s>\\)" 'newline 1)
(define-lex-regex-analyzer semantic-lex-newline-as-whitespace
"Detect and create newline tokens.
Index: wisent/wisent-python.el
===================================================================
RCS file: /cvsroot/cedet/cedet/semantic/wisent/wisent-python.el,v
retrieving revision 1.11
diff -u -r1.11 wisent-python.el
--- wisent/wisent-python.el 2 Jul 2002 04:07:02 -0000 1.11
+++ wisent/wisent-python.el 11 Jul 2002 07:55:19 -0000
@@ -150,6 +150,72 @@
((looking-at "\\s-*\\(#\\|$\\)") -1)
(t (current-indentation)))))
+;;;****************************************************************************
+;;;@ New Lexer
+;;;****************************************************************************
+
+(define-lex-regex-analyzer semantic-lex-python-triple-quotes
+ "Create a 'string token from strings quoted with triple double-quotes."
+ "\\(\\<r\\)?\"\"\""
+ (let ((raw-p (looking-at "r"))
+ beg end)
+ (setq beg (point))
+ (forward-char 3)
+ (search-forward "\"\"\"")
+ (setq end (point))
+ (semantic-lex-token (if raw-p
+ 'raw-string
+ 'string)
+ beg end)))
+
+(define-lex-regex-analyzer semantic-lex-python-raw-string
+ "Handle python raw-strings. Return a 'raw-string syntactic token."
+ wisent-python-raw-string-re
+ (let* ((b (point))
+ (e (condition-case nil
+ (progn
+ (forward-char) ;; skip 'r'
+ (cond
+ ((looking-at "\"\"\"")
+ (forward-char 3)
+ (search-forward "\"\"\""))
+ (t
+ (forward-sexp 1)))
+ (point))
+ ;; This case makes flex
+ ;; robust to broken strings.
+ (error
+ (progn
+ (goto-char
+ (funcall
+ semantic-flex-unterminated-syntax-end-function
+ 'string
+ start end))
+ (point))))))
+ (semantic-lex-token 'raw-string b e)))
+
+(define-lex semantic-python-lexer
+ "Lexical Analyzer for Python code."
+ semantic-lex-beginning-of-line
+ semantic-lex-python-triple-quotes
+ semantic-lex-python-raw-string
+ semantic-lex-newline
+ semantic-lex-ignore-whitespace
+ semantic-lex-number
+ semantic-lex-symbol-or-keyword
+ semantic-lex-charquote
+ semantic-lex-paren-or-list
+ semantic-lex-close-paren
+ semantic-lex-string
+ semantic-lex-ignore-comments
+ semantic-lex-punctuation
+ semantic-lex-default-action
+ )
+
+;;;****************************************************************************
+;;;@ Old Lexer
+;;;****************************************************************************
+
(defun wisent-python-lex-bol ()
"Handle BOL syntactic tokens.
Produce corresponding INDENT or DEDENT python's lexical tokens."
@@ -342,7 +408,7 @@
(defconst wisent-python-parser-tables
(eval-when-compile
-;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-01 20:51-0700
+;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-11 00:48-0700
(wisent-compile-grammar
'((NEWLINE LPAREN RPAREN LBRACE RBRACE LBRACK RBRACK LTLTEQ GTGTEQ EXPEQ DIVDIVEQ DIVDIV LTLT GTGT EXPONENT EQ GE LE PLUSEQ MINUSEQ MULTEQ DIVEQ MODEQ AMPEQ OREQ HATEQ LTGT NE HAT LT GT AMP MULT DIV MOD PLUS MINUS PERIOD TILDE BAR COLON SEMICOLON COMMA ASSIGN BACKQUOTE BACKSLASH STRING_LITERAL NUMBER_LITERAL NAME INDENT DEDENT RAW_STRING_LITERAL AND ASSERT BREAK CLASS CONTINUE DEF DEL ELIF ELSE EXCEPT EXEC FINALLY FOR FROM GLOBAL IF IMPORT IN IS LAMBDA NOT OR PASS PRINT RAISE RETURN TRY WHILE YIELD)
nil
@@ -948,7 +1014,7 @@
(defconst wisent-python-keywords
(identity
-;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-01 20:51-0700
+;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-11 00:48-0700
(semantic-flex-make-keyword-table
'(("and" . AND)
("assert" . ASSERT)
@@ -1012,7 +1078,7 @@
(defconst wisent-python-tokens
(identity
-;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-01 20:51-0700
+;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-11 00:48-0700
(wisent-flex-make-token-table
'(("raw-string"
(RAW_STRING_LITERAL))
@@ -1093,7 +1159,7 @@
(defun wisent-python-default-setup ()
"Setup buffer for parse."
-;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-01 20:51-0700
+;;DO NOT EDIT! Generated from wisent-python.wy - 2002-07-11 00:48-0700
(progn
(setq semantic-bovinate-parser 'wisent-bovinate-nonterminal
semantic-bovinate-parser-name "LALR"
@@ -1129,6 +1195,8 @@
(cons wisent-python-raw-string-re 'wisent-python-flex-raw-string))
semantic-flex-extensions semantic-flex-python-extensions
+
+ semantic-lex-analyzer #'semantic-python-lexer
))
)
Index: wisent/wisent-python.wy
===================================================================
RCS file: /cvsroot/cedet/cedet/semantic/wisent/wisent-python.wy,v
retrieving revision 1.11
diff -u -r1.11 wisent-python.wy
--- wisent/wisent-python.wy 25 Jun 2002 05:13:11 -0000 1.11
+++ wisent/wisent-python.wy 11 Jul 2002 07:56:08 -0000
@@ -58,6 +58,8 @@
(cons wisent-python-raw-string-re 'wisent-python-flex-raw-string))
semantic-flex-extensions semantic-flex-python-extensions
+
+ semantic-lex-analyzer #'semantic-python-lexer
)
%}
|