[javascriptlint-commit] SF.net SVN: javascriptlint:[346] trunk/jsengine/tokenizer

[<mat...@us...>]

Revision: 346
          http://sourceforge.net/p/javascriptlint/code/346
Author:   matthiasmiller
Date:     2013-10-10 15:28:32 +0000 (Thu, 10 Oct 2013)
Log Message:
-----------
Refactor punctuator/keyword handling.

Modified Paths:
--------------
    trunk/jsengine/tokenizer/__init__.py
    trunk/jsengine/tokenizer/tok.py

Modified: trunk/jsengine/tokenizer/__init__.py
===================================================================
--- trunk/jsengine/tokenizer/__init__.py	2013-10-10 15:25:47 UTC (rev 345)
+++ trunk/jsengine/tokenizer/__init__.py	2013-10-10 15:28:32 UTC (rev 346)
@@ -5,15 +5,11 @@
 _WHITESPACE = u'\u0020\t\u000B\u000C\u00A0\uFFFF'
 _LINETERMINATOR = u'\u000A\u000D\u2028\u2029'
 _DIGITS = u'0123456789'
-_DOT_DIGITS = [u'.%s' % digit for digit in _DIGITS]
 _HEX_DIGITS = _DIGITS + u'abcdefABCDEF'
 _IDENT = u'abcdefghijklmnopqrstuvwxyz' + \
          u'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + \
          u'$_'
 
-_KEYWORDS = tok.getkeywords()
-_PUNCTUATOR_TREE = tok.get_punctuator_tree()
-
 class _Char(object):
     def __init__(self, u):
         assert isinstance(u, int) or u is None, u
@@ -179,7 +175,7 @@
 
     def expect_identifiername(self):
         encountered = self.advance()
-        if encountered.tok in list(_KEYWORDS.values()):
+        if tok.keywords.has(encountered.tok) != -1:
             encountered.tok = tok.NAME
         if encountered.tok != tok.NAME:
             raise JSSyntaxError(encountered.start_offset, 'syntax_error')
@@ -332,29 +328,28 @@
             atom = stream.get_watched_reads()
             return Token(tok.NUMBER, atom=atom)
 
-        if c.uval in _PUNCTUATOR_TREE:
-            d = _PUNCTUATOR_TREE[c.uval]
+        if tok.punctuators.hasprefix(c.tostr()):
+            s = c.tostr()
             while True:
                 c = stream.peekchr()
-                if c and c.uval in d:
+                if c and tok.punctuators.hasprefix(s + c.tostr()):
+                    s += c.tostr()
                     stream.readchr()
-                    d = d[c.uval]
                 else:
                     break
-            try:
-                return Token(d[-1])
-            except KeyError:
-                print('oops')
+            d = tok.punctuators.get(s)
+            if not d:
                 raise JSSyntaxError(stream.get_offset(), 'syntax_error')
-
+            return Token(d)
         if c.instr(_IDENT):
             while stream.readchrin(_IDENT + _DIGITS):
                 pass
 
             atom = stream.get_watched_reads()
-            if atom in _KEYWORDS:
-                return Token(_KEYWORDS[atom], atom=atom)
-            return Token(tok.NAME, atom=atom)
+            tt = tok.keywords.get(atom, tok.NAME)
+            t = Token(tt)
+            t.atom = atom
+            return t
 
         raise JSSyntaxError(stream.get_offset(), 'unexpected_char',
                             { 'char': c.tostr() })

Modified: trunk/jsengine/tokenizer/tok.py
===================================================================
--- trunk/jsengine/tokenizer/tok.py	2013-10-10 15:25:47 UTC (rev 345)
+++ trunk/jsengine/tokenizer/tok.py	2013-10-10 15:28:32 UTC (rev 346)
@@ -10,12 +10,10 @@
     def __repr__(self):
         return 'TokenType(%r, %r)' % (self._category, self._literal)
 
-    @property
-    def category(self):
+    def getcategory(self):
         return self._category
 
-    @property
-    def literal(self):
+    def getliteral(self):
         return self._literal
 
 # Symbols
@@ -113,16 +111,44 @@
 SPACE = TokenType('other', '(sp)')
 STRING = TokenType('other', '(str)')
 
-def getkeywords():
-    return dict((t.literal, t) for t in _ALL_TOKENS if t.category == 'kw')
+# Freeze the list of keywords
+_ALL_TOKENS = tuple(_ALL_TOKENS)
 
-def get_punctuator_tree():
-    tree = {}
-    for punctuator in (t for t in _ALL_TOKENS if t.category == 'sym'):
-        leaf = tree
-        for c in punctuator.literal:
-            leaf = leaf.setdefault(ord(c), {})
-        assert not None in leaf, punctuator.literal
-        leaf[-1] = punctuator
-    return tree
+class _Keywords(object):
+    def __init__(self):
+        self._d = {}
+        for tt in _ALL_TOKENS:
+            if tt.getcategory() == 'kw':
+                self._d[tt.getliteral()] = tt
 
+    def get(self, literal, default):
+        return self._d.get(literal, default)
+
+    def has(self, tok):
+        for iter in self._d.values():
+            if iter == tok:
+                return True
+        return False
+
+keywords = _Keywords()
+
+class _Punctuators(object):
+    def __init__(self):
+        self._prefixes = {}
+        self._punctuators = {}
+
+        for t in _ALL_TOKENS:
+            if t.getcategory() == 'sym':
+                literal = t.getliteral()
+                for i in range(len(literal)):
+                    prefix = literal[:i+1]
+                    self._prefixes[prefix] = True
+                self._punctuators[literal] = t
+
+    def hasprefix(self, prefix):
+        return self._prefixes.get(prefix, False)
+
+    def get(self, literal):
+        return self._punctuators.get(literal)
+
+punctuators = _Punctuators()

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.