Thread: [javascriptlint-commit] SF.net SVN: javascriptlint:[335] trunk/jsengine/tokenizer/__init__.py
Status: Beta
Brought to you by:
matthiasmiller
From: <mat...@us...> - 2013-10-09 13:52:55
|
Revision: 335 http://sourceforge.net/p/javascriptlint/code/335 Author: matthiasmiller Date: 2013-10-09 13:52:50 +0000 (Wed, 09 Oct 2013) Log Message: ----------- Simplify TokenStream. Modified Paths: -------------- trunk/jsengine/tokenizer/__init__.py Modified: trunk/jsengine/tokenizer/__init__.py =================================================================== --- trunk/jsengine/tokenizer/__init__.py 2013-10-08 16:36:44 UTC (rev 334) +++ trunk/jsengine/tokenizer/__init__.py 2013-10-09 13:52:50 UTC (rev 335) @@ -178,24 +178,27 @@ return self._content[self._offset - 1] raise JSSyntaxError(self.get_offset(-1), 'unexpected_eof') - def readif(self, len_, seq): - s = self.peekif(len_, seq) + def readchrif(self, seq): + s = self.peekchrif(seq) if s: - assert len(s) == len_ - self._offset += len_ + assert len(s) == 1 + self._offset += 1 return s - def peekchr(self, seq): - if self._offset < len(self._content) and self._content[self._offset] in seq: + def peekchrif(self, seq): + if self._offset < len(self._content) and \ + self._content[self._offset] in seq: return self._content[self._offset] - def peekif(self, len_, seq): + def readtextif(self, text): """ Returns the string if found. Otherwise returns None. """ + len_ = len(text) if self._offset + len_ <= len(self._content): peeked = self._content[self._offset:self._offset+len_] - if peeked in seq: - return peeked + if peeked == text: + self._offset += len_ + return text class Tokenizer: def __init__(self, stream): @@ -308,7 +311,7 @@ # TODO: Validate and save while True: - c = stream.readif(1, _IDENT) + c = stream.readchrif(_IDENT) if not c: break @@ -326,9 +329,9 @@ if c in _WHITESPACE or c in _LINETERMINATOR: linebreak = c in _LINETERMINATOR while True: - if stream.readif(1, _LINETERMINATOR): + if stream.readchrif(_LINETERMINATOR): linebreak = True - elif stream.readif(1, _WHITESPACE): + elif stream.readchrif(_WHITESPACE): pass else: break @@ -339,11 +342,11 @@ # COMMENTS if c == '/': - if stream.peekchr("/"): - while not stream.eof() and not stream.peekif(1, _LINETERMINATOR): + if stream.peekchrif("/"): + while not stream.eof() and not stream.peekchrif(_LINETERMINATOR): stream.readchr() return Token(tok.CPP_COMMENT) - if stream.peekchr("*"): + if stream.peekchrif("*"): linebreak = False while True: if stream.eof(): @@ -351,12 +354,12 @@ c = stream.readchr() if c in _LINETERMINATOR: linebreak = True - elif c == '*' and stream.readif(1, '/'): + elif c == '*' and stream.readchrif('/'): return Token(tok.C_COMMENT) return Token(tok.EOF) elif c == '<': - if stream.readif(3, ('!--',)): - while not stream.eof() and not stream.peekif(1, _LINETERMINATOR): + if stream.readtextif('!--'): + while not stream.eof() and not stream.peekchrif(_LINETERMINATOR): stream.readchr() return Token(tok.HTML_COMMENT) @@ -374,35 +377,35 @@ s += c # NUMBERS - if c in _DIGITS or (c == '.' and stream.peekchr(_DIGITS)): + if c in _DIGITS or (c == '.' and stream.peekchrif(_DIGITS)): s = c # TODO stream.watch_reads() - if c == '0' and stream.readif(1, 'xX'): + if c == '0' and stream.readchrif('xX'): # Hex - while stream.readif(1, _HEX_DIGITS): + while stream.readchrif(_HEX_DIGITS): pass - elif c == '0' and stream.readif(1, _DIGITS): + elif c == '0' and stream.readchrif(_DIGITS): # Octal - while stream.readif(1, _DIGITS): + while stream.readchrif(_DIGITS): pass else: # Decimal if c != '.': - while stream.readif(1, _DIGITS): + while stream.readchrif(_DIGITS): pass - stream.readif(1, '.') + stream.readchrif('.') - while stream.readif(1, _DIGITS): + while stream.readchrif(_DIGITS): pass - if stream.readif(1, 'eE'): - stream.readif(1, '+-') - if not stream.readif(1, _DIGITS): + if stream.readchrif('eE'): + stream.readchrif('+-') + if not stream.readchrif(_DIGITS): raise JSSyntaxError(stream.get_offset(), 'syntax_error') - while stream.readif(1, _DIGITS): + while stream.readchrif(_DIGITS): pass - if stream.peekchr(_IDENT): + if stream.peekchrif(_IDENT): return Token(tok.ERROR) atom = s + stream.get_watched_reads() @@ -411,7 +414,7 @@ if c in _PUNCTUATOR_TREE: d = _PUNCTUATOR_TREE[c] while True: - c = stream.readif(1, list(d.keys())) + c = stream.readchrif(list(d.keys())) if c: d = d[c] else: @@ -426,7 +429,7 @@ s = '' while c: s += c - c = stream.readif(1, _IDENT + _DIGITS) + c = stream.readchrif(_IDENT + _DIGITS) if s in _KEYWORDS: return Token(_KEYWORDS[s], atom=s) elif s: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mat...@us...> - 2013-10-09 20:21:35
|
Revision: 337 http://sourceforge.net/p/javascriptlint/code/337 Author: matthiasmiller Date: 2013-10-09 20:21:32 +0000 (Wed, 09 Oct 2013) Log Message: ----------- Attempt to simplify token definitions. Modified Paths: -------------- trunk/jsengine/tokenizer/__init__.py Modified: trunk/jsengine/tokenizer/__init__.py =================================================================== --- trunk/jsengine/tokenizer/__init__.py 2013-10-09 19:41:02 UTC (rev 336) +++ trunk/jsengine/tokenizer/__init__.py 2013-10-09 20:21:32 UTC (rev 337) @@ -10,139 +10,131 @@ u'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + \ u'$_' -_PUNCTUATORS = { - "<<<=": "ASSIGN_ULSHIFT", - ">>>=": "ASSIGN_URSHIFT", - "===": "EQ_STRICT", - "!==": "NE_STRICT", - ">>>": "URSHIFT", - "<<=": "ASSIGN_LSHIFT", - ">>=": "ASSIGN_RSHIFT", - "<=": "LE", - ">=": "GE", - "==": "EQ", - "!=": "NE", - "++": "INC", - "--": "DEC", - "<<": "LSHIFT", - ">>": "RSHIFT", - "&&": "LOGICAL_AND", - "||": "LOGICAL_OR", - "+=": "ASSIGN_ADD", - "-=": "ASSIGN_SUB", - "*=": "ASSIGN_MUL", - "%=": "ASSIGN_MOD", - "&=": "ASSIGN_BIT_AND", - "|=": "ASSIGN_BIT_OR", - "^=": "ASSIGN_BIT_XOR", - "/=": "ASSIGN_DIV", - "{": "LBRACE", - "}": "RBRACE", - "(": "LPAREN", - ")": "RPAREN", - "[": "LBRACKET", - "]": "RBRACKET", - ".": "DOT", - ";": "SEMI", - ",": "COMMA", - "<": "LT", - ">": "GT", - "+": "ADD", - "-": "SUB", - "*": "MUL", - "%": "MOD", - "|": "BIT_OR", - "&": "BIT_AND", - "^": "BIT_XOR", - "!": "LOGICAL_NOT", - "~": "BIT_NOT", - "?": "QUESTION", - ":": "COLON", - "=": "ASSIGN", - "/": "DIV", - "!": "LOGICAL_NOT", -} +_ALL_TOKENS = [] -_KEYWORDS = dict((keyword, keyword.upper()) for keyword in [ - 'break', - 'case', - 'catch', - 'continue', - 'default', - 'delete', - 'do', - 'else', - 'false', - 'finally', - 'for', - 'function', - 'if', - 'in', - 'instanceof', - 'new', - 'null', - 'return', - 'switch', - 'this', - 'throw', - 'true', - 'typeof', - 'try', - 'var', - 'void', - 'while', - 'with', -]) +class _Token(object): + def __init__(self, category, literal): + self._category = category + self._literal = literal + _ALL_TOKENS.append(self) -_TOKENS = [ - 'C_COMMENT', - 'CPP_COMMENT', - 'HTML_COMMENT', - 'ERROR', - 'EOF', - 'EOL', - 'NAME', - 'NUMBER', - 'OPERATOR', - 'REGEXP', - 'SPACE', - 'STRING', -] + def __repr__(self): + return '_Token(%r, %r)' % (self._category, self._literal) -class _Token(object): - def __init__(self, name): - self._name = name + @property + def category(self): + return self._category - def __eq__(self, other): - assert isinstance(other, _Token) - return self is other + @property + def literal(self): + return self._literal - def __repr__(self): - return 'tok.%s' % self._name - -class _Tokens: +class _Tokens(object): def __init__(self): - for token in _TOKENS: - setattr(self, token, _Token(token)) + # Load symbols + self.ASSIGN_ULSHIFT = _Token('sym', '<<<=') + self.ASSIGN_URSHIFT = _Token('sym', '>>>=') + self.EQ_STRICT = _Token('sym', '===') + self.NE_STRICT = _Token('sym', '!==') + self.URSHIFT = _Token('sym', '>>>') + self.ASSIGN_LSHIFT = _Token('sym', '<<=') + self.ASSIGN_RSHIFT = _Token('sym', '>>=') + self.LE = _Token('sym', '<=') + self.GE = _Token('sym', '>=') + self.EQ = _Token('sym', '==') + self.NE = _Token('sym', '!=') + self.INC = _Token('sym', '++') + self.DEC = _Token('sym', '--') + self.LSHIFT = _Token('sym', '<<') + self.RSHIFT = _Token('sym', '>>') + self.LOGICAL_AND = _Token('sym', '&&') + self.LOGICAL_OR = _Token('sym', '||') + self.ASSIGN_ADD = _Token('sym', '+=') + self.ASSIGN_SUB = _Token('sym', '-=') + self.ASSIGN_MUL = _Token('sym', '*=') + self.ASSIGN_MOD = _Token('sym', '%=') + self.ASSIGN_BIT_AND = _Token('sym', '&=') + self.ASSIGN_BIT_OR = _Token('sym', '|=') + self.ASSIGN_BIT_XOR = _Token('sym', '^=') + self.ASSIGN_DIV = _Token('sym', '/=') + self.LBRACE = _Token('sym', '{') + self.RBRACE = _Token('sym', '}') + self.LPAREN = _Token('sym', '(') + self.RPAREN = _Token('sym', ')') + self.LBRACKET = _Token('sym', '[') + self.RBRACKET = _Token('sym', ']') + self.DOT = _Token('sym', '.') + self.SEMI = _Token('sym', ';') + self.COMMA = _Token('sym', ',') + self.LT = _Token('sym', '<') + self.GT = _Token('sym', '>') + self.ADD = _Token('sym', '+') + self.SUB = _Token('sym', '-') + self.MUL = _Token('sym', '*') + self.MOD = _Token('sym', '%') + self.BIT_OR = _Token('sym', '|') + self.BIT_AND = _Token('sym', '&') + self.BIT_XOR = _Token('sym', '^') + self.LOGICAL_NOT = _Token('sym', '!') + self.BIT_NOT = _Token('sym', '~') + self.QUESTION = _Token('sym', '?') + self.COLON = _Token('sym', ':') + self.ASSIGN = _Token('sym', '=') + self.DIV = _Token('sym', '/') - for key, name in list(_KEYWORDS.items()): - _KEYWORDS[key] = _Token(name) - setattr(self, name, _KEYWORDS[key]) + # Load keywords + self.BREAK = _Token('kw', 'break') + self.CASE = _Token('kw', 'case') + self.CATCH = _Token('kw', 'catch') + self.CONTINUE = _Token('kw', 'continue') + self.DEFAULT = _Token('kw', 'default') + self.DELETE = _Token('kw', 'delete') + self.DO = _Token('kw', 'do') + self.ELSE = _Token('kw', 'else') + self.FALSE = _Token('kw', 'false') + self.FINALLY = _Token('kw', 'finally') + self.FOR = _Token('kw', 'for') + self.FUNCTION = _Token('kw', 'function') + self.IF = _Token('kw', 'if') + self.IN = _Token('kw', 'in') + self.INSTANCEOF = _Token('kw', 'instanceof') + self.NEW = _Token('kw', 'new') + self.NULL = _Token('kw', 'null') + self.RETURN = _Token('kw', 'return') + self.SWITCH = _Token('kw', 'switch') + self.THIS = _Token('kw', 'this') + self.THROW = _Token('kw', 'throw') + self.TRUE = _Token('kw', 'true') + self.TYPEOF = _Token('kw', 'typeof') + self.TRY = _Token('kw', 'try') + self.VAR = _Token('kw', 'var') + self.VOID = _Token('kw', 'void') + self.WHILE = _Token('kw', 'while') + self.WITH = _Token('kw', 'with') - for key, name in list(_PUNCTUATORS.items()): - _PUNCTUATORS[key] = _Token(name) - setattr(self, name, _PUNCTUATORS[key]) + # Load other tokens + self.C_COMMENT = _Token('other', '/*') + self.CPP_COMMENT = _Token('other', '//') + self.HTML_COMMENT = _Token('other', '<!--') + self.ERROR = _Token('other', 'err') + self.EOF = _Token('other', 'eof') + self.EOL = _Token('other', 'eol') + self.NAME = _Token('other', '(name)') + self.NUMBER = _Token('other', '(num)') + self.OPERATOR = _Token('other', '(op)') + self.REGEXP = _Token('other', '(re)') + self.SPACE = _Token('other', '(sp)') + self.STRING = _Token('other', '(str)') tok = _Tokens() - - +_KEYWORDS = dict((t.literal, t) for t in _ALL_TOKENS if t.category == 'kw') _PUNCTUATOR_TREE = {} -for punctuator in _PUNCTUATORS: +for punctuator in (t for t in _ALL_TOKENS if t.category == 'sym'): d = _PUNCTUATOR_TREE - for c in punctuator: + for c in punctuator.literal: d = d.setdefault(c, {}) - assert not None in d - d[None] = _PUNCTUATORS[punctuator] + assert not None in d, punctuator.literal + d[None] = punctuator class Token: def __init__(self, tok, atom=None): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mat...@us...> - 2013-10-09 20:30:38
|
Revision: 338 http://sourceforge.net/p/javascriptlint/code/338 Author: matthiasmiller Date: 2013-10-09 20:30:33 +0000 (Wed, 09 Oct 2013) Log Message: ----------- Tweak to _Tokens object Modified Paths: -------------- trunk/jsengine/tokenizer/__init__.py Modified: trunk/jsengine/tokenizer/__init__.py =================================================================== --- trunk/jsengine/tokenizer/__init__.py 2013-10-09 20:21:32 UTC (rev 337) +++ trunk/jsengine/tokenizer/__init__.py 2013-10-09 20:30:33 UTC (rev 338) @@ -30,101 +30,100 @@ return self._literal class _Tokens(object): - def __init__(self): - # Load symbols - self.ASSIGN_ULSHIFT = _Token('sym', '<<<=') - self.ASSIGN_URSHIFT = _Token('sym', '>>>=') - self.EQ_STRICT = _Token('sym', '===') - self.NE_STRICT = _Token('sym', '!==') - self.URSHIFT = _Token('sym', '>>>') - self.ASSIGN_LSHIFT = _Token('sym', '<<=') - self.ASSIGN_RSHIFT = _Token('sym', '>>=') - self.LE = _Token('sym', '<=') - self.GE = _Token('sym', '>=') - self.EQ = _Token('sym', '==') - self.NE = _Token('sym', '!=') - self.INC = _Token('sym', '++') - self.DEC = _Token('sym', '--') - self.LSHIFT = _Token('sym', '<<') - self.RSHIFT = _Token('sym', '>>') - self.LOGICAL_AND = _Token('sym', '&&') - self.LOGICAL_OR = _Token('sym', '||') - self.ASSIGN_ADD = _Token('sym', '+=') - self.ASSIGN_SUB = _Token('sym', '-=') - self.ASSIGN_MUL = _Token('sym', '*=') - self.ASSIGN_MOD = _Token('sym', '%=') - self.ASSIGN_BIT_AND = _Token('sym', '&=') - self.ASSIGN_BIT_OR = _Token('sym', '|=') - self.ASSIGN_BIT_XOR = _Token('sym', '^=') - self.ASSIGN_DIV = _Token('sym', '/=') - self.LBRACE = _Token('sym', '{') - self.RBRACE = _Token('sym', '}') - self.LPAREN = _Token('sym', '(') - self.RPAREN = _Token('sym', ')') - self.LBRACKET = _Token('sym', '[') - self.RBRACKET = _Token('sym', ']') - self.DOT = _Token('sym', '.') - self.SEMI = _Token('sym', ';') - self.COMMA = _Token('sym', ',') - self.LT = _Token('sym', '<') - self.GT = _Token('sym', '>') - self.ADD = _Token('sym', '+') - self.SUB = _Token('sym', '-') - self.MUL = _Token('sym', '*') - self.MOD = _Token('sym', '%') - self.BIT_OR = _Token('sym', '|') - self.BIT_AND = _Token('sym', '&') - self.BIT_XOR = _Token('sym', '^') - self.LOGICAL_NOT = _Token('sym', '!') - self.BIT_NOT = _Token('sym', '~') - self.QUESTION = _Token('sym', '?') - self.COLON = _Token('sym', ':') - self.ASSIGN = _Token('sym', '=') - self.DIV = _Token('sym', '/') + # Symbols + ASSIGN_ULSHIFT = _Token('sym', '<<<=') + ASSIGN_URSHIFT = _Token('sym', '>>>=') + EQ_STRICT = _Token('sym', '===') + NE_STRICT = _Token('sym', '!==') + URSHIFT = _Token('sym', '>>>') + ASSIGN_LSHIFT = _Token('sym', '<<=') + ASSIGN_RSHIFT = _Token('sym', '>>=') + LE = _Token('sym', '<=') + GE = _Token('sym', '>=') + EQ = _Token('sym', '==') + NE = _Token('sym', '!=') + INC = _Token('sym', '++') + DEC = _Token('sym', '--') + LSHIFT = _Token('sym', '<<') + RSHIFT = _Token('sym', '>>') + LOGICAL_AND = _Token('sym', '&&') + LOGICAL_OR = _Token('sym', '||') + ASSIGN_ADD = _Token('sym', '+=') + ASSIGN_SUB = _Token('sym', '-=') + ASSIGN_MUL = _Token('sym', '*=') + ASSIGN_MOD = _Token('sym', '%=') + ASSIGN_BIT_AND = _Token('sym', '&=') + ASSIGN_BIT_OR = _Token('sym', '|=') + ASSIGN_BIT_XOR = _Token('sym', '^=') + ASSIGN_DIV = _Token('sym', '/=') + LBRACE = _Token('sym', '{') + RBRACE = _Token('sym', '}') + LPAREN = _Token('sym', '(') + RPAREN = _Token('sym', ')') + LBRACKET = _Token('sym', '[') + RBRACKET = _Token('sym', ']') + DOT = _Token('sym', '.') + SEMI = _Token('sym', ';') + COMMA = _Token('sym', ',') + LT = _Token('sym', '<') + GT = _Token('sym', '>') + ADD = _Token('sym', '+') + SUB = _Token('sym', '-') + MUL = _Token('sym', '*') + MOD = _Token('sym', '%') + BIT_OR = _Token('sym', '|') + BIT_AND = _Token('sym', '&') + BIT_XOR = _Token('sym', '^') + LOGICAL_NOT = _Token('sym', '!') + BIT_NOT = _Token('sym', '~') + QUESTION = _Token('sym', '?') + COLON = _Token('sym', ':') + ASSIGN = _Token('sym', '=') + DIV = _Token('sym', '/') - # Load keywords - self.BREAK = _Token('kw', 'break') - self.CASE = _Token('kw', 'case') - self.CATCH = _Token('kw', 'catch') - self.CONTINUE = _Token('kw', 'continue') - self.DEFAULT = _Token('kw', 'default') - self.DELETE = _Token('kw', 'delete') - self.DO = _Token('kw', 'do') - self.ELSE = _Token('kw', 'else') - self.FALSE = _Token('kw', 'false') - self.FINALLY = _Token('kw', 'finally') - self.FOR = _Token('kw', 'for') - self.FUNCTION = _Token('kw', 'function') - self.IF = _Token('kw', 'if') - self.IN = _Token('kw', 'in') - self.INSTANCEOF = _Token('kw', 'instanceof') - self.NEW = _Token('kw', 'new') - self.NULL = _Token('kw', 'null') - self.RETURN = _Token('kw', 'return') - self.SWITCH = _Token('kw', 'switch') - self.THIS = _Token('kw', 'this') - self.THROW = _Token('kw', 'throw') - self.TRUE = _Token('kw', 'true') - self.TYPEOF = _Token('kw', 'typeof') - self.TRY = _Token('kw', 'try') - self.VAR = _Token('kw', 'var') - self.VOID = _Token('kw', 'void') - self.WHILE = _Token('kw', 'while') - self.WITH = _Token('kw', 'with') + # Keywords + BREAK = _Token('kw', 'break') + CASE = _Token('kw', 'case') + CATCH = _Token('kw', 'catch') + CONTINUE = _Token('kw', 'continue') + DEFAULT = _Token('kw', 'default') + DELETE = _Token('kw', 'delete') + DO = _Token('kw', 'do') + ELSE = _Token('kw', 'else') + FALSE = _Token('kw', 'false') + FINALLY = _Token('kw', 'finally') + FOR = _Token('kw', 'for') + FUNCTION = _Token('kw', 'function') + IF = _Token('kw', 'if') + IN = _Token('kw', 'in') + INSTANCEOF = _Token('kw', 'instanceof') + NEW = _Token('kw', 'new') + NULL = _Token('kw', 'null') + RETURN = _Token('kw', 'return') + SWITCH = _Token('kw', 'switch') + THIS = _Token('kw', 'this') + THROW = _Token('kw', 'throw') + TRUE = _Token('kw', 'true') + TYPEOF = _Token('kw', 'typeof') + TRY = _Token('kw', 'try') + VAR = _Token('kw', 'var') + VOID = _Token('kw', 'void') + WHILE = _Token('kw', 'while') + WITH = _Token('kw', 'with') - # Load other tokens - self.C_COMMENT = _Token('other', '/*') - self.CPP_COMMENT = _Token('other', '//') - self.HTML_COMMENT = _Token('other', '<!--') - self.ERROR = _Token('other', 'err') - self.EOF = _Token('other', 'eof') - self.EOL = _Token('other', 'eol') - self.NAME = _Token('other', '(name)') - self.NUMBER = _Token('other', '(num)') - self.OPERATOR = _Token('other', '(op)') - self.REGEXP = _Token('other', '(re)') - self.SPACE = _Token('other', '(sp)') - self.STRING = _Token('other', '(str)') + # Other tokens + C_COMMENT = _Token('other', '/*') + CPP_COMMENT = _Token('other', '//') + HTML_COMMENT = _Token('other', '<!--') + ERROR = _Token('other', 'err') + EOF = _Token('other', 'eof') + EOL = _Token('other', 'eol') + NAME = _Token('other', '(name)') + NUMBER = _Token('other', '(num)') + OPERATOR = _Token('other', '(op)') + REGEXP = _Token('other', '(re)') + SPACE = _Token('other', '(sp)') + STRING = _Token('other', '(str)') tok = _Tokens() _KEYWORDS = dict((t.literal, t) for t in _ALL_TOKENS if t.category == 'kw') This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mat...@us...> - 2013-10-09 20:33:14
|
Revision: 339 http://sourceforge.net/p/javascriptlint/code/339 Author: matthiasmiller Date: 2013-10-09 20:33:12 +0000 (Wed, 09 Oct 2013) Log Message: ----------- Simplify TokenStream.get_offset Modified Paths: -------------- trunk/jsengine/tokenizer/__init__.py Modified: trunk/jsengine/tokenizer/__init__.py =================================================================== --- trunk/jsengine/tokenizer/__init__.py 2013-10-09 20:30:33 UTC (rev 338) +++ trunk/jsengine/tokenizer/__init__.py 2013-10-09 20:33:12 UTC (rev 339) @@ -156,8 +156,8 @@ self._offset = 0 self._watched_offset = None - def get_offset(self, offset=0): - return self._start_offset + self._offset + offset + def get_offset(self): + return self._start_offset + self._offset def watch_reads(self): self._watched_offset = self._offset @@ -175,7 +175,7 @@ if self._offset < len(self._content): self._offset += 1 return self._content[self._offset - 1] - raise JSSyntaxError(self.get_offset(-1), 'unexpected_eof') + raise JSSyntaxError(self.get_offset()-1, 'unexpected_eof') def readchrif(self, seq): s = self.peekchrif(seq) @@ -240,7 +240,7 @@ self._readahead() if self._peeked[-1].tok == tok.DIV: token = self._parse_rest_of_regexp() - token.set_offset(self._peeked[-1].start_offset, self._stream.get_offset(-1)) + token.set_offset(self._peeked[-1].start_offset, self._stream.get_offset()-1) self._peeked = [] if token.tok == tok.ERROR: self._error = True @@ -275,7 +275,7 @@ while True: start_offset = self._stream.get_offset() peek = self._next() - end_offset = self._stream.get_offset(-1) + end_offset = self._stream.get_offset()-1 if peek.tok == tok.ERROR: peek.set_offset(end_offset, end_offset) else: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <mat...@us...> - 2013-10-09 21:51:05
|
Revision: 341 http://sourceforge.net/p/javascriptlint/code/341 Author: matthiasmiller Date: 2013-10-09 21:51:03 +0000 (Wed, 09 Oct 2013) Log Message: ----------- Remove unused parameters from Tokenizer.advance Modified Paths: -------------- trunk/jsengine/tokenizer/__init__.py Modified: trunk/jsengine/tokenizer/__init__.py =================================================================== --- trunk/jsengine/tokenizer/__init__.py 2013-10-09 21:34:59 UTC (rev 340) +++ trunk/jsengine/tokenizer/__init__.py 2013-10-09 21:51:03 UTC (rev 341) @@ -96,23 +96,16 @@ else: return peek - def advance(self, skipspace=True, skipcomments=True): + def advance(self): assert not self._error self._readahead() - for i, peek in enumerate(self._peeked): - if not skipspace and peek.tok in (tok.EOL, tok.SPACE): - self._peeked = self._peeked[i+1:] - return peek - elif not skipcomments and peek.tok in (tok.C_COMMENT, tok.CPP_COMMENT, tok.HTML_COMMENT): - self._peeked = self._peeked[i+1:] - return peek - else: - self._peeked = [] - if peek.tok == tok.ERROR: - self._error = True - raise JSSyntaxError(peek.start_offset, peek.atom or 'syntax_error') - return peek + peek = self._peeked[-1] + self._peeked = [] + if peek.tok == tok.ERROR: + self._error = True + raise JSSyntaxError(peek.start_offset, peek.atom or 'syntax_error') + return peek def next_withregexp(self): assert not self._error This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |