[javascriptlint-commit] SF.net SVN: javascriptlint:[334] trunk
Status: Beta
Brought to you by:
matthiasmiller
Menu
▾
▴
[javascriptlint-commit] SF.net SVN: javascriptlint:[334] trunk
|
From: <mat...@us...> - 2013-10-08 16:36:50
|
Revision: 334
http://sourceforge.net/p/javascriptlint/code/334
Author: matthiasmiller
Date: 2013-10-08 16:36:44 +0000 (Tue, 08 Oct 2013)
Log Message:
-----------
Change the parser to only use offsets internally and to convert offsets to line/col only when reporting errors.
Modified Paths:
--------------
trunk/javascriptlint/htmlparse.py
trunk/javascriptlint/jsparse.py
trunk/javascriptlint/lint.py
trunk/javascriptlint/warnings.py
trunk/jsengine/__init__.py
trunk/jsengine/parser/__init__.py
trunk/jsengine/structs.py
trunk/jsengine/tokenizer/__init__.py
trunk/test.py
Modified: trunk/javascriptlint/htmlparse.py
===================================================================
--- trunk/javascriptlint/htmlparse.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/javascriptlint/htmlparse.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -2,18 +2,25 @@
import HTMLParser
import unittest
+from jsengine.structs import NodePos, NodePositions
+
class _Parser(HTMLParser.HTMLParser):
def __init__(self):
HTMLParser.HTMLParser.__init__(self)
self._tags = []
+ self._node_positions = None
+ def feed(self, data):
+ # Reset line numbers whenever we get data.
+ self._node_positions = None
+ HTMLParser.HTMLParser.feed(self, data)
+
def handle_starttag(self, tag, attributes):
if tag.lower() == 'script':
attr = dict(attributes)
self._tags.append({
'type': 'start',
- 'lineno': self.lineno,
- 'offset': self.offset,
+ 'offset': self._getoffset(),
'len': len(self.get_starttag_text()),
'attr': attr
})
@@ -22,8 +29,7 @@
if tag.lower() == 'script':
self._tags.append({
'type': 'end',
- 'lineno': self.lineno,
- 'offset': self.offset,
+ 'offset': self._getoffset(),
})
def unknown_decl(self, data):
@@ -33,9 +39,16 @@
def gettags(self):
return self._tags
+ def _getoffset(self):
+ # htmlparse returns 1-based line numbers. Calculate the offset of the
+ # script's contents.
+ if self._node_positions is None:
+ self._node_positions = NodePositions(self.rawdata)
+ pos = NodePos(self.lineno - 1, self.offset)
+ return self._node_positions.to_offset(pos)
+
+
def findscripttags(s):
- """ Note that the lineno is 1-based.
- """
parser = _Parser()
parser.feed(s)
parser.close()
Modified: trunk/javascriptlint/jsparse.py
===================================================================
--- trunk/javascriptlint/jsparse.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/javascriptlint/jsparse.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -16,7 +16,8 @@
return True
return jsengine.parser.is_valid_version(jsversion.version)
-def findpossiblecomments(script, node_positions):
+def findpossiblecomments(script, script_offset):
+ assert not script_offset is None
pos = 0
single_line_re = r"//[^\r\n]*"
multi_line_re = r"/\*(.*?)\*/"
@@ -41,38 +42,34 @@
start_offset = match.start()
end_offset = match.end()-1
- start_pos = node_positions.from_offset(start_offset)
- end_pos = node_positions.from_offset(end_offset)
- comment_node = ParseNode(kind.COMMENT, opcode, start_pos, end_pos,
- comment_text, [])
+ comment_node = ParseNode(kind.COMMENT, opcode,
+ script_offset + start_offset,
+ script_offset + end_offset, comment_text, [])
comments.append(comment_node)
# Start searching immediately after the start of the comment in case
# this one was within a string or a regexp.
pos = match.start()+1
-def parse(script, jsversion, error_callback, startpos=None):
- """ All node positions will be relative to startpos. This allows scripts
- to be embedded in a file (for example, HTML).
+def parse(script, jsversion, error_callback, start_offset=0):
+ """ All node positions will be relative to start_offset. This allows
+ scripts to be embedded in a file (for example, HTML).
"""
- startpos = startpos or NodePos(0, 0)
+ assert not start_offset is None
jsversion = jsversion or JSVersion.default()
assert isvalidversion(jsversion), jsversion
if jsversion.e4x:
- error_callback(startpos.line, startpos.col, 'e4x_deprecated', {})
+ error_callback(start_offset, 'e4x_deprecated', {})
return jsengine.parser.parse(script, jsversion.version,
- error_callback,
- startpos)
+ error_callback, start_offset)
-def filtercomments(possible_comments, node_positions, root_node):
+def filtercomments(possible_comments, root_node):
comment_ignore_ranges = NodeRanges()
def process(node):
if node.kind == tok.STRING or \
(node.kind == tok.OBJECT and node.opcode == op.REGEXP):
- start_offset = node_positions.to_offset(node.start_pos())
- end_offset = node_positions.to_offset(node.end_pos())
- comment_ignore_ranges.add(start_offset, end_offset)
+ comment_ignore_ranges.add(node.start_offset, node.end_offset)
for kid in node.kids:
if kid:
process(kid)
@@ -80,25 +77,22 @@
comments = []
for comment in possible_comments:
- start_offset = node_positions.to_offset(comment.start_pos())
- end_offset = node_positions.to_offset(comment.end_pos())
- if comment_ignore_ranges.has(start_offset):
+ if comment_ignore_ranges.has(comment.start_offset):
continue
- comment_ignore_ranges.add(start_offset, end_offset)
+ comment_ignore_ranges.add(comment.start_offset, comment.end_offset)
comments.append(comment)
return comments
-def findcomments(script, root_node, start_pos=None):
- node_positions = NodePositions(script, start_pos)
- possible_comments = findpossiblecomments(script, node_positions)
- return filtercomments(possible_comments, node_positions, root_node)
+def findcomments(script, root_node, start_offset=0):
+ possible_comments = findpossiblecomments(script, start_offset)
+ return filtercomments(possible_comments, root_node)
def is_compilable_unit(script, jsversion):
jsversion = jsversion or JSVersion.default()
assert isvalidversion(jsversion)
return jsengine.parser.is_compilable_unit(script, jsversion.version)
-def _dump_node(node, depth=0):
+def _dump_node(node, node_positions, depth=0):
if node is None:
print ' '*depth,
print '(None)'
@@ -107,7 +101,8 @@
print ' '*depth,
print '%s, %s' % (repr(node.kind), repr(node.opcode))
print ' '*depth,
- print '%s - %s' % (node.start_pos(), node.end_pos())
+ print '%s - %s' % (node_positions.from_offset(node.start_offset),
+ node_positions.from_offset(node.end_offset))
if hasattr(node, 'atom'):
print ' '*depth,
print 'atom: %s' % node.atom
@@ -116,13 +111,14 @@
print '(no semicolon)'
print
for node in node.kids:
- _dump_node(node, depth+1)
+ _dump_node(node, node_positions, depth+1)
def dump_tree(script):
def error_callback(line, col, msg, msg_args):
print '(%i, %i): %s', (line, col, msg)
node = parse(script, None, error_callback)
- _dump_node(node)
+ node_positions = NodePositions(script)
+ _dump_node(node, node_positions)
class TestComments(unittest.TestCase):
def _test(self, script, expected_comments):
@@ -230,42 +226,42 @@
class TestLineOffset(unittest.TestCase):
def testErrorPos(self):
- def geterror(script, startpos):
+ def geterror(script, start_offset):
errors = []
- def onerror(line, col, msg, msg_args):
- errors.append((line, col, msg, msg_args))
- parse(script, None, onerror, startpos)
+ def onerror(offset, msg, msg_args):
+ errors.append((offset, msg, msg_args))
+ parse(script, None, onerror, start_offset)
self.assertEquals(len(errors), 1)
return errors[0]
- self.assertEquals(geterror(' ?', None), (0, 1, 'syntax_error', {}))
- self.assertEquals(geterror('\n ?', None), (1, 1, 'syntax_error', {}))
- self.assertEquals(geterror(' ?', NodePos(1, 1)), (1, 2, 'syntax_error', {}))
- self.assertEquals(geterror('\n ?', NodePos(1, 1)), (2, 1, 'syntax_error', {}))
+ self.assertEquals(geterror(' ?', 0), (1, 'syntax_error', {}))
+ self.assertEquals(geterror('\n ?', 0), (2, 'syntax_error', {}))
+ self.assertEquals(geterror(' ?', 2), (3, 'syntax_error', {}))
+ self.assertEquals(geterror('\n ?', 2), (4, 'syntax_error', {}))
def testNodePos(self):
- def getnodepos(script, startpos):
- root = parse(script, None, None, startpos)
+ def getnodepos(script, start_offset):
+ root = parse(script, None, None, start_offset)
self.assertEquals(root.kind, tok.LC)
var, = root.kids
self.assertEquals(var.kind, tok.VAR)
- return var.start_pos()
- self.assertEquals(getnodepos('var x;', None), NodePos(0, 0))
- self.assertEquals(getnodepos(' var x;', None), NodePos(0, 1))
- self.assertEquals(getnodepos('\n\n var x;', None), NodePos(2, 1))
- self.assertEquals(getnodepos('var x;', NodePos(3, 4)), NodePos(3, 4))
- self.assertEquals(getnodepos(' var x;', NodePos(3, 4)), NodePos(3, 5))
- self.assertEquals(getnodepos('\n\n var x;', NodePos(3, 4)), NodePos(5, 1))
+ return var.start_offset
+ self.assertEquals(getnodepos('var x;', 0), 0)
+ self.assertEquals(getnodepos(' var x;', 0), 1)
+ self.assertEquals(getnodepos('\n\n var x;', 0), 3)
+ self.assertEquals(getnodepos('var x;', 7), 7)
+ self.assertEquals(getnodepos(' var x;', 7), 8)
+ self.assertEquals(getnodepos('\n\n var x;', 7), 10)
def testComments(self):
- def testcomment(comment, startpos, expectedpos):
+ def testcomment(comment, startpos, expected_offset):
root = parse(comment, None, None, startpos)
comment, = findcomments(comment, root, startpos)
- self.assertEquals(comment.start_pos(), expectedpos)
+ self.assertEquals(comment.start_offset, expected_offset)
for comment in ('/*comment*/', '//comment'):
- testcomment(comment, None, NodePos(0, 0))
- testcomment(' %s' % comment, None, NodePos(0, 1))
- testcomment('\n\n %s' % comment, None, NodePos(2, 1))
- testcomment('%s' % comment, NodePos(3, 4), NodePos(3, 4))
- testcomment(' %s' % comment, NodePos(3, 4), NodePos(3, 5))
- testcomment('\n\n %s' % comment, NodePos(3, 4), NodePos(5, 1))
+ testcomment(comment, 0, 0)
+ testcomment(' %s' % comment, 0, 1)
+ testcomment('\n\n %s' % comment, 0, 3)
+ testcomment('%s' % comment, 7, 7)
+ testcomment(' %s' % comment, 7, 8)
+ testcomment('\n\n %s' % comment, 7, 10)
if __name__ == '__main__':
unittest.main()
Modified: trunk/javascriptlint/lint.py
===================================================================
--- trunk/javascriptlint/lint.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/javascriptlint/lint.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -140,7 +140,7 @@
# sorted by node position.
unreferenced = [(key[0], key[1], node) for key, node
in unreferenced.items()]
- unreferenced.sort(key=lambda x: x[2].start_pos())
+ unreferenced.sort(key=lambda x: x[2].start_offset)
return {
'unreferenced': unreferenced,
@@ -214,8 +214,8 @@
# Conditionally add it to an inner scope.
assert self._node
- if (node.start_pos() >= self._node.start_pos() and \
- node.end_pos() <= self._node.end_pos()):
+ if (node.start_offset >= self._node.start_offset and \
+ node.end_offset <= self._node.end_offset):
return self
class _Script:
@@ -245,7 +245,6 @@
def _findhtmlscripts(contents, default_version):
starttag = None
- nodepos = jsparse.NodePositions(contents)
for tag in htmlparse.findscripttags(contents):
if tag['type'] == 'start':
# Ignore nested start tags.
@@ -265,13 +264,9 @@
# htmlparse returns 1-based line numbers. Calculate the
# position of the script's contents.
- tagpos = jsparse.NodePos(starttag['lineno']-1, starttag['offset'])
- tagoffset = nodepos.to_offset(tagpos)
- startoffset = tagoffset + starttag['len']
- startpos = nodepos.from_offset(startoffset)
- endpos = jsparse.NodePos(tag['lineno']-1, tag['offset'])
- endoffset = nodepos.to_offset(endpos)
- script = contents[startoffset:endoffset]
+ start_offset = starttag['offset'] + starttag['len']
+ end_offset = tag['offset']
+ script = contents[start_offset:end_offset]
if not jsparse.isvalidversion(starttag['jsversion']) or \
jsparse.is_compilable_unit(script, starttag['jsversion']):
@@ -279,7 +274,7 @@
yield {
'type': 'inline',
'jsversion': starttag['jsversion'],
- 'pos': startpos,
+ 'offset': start_offset,
'contents': script,
}
starttag = None
@@ -294,8 +289,9 @@
import_path = import_path.replace('\\', os.sep)
import_path = os.path.join(os.path.dirname(path), import_path)
return lint_file(import_path, 'js', jsversion, encoding)
- def _lint_error(*args):
- return lint_error(normpath, *args)
+ def _lint_error(offset, errname, errdesc):
+ pos = node_positions.from_offset(offset)
+ return lint_error(normpath, pos.line, pos.col, errname, errdesc)
normpath = fs.normpath(path)
if normpath in lint_cache:
@@ -307,12 +303,13 @@
try:
contents = fs.readfile(path, encoding)
except IOError, error:
- _lint_error(0, 0, 'io_error', unicode(error))
+ lint_error(normpath, 0, 0, 'io_error', unicode(error))
return lint_cache[normpath]
+ node_positions = jsparse.NodePositions(contents)
script_parts = []
if kind == 'js':
- script_parts.append((None, jsversion or conf['default-version'], contents))
+ script_parts.append((0, jsversion or conf['default-version'], contents))
elif kind == 'html':
assert jsversion is None
for script in _findhtmlscripts(contents, conf['default-version']):
@@ -324,7 +321,7 @@
other = import_script(script['src'], script['jsversion'])
lint_cache[normpath].importscript(other)
elif script['type'] == 'inline':
- script_parts.append((script['pos'], script['jsversion'],
+ script_parts.append((script['offset'], script['jsversion'],
script['contents']))
else:
assert False, 'Invalid internal script type %s' % \
@@ -343,18 +340,18 @@
else:
lint_file(path, 'js', None, encoding)
-def _lint_script_part(scriptpos, jsversion, script, script_cache, conf,
+def _lint_script_part(script_offset, jsversion, script, script_cache, conf,
ignores, report_native, report_lint, import_callback):
- def parse_error(row, col, msg, msg_args):
+ def parse_error(offset, msg, msg_args):
if not msg in ('anon_no_return_value', 'no_return_value',
'redeclared_var', 'var_hides_arg'):
- parse_errors.append((jsparse.NodePos(row, col), msg, msg_args))
+ parse_errors.append((offset, msg, msg_args))
- def report(node, errname, pos=None, **errargs):
+ def report(node, errname, offset=0, **errargs):
if errname == 'empty_statement' and node.kind == tok.LC:
for pass_ in passes:
- if pass_.start_pos() > node.start_pos() and \
- pass_.end_pos() < node.end_pos():
+ if pass_.start_offset > node.start_offset and \
+ pass_.end_offset < node.end_offset:
passes.remove(pass_)
return
@@ -363,12 +360,12 @@
# the next case/default.
assert node.kind in (tok.CASE, tok.DEFAULT)
prevnode = node.parent.kids[node.node_index-1]
- expectedfallthru = prevnode.end_pos(), node.start_pos()
+ expectedfallthru = prevnode.end_offset, node.start_offset
elif errname == 'missing_break_for_last_case':
# Find the end of the current case/default and the end of the
# switch.
assert node.parent.kind == tok.LC
- expectedfallthru = node.end_pos(), node.parent.end_pos()
+ expectedfallthru = node.end_offset, node.parent.end_offset
else:
expectedfallthru = None
@@ -377,11 +374,11 @@
for fallthru in fallthrus:
# Look for a fallthru between the end of the current case or
# default statement and the beginning of the next token.
- if fallthru.start_pos() > start and fallthru.end_pos() < end:
+ if fallthru.start_offset > start and fallthru.end_offset < end:
fallthrus.remove(fallthru)
return
- report_lint(node, errname, pos, **errargs)
+ report_lint(node, errname, offset, **errargs)
parse_errors = []
declares = []
@@ -390,8 +387,7 @@
fallthrus = []
passes = []
- node_positions = jsparse.NodePositions(script, scriptpos)
- possible_comments = jsparse.findpossiblecomments(script, node_positions)
+ possible_comments = jsparse.findpossiblecomments(script, script_offset)
# Check control comments for the correct version. It may be this comment
# isn't a valid comment (for example, it might be inside a string literal)
@@ -410,18 +406,18 @@
report(node, 'unsupported_version', version=parms)
if not jsparse.isvalidversion(jsversion):
- report_lint(jsversionnode, 'unsupported_version', scriptpos,
+ report_lint(jsversionnode, 'unsupported_version', script_offset,
version=jsversion.version)
return
- root = jsparse.parse(script, jsversion, parse_error, scriptpos)
+ root = jsparse.parse(script, jsversion, parse_error, script_offset)
if not root:
# Report errors and quit.
- for pos, msg, msg_args in parse_errors:
- report_native(pos, msg, msg_args)
+ for offset, msg, msg_args in parse_errors:
+ report_native(offset, msg, msg_args)
return
- comments = jsparse.filtercomments(possible_comments, node_positions, root)
+ comments = jsparse.filtercomments(possible_comments, root)
if jsversionnode is not None and jsversionnode not in comments:
# TODO
@@ -449,7 +445,7 @@
start_ignore = node
elif keyword == 'end':
if start_ignore:
- ignores.append((start_ignore.start_pos(), node.end_pos()))
+ ignores.append((start_ignore.start_offset, node.end_offset))
start_ignore = None
else:
report(node, 'mismatch_ctrl_comments')
@@ -471,8 +467,8 @@
# Report at the actual error of the location. Add two
# characters for the opening two characters.
if nested_comment >= 0:
- pos = node_positions.from_offset(node_positions.to_offset(comment.start_pos()) + 2 + nested_comment)
- report(comment, 'nested_comment', pos=pos)
+ offset = comment.start_offset + 2 + nested_comment
+ report(comment, 'nested_comment', offset=offset)
if comment.atom.lower().startswith('jsl:'):
report(comment, 'jsl_cc_not_understood')
elif comment.atom.startswith('@'):
@@ -481,8 +477,8 @@
report(start_ignore, 'mismatch_ctrl_comments')
# Wait to report parse errors until loading jsl:ignore directives.
- for pos, msg in parse_errors:
- report_native(pos, msg)
+ for offset, msg in parse_errors:
+ report_native(offset, msg)
# Find all visitors and convert them into "onpush" callbacks that call "report"
visitors = {
@@ -516,15 +512,15 @@
unused_scope.set_unused(name, node)
def _lint_script_parts(script_parts, script_cache, lint_error, conf, import_callback):
- def report_lint(node, errname, pos=None, **errargs):
+ def report_lint(node, errname, offset=0, **errargs):
errdesc = warnings.format_error(errname, **errargs)
- _report(pos or node.start_pos(), errname, errdesc, True)
+ _report(offset or node.start_offset, errname, errdesc, True)
- def report_native(pos, errname, errargs):
+ def report_native(offset, errname, errargs):
errdesc = warnings.format_error(errname, **errargs)
- _report(pos, errname, errdesc, False)
+ _report(offset, errname, errdesc, False)
- def _report(pos, errname, errdesc, require_key):
+ def _report(offset, errname, errdesc, require_key):
try:
if not conf[errname]:
return
@@ -533,14 +529,14 @@
raise
for start, end in ignores:
- if pos >= start and pos <= end:
+ if offset >= start and offset <= end:
return
- return lint_error(pos.line, pos.col, errname, errdesc)
+ return lint_error(offset, errname, errdesc)
- for scriptpos, jsversion, script in script_parts:
+ for script_offset, jsversion, script in script_parts:
ignores = []
- _lint_script_part(scriptpos, jsversion, script, script_cache, conf, ignores,
+ _lint_script_part(script_offset, jsversion, script, script_cache, conf, ignores,
report_native, report_lint, import_callback)
scope = script_cache.scope
@@ -576,10 +572,10 @@
# TODO: This is ugly hardcoding to improve the error positioning of
# "missing_semicolon" errors.
if visitor.warning in ('missing_semicolon', 'missing_semicolon_for_lambda'):
- pos = warning.node.end_pos()
+ offset = warning.node.end_offset
else:
- pos = None
- report(warning.node, visitor.warning, pos=pos, **warning.errargs)
+ offset = None
+ report(warning.node, visitor.warning, offset=offset, **warning.errargs)
return onpush
def _warn_or_declare(scope, name, type_, node, report):
Modified: trunk/javascriptlint/warnings.py
===================================================================
--- trunk/javascriptlint/warnings.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/javascriptlint/warnings.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -584,7 +584,7 @@
if filter(is_return_with_val, exit_points):
# If the function returns a value, find all returns without a value.
returns = filter(is_return_without_val, exit_points)
- returns.sort(key=lambda node: node.start_pos())
+ returns.sort(key=lambda node: node.start_offset)
if returns:
raise LintWarning(returns[0], name=name)
# Warn if the function sometimes exits naturally.
Modified: trunk/jsengine/__init__.py
===================================================================
--- trunk/jsengine/__init__.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/jsengine/__init__.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -10,12 +10,12 @@
)
class JSSyntaxError(BaseException):
- def __init__(self, pos, msg, msg_args=None):
+ def __init__(self, offset, msg, msg_args=None):
assert msg in _MESSAGES, msg
- self.pos = pos
+ self.offset = offset
self.msg = msg
self.msg_args = msg_args or {}
def __unicode__(self):
- return '%s: %s' % (self.pos, self.msg)
+ return '%s: %s' % (self.offset, self.msg)
def __repr__(self):
- return 'JSSyntaxError(%r, %r, %r)' % (self.pos, self.msg. self.msg_args)
+ return 'JSSyntaxError(%r, %r, %r)' % (self.offset, self.msg. self.msg_args)
Modified: trunk/jsengine/parser/__init__.py
===================================================================
--- trunk/jsengine/parser/__init__.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/jsengine/parser/__init__.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -22,16 +22,16 @@
"1.7",
]
-def _auto_semicolon(t, kind_, op_, startpos, endpos, atom, kids):
+def _auto_semicolon(t, kind_, op_, start_offset, end_offset, atom, kids):
nosemi = False
if t.peek_sameline().tok not in (tok.EOF, tok.EOL, tok.RBRACE):
x = t.advance()
if x.tok != tok.SEMI:
- raise JSSyntaxError(x.startpos, 'semi_before_stmnt')
- endpos = x.endpos
+ raise JSSyntaxError(x.start_offset, 'semi_before_stmnt')
+ end_offset = x.end_offset
else:
nosemi = True
- return ParseNode(kind_, op_, startpos, endpos, atom, kids, nosemi)
+ return ParseNode(kind_, op_, start_offset, end_offset, atom, kids, nosemi)
def _function_arglist(t):
fn_args = []
@@ -39,8 +39,8 @@
while True:
x = t.expect(tok.NAME)
fn_args.append(ParseNode(kind.NAME, op.ARGNAME,
- x.startpos,
- x.endpos, x.atom, []))
+ x.start_offset,
+ x.end_offset, x.atom, []))
if t.peek().tok == tok.COMMA:
t.advance()
else:
@@ -50,23 +50,23 @@
def _primary_expression(t):
x = t.next_withregexp()
if x.tok == tok.THIS:
- return ParseNode(kind.PRIMARY, op.THIS, x.startpos, x.endpos, None, [])
+ return ParseNode(kind.PRIMARY, op.THIS, x.start_offset, x.end_offset, None, [])
elif x.tok == tok.NAME:
- return ParseNode(kind.NAME, op.NAME, x.startpos, x.endpos, x.atom, [None])
+ return ParseNode(kind.NAME, op.NAME, x.start_offset, x.end_offset, x.atom, [None])
elif x.tok == tok.NULL:
- return ParseNode(kind.PRIMARY, op.NULL, x.startpos, x.endpos, None, [])
+ return ParseNode(kind.PRIMARY, op.NULL, x.start_offset, x.end_offset, None, [])
elif x.tok == tok.TRUE:
- return ParseNode(kind.PRIMARY, op.TRUE, x.startpos, x.endpos, None, [])
+ return ParseNode(kind.PRIMARY, op.TRUE, x.start_offset, x.end_offset, None, [])
elif x.tok == tok.FALSE:
- return ParseNode(kind.PRIMARY, op.FALSE, x.startpos, x.endpos, None, [])
+ return ParseNode(kind.PRIMARY, op.FALSE, x.start_offset, x.end_offset, None, [])
elif x.tok == tok.STRING:
- return ParseNode(kind.STRING, op.STRING, x.startpos, x.endpos, x.atom, [])
+ return ParseNode(kind.STRING, op.STRING, x.start_offset, x.end_offset, x.atom, [])
elif x.tok == tok.REGEXP:
- return ParseNode(kind.OBJECT, op.REGEXP, x.startpos, x.endpos, None, [])
+ return ParseNode(kind.OBJECT, op.REGEXP, x.start_offset, x.end_offset, None, [])
elif x.tok == tok.NUMBER:
- return ParseNode(kind.NUMBER, None, x.startpos, x.endpos, x.atom, [])
+ return ParseNode(kind.NUMBER, None, x.start_offset, x.end_offset, x.atom, [])
elif x.tok == tok.LBRACKET:
- startpos = x.startpos
+ start_offset = x.start_offset
items = []
end_comma = None
if t.peek().tok != tok.RBRACKET:
@@ -83,18 +83,18 @@
# Expect a comma and use it if the value was missing.
x = t.expect(tok.COMMA)
comma = ParseNode(kind.COMMA, None,
- x.startpos, x.endpos, None, [])
+ x.start_offset, x.end_offset, None, [])
items[-1] = items[-1] or comma
# Check for the end.
if t.peek().tok == tok.RBRACKET:
end_comma = comma
break
- endpos = t.expect(tok.RBRACKET).endpos
- return ParseNode(kind.RB, None, startpos, endpos, None, items,
+ end_offset = t.expect(tok.RBRACKET).end_offset
+ return ParseNode(kind.RB, None, start_offset, end_offset, None, items,
end_comma=end_comma)
elif x.tok == tok.LBRACE:
- startpos = x.startpos
+ start_offset = x.start_offset
kids = []
# TODO: get/set
end_comma = None
@@ -104,50 +104,50 @@
break
elif x.tok == tok.STRING:
t.expect(tok.STRING)
- key = ParseNode(kind.STRING, None, x.startpos,
- x.endpos, x.atom, [])
+ key = ParseNode(kind.STRING, None, x.start_offset,
+ x.end_offset, x.atom, [])
elif x.tok == tok.NUMBER:
t.expect(tok.NUMBER)
- key = ParseNode(kind.NUMBER, None, x.startpos,
- x.endpos, x.atom, [])
+ key = ParseNode(kind.NUMBER, None, x.start_offset,
+ x.end_offset, x.atom, [])
else:
x = t.expect_identifiername()
- key = ParseNode(kind.NAME, None, x.startpos, x.endpos,
+ key = ParseNode(kind.NAME, None, x.start_offset, x.end_offset,
x.atom, [])
t.expect(tok.COLON)
value = _assignment_expression(t, True)
- kids.append(ParseNode(kind.COLON, None, key.startpos,
- value.endpos, None, [key, value]))
+ kids.append(ParseNode(kind.COLON, None, key.start_offset,
+ value.end_offset, None, [key, value]))
if t.peek().tok == tok.COMMA:
x = t.advance()
end_comma = ParseNode(kind.COMMA, None,
- x.startpos, x.endpos, None, [])
+ x.start_offset, x.end_offset, None, [])
else:
end_comma = None
break
- endpos = t.expect(tok.RBRACE).endpos
- return ParseNode(kind.RC, None, startpos, endpos, None, kids,
+ end_offset = t.expect(tok.RBRACE).end_offset
+ return ParseNode(kind.RC, None, start_offset, end_offset, None, kids,
end_comma=end_comma)
elif x.tok == tok.LPAREN:
- startpos = x.startpos
+ start_offset = x.start_offset
kid = _expression(t, True)
- endpos = t.expect(tok.RPAREN).endpos
- return ParseNode(kind.RP, None, startpos, endpos, None, [kid])
+ end_offset = t.expect(tok.RPAREN).end_offset
+ return ParseNode(kind.RP, None, start_offset, end_offset, None, [kid])
else:
- raise JSSyntaxError(x.startpos, 'syntax_error')
+ raise JSSyntaxError(x.start_offset, 'syntax_error')
def _function_declaration(t, named_opcode):
node = _function_expression(t, named_opcode)
# Convert anonymous functions in expressions.
if node.opcode == op.ANONFUNOBJ:
- node = _auto_semicolon(t, kind.SEMI, None, node.startpos, node.endpos,
+ node = _auto_semicolon(t, kind.SEMI, None, node.start_offset, node.end_offset,
None, [node])
return node
def _function_expression(t, named_opcode):
- startpos = t.expect(tok.FUNCTION).startpos
+ start_offset = t.expect(tok.FUNCTION).start_offset
if t.peek().tok == tok.NAME:
fn_name = t.expect(tok.NAME).atom
opcode = named_opcode
@@ -157,12 +157,12 @@
t.expect(tok.LPAREN)
fn_args = _function_arglist(t)
t.expect(tok.RPAREN)
- fn_body_startpos = t.expect(tok.LBRACE).startpos
+ fn_body_start_offset = t.expect(tok.LBRACE).start_offset
kids = _sourceelements(t, tok.RBRACE)
- fn_body_endpos = t.expect(tok.RBRACE).endpos
- fn_body = ParseNode(kind.LC, None, fn_body_startpos,
- fn_body_endpos, None, kids)
- return ParseNode(kind.FUNCTION, opcode, startpos, fn_body.endpos,
+ fn_body_end_offset = t.expect(tok.RBRACE).end_offset
+ fn_body = ParseNode(kind.LC, None, fn_body_start_offset,
+ fn_body_end_offset, None, kids)
+ return ParseNode(kind.FUNCTION, opcode, start_offset, fn_body.end_offset,
fn_name, [fn_body], fn_args=fn_args)
def _argument_list(t):
@@ -177,17 +177,17 @@
return args
def _new_expression(t):
- startpos = t.expect(tok.NEW).startpos
+ start_offset = t.expect(tok.NEW).start_offset
expr = _member_expression(t)
# If no (), this is a variant of the NewExpression
if t.peek().tok == tok.LPAREN:
t.expect(tok.LPAREN)
args = _argument_list(t)
- endpos = t.expect(tok.RPAREN).endpos
+ end_offset = t.expect(tok.RPAREN).end_offset
else:
args = []
- endpos = expr.endpos
- return ParseNode(kind.NEW, op.NEW, startpos, endpos,
+ end_offset = expr.end_offset
+ return ParseNode(kind.NEW, op.NEW, start_offset, end_offset,
None, [expr] + args)
def _member_expression(t, _recurse=True):
@@ -203,13 +203,13 @@
if t.peek().tok == tok.LBRACKET:
t.advance()
expr = _expression(t, True)
- endpos = t.expect(tok.RBRACKET).endpos
- kid = ParseNode(kind.LB, op.GETELEM, kid.startpos, endpos,
+ end_offset = t.expect(tok.RBRACKET).end_offset
+ kid = ParseNode(kind.LB, op.GETELEM, kid.start_offset, end_offset,
None, [kid, expr])
elif t.peek().tok == tok.DOT:
t.advance()
expr = t.expect_identifiername()
- kid = ParseNode(kind.DOT, op.GETPROP, kid.startpos, expr.endpos,
+ kid = ParseNode(kind.DOT, op.GETPROP, kid.start_offset, expr.end_offset,
expr.atom, [kid])
else:
return kid
@@ -224,21 +224,21 @@
if x.tok == tok.LPAREN:
t.expect(tok.LPAREN)
args = _argument_list(t)
- endpos = t.expect(tok.RPAREN).endpos
- expr = ParseNode(kind.LP, op.CALL, expr.startpos,
- endpos, None, [expr] + args)
+ end_offset = t.expect(tok.RPAREN).end_offset
+ expr = ParseNode(kind.LP, op.CALL, expr.start_offset,
+ end_offset, None, [expr] + args)
elif x.tok == tok.LBRACKET:
t.expect(tok.LBRACKET)
lookup = _expression(t, True)
- endpos = t.expect(tok.RBRACKET).endpos
+ end_offset = t.expect(tok.RBRACKET).end_offset
expr = ParseNode(kind.LB, op.GETELEM,
- expr.startpos, endpos,
+ expr.start_offset, end_offset,
None, [expr, lookup])
elif x.tok == tok.DOT:
t.expect(tok.DOT)
lookup = t.expect_identifiername()
expr = ParseNode(kind.DOT, op.GETPROP,
- expr.startpos, lookup.endpos,
+ expr.start_offset, lookup.end_offset,
lookup.atom, [expr])
else:
return expr
@@ -251,17 +251,17 @@
def _postfix_expression(t):
kid = _lefthandside_expression(t)
if t.peek_sameline().tok == tok.INC:
- endpos = t.expect(tok.INC).endpos
+ end_offset = t.expect(tok.INC).end_offset
if kid.kind == kind.DOT and kid.opcode == op.GETPROP:
opcode = op.PROPINC
else:
opcode = op.NAMEINC
return ParseNode(kind.INC, opcode,
- kid.startpos, endpos, None, [kid])
+ kid.start_offset, end_offset, None, [kid])
elif t.peek_sameline().tok == tok.DEC:
- endpos = t.expect(tok.DEC).endpos
+ end_offset = t.expect(tok.DEC).end_offset
return ParseNode(kind.DEC, op.NAMEDEC,
- kid.startpos, endpos, None, [kid])
+ kid.start_offset, end_offset, None, [kid])
else:
return kid
@@ -280,9 +280,9 @@
x = t.peek()
if x.tok in _UNARY:
kind_, op_ = _UNARY[x.tok]
- startpos = t.advance().startpos
+ start_offset = t.advance().start_offset
kid = _unary_expression(t)
- return ParseNode(kind_, op_, startpos, kid.endpos, None, [kid])
+ return ParseNode(kind_, op_, start_offset, kid.end_offset, None, [kid])
else:
return _postfix_expression(t)
@@ -300,7 +300,7 @@
t.advance()
kids.append(child_expr_callback(t))
expr = ParseNode(kind_, op_,
- kids[0].startpos, kids[1].endpos,
+ kids[0].start_offset, kids[1].end_offset,
None, kids)
_MULTIPLICATIVE = {
@@ -359,7 +359,7 @@
t.advance()
right = _equality_expression(t, allowin)
left = ParseNode(kind.BITAND, op.BITAND,
- left.startpos, right.endpos,
+ left.start_offset, right.end_offset,
None, [left, right])
return left
@@ -369,7 +369,7 @@
t.advance()
right = _bitwise_and_expression(t, allowin)
left = ParseNode(kind.BITXOR, op.BITXOR,
- left.startpos, right.endpos,
+ left.start_offset, right.end_offset,
None, [left, right])
return left
@@ -379,7 +379,7 @@
t.advance()
right = _bitwise_xor_expression(t, allowin)
left = ParseNode(kind.BITOR, op.BITOR,
- left.startpos, right.endpos,
+ left.start_offset, right.end_offset,
None, [left, right])
return left
@@ -396,7 +396,7 @@
right = exprs.pop()
left = exprs[-1]
exprs[-1] = ParseNode(kind.AND, op.AND,
- left.startpos, right.endpos,
+ left.start_offset, right.end_offset,
None, [left, right])
return exprs[0]
@@ -413,7 +413,7 @@
right = exprs.pop()
left = exprs[-1]
exprs[-1] = ParseNode(kind.OR, op.OR,
- left.startpos, right.endpos,
+ left.start_offset, right.end_offset,
None, [left, right])
return exprs[0]
@@ -425,7 +425,7 @@
t.expect(tok.COLON)
else_ = _assignment_expression(t, allowin)
return ParseNode(kind.HOOK, None,
- kid.startpos, else_.endpos,
+ kid.start_offset, else_.end_offset,
None, [kid, if_, else_])
else:
return kid
@@ -463,12 +463,12 @@
assert kid.opcode == op.CALL
kid.opcode = op.SETCALL
else:
- raise JSSyntaxError(left.startpos, 'invalid_assign')
+ raise JSSyntaxError(left.start_offset, 'invalid_assign')
kind_, op_ = _ASSIGNS[t.peek().tok]
t.advance()
right = _assignment_expression(t, allowin)
return ParseNode(kind_, op_,
- left.startpos, right.endpos, None, [left, right])
+ left.start_offset, right.end_offset, None, [left, right])
else:
return left
@@ -479,8 +479,8 @@
t.advance()
items.append(_assignment_expression(t, allowin))
if len(items) > 1:
- return ParseNode(kind.COMMA, None, items[0].startpos,
- items[-1].endpos, None, items)
+ return ParseNode(kind.COMMA, None, items[0].start_offset,
+ items[-1].end_offset, None, items)
else:
return items[0]
@@ -493,8 +493,8 @@
t.advance()
value = _assignment_expression(t, allowin)
nodes.append(ParseNode(kind.NAME, op.SETNAME if value else op.NAME,
- x.startpos,
- value.endpos if value else x.endpos,
+ x.start_offset,
+ value.end_offset if value else x.end_offset,
x.atom, [value]))
if t.peek().tok == tok.COMMA:
@@ -504,27 +504,27 @@
def _block_statement(t):
kids = []
- startpos = t.expect(tok.LBRACE).startpos
+ start_offset = t.expect(tok.LBRACE).start_offset
while t.peek().tok != tok.RBRACE:
kids.append(_statement(t))
- endpos = t.expect(tok.RBRACE).endpos
- return ParseNode(kind.LC, None, startpos, endpos, None, kids)
+ end_offset = t.expect(tok.RBRACE).end_offset
+ return ParseNode(kind.LC, None, start_offset, end_offset, None, kids)
def _empty_statement(t):
# EMPTY STATEMENT
x = t.expect(tok.SEMI)
- return ParseNode(kind.SEMI, None, x.startpos, x.endpos, None, [None])
+ return ParseNode(kind.SEMI, None, x.start_offset, x.end_offset, None, [None])
def _var_statement(t):
# VARIABLE STATEMENT
- startpos = t.expect(tok.VAR).startpos
+ start_offset = t.expect(tok.VAR).start_offset
nodes = _variable_declaration(t, True)
return _auto_semicolon(t, kind.VAR, op.DEFVAR,
- startpos, nodes[-1].endpos, None, nodes)
+ start_offset, nodes[-1].end_offset, None, nodes)
def _if_statement(t):
# IF STATEMENT
- startpos = t.expect(tok.IF).startpos
+ start_offset = t.expect(tok.IF).start_offset
t.expect(tok.LPAREN)
condition = _expression(t, True)
t.expect(tok.RPAREN)
@@ -534,38 +534,38 @@
else_body = _statement(t)
else:
else_body = None
- endpos = else_body.endpos if else_body else if_body.endpos
- return ParseNode(kind.IF, None, startpos,
- endpos, None, [condition, if_body, else_body])
+ end_offset = else_body.end_offset if else_body else if_body.end_offset
+ return ParseNode(kind.IF, None, start_offset,
+ end_offset, None, [condition, if_body, else_body])
def _do_statement(t):
- startpos = t.expect(tok.DO).startpos
+ start_offset = t.expect(tok.DO).start_offset
code = _statement(t)
t.expect(tok.WHILE)
t.expect(tok.LPAREN)
expr = _expression(t, True)
endtoken = t.expect(tok.RPAREN)
return _auto_semicolon(t, kind.DO, None,
- startpos, endtoken.endpos, None, [code, expr])
+ start_offset, endtoken.end_offset, None, [code, expr])
def _while_statement(t):
- startpos = t.expect(tok.WHILE).startpos
+ start_offset = t.expect(tok.WHILE).start_offset
t.expect(tok.LPAREN)
expr = _expression(t, True)
t.expect(tok.RPAREN)
code = _statement(t)
return ParseNode(kind.WHILE, None,
- startpos, code.endpos, None, [expr, code])
+ start_offset, code.end_offset, None, [expr, code])
def _for_statement(t):
- for_startpos = t.expect(tok.FOR).startpos
+ for_start_offset = t.expect(tok.FOR).start_offset
t.expect(tok.LPAREN)
for_exprs = []
if t.peek().tok == tok.VAR:
- var_startpos = t.advance().startpos
+ var_start_offset = t.advance().start_offset
kids = _variable_declaration(t, False)
- vars = ParseNode(kind.VAR, op.DEFVAR, var_startpos, kids[-1].endpos,
+ vars = ParseNode(kind.VAR, op.DEFVAR, var_start_offset, kids[-1].end_offset,
None, kids)
if t.peek().tok == tok.IN:
@@ -589,8 +589,8 @@
for_exprs = [expr, None, None]
if len(for_exprs) == 2:
- condition = ParseNode(kind.IN, None, for_exprs[0].startpos,
- for_exprs[-1].endpos, None, for_exprs)
+ condition = ParseNode(kind.IN, None, for_exprs[0].start_offset,
+ for_exprs[-1].end_offset, None, for_exprs)
else:
x = t.expect(tok.SEMI)
if t.peek().tok != tok.SEMI:
@@ -605,12 +605,12 @@
body = _statement(t)
return ParseNode(kind.FOR,
op.FORIN if condition.kind == kind.IN else None,
- for_startpos, body.endpos,
+ for_start_offset, body.end_offset,
None, [condition, body])
def _continue_statement(t):
endtoken = t.expect(tok.CONTINUE)
- startpos = endtoken.startpos
+ start_offset = endtoken.start_offset
if t.peek_sameline().tok == tok.NAME:
endtoken = t.expect(tok.NAME)
@@ -618,11 +618,11 @@
else:
name = None
# TODO: Validate Scope Labels
- return _auto_semicolon(t, kind.CONTINUE, None, startpos, endtoken.endpos, name, [])
+ return _auto_semicolon(t, kind.CONTINUE, None, start_offset, endtoken.end_offset, name, [])
def _break_statement(t):
endtoken = t.expect(tok.BREAK)
- startpos = endtoken.startpos
+ start_offset = endtoken.start_offset
if t.peek_sameline().tok == tok.NAME:
endtoken = t.expect(tok.NAME)
@@ -630,11 +630,11 @@
else:
name = None
# TODO: Validate Scope Labels
- return _auto_semicolon(t, kind.BREAK, None, startpos, endtoken.endpos, name, [])
+ return _auto_semicolon(t, kind.BREAK, None, start_offset, endtoken.end_offset, name, [])
def _return_statement(t):
endtoken = t.expect(tok.RETURN)
- startpos = endtoken.startpos
+ start_offset = endtoken.start_offset
if t.peek_sameline().tok not in (tok.EOF, tok.EOL, tok.SEMI, tok.RBRACE):
expr = _expression(t, True)
@@ -642,108 +642,108 @@
else:
expr = None
# TODO: Validate Scope Labels
- return _auto_semicolon(t, kind.RETURN, None, startpos, endtoken.endpos,
+ return _auto_semicolon(t, kind.RETURN, None, start_offset, endtoken.end_offset,
None, [expr])
def _with_statement(t):
- startpos = t.expect(tok.WITH).startpos
+ start_offset = t.expect(tok.WITH).start_offset
t.expect(tok.LPAREN)
expr = _expression(t, True)
t.expect(tok.RPAREN)
body = _statement(t)
- return ParseNode(kind.WITH, None, startpos, body.endpos, None, [expr, body])
+ return ParseNode(kind.WITH, None, start_offset, body.end_offset, None, [expr, body])
def _switch_statement(t):
- switch_startpos = t.expect(tok.SWITCH).startpos
+ switch_start_offset = t.expect(tok.SWITCH).start_offset
t.expect(tok.LPAREN)
expr = _expression(t, True)
t.expect(tok.RPAREN)
- lc_startpos = t.expect(tok.LBRACE).startpos
+ lc_start_offset = t.expect(tok.LBRACE).start_offset
cases = []
while t.peek().tok != tok.RBRACE:
case_kind = None
case_expr = None
if t.peek().tok == tok.CASE:
- case_startpos = t.advance().startpos
+ case_start_offset = t.advance().start_offset
case_kind = kind.CASE
case_expr = _expression(t, True)
elif t.peek().tok == tok.DEFAULT:
- case_startpos = t.advance().startpos
+ case_start_offset = t.advance().start_offset
case_kind = kind.DEFAULT
else:
- raise JSSyntaxError(t.peek().startpos, 'invalid_case')
+ raise JSSyntaxError(t.peek().start_offset, 'invalid_case')
- case_endpos = t.expect(tok.COLON).endpos
+ case_end_offset = t.expect(tok.COLON).end_offset
statements = []
while t.peek().tok not in (tok.DEFAULT, tok.CASE, tok.RBRACE):
statements.append(_statement(t))
if statements:
- statements_startpos = statements[0].startpos
- statements_endpos = statements[-1].endpos
- case_endpos = statements[-1].endpos
+ statements_start_offset = statements[0].start_offset
+ statements_end_offset = statements[-1].end_offset
+ case_end_offset = statements[-1].end_offset
else:
- statements_startpos = case_endpos
- statements_endpos = case_endpos
+ statements_start_offset = case_end_offset
+ statements_end_offset = case_end_offset
- cases.append(ParseNode(case_kind, None, case_startpos, case_endpos,
+ cases.append(ParseNode(case_kind, None, case_start_offset, case_end_offset,
None, [
case_expr,
- ParseNode(kind.LC, None, statements_startpos,
- statements_endpos, None, statements)
+ ParseNode(kind.LC, None, statements_start_offset,
+ statements_end_offset, None, statements)
]))
- rc_endpos = t.expect(tok.RBRACE).endpos
- return ParseNode(kind.SWITCH, None, switch_startpos, rc_endpos,
+ rc_end_offset = t.expect(tok.RBRACE).end_offset
+ return ParseNode(kind.SWITCH, None, switch_start_offset, rc_end_offset,
None, [expr,
- ParseNode(kind.LC, None, lc_startpos, rc_endpos, None, cases)])
+ ParseNode(kind.LC, None, lc_start_offset, rc_end_offset, None, cases)])
def _throw_statement(t):
# TODO: Validate Scope
- startpos = t.expect(tok.THROW).startpos
+ start_offset = t.expect(tok.THROW).start_offset
if t.peek_sameline().tok == tok.EOL:
- raise JSSyntaxError(t.peek_sameline().startpos, 'expected_statement')
+ raise JSSyntaxError(t.peek_sameline().start_offset, 'expected_statement')
expr = _expression(t, True)
- return _auto_semicolon(t, kind.THROW, op.THROW, startpos, expr.endpos,
+ return _auto_semicolon(t, kind.THROW, op.THROW, start_offset, expr.end_offset,
None, [expr])
def _try_statement(t):
- try_startpos = t.expect(tok.TRY).startpos
+ try_start_offset = t.expect(tok.TRY).start_offset
try_node = _block_statement(t)
catch_node = None
finally_node = None
- try_endpos = None
+ try_end_offset = None
if t.peek().tok == tok.CATCH:
- catch_startpos = t.advance().startpos
+ catch_start_offset = t.advance().start_offset
t.expect(tok.LPAREN)
x = t.expect(tok.NAME)
- catch_expr = ParseNode(kind.NAME, None, x.startpos, x.endpos,
+ catch_expr = ParseNode(kind.NAME, None, x.start_offset, x.end_offset,
x.atom, [None])
t.expect(tok.RPAREN)
catch_block = _block_statement(t)
- catch_endpos = catch_block.endpos
+ catch_end_offset = catch_block.end_offset
catch_node = \
ParseNode(kind.RESERVED, None, None, None, None, [
ParseNode(kind.LEXICALSCOPE, op.LEAVEBLOCK,
- catch_startpos, catch_endpos, None, [
- ParseNode(kind.CATCH, None, catch_startpos,
- catch_endpos, None,
+ catch_start_offset, catch_end_offset, None, [
+ ParseNode(kind.CATCH, None, catch_start_offset,
+ catch_end_offset, None,
[catch_expr, None, catch_block])
])
])
- try_endpos = catch_endpos
+ try_end_offset = catch_end_offset
if t.peek().tok == tok.FINALLY:
t.advance()
finally_node = _block_statement(t)
- try_endpos = finally_node.endpos
+ try_end_offset = finally_node.end_offset
if not catch_node and not finally_node:
- raise JSSyntaxError(try_endpos, 'invalid_catch')
+ raise JSSyntaxError(try_end_offset, 'invalid_catch')
- return ParseNode(kind.TRY, None, try_startpos, try_endpos,
+ return ParseNode(kind.TRY, None, try_start_offset, try_end_offset,
None,
[try_node, catch_node, finally_node])
@@ -779,7 +779,7 @@
elif x.tok == tok.TRY:
return _try_statement(t)
elif x.tok == tok.EOF:
- raise JSSyntaxError(x.startpos, 'unexpected_eof')
+ raise JSSyntaxError(x.start_offset, 'unexpected_eof')
elif x.tok == tok.FUNCTION:
return _function_declaration(t, op.CLOSURE) #TODO: warn, since this is not reliable
@@ -788,13 +788,13 @@
if expr.kind == tok.NAME and t.peek().tok == tok.COLON:
t.expect(tok.COLON)
stmt = _statement(t)
- return ParseNode(kind.COLON, op.NAME, expr.startpos,
- stmt.endpos, expr.atom, [stmt])
+ return ParseNode(kind.COLON, op.NAME, expr.start_offset,
+ stmt.end_offset, expr.atom, [stmt])
- return _auto_semicolon(t, kind.SEMI, None, expr.startpos, expr.endpos,
+ return _auto_semicolon(t, kind.SEMI, None, expr.start_offset, expr.end_offset,
None, [expr])
else:
- raise JSSyntaxError(x.startpos, 'syntax_error')
+ raise JSSyntaxError(x.start_offset, 'syntax_error')
def _sourceelements(t, end_tok):
nodes = []
@@ -807,13 +807,14 @@
else:
nodes.append(_statement(t))
-def parsestring(s, startpos=None):
- stream = tokenizer.TokenStream(s, startpos)
+def parsestring(s, start_offset=0):
+ assert not start_offset is None
+ stream = tokenizer.TokenStream(s, start_offset)
t = tokenizer.Tokenizer(stream)
nodes = _sourceelements(t, tok.EOF)
- lc_endpos = t.expect(tok.EOF).endpos
- lc_startpos = nodes[-1].startpos if nodes else lc_endpos
- return ParseNode(kind.LC, None, lc_startpos, lc_endpos, None, nodes)
+ lc_end_offset = t.expect(tok.EOF).end_offset
+ lc_start_offset = nodes[-1].start_offset if nodes else lc_end_offset
+ return ParseNode(kind.LC, None, lc_start_offset, lc_end_offset, None, nodes)
def is_valid_version(version):
return version in _VERSIONS
@@ -824,14 +825,13 @@
assert kid.parent is node
_validate(kid, depth+1)
-def parse(script, jsversion,
- error_callback, startpos):
+def parse(script, jsversion, error_callback, start_offset):
# TODO: respect version
assert is_valid_version(jsversion)
try:
- root = parsestring(script, startpos)
+ root = parsestring(script, start_offset)
except JSSyntaxError as error:
- error_callback(error.pos.line, error.pos.col, error.msg, error.msg_args)
+ error_callback(error.offset, error.msg, error.msg_args)
return None
_validate(root)
return root
@@ -868,8 +868,8 @@
self.assertEquals(right.kind, kind.RC)
node = right.end_comma
self.assertEquals(node.kind, tok.COMMA)
- self.assertEquals(node.startpos, NodePos(0, 6))
- self.assertEquals(node.endpos, NodePos(0, 6))
+ self.assertEquals(node.start_offset, NodePos(0, 6))
+ self.assertEquals(node.end_offset, NodePos(0, 6))
def _testArrayEndComma(self, script, col):
root = parsestring(script)
node, = root.kids
@@ -885,8 +885,8 @@
self.assert_(node is None)
else:
self.assertEquals(node.kind, tok.COMMA)
- self.assertEquals(node.startpos, NodePos(0, col))
- self.assertEquals(node.endpos, NodePos(0, col))
+ self.assertEquals(node.start_offset, NodePos(0, col))
+ self.assertEquals(node.end_offset, NodePos(0, col))
def testArrayEndComma(self):
self._testArrayEndComma('a=[,]', 3)
self._testArrayEndComma('a=[a,]', 4)
Modified: trunk/jsengine/structs.py
===================================================================
--- trunk/jsengine/structs.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/jsengine/structs.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -90,26 +90,27 @@
class ParseNode:
node_index = None
parent = None
- def __init__(self, kind_, op_, start_pos, end_pos, atom, kids,
+ def __init__(self, kind_, op_, start_offset, end_offset, atom, kids,
no_semi=False, end_comma=None, fn_args=None):
assert not kids is None
assert kind.contains(kind_)
assert op_ is None or op.contains(op_)
if kind_ == kind.RESERVED:
- assert start_pos is None
- assert end_pos is None
+ assert start_offset is None
+ assert end_offset is None
else:
- assert isinstance(start_pos, NodePos), repr(start_pos)
- assert isinstance(end_pos, NodePos), repr(end_pos)
+ assert isinstance(start_offset, int), repr(start_offset)
+ assert isinstance(end_offset, int), repr(end_offset)
assert end_comma is None or isinstance(end_comma, ParseNode)
- assert (start_pos is None and end_pos is None) or start_pos <= end_pos
+ assert (start_offset is None and end_offset is None) or start_offset <= end_offset, \
+ (start_offset, end_offset)
self.kind = kind_
self.opcode = op_
self.atom = atom
self.kids = kids
self._lefthandside = False
- self.startpos = start_pos
- self.endpos = end_pos
+ self.start_offset = start_offset
+ self.end_offset = end_offset
self.no_semi = no_semi
self.end_comma = end_comma
@@ -132,11 +133,6 @@
else:
self.dval = float(self.atom)
- def start_pos(self):
- return self.startpos
- def end_pos(self):
- return self.endpos
-
def is_equivalent(self, other, are_functions_equiv=False):
if not other:
return False
Modified: trunk/jsengine/tokenizer/__init__.py
===================================================================
--- trunk/jsengine/tokenizer/__init__.py 2013-10-03 20:13:37 UTC (rev 333)
+++ trunk/jsengine/tokenizer/__init__.py 2013-10-08 16:36:44 UTC (rev 334)
@@ -1,6 +1,5 @@
# vim: sw=4 ts=4 et
from jsengine import JSSyntaxError
-from jsengine.structs import NodePositions
_WHITESPACE = u'\u0020\t\u000B\u000C\u00A0\uFFFF'
_LINETERMINATOR = u'\u000A\u000D\u2028\u2029'
@@ -141,59 +140,60 @@
def __init__(self, tok, atom=None):
self.tok = tok
self.atom = atom
- self.startpos = None
- self.endpos = None
- def setpos(self, startpos, endpos):
- self.startpos = startpos
- self.endpos = endpos
+ self.start_offset = None
+ self.end_offset = None
+ def set_offset(self, start_offset, end_offset):
+ self.start_offset = start_offset
+ self.end_offset = end_offset
def __repr__(self):
return 'Token(%r, %r)' % \
(self.tok, self.atom)
class TokenStream:
- def __init__(self, content, startpos=None):
+ def __init__(self, content, start_offset=0):
+ assert isinstance(start_offset, int)
self._content = content
- self._pos = 0
- self._watched_pos = None
- self._nodepositions = NodePositions(content, startpos)
+ self._start_offset = start_offset
+ self._offset = 0
+ self._watched_offset = None
- def getpos(self, offset=0):
- return self._nodepositions.from_offset(self._pos+offset)
+ def get_offset(self, offset=0):
+ return self._start_offset + self._offset + offset
def watch_reads(self):
- self._watched_pos = self._pos
+ self._watched_offset = self._offset
def get_watched_reads(self):
- assert not self._watched_pos == None
- s = self._content[self._watched_pos:self._pos]...
[truncated message content] |
