From: <fwi...@us...> - 2009-10-28 16:01:33
|
Revision: 6925 http://jython.svn.sourceforge.net/jython/?rev=6925&view=rev Author: fwierzbicki Date: 2009-10-28 16:01:21 +0000 (Wed, 28 Oct 2009) Log Message: ----------- Bringing in pypy's ast.py generator to see if I can munge it to work for Jython. Added Paths: ----------- trunk/jython/ast/asdl_py.py Added: trunk/jython/ast/asdl_py.py =================================================================== --- trunk/jython/ast/asdl_py.py (rev 0) +++ trunk/jython/ast/asdl_py.py 2009-10-28 16:01:21 UTC (rev 6925) @@ -0,0 +1,642 @@ +""" +Generate AST node definitions from an ASDL description. +""" + +import sys +import os +import asdl + + +class ASDLVisitor(asdl.VisitorBase): + + def __init__(self, stream, data): + super(ASDLVisitor, self).__init__() + self.stream = stream + self.data = data + + def visitModule(self, mod, *args): + for df in mod.dfns: + self.visit(df, *args) + + def visitSum(self, sum, *args): + for tp in sum.types: + self.visit(tp, *args) + + def visitType(self, tp, *args): + self.visit(tp.value, *args) + + def visitProduct(self, prod, *args): + for field in prod.fields: + self.visit(field, *args) + + def visitConstructor(self, cons, *args): + for field in cons.fields: + self.visit(field, *args) + + def visitField(self, field): + pass + + def emit(self, line, level=0): + indent = " "*level + self.stream.write(indent + line + "\n") + + +def is_simple_sum(sum): + assert isinstance(sum, asdl.Sum) + for constructor in sum.types: + if constructor.fields: + return False + return True + + +class ASTNodeVisitor(ASDLVisitor): + + def visitType(self, tp): + self.visit(tp.value, tp.name) + + def visitSum(self, sum, base): + if is_simple_sum(sum): + self.emit("class %s(AST):" % (base,)) + self.emit("") + self.emit("def to_simple_int(self, space):", 1) + self.emit("w_msg = space.wrap(\"not a valid %s\")" % (base,), 2) + self.emit("raise OperationError(space.w_TypeError, w_msg)", 2) + self.emit("") + for i, cons in enumerate(sum.types): + self.emit("class _%s(%s):" % (cons.name, base)) + self.emit("") + self.emit("def to_simple_int(self, space):", 1) + self.emit("return %i" % (i + 1,), 2) + self.emit("") + for i, cons in enumerate(sum.types): + self.emit("%s = %i" % (cons.name, i + 1)) + self.emit("") + self.emit("%s_to_class = [" % (base,)) + for cons in sum.types: + self.emit("_%s," % (cons.name,), 1) + self.emit("]") + self.emit("") + else: + self.emit("class %s(AST):" % (base,)) + self.emit("") + slots = ", ".join(repr(attr.name.value) for attr in sum.attributes) + self.emit("__slots__ = (%s)" % (slots,), 1) + self.emit("") + if sum.attributes: + args = ", ".join(attr.name.value for attr in sum.attributes) + self.emit("def __init__(self, %s):" % (args,), 1) + for attr in sum.attributes: + self.visit(attr) + self.emit("") + for cons in sum.types: + self.visit(cons, base, sum.attributes) + self.emit("") + + def visitProduct(self, product, name): + self.emit("class %s(AST):" % (name,)) + self.emit("") + slots = self.make_slots(product.fields) + self.emit("__slots__ = (%s)" % (slots,), 1) + self.emit("") + self.make_constructor(product.fields, product) + self.emit("") + self.emit("def walkabout(self, visitor):", 1) + self.emit("visitor.visit_%s(self)" % (name,), 2) + self.emit("") + self.make_var_syncer(product.fields, product, name) + + def make_slots(self, fields): + slots = [] + for field in fields: + name = repr(field.name.value) + slots.append(name) + if field.seq: + slots.append("'w_%s'" % (field.name,)) + return ", ".join(slots) + + def make_var_syncer(self, fields, node, name): + self.emit("def sync_app_attrs(self, space):", 1) + config = (self.data.optional_masks[node], + self.data.required_masks[node]) + self.emit("if (self.initialization_state & ~%i) ^ %i:" % config, 2) + names = [] + for field in fields: + if field.opt: + names.append("None") + else: + names.append(repr(field.name.value)) + sub = (", ".join(names), name.value) + self.emit("missing_field(space, self.initialization_state, [%s], %r)" + % sub, 3) + self.emit("else:", 2) + # Fill in all the default fields. + doing_something = False + for field in fields: + if field.opt: + doing_something = True + flag = self.data.field_masks[field] + self.emit("if not self.initialization_state & %i:" % (flag,), 3) + default = "0" if field.type.value == "int" else "None" + self.emit("self.%s = %s" % (field.name, default), 4) + if not doing_something: + self.emit("pass", 3) + for attr in fields: + if attr.seq: + self.emit("w_list = self.w_%s" % (attr.name,), 2) + self.emit("if w_list is not None:", 2) + self.emit("list_w = space.viewiterable(w_list)", 3) + self.emit("if list_w:", 3) + unwrapper = get_unwrapper(attr.type.value, "w_obj", + self.data.simple_types) + config = (attr.name, unwrapper) + self.emit("self.%s = [%s for w_obj in list_w]" % config, + 4), + self.emit("else:", 3) + self.emit("self.%s = None" % (attr.name,), 4) + if attr.type.value not in asdl.builtin_types and \ + attr.type.value not in self.data.simple_types: + self.emit("if self.%s is not None:" % (attr.name,), 2) + self.emit("for node in self.%s:" % (attr.name,), 3) + self.emit("node.sync_app_attrs(space)", 4) + elif attr.type.value not in asdl.builtin_types and \ + attr.type.value not in self.data.simple_types: + doing_something = True + level = 2 + if attr.opt: + self.emit("if self.%s:" % (attr.name,), 2) + level += 1 + self.emit("self.%s.sync_app_attrs(space)" % (attr.name,), level) + self.emit("") + + def make_constructor(self, fields, node, extras=None, base=None): + if fields or extras: + arg_fields = fields + extras if extras else fields + args = ", ".join(str(field.name) for field in arg_fields) + self.emit("def __init__(self, %s):" % args, 1) + for field in fields: + self.visit(field) + if extras: + base_args = ", ".join(str(field.name) for field in extras) + self.emit("%s.__init__(self, %s)" % (base, base_args), 2) + else: + self.emit("def __init__(self):", 1) + have_everything = self.data.required_masks[node] | \ + self.data.optional_masks[node] + self.emit("self.initialization_state = %i" % (have_everything,), 2) + + def visitConstructor(self, cons, base, extra_attributes): + self.emit("class %s(%s):" % (cons.name, base)) + self.emit("") + slots = self.make_slots(cons.fields) + self.emit("__slots__ = (%s)" % (slots,), 1) + self.emit("") + for field in self.data.cons_attributes[cons]: + subst = (field.name, self.data.field_masks[field]) + self.emit("_%s_mask = %i" % subst, 1) + self.emit("") + self.make_constructor(cons.fields, cons, extra_attributes, base) + self.emit("") + self.emit("def walkabout(self, visitor):", 1) + self.emit("visitor.visit_%s(self)" % (cons.name,), 2) + self.emit("") + self.emit("def mutate_over(self, visitor):", 1) + for field in cons.fields: + if field.type.value not in asdl.builtin_types and \ + field.type.value not in self.data.prod_simple: + if field.opt or field.seq: + level = 3 + self.emit("if self.%s:" % (field.name,), 2) + else: + level = 2 + if field.seq: + sub = (field.name,) + self.emit("visitor._mutate_sequence(self.%s)" % sub, level) + else: + sub = (field.name, field.name) + self.emit("self.%s = self.%s.mutate_over(visitor)" % sub, + level) + self.emit("return visitor.visit_%s(self)" % (cons.name,), 2) + self.emit("") + self.make_var_syncer(cons.fields + self.data.cons_attributes[cons], + cons, cons.name) + + def visitField(self, field): + self.emit("self.%s = %s" % (field.name, field.name), 2) + if field.seq: + self.emit("self.w_%s = None" % (field.name,), 2) + + +class ASTVisitorVisitor(ASDLVisitor): + """A meta visitor! :)""" + + def visitModule(self, mod): + self.emit("class ASTVisitor(object):") + self.emit("") + self.emit("def visit_sequence(self, seq):", 1) + self.emit("for node in seq:", 2) + self.emit("node.walkabout(self)", 3) + self.emit("") + self.emit("def default_visitor(self, node):", 1) + self.emit("raise NodeVisitorNotImplemented", 2) + self.emit("") + self.emit("def _mutate_sequence(self, seq):", 1) + self.emit("for i in range(len(seq)):", 2) + self.emit("seq[i] = seq[i].mutate_over(self)", 3) + self.emit("") + super(ASTVisitorVisitor, self).visitModule(mod) + self.emit("") + + def visitType(self, tp): + if not (isinstance(tp.value, asdl.Sum) and + is_simple_sum(tp.value)): + super(ASTVisitorVisitor, self).visitType(tp, tp.name) + + def visitProduct(self, prod, name): + self.emit("def visit_%s(self, node):" % (name,), 1) + self.emit("return self.default_visitor(node)", 2) + + def visitConstructor(self, cons, _): + self.emit("def visit_%s(self, node):" % (cons.name,), 1) + self.emit("return self.default_visitor(node)", 2) + + +class GenericASTVisitorVisitor(ASDLVisitor): + + def visitModule(self, mod): + self.emit("class GenericASTVisitor(ASTVisitor):") + self.emit("") + super(GenericASTVisitorVisitor, self).visitModule(mod) + self.emit("") + + def visitType(self, tp): + if not (isinstance(tp.value, asdl.Sum) and + is_simple_sum(tp.value)): + super(GenericASTVisitorVisitor, self).visitType(tp, tp.name) + + def visitProduct(self, prod, name): + self.make_visitor(name, prod.fields) + + def visitConstructor(self, cons, _): + self.make_visitor(cons.name, cons.fields) + + def make_visitor(self, name, fields): + self.emit("def visit_%s(self, node):" % (name,), 1) + have_body = False + for field in fields: + if self.visitField(field): + have_body = True + if not have_body: + self.emit("pass", 2) + self.emit("") + + def visitField(self, field): + if field.type.value not in asdl.builtin_types and \ + field.type.value not in self.data.simple_types: + if field.seq or field.opt: + self.emit("if node.%s:" % (field.name,), 2) + level = 3 + else: + level = 2 + if field.seq: + template = "self.visit_sequence(node.%s)" + else: + template = "node.%s.walkabout(self)" + self.emit(template % (field.name,), level) + return True + return False + + +asdl_type_map = { + "int" : "int_w", + "identifier" : "str_w", + "bool" : "bool_w" +} + +def get_unwrapper(tp, name, simple_types): + if tp in asdl.builtin_types: + return "space.%s(%s)" % (asdl_type_map[tp], name) + elif tp in simple_types: + return "space.interp_w(%s, %s).to_simple_int(space)" % (tp, name) + else: + return "space.interp_w(%s, %s)" % (tp, name) + + +# CPython lets blank AST nodes (no constructor arguments) be created +# and the attributes added later. In CPython, it is implemented by +# implementing applevel and c level AST as different structures and +# copying between them. This is hideous, so we use a slightly less +# ugly hack in PyPy. Each field has a bitmask which is set on the +# initialization_state attribute when the field type is set. When +# sync_app_attrs() is called, it's a simple matter of removing the +# optional field flags from initialization_state, and using XOR to +# test if all the required fields have been set. +class AppExposeVisitor(ASDLVisitor): + + def visitType(self, tp): + super(AppExposeVisitor, self).visitType(tp, tp.name) + + def visitSum(self, sum, name): + for field in sum.attributes: + self.make_property(field, name, True) + self.make_typedef(name, "AST", sum.attributes, + fields_name="_attributes") + if not is_simple_sum(sum): + super(AppExposeVisitor, self).visitSum(sum, name) + else: + for cons in sum.types: + self.make_typedef("_" + cons.name.value, name, (), cons.name, + concrete=True) + + def make_typedef(self, name, base, fields, display_name=None, + fields_name="_fields", concrete=False, needs_init=False): + if display_name is None: + display_name = name + self.emit("%s.typedef = typedef.TypeDef(\"%s\"," % (name, display_name)) + self.emit("%s.typedef," % (base,), 1) + comma_fields = ", ".join(repr(field.name.value) for field in fields) + self.emit("%s=_FieldsWrapper([%s])," % (fields_name, comma_fields), 1) + for field in fields: + getter = "%s_get_%s" % (name, field.name) + setter = "%s_set_%s" % (name, field.name) + config = (field.name, getter, setter, name) + self.emit("%s=typedef.GetSetProperty(%s, %s, cls=%s)," % config, 1) + # CPython lets you create instances of "abstract" AST nodes + # like ast.expr or even ast.AST. This doesn't seem to useful + # and would be a pain to implement safely, so we don't allow + # it. + if concrete: + self.emit("__new__=interp2app(get_AST_new(%s))," % (name,), 1) + if needs_init: + self.emit("__init__=interp2app(%s_init)," % (name,), 1) + self.emit(")") + self.emit("%s.typedef.acceptable_as_base_class = False" % (name,)) + self.emit("") + + def make_init(self, name, fields): + comma_fields = ", ".join(repr(field.name.value) for field in fields) + config = (name, comma_fields) + self.emit("_%s_field_unroller = unrolling_iterable([%s])" % config) + self.emit("def %s_init(space, w_self, args):" % (name,)) + self.emit("w_self = space.descr_self_interp_w(%s, w_self)" % (name,), 1) + for field in fields: + if field.seq: + self.emit("w_self.w_%s = None" % (field.name,), 1) + self.emit("args_w, kwargs_w = args.unpack()", 1) + self.emit("if args_w:", 1) + arity = len(fields) + if arity: + self.emit("if len(args_w) != %i:" % (arity,), 2) + self.emit("w_err = space.wrap(\"%s constructor takes 0 or %i " \ + "positional arguments\")" % (name, arity), 3) + self.emit("raise OperationError(space.w_TypeError, w_err)", 3) + self.emit("i = 0", 2) + self.emit("for field in _%s_field_unroller:" % (name,), 2) + self.emit("space.setattr(w_self, space.wrap(field), args_w[i])", 3) + self.emit("i += 1", 3) + else: + self.emit("w_err = space.wrap(\"%s constructor takes no " \ + " arguments\")" % (name,), 2) + self.emit("raise OperationError(space.w_TypeError, w_err)", 2) + self.emit("for field, w_value in kwargs_w.iteritems():", 1) + self.emit("space.setattr(w_self, space.wrap(field), w_value)", 2) + self.emit("%s_init.unwrap_spec = [ObjSpace, W_Root, Arguments]" + % (name,)) + self.emit("") + + def visitConstructor(self, cons, base): + super(AppExposeVisitor, self).visitConstructor(cons, cons.name) + self.make_init(cons.name, cons.fields + self.data.cons_attributes[cons]) + self.make_typedef(cons.name, base, cons.fields, concrete=True, + needs_init=True) + + def visitProduct(self, product, name): + super(AppExposeVisitor, self).visitProduct(product, name) + self.make_init(name, product.fields) + self.make_typedef(name, "AST", product.fields, concrete=True, + needs_init=True) + + def visitField(self, field, name): + self.make_property(field, name) + + def make_property(self, field, name, different_masks=False): + func = "def %s_get_%s(space, w_self):" % (name, field.name) + self.emit(func) + if different_masks: + flag = "w_self._%s_mask" % (field.name,) + else: + flag = self.data.field_masks[field] + self.emit("if not w_self.initialization_state & %s:" % (flag,), 1) + self.emit("w_err = space.wrap(\"attribute '%s' has not been set\")" % + (field.name,), 2) + self.emit("raise OperationError(space.w_AttributeError, w_err)", 2) + if field.seq: + self.emit("if w_self.w_%s is None:" % (field.name,), 1) + self.emit("if w_self.%s is None:" % (field.name,), 2) + self.emit("w_list = space.newlist([])", 3) + self.emit("else:", 2) + if field.type.value in self.data.simple_types: + wrapper = "%s_to_class[node - 1]()" % (field.type,) + else: + wrapper = "space.wrap(node)" + self.emit("list_w = [%s for node in w_self.%s]" % + (wrapper, field.name), 3) + self.emit("w_list = space.newlist(list_w)", 3) + self.emit("w_self.w_%s = w_list" % (field.name,), 2) + self.emit("return w_self.w_%s" % (field.name,), 1) + elif field.type.value in self.data.simple_types: + config = (field.type, field.name) + self.emit("return %s_to_class[w_self.%s - 1]()" % config, 1) + elif field.type.value in ("object", "string"): + self.emit("return w_self.%s" % (field.name,), 1) + else: + self.emit("return space.wrap(w_self.%s)" % (field.name,), 1) + self.emit("") + + func = "def %s_set_%s(space, w_self, w_new_value):" % (name, field.name) + self.emit(func) + if field.seq: + self.emit("w_self.w_%s = w_new_value" % (field.name,), 1) + elif field.type.value not in asdl.builtin_types: + # These are always other AST nodes. + if field.type.value in self.data.simple_types: + self.emit("obj = space.interp_w(%s, w_new_value)" % \ + (field.type,), 1) + self.emit("w_self.%s = obj.to_simple_int(space)" % + (field.name,), 1) + else: + config = (field.name, field.type, repr(field.opt)) + self.emit("w_self.%s = space.interp_w(%s, w_new_value, %s)" % + config, 1) + else: + level = 1 + if field.opt and field.type.value != "int": + self.emit("if space.is_w(w_new_value, space.w_None):", 1) + self.emit("w_self.%s = None" % (field.name,), 2) + level += 1 + self.emit("else:", 1) + if field.type.value == "object": + self.emit("w_self.%s = w_new_value" % (field.name,), level) + elif field.type.value == "string": + self.emit("if not space.is_true(space.isinstance(" \ + "w_new_value, space.w_basestring)):", level) + line = "w_err = space.wrap(\"some kind of string required\")" + self.emit(line, level + 1) + self.emit("raise OperationError(space.w_TypeError, w_err)", + level + 1) + self.emit("w_self.%s = w_new_value" % (field.name,), level) + else: + space_method = asdl_type_map[field.type.value] + config = (field.name, space_method) + self.emit("w_self.%s = space.%s(w_new_value)" % config, level) + self.emit("w_self.initialization_state |= %s" % (flag,), 1) + self.emit("") + + +def copy_field(field): + return asdl.Field(field.type, field.name, field.seq, field.opt) + + +class ASDLData(object): + + def __init__(self, tree): + simple_types = set() + prod_simple = set() + field_masks = {} + required_masks = {} + optional_masks = {} + cons_attributes = {} + def add_masks(fields, node): + required_mask = 0 + optional_mask = 0 + for i, field in enumerate(fields): + flag = 1 << i + field_masks[field] = flag + if field.opt: + optional_mask |= flag + else: + required_mask |= flag + required_masks[node] = required_mask + optional_masks[node] = optional_mask + for tp in tree.dfns: + if isinstance(tp.value, asdl.Sum): + sum = tp.value + if is_simple_sum(sum): + simple_types.add(tp.name.value) + else: + for cons in sum.types: + attrs = [copy_field(field) for field in sum.attributes] + add_masks(cons.fields + attrs, cons) + cons_attributes[cons] = attrs + else: + prod = tp.value + prod_simple.add(tp.name.value) + add_masks(prod.fields, prod) + prod_simple.update(simple_types) + self.cons_attributes = cons_attributes + self.simple_types = simple_types + self.prod_simple = prod_simple + self.field_masks = field_masks + self.required_masks = required_masks + self.optional_masks = optional_masks + + +HEAD = """# Generated by tools/asdl_py.py +from pypy.interpreter.baseobjspace import Wrappable, ObjSpace, W_Root +from pypy.interpreter import typedef +from pypy.interpreter.gateway import interp2app +from pypy.interpreter.argument import Arguments +from pypy.interpreter.error import OperationError +from pypy.rlib.unroll import unrolling_iterable +from pypy.tool.pairtype import extendabletype +from pypy.tool.sourcetools import func_with_new_name + + +class AST(Wrappable): + + __slots__ = ("initialization_state",) + + __metaclass__ = extendabletype + + def walkabout(self, visitor): + raise AssertionError("walkabout() implementation not provided") + + def mutate_over(self, visitor): + raise AssertionError("mutate_over() implementation not provided") + + def sync_app_attrs(self, space): + raise NotImplementedError + + +class NodeVisitorNotImplemented(Exception): + pass + + +class _FieldsWrapper(Wrappable): + "Hack around the fact we can't store tuples on a TypeDef." + + def __init__(self, fields): + self.fields = fields + + def __spacebind__(self, space): + return space.newtuple([space.wrap(field) for field in self.fields]) + + +def get_AST_new(node_class): + def generic_AST_new(space, w_type, __args__): + node = space.allocate_instance(node_class, w_type) + node.initialization_state = 0 + return space.wrap(node) + generic_AST_new.unwrap_spec = [ObjSpace, W_Root, Arguments] + return func_with_new_name(generic_AST_new, "new_%s" % node_class.__name__) + + +AST.typedef = typedef.TypeDef("AST", + _fields=_FieldsWrapper([]), + _attributes=_FieldsWrapper([]), +) +AST.typedef.acceptable_as_base_class = False + + +def missing_field(space, state, required, host): + "Find which required field is missing." + for i in range(len(required)): + if not (state >> i) & 1: + missing = required[i] + if missing is not None: + err = "required attribute '%s' missing from %s" + err = err % (missing, host) + w_err = space.wrap(err) + raise OperationError(space.w_TypeError, w_err) + raise AssertionError("should not reach here") + + +""" + +visitors = [ASTNodeVisitor, ASTVisitorVisitor, GenericASTVisitorVisitor, + AppExposeVisitor] + + +def main(argv): + if len(argv) == 3: + def_file, out_file = argv[1:] + elif len(argv) == 1: + print "Assuming default values of Python.asdl and ast.py" + here = os.path.dirname(__file__) + def_file = os.path.join(here, "Python.asdl") + out_file = os.path.join(here, "..", "ast.py") + else: + print >> sys.stderr, "invalid arguments" + return 2 + mod = asdl.parse(def_file) + data = ASDLData(mod) + fp = open(out_file, "w") + try: + fp.write(HEAD) + for visitor in visitors: + visitor(fp, data).visit(mod) + finally: + fp.close() + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) Property changes on: trunk/jython/ast/asdl_py.py ___________________________________________________________________ Added: svn:keywords + Id Added: svn:eol-style + native This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |