Thread: [Jython-checkins] SF.net SVN: jython:[5272] branches/advanced

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Revision: 5272
          http://jython.svn.sourceforge.net/jython/?rev=5272&view=rev
Author:   thobes
Date:     2008-08-29 21:59:51 +0000 (Fri, 29 Aug 2008)

Log Message:
-----------
Committing a large portion of code to the advanced compiler project.
The advanced compiler is mostly outlined, there quite a few implementation details to complete, but I have a good idea and direction for implementing these.

Further details on content, direction and status of the project can be found in BRANCH.txt

Modified Paths:
--------------
    branches/advanced/.classpath
    branches/advanced/BRANCH.txt
    branches/advanced/Lib/_ast.py
    branches/advanced/build.xml
    branches/advanced/src/org/python/antlr/Visitor.java
    branches/advanced/src/org/python/compiler/Future.java
    branches/advanced/src/org/python/core/CompilerFacade.java
    branches/advanced/src/org/python/core/FutureFeature.java
    branches/advanced/src/org/python/core/Py.java
    branches/advanced/src/org/python/core/PyEllipsis.java
    branches/advanced/src/org/python/core/PyNone.java
    branches/advanced/src/org/python/core/PyNotImplemented.java
    branches/advanced/src/org/python/core/PythonCompiler.java

Added Paths:
-----------
    branches/advanced/compiler/org/python/code/ArgumentParser.java
    branches/advanced/compiler/org/python/compiler/AdvancedCompiler.java
    branches/advanced/compiler/org/python/compiler/AdvancedPreferences.java
    branches/advanced/compiler/org/python/compiler/CodeGenerator.java
    branches/advanced/compiler/org/python/compiler/CompilerDirector.java
    branches/advanced/compiler/org/python/compiler/ConstantChecker.java
    branches/advanced/compiler/org/python/compiler/ConstraintsDefinition.java
    branches/advanced/compiler/org/python/compiler/FlowGraphBundle.java
    branches/advanced/compiler/org/python/compiler/FlowGraphGenerator.java
    branches/advanced/compiler/org/python/compiler/GeneratedCodeState.java
    branches/advanced/compiler/org/python/compiler/IntermediateCodeGenerator.java
    branches/advanced/compiler/org/python/compiler/IntermediateCodeGeneratorFactory.java
    branches/advanced/compiler/org/python/compiler/PragmaParser.java
    branches/advanced/compiler/org/python/compiler/ScopeBuilder.java
    branches/advanced/compiler/org/python/compiler/ScopeFactory.java
    branches/advanced/compiler/org/python/compiler/ScopeInformation.java
    branches/advanced/compiler/org/python/compiler/ScopesBuilder.java
    branches/advanced/compiler/org/python/compiler/SyntaxErrorPolicy.java
    branches/advanced/compiler/org/python/compiler/flowgraph/
    branches/advanced/compiler/org/python/compiler/flowgraph/Block.java
    branches/advanced/compiler/org/python/compiler/flowgraph/CodeGraph.java
    branches/advanced/compiler/org/python/compiler/flowgraph/Constant.java
    branches/advanced/compiler/org/python/compiler/flowgraph/FilteredGraph.java
    branches/advanced/compiler/org/python/compiler/flowgraph/IfGroup.java
    branches/advanced/compiler/org/python/compiler/flowgraph/Link.java
    branches/advanced/compiler/org/python/compiler/flowgraph/LinkGroup.java
    branches/advanced/compiler/org/python/compiler/flowgraph/LoopGroup.java
    branches/advanced/compiler/org/python/compiler/flowgraph/SuperBlock.java
    branches/advanced/compiler/org/python/compiler/flowgraph/Transformer.java
    branches/advanced/compiler/org/python/compiler/flowgraph/TryFinallyGroup.java
    branches/advanced/compiler/org/python/compiler/flowgraph/ValueOperation.java
    branches/advanced/compiler/org/python/compiler/flowgraph/Variable.java
    branches/advanced/compiler/org/python/compiler/flowgraph/VoidOperation.java
    branches/advanced/src/org/python/compiler/LegacyCompiler.java
    branches/advanced/src/org/python/core/Pragma.java
    branches/advanced/src/org/python/core/PragmaReceiver.java
    branches/advanced/src/org/python/core/PythonCodeBundle.java

Modified: branches/advanced/.classpath
===================================================================

--- branches/advanced/.classpath	2008-08-29 21:08:52 UTC (rev 5271)
+++ branches/advanced/.classpath	2008-08-29 21:59:51 UTC (rev 5272)
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <classpath>
 	<classpathentry kind="lib" path="build/exposed"/>
-	<classpathentry excluding="com/ziclix/python/sql/handler/InformixDataHandler.java|com/ziclix/python/sql/handler/OracleDataHandler.java|org/python/compiler/TentativeCompiler.java" kind="src" output="build/classes" path="src"/>
+	<classpathentry excluding="com/ziclix/python/sql/handler/InformixDataHandler.java|com/ziclix/python/sql/handler/OracleDataHandler.java|org/python/compiler/TentativeCompiler.java|org/python/util/MultipleEnumSet.java" kind="src" output="build/classes" path="src"/>
 	<classpathentry kind="src" output="build/classes" path="build/gensrc"/>
 	<classpathentry kind="src" output="build/classes" path="tests/java"/>
 	<classpathentry kind="src" path="bugtests/classes"/>

Modified: branches/advanced/BRANCH.txt
===================================================================
--- branches/advanced/BRANCH.txt	2008-08-29 21:08:52 UTC (rev 5271)
+++ branches/advanced/BRANCH.txt	2008-08-29 21:59:51 UTC (rev 5272)
@@ -2,4 +2,89 @@
 ========================
 
 The advanced compiler is a project that strives to add an optimizing, advanced
-compiler to Jython.
\ No newline at end of file
+compiler to Jython.
+
+The content of this branch:
+ [Most of the code relevant for the advanced compiler package is in the 
+  compiler source directory, only minor modifications has been made in the
+  src source directory]
+ * org.python.compiler.bytecode and org.python.bytecode
+   The first approach for implementing the Advanced compiler, an abstraction
+   utilizing Python Bytecode as an API and a model for generating and
+   transforming code before handing it over to the JVM.
+   This prooved to be too hard to work with in terms of modeling aspects
+   important for performance on the JVM.
+   This could still be useful for the ability to import pbc files to Jython,
+   as made possible during Summer of Code 2007. It might also be useful for
+   implementing dynamic evaluation of Python code in environments where
+   it is not possible to generate and load Java byte code, such as in
+   restricted environments (for example applets) or on other platforms,
+   such as Android.
+ * org.python.compiler.advanced
+   The second attempt at implementing the Advanced compiler, or at least the
+   API part of it.
+   This suffered from trying to be too general, extensible and decoupled.
+   It was a good experience though.
+ * org.python.compiler.flowgrah
+   The third and current implementation of the Advanced compiler.
+   Using a flow graph as an intermediate representation.
+   The code for generating the flow graph is in the org.python.compiler package.
+   The state of this implementation is:
+   * The guts of the flow graph representation has not been implemented yet,
+     it is at an API design stage.
+   * The code for generating the data flow of the flow graph has been
+     implemented. Although it might have to be refactored when the internals
+     of the flow graph is implemented, since that might imply some API changes.
+   * The code for generating the control flow of the flow graph is being
+     implemented. It started out as being a simple flow between arbitrary
+     blocks in the graph, but it turns out that there will be some gain in
+     having the flow graph be built from graph patterns. This lead to quite a
+     large refactoring of the control flow aspects of the flow graph API.
+   * The code for annotating the flow graph and for turning the flow graph into
+     Java bytecode is only at an "ideas on a piece of paper" stage at the
+     moment.
+
+The flow of the compiler will be:
+ Step 1. Parse the source text and build an abstract syntax tree.
+    This is already implemented, the advanced compiler uses the same AST as
+    the legacy compiler.
+ * Optionally any AST transformations that the user has added to the compiler
+   will be applied after step 1.
+ Step 2. Verify AST constraints and build up information about the scopes.
+    This has been mostly implemented, but some pieces of the representation
+    of the scope information data structure is still under development.
+    The AST verification adds constraints that are expensive to apply in the
+    parsing step. Such as scope rules, restricted values, illegal combinations
+    of nodes and such things.
+ Step 3. Transform the AST to a flow graph representation.
+    This is being implemented, see above.
+ * Optionally apply any flow graph transformations here.
+   These transformations will typically be type annotations and optimizations.
+   For this a transformation API for the flow graph is required, this will be
+   designed when the first transformations are implemented. It will need to
+   have good support for things such as inlining, loop unrolling and
+   code substitution. This is where it makes a large difference to have the
+   controll flow of the flow graph be built from a set of graph patterns
+   instead of an arbitrary flow structure.
+ Step 4. Generate target code (Java byte code) from the flow graph.
+   This has not been implemented yet, see above.
+
+Annother aspect that needs to be implemeted is serialization of the flow graph,
+this is needed to enable reloading it at runtime and specialize it given the
+runtime data types. This would be a Python aware pre-JIT optimizer (the JVM
+JIT is good, but it can get a lot of help from a layer of language awareness).
+Serializability has been in mind when designing the flow graph representation,
+but it has not been the main focus.
+These optimizations could be more agressive than the once in the transformation
+step in between steps 3 and 4, but would use the same API. Perhaps triggers
+could be added in the "static transformation" to alert the runtime system when
+certain optimizations might be applicable in the "JIT transformation" step.
+
+A summary of the things left to do (in chronological order):
+ * Design the control flow aspect of the flow graph API.
+ * Finish the flow graph generation given the control flow API.
+ * Implement the internals of the flow graph.
+ * Implement byte code generation from the flow graph.
+ * Implement transformations of the flow graph.
+Here the advanced compiler will be working. After this comes the just in time
+re-compilation project.

Modified: branches/advanced/Lib/_ast.py
===================================================================
--- branches/advanced/Lib/_ast.py	2008-08-29 21:08:52 UTC (rev 5271)
+++ branches/advanced/Lib/_ast.py	2008-08-29 21:59:51 UTC (rev 5272)
@@ -2,7 +2,7 @@
 from org.python.antlr.ast.operatorType import Add,Sub,Mult,Div,FloorDiv,Mod,LShift,RShift,BitOr,BitAnd,BitXor,Pow
 from org.python.antlr.ast.cmpopType import Eq,Gt,GtE,In,Is,IsNot,Lt,LtE,NotEq,NotIn
 from org.python.antlr.ast.unaryopType import Invert,Not,UAdd,USub
-from org.python.core.PyTableCode import PyCF_ONLY_AST
+from org.python.core.CompilerFlags import PyCF_ONLY_AST
 from org.python.antlr.ast.expr_contextType import Load, Store, Del, AugLoad, AugStore, Param
 
 from org.python.antlr import AST

Modified: branches/advanced/build.xml
===================================================================
--- branches/advanced/build.xml	2008-08-29 21:08:52 UTC (rev 5271)
+++ branches/advanced/build.xml	2008-08-29 21:59:51 UTC (rev 5272)
@@ -480,6 +480,7 @@
             <src path="${interpreter.source.dir}"/>
 
             <exclude name="org/python/parser/python.java" />
+		  <exclude name="**/MultipleEnumSet.java" />
             <exclude name="**/handler/InformixDataHandler.java" unless="informix.present" />
             <exclude name="**/handler/OracleDataHandler.java" unless="oracle.present" />
             <classpath refid="main.classpath" />

Added: branches/advanced/compiler/org/python/code/ArgumentParser.java
===================================================================
--- branches/advanced/compiler/org/python/code/ArgumentParser.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/code/ArgumentParser.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,48 @@
+package org.python.code;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.python.core.PyDictionary;
+import org.python.core.PyObject;
+import org.python.core.PyTuple;
+
+/**
+ * Work in progress: an outline for an argument parser based on a perfect hash.
+ * 
+ * @author Tobias Ivarsson
+ */
+public class ArgumentParser {
+
+    public static PyObject[] parse(ArgumentParser spec, PyObject[] arguments,
+            String[] keywords, PyObject star, PyDictionary starstar) {
+        PyObject[] result = new PyObject[totalArgSize(spec)];
+        List<PyObject> stararg;
+        PyDictionary kwstararg;
+        if (spec.has_kwstararg) {
+            kwstararg = spec.has_kwstararg ? new PyDictionary() : null;
+            result[result.length - 1] = kwstararg;
+        }
+
+        stararg = spec.has_stararg ? new LinkedList<PyObject>() : null;
+
+        if (stararg != null) {
+            PyObject[] elements = new PyObject[stararg.size()];
+            elements = stararg.toArray(elements);
+            result[result.length - (1 + (spec.has_kwstararg ? 1 : 0))] = new PyTuple(
+                    elements);
+        }
+        return result;
+    }
+
+    private int argcount;
+    private boolean has_stararg;
+    private boolean has_kwstararg;
+    private int positionalonlycount;
+
+    private static int totalArgSize(ArgumentParser spec) {
+        return spec.argcount + (spec.has_stararg ? 1 : 0)
+                + spec.positionalonlycount + (spec.has_kwstararg ? 1 : 0);
+    }
+
+}

Added: branches/advanced/compiler/org/python/compiler/AdvancedCompiler.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/AdvancedCompiler.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/AdvancedCompiler.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,73 @@
+package org.python.compiler;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.python.antlr.PythonTree;
+import org.python.antlr.ast.Module;
+import org.python.antlr.ast.VisitorBase;
+import org.python.antlr.ast.VisitorIF;
+import org.python.antlr.ast.modType;
+import org.python.compiler.flowgraph.Transformer;
+import org.python.compiler.flowgraph.Variable;
+import org.python.core.CompilerFlags;
+import org.python.core.ParserFacade;
+import org.python.core.PythonCodeBundle;
+import org.python.core.PythonCompiler;
+
+public class AdvancedCompiler implements PythonCompiler {
+
+    private VisitorIF<?> astTransformer;
+    private final Transformer flowgraphTransformations = null; // TODO
+    private final AdvancedPreferences preferences = null;
+    private final CodeGenerator codegen = new CodeGenerator() {
+        // This depends on the preferences
+    };
+
+    public PythonCodeBundle compile(modType node, String name, String filename,
+            boolean linenumbers, boolean printResults, boolean setFile,
+            CompilerFlags cflags) throws Exception {
+        if (astTransformer != null) {
+            node.accept(astTransformer);
+        }
+        FlowGraphBundle bundle = new FlowGraphBundle(name, filename);
+        CompilerDirector.compile(bundle, preferences, cflags, linenumbers,
+                printResults, setFile, node);
+        bundle.acceptTransformations(flowgraphTransformations);
+        return bundle.generateCode(codegen);
+    }
+    
+    public static Map<PythonTree, ScopeInformation> test(String source) throws Exception {
+        return testScopeAnalyzer(ParserFacade.parse(source, "exec"));
+    }
+
+    private static Map<PythonTree, ScopeInformation> testScopeAnalyzer(
+            PythonTree parse) throws Exception {
+        return parse.accept(new VisitorBase<Map<PythonTree, ScopeInformation>>() {
+
+            @Override
+            public void traverse(PythonTree node) throws Exception {
+            }
+
+            @Override
+            protected Map<PythonTree, ScopeInformation> unhandled_node(
+                    PythonTree node) throws Exception {
+                return null;
+            }
+
+            @Override
+            public Map<PythonTree, ScopeInformation> visitModule(Module node)
+                    throws Exception {
+                FlowGraphBundle bundle = new FlowGraphBundle("data", "data.fil");
+                Map<PythonTree, ScopeInformation> scopes = new HashMap<PythonTree, ScopeInformation>();
+                ScopeInformation scope = ScopesBuilder.scan(
+                        new SyntaxErrorPolicy.ErrorCollectingPolicy(), null,
+                        new HashMap<String, ConstantChecker<Variable>>(),
+                        scopes, bundle, node.body);
+                scopes.put(node, scope);
+                return scopes;
+            }
+        });
+    }
+
+}

Added: branches/advanced/compiler/org/python/compiler/AdvancedPreferences.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/AdvancedPreferences.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/AdvancedPreferences.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,121 @@
+package org.python.compiler;
+
+import java.util.Map;
+
+import org.python.antlr.ast.Name;
+import org.python.antlr.ast.Num;
+import org.python.antlr.ast.exprType;
+import org.python.compiler.flowgraph.CodeGraph;
+import org.python.compiler.flowgraph.Variable;
+import org.python.core.CompilerFlags;
+import org.python.core.Py;
+
+public class AdvancedPreferences implements ConstraintsDefinition<Variable> {
+
+    private interface Setting<T> {
+
+    }
+
+    private enum Booleans implements Setting<Boolean> {
+        BOOLEAN_CONSTANTS, CORE_CONSTANTS
+
+    }
+
+    private enum Strings implements Setting<String> {
+
+    }
+
+    private enum Numbers implements Setting<Number> {
+
+    }
+
+    private enum Constant implements ConstantChecker<Variable> {
+        NONE {
+
+            @Override
+            public boolean acceptValue(exprType value) {
+                return (value instanceof Name)
+                        && ((Name) value).id.equals("None");
+            }
+
+            @Override
+            public Variable getConstantValue() {
+                return CodeGraph.None();
+            }
+        },
+        ELLIPSIS {
+
+            @Override
+            public boolean acceptValue(exprType value) {
+                return (value instanceof Name)
+                        && ((Name) value).id.equals("Ellipsis");
+            }
+
+            @Override
+            public Variable getConstantValue() {
+                return CodeGraph.Ellipsis();
+            }
+        },
+        TRUE {
+
+            @Override
+            public boolean acceptValue(exprType value) {
+                if (value instanceof Num) {
+                    Num num = (Num) value;
+                    return num.equals(1) || num.equals(Py.newInteger(1));
+                } else if (value instanceof Name) {
+                    Name name = (Name) value;
+                    return name.id.equals("True");
+                }
+                return false;
+            }
+
+            @Override
+            public Variable getConstantValue() {
+                return CodeGraph.True();
+            }
+        },
+        FALSE {
+
+            @Override
+            public boolean acceptValue(exprType value) {
+                if (value instanceof Num) {
+                    Num num = (Num) value;
+                    return num.equals(0) || num.equals(Py.newInteger(0));
+                } else if (value instanceof Name) {
+                    Name name = (Name) value;
+                    return name.id.equals("False");
+                }
+                return false;
+            }
+
+            @Override
+            public Variable getConstantValue() {
+                return CodeGraph.False();
+            }
+        };
+
+        public abstract boolean acceptValue(exprType value);
+
+        public abstract Variable getConstantValue();
+
+    }
+
+    private <T> T getSetting(Setting<T> boolean_constants, CompilerFlags flags) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public void update(CompilerFlags flags,
+            Map<String, ConstantChecker<Variable>> constNames) {
+        // TODO
+        if (getSetting(Booleans.CORE_CONSTANTS, flags)) {
+            constNames.put("None", Constant.NONE);
+        }
+        if (getSetting(Booleans.BOOLEAN_CONSTANTS, flags)) {
+            constNames.put("True", Constant.TRUE);
+            constNames.put("False", Constant.FALSE);
+        }
+    }
+
+}

Added: branches/advanced/compiler/org/python/compiler/CodeGenerator.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/CodeGenerator.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/CodeGenerator.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,10 @@
+package org.python.compiler;
+
+/**
+ * Generates executable code from the Intermediate Representation.
+ * 
+ * @author Tobias Ivarsson
+ */
+interface CodeGenerator {
+
+}

Added: branches/advanced/compiler/org/python/compiler/CompilerDirector.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/CompilerDirector.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/CompilerDirector.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,177 @@
+package org.python.compiler;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.python.antlr.PythonTree;
+import org.python.antlr.ast.ClassDef;
+import org.python.antlr.ast.Expression;
+import org.python.antlr.ast.FunctionDef;
+import org.python.antlr.ast.GeneratorExp;
+import org.python.antlr.ast.Interactive;
+import org.python.antlr.ast.Lambda;
+import org.python.antlr.ast.Return;
+import org.python.antlr.ast.VisitorBase;
+import org.python.antlr.ast.modType;
+import org.python.antlr.ast.stmtType;
+import org.python.core.CompilerFlags;
+import org.python.core.FutureFeature;
+
+class CompilerDirector<E, S, H> extends VisitorBase<E> {
+
+    private static final PragmaParser interactivePragmas = new PragmaParser(
+            FutureFeature.PRAGMA_MODULE);
+    private static final PragmaParser modulePragmas = new PragmaParser(
+            FutureFeature.PRAGMA_MODULE);
+    private final CompilerFlags flags;
+    private final SyntaxErrorPolicy errorPolicy = null;
+    private final Map<PythonTree, S> scopes = new HashMap<PythonTree, S>();
+    private IntermediateCodeGenerator<E, S, H> codegen;
+    private final boolean printResults;
+    private final boolean linenumbers; // TODO
+    private final boolean setFile; // TODO
+    private final IntermediateCodeGeneratorFactory<E, S, H> factory;
+    private final ConstraintsDefinition<E> constraints;
+    private final Map<String, ConstantChecker<E>> constNames = new HashMap<String, ConstantChecker<E>>();
+
+    public static <E, S, H> void compile(
+            IntermediateCodeGeneratorFactory<E, S, H> factory,
+            ConstraintsDefinition<E> constraints, CompilerFlags cflags,
+            boolean linenumbers, boolean printResults, boolean setFile,
+            modType base) throws Exception {
+        base.accept(new CompilerDirector<E, S, H>(factory, constraints, cflags,
+                linenumbers, printResults, setFile));
+    }
+
+    private CompilerDirector(IntermediateCodeGeneratorFactory<E, S, H> factory,
+            ConstraintsDefinition<E> constraints, CompilerFlags cflags,
+            boolean linenumbers, boolean printResults, boolean setFile) {
+        this.factory = factory;
+        this.constraints = constraints;
+        this.printResults = printResults;
+        this.linenumbers = linenumbers;
+        this.setFile = setFile;
+        // TODO copy the compiler flags
+        flags = new CompilerFlags();
+    }
+
+    @Override
+    public void traverse(PythonTree node) throws Exception {
+        // Don't do anything, the traversal is handled by the code generator
+    }
+
+    @Override
+    protected E unhandled_node(PythonTree node) throws Exception {
+        // Not explicitly handled by this? then it has to be handled by the code generator
+        return node.accept(codegen);
+    }
+
+    // Top level elements - initialize the compiler
+
+    @Override
+    public E visitModule(org.python.antlr.ast.Module node) throws Exception {
+        compile(modulePragmas, false, node.body);
+        return null;
+    }
+
+    @Override
+    public E visitInteractive(Interactive node) throws Exception {
+        compile(interactivePragmas, printResults, node.body);
+        return null;
+    }
+
+    @Override
+    public E visitExpression(Expression node) throws Exception {
+        compile(null, false, new Return(node, node.body));
+        return null;
+    }
+
+    private void compile(PragmaParser pragmas, boolean printResults,
+            stmtType... body) throws Exception {
+        if (pragmas != null) {
+            pragmas.parse(body, flags);
+        }
+        constraints.update(flags, constNames);
+        S scope = ScopesBuilder.scan(errorPolicy, pragmas, constNames, scopes,
+                factory, body);
+        IntermediateCodeGenerator<E, S, H> old = codegen;
+        try {
+            // TODO: pass on the flags as well...
+            codegen = factory.createCodeGenerator(printResults, constNames,
+                    this, scope);
+            for (stmtType stmt : body) {
+                stmt.accept(this);
+            }
+        } finally {
+            codegen = old;
+        }
+    }
+
+    // Scope defining elements
+
+    @Override
+    public E visitClassDef(ClassDef node) throws Exception {
+        S scope = scopes.get(node);
+        H state = codegen.beforeClassDef(node, scope);
+        IntermediateCodeGenerator<E, S, H> old = codegen;
+        E result;
+        try {
+            codegen = factory.createCodeGenerator(false, constNames, this,
+                    scope);
+            result = codegen.visitClassDef(node);
+        } finally {
+            codegen = old;
+        }
+        codegen.afterClassDef(node, state, result);
+        return result;
+    }
+
+    @Override
+    public E visitFunctionDef(FunctionDef node) throws Exception {
+        S scope = scopes.get(node);
+        H state = codegen.beforeFunctionDef(node, scope);
+        IntermediateCodeGenerator<E, S, H> old = codegen;
+        E result;
+        try {
+            codegen = factory.createCodeGenerator(false, constNames, this,
+                    scope);
+            result = codegen.visitFunctionDef(node);
+        } finally {
+            codegen = old;
+        }
+        codegen.afterFunctionDef(node, state, result);
+        return result;
+    }
+
+    @Override
+    public E visitLambda(Lambda node) throws Exception {
+        S scope = scopes.get(node);
+        H state = codegen.beforeLambda(node, scope);
+        IntermediateCodeGenerator<E, S, H> old = codegen;
+        E result;
+        try {
+            codegen = factory.createCodeGenerator(false, constNames, this,
+                    scope);
+            result = codegen.visitLambda(node);
+        } finally {
+            codegen = old;
+        }
+        return codegen.afterLambda(node, state, result);
+    }
+
+    @Override
+    public E visitGeneratorExp(GeneratorExp node) throws Exception {
+        S scope = scopes.get(node);
+        H state = codegen.beforeGeneratorExp(node, scope);
+        IntermediateCodeGenerator<E, S, H> old = codegen;
+        E result;
+        try {
+            codegen = factory.createCodeGenerator(false, constNames, this,
+                    scope);
+            result = codegen.visitGeneratorExp(node);
+        } finally {
+            codegen = old;
+        }
+        return codegen.afterGeneratorExp(node, state, result);
+    }
+}
\ No newline at end of file

Added: branches/advanced/compiler/org/python/compiler/ConstantChecker.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/ConstantChecker.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/ConstantChecker.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,11 @@
+package org.python.compiler;
+
+import org.python.antlr.ast.exprType;
+
+public interface ConstantChecker<T> {
+
+    boolean acceptValue(exprType value);
+
+    T getConstantValue();
+
+}

Added: branches/advanced/compiler/org/python/compiler/ConstraintsDefinition.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/ConstraintsDefinition.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/ConstraintsDefinition.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,11 @@
+package org.python.compiler;
+
+import java.util.Map;
+
+import org.python.core.CompilerFlags;
+
+interface ConstraintsDefinition<E> {
+
+    void update(CompilerFlags flags, Map<String, ConstantChecker<E>> constNames);
+
+}

Added: branches/advanced/compiler/org/python/compiler/FlowGraphBundle.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/FlowGraphBundle.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/FlowGraphBundle.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,71 @@
+package org.python.compiler;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.python.antlr.ast.VisitorIF;
+import org.python.compiler.flowgraph.CodeGraph;
+import org.python.compiler.flowgraph.Transformer;
+import org.python.compiler.flowgraph.Variable;
+import org.python.core.PythonCodeBundle;
+
+/**
+ * Represents the super graph of a compilation unit.
+ * 
+ * @author Tobias Ivarsson
+ */
+final class FlowGraphBundle
+        implements
+        IntermediateCodeGeneratorFactory<Variable, ScopeInformation, GeneratedCodeState> {
+
+    private final String name;
+    private final String filename;
+    private final Map<Object, Variable> constantPool = new HashMap<Object, Variable>();
+
+    public FlowGraphBundle(String name, String filename) {
+        this.name = name;
+        this.filename = filename;
+    }
+
+    void acceptTransformations(Transformer transformer) {
+        // TODO Auto-generated method stub
+
+    }
+
+    PythonCodeBundle generateCode(CodeGenerator codegen) {
+        return null;
+    }
+
+    public IntermediateCodeGenerator<Variable, ScopeInformation, GeneratedCodeState> createCodeGenerator(
+            boolean printExpr,
+            Map<String, ConstantChecker<Variable>> constants,
+            VisitorIF<Variable> driver, ScopeInformation scope) {
+        CodeGraph graph = new CodeGraph();
+        return new FlowGraphGenerator(printExpr, driver, constants, graph);
+    }
+
+    public ScopeInformation createClass(String name, String[] locals,
+            String[] explicitlyGlobal, String[] explicitlyClosure,
+            String[] free, String[] scopeRequired, boolean hasStarImport,
+            List<ScopeInformation> children) {
+        return ScopeInformation.makeClassScope(name, locals, explicitlyGlobal,
+                explicitlyClosure, free, scopeRequired, hasStarImport, children);
+    }
+
+    public ScopeInformation createFunction(String name, String[] parameters,
+            String[] locals, String[] explicitlyGlobal,
+            String[] explicitlyClosure, String[] free, String[] scopeRequired,
+            boolean isGenerator, boolean hasStarImport,
+            List<ScopeInformation> children) {
+        return ScopeInformation.makeFunctionScope(name, parameters, locals,
+                explicitlyGlobal, explicitlyClosure, free, scopeRequired,
+                isGenerator, hasStarImport, children);
+    }
+
+    public ScopeInformation createGlobal(String[] locals, String[] cell,
+            boolean hasStarImport, List<ScopeInformation> children) {
+        return ScopeInformation.makeGlobalScope(locals, cell, hasStarImport,
+                children);
+    }
+}

Added: branches/advanced/compiler/org/python/compiler/FlowGraphGenerator.java
===================================================================
--- branches/advanced/compiler/org/python/compiler/FlowGraphGenerator.java	                        (rev 0)
+++ branches/advanced/compiler/org/python/compiler/FlowGraphGenerator.java	2008-08-29 21:59:51 UTC (rev 5272)
@@ -0,0 +1,1030 @@
+package org.python.compiler;
+
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Stack;
+import java.util.Vector;
+
+import org.python.antlr.ParseException;
+import org.python.antlr.PythonTree;
+import org.python.antlr.ast.Assert;
+import org.python.antlr.ast.Assign;
+import org.python.antlr.ast.Attribute;
+import org.python.antlr.ast.AugAssign;
+import org.python.antlr.ast.BinOp;
+import org.python.antlr.ast.BoolOp;
+import org.python.antlr.ast.Break;
+import org.python.antlr.ast.Call;
+import org.python.antlr.ast.ClassDef;
+import org.python.antlr.ast.Compare;
+import org.python.antlr.ast.Continue;
+import org.python.antlr.ast.Delete;
+import org.python.antlr.ast.Dict;
+import org.python.antlr.ast.Ellipsis;
+import org.python.antlr.ast.Exec;
+import org.python.antlr.ast.Expr;
+import org.python.antlr.ast.ExtSlice;
+import org.python.antlr.ast.For;
+import org.python.antlr.ast.FunctionDef;
+import org.python.antlr.ast.GeneratorExp;
+import org.python.antlr.ast.Global;
+import org.python.antlr.ast.If;
+import org.python.antlr.ast.IfExp;
+import org.python.antlr.ast.Import;
+import org.python.antlr.ast.ImportFrom;
+import org.python.antlr.ast.Index;
+import org.python.antlr.ast.Lambda;
+import org.python.antlr.ast.List;
+import org.python.antlr.ast.ListComp;
+import org.python.antlr.ast.Name;
+import org.python.antlr.ast.Num;
+import org.python.antlr.ast.Pass;
+import org.python.antlr.ast.Print;
+import org.python.antlr.ast.Raise;
+import org.python.antlr.ast.Repr;
+import org.python.antlr.ast.Return;
+import org.python.antlr.ast.Slice;
+import org.python.antlr.ast.Str;
+import org.python.antlr.ast.Subscript;
+import org.python.antlr.ast.Suite;
+import org.python.antlr.ast.TryExcept;
+import org.python.antlr.ast.TryFinally;
+import org.python.antlr.ast.Tuple;
+import org.python.antlr.ast.UnaryOp;
+import org.python.antlr.ast.Unicode;
+import org.python.antlr.ast.VisitorBase;
+import org.python.antlr.ast.VisitorIF;
+import org.python.antlr.ast.While;
+import org.python.antlr.ast.With;
+import org.python.antlr.ast.Yield;
+import org.python.antlr.ast.aliasType;
+import org.python.antlr.ast.cmpopType;
+import org.python.antlr.ast.comprehensionType;
+import org.python.antlr.ast.exprType;
+import org.python.antlr.ast.operatorType;
+import org.python.antlr.ast.sliceType;
+import org.python.antlr.ast.stmtType;
+import org.python.antlr.ast.unaryopType;
+import org.python.compiler.flowgraph.Block;
+import org.python.compiler.flowgraph.CodeGraph;
+import org.python.compiler.flowgraph.Link;
+import org.python.compiler.flowgraph.LinkGroup;
+import org.python.compiler.flowgraph.ValueOperation;
+import org.python.compiler.flowgraph.Variable;
+import org.python.compiler.flowgraph.VoidOperation;
+
+/**
+ * @author Tobias Ivarsson
+ */
+public class FlowGraphGenerator extends VisitorBase<Variable>
+        implements
+        IntermediateCodeGenerator<Variable, ScopeInformation, GeneratedCodeState> {
+
+    private static class BreakAction {
+
+        private final Block onBreak;
+        private final Block onContinue;
+        private final LinkGroup loop;
+
+        public BreakAction(LinkGroup loop, Block onBreak, Block onContinue) {
+            this.loop = loop;
+            this.onBreak = onBreak;
+            this.onContinue = onContinue;
+        }
+
+    }
+
+    //private final CompilerDirector<Variable, ScopeInformation, GeneratedCodeState> compiler;
+    private final VisitorIF<Variable> compiler;
+    private final CodeGraph graph;
+    private final boolean printResult;
+    private Block block;
+    private final Map<String, ConstantChecker<Variable>> constants;
+    private final Stack<BreakAction> breakStack = new Stack<BreakAction>();
+    private static final Map<operatorType, ValueOperation> binaryOperators;
+    static {
+        Map<operatorType, ValueOperation> ops = new EnumMap<operatorType, ValueOperation>(
+                operatorType.class);
+        ops.put(operatorType.Add, ValueOperation.ADD);
+        ops.put(operatorType.Sub, ValueOperation.SUBTRACT);
+        ops.put(operatorType.Mult, ValueOperation.MULTIPLY);
+        ops.put(operatorType.Div, ValueOperation.TRUE_DIVIDE);
+        ops.put(operatorType.Mod, ValueOperation.MODULO);
+        ops.put(operatorType.Pow, ValueOperation.POWER);
+        ops.put(operatorType.LShift, ValueOperation.SHIFT_LEFT);
+        ops.put(operatorType.RShift, ValueOperation.SHIFT_RIGHT);
+        ops.put(operatorType.BitOr, ValueOperation.OR);
+        ops.put(operatorType.BitXor, ValueOperation.XOR);
+        ops.put(operatorType.BitAnd, ValueOperation.AND);
+        ops.put(operatorType.FloorDiv, ValueOperation.FLOOR_DIVIDE);
+        binaryOperators = Collections.unmodifiableMap(ops);
+    }
+    private static final Map<operatorType, ValueOperation> augOperators;
+    static {
+        Map<operatorType, ValueOperation> ops = new EnumMap<operatorType, ValueOperation>(
+                operatorType.class);
+        ops.put(operatorType.Add, ValueOperation.AUG_ADD);
+        ops.put(operatorType.Sub, ValueOperation.AUG_SUBTRACT);
+        ops.put(operatorType.Mult, ValueOperation.AUG_MULTIPLY);
+        ops.put(operatorType.Div, ValueOperation.AUG_TRUE_DIVIDE);
+        ops.put(operatorType.Mod, ValueOperation.AUG_MODULO);
+        ops.put(operatorType.Pow, ValueOperation.AUG_POWER);
+        ops.put(operatorType.LShift, ValueOperation.AUG_SHIFT_LEFT);
+        ops.put(operatorType.RShift, ValueOperation.AUG_SHIFT_RIGHT);
+        ops.put(operatorType.BitOr, ValueOperation.AUG_OR);
+        ops.put(operatorType.BitXor, ValueOperation.AUG_XOR);
+        ops.put(operatorType.BitAnd, ValueOperation.AUG_AND);
+        ops.put(operatorType.FloorDiv, ValueOperation.AUG_FLOOR_DIVIDE);
+        augOperators = Collections.unmodifiableMap(ops);
+    }
+    private static final Map<unaryopType, ValueOperation> unaryOperators;
+    static {
+        Map<unaryopType, ValueOperation> ops = new EnumMap<unaryopType, ValueOperation>(
+                unaryopType.class);
+        ops.put(unaryopType.Invert, ValueOperation.INVERT);
+        ops.put(unaryopType.Not, ValueOperation.NOT);
+        ops.put(unaryopType.UAdd, ValueOperation.POSITIVE);
+        ops.put(unaryopType.USub, ValueOperation.NEGATIVE);
+        unaryOperators = ops;
+    }
+    private static final Map<cmpopType, ValueOperation> comparators;
+    static {
+        Map<cmpopType, ValueOperation> ops = new EnumMap<cmpopType, ValueOperation>(
+                cmpopType.class);
+        ops.put(cmpopType.Eq, ValueOperation.EQUAL);
+        ops.put(cmpopType.NotEq, ValueOperation.NOT_EQUAL);
+        ops.put(cmpopType.Lt, ValueOperation.LESS_THAN);
+        ops.put(cmpopType.LtE, ValueOperation.LESS_THAN_OR_EQUAL_TO);
+        ops.put(cmpopType.Gt, ValueOperation.GREATER_THAN);
+        ops.put(cmpopType.GtE, ValueOperation.GREATER_THAN_OR_EQUAL_TO);
+        ops.put(cmpopType.Is, ValueOperation.IS);
+        ops.put(cmpopType.IsNot, ValueOperation.IS_NOT);
+        ops.put(cmpopType.In, ValueOperation.CONTAINED_IN);
+        ops.put(cmpopType.NotIn, ValueOperation.NOT_CONTAINED_IN);
+        comparators = Collections.unmodifiableMap(ops);
+    }
+
+    /**
+     * @param compiler
+     * @param constants 
+     * @param graph
+     */
+    public FlowGraphGenerator(VisitorIF<Variable> compiler,
+            Map<String, ConstantChecker<Variable>> constants, CodeGraph graph) {
+        this(false, compiler, constants, graph);
+    }
+
+    /**
+     * @param printResult
+     * @param compiler
+     * @param constants 
+     * @param graph
+     */
+    public FlowGraphGenerator(boolean printResult,
+            VisitorIF<Variable> compiler,
+            Map<String, ConstantChecker<Variable>> constants, CodeGraph graph) {
+        this.printResult = printResult;
+        this.compiler = compiler;
+        this.constants = constants;
+        this.graph = graph;
+    }
+
+    // Handle undefined
+
+    @Override
+    public void traverse(PythonTree node) throws Exception {
+        // Delegate to the director
+        node.traverse(compiler);
+    }
+
+    @Override
+    protected Variable unhandled_node(PythonTree node) throws Exception {
+        throw new ParseException("Cannot generate code for "
+                + node.getClass().getName(), node);
+    }
+
+    // Helper method
+
+    private Variable loadClosure(ScopeInformation scope) {
+        java.util.List<Variable> freeVars = new LinkedList<Variable>();
+        for (String free : scope.freeVariables()) {
+            freeVars.add(graph.getVariable(free));
+        }
+        Variable[] closure = freeVars.toArray(new Variable[] {});
+        return block.valueOperation(ValueOperation.MAKE_ARRAY, closure);
+    }
+
+    private Variable loadDefaults(exprType[] defaults) throws Exception {
+        Variable[] vars = evalAll(defaults);
+        return block.valueOperation(ValueOperation.MAKE_ARRAY, vars);
+    }
+
+    private Variable[] loadDecorators(exprType[] decorators) throws Exception {
+        Variable[] loaded = evalAll(decorators);
+        // reverse the result, decorators are declared and loaded top-down,
+        // but applied bottom-up
+        Variable[] result = new Variable[loaded.length];
+        for (int i = 0; i < loaded.length; i--) {
+            result[result.length - i] = loaded[i];
+        }
+        return result;
+    }
+
+    private Variable applyDecorators(Variable target, Variable[] decorators) {
+        if (decorators == null || decorators.length == 0) {
+            return target;
+        }
+        for (Variable decorator : decorators) {
+            target = block.valueOperation(ValueOperation.CALL, decorator,
+                    target);
+        }
+        return target;
+    }
+
+    private Variable eval(exprType value, Variable... preserve)
+            throws Exception {
+        return value.accept(this);
+    }
+
+    private Variable eval(sliceType slice, Variable... preserve)
+            throws Exception {
+        return slice.accept(this);
+    }
+
+    private Variable[] evalAll(exprType... values) throws Exception {
+        Variable[] result = new Variable[values.length];
+        for (int i = 0; i < values.length; i++) {
+            result[i] = eval(values[i], result);
+        }
+        return result;
+    }
+
+    private void compile(stmtType... body) throws Exception {
+        for (stmtType stmt : body) {
+            stmt.accept(compiler);
+        }
+    }
+
+    private class Assignment extends VisitorBase<Void> {
+
+        private final Variable value;
+
+        Assignment(Variable value) {
+            this.value = value;
+        }
+
+        @Override
+        public void traverse(PythonTree node) throws Exception {
+        }
+
+        @Override
+        protected Void unhandled_node(PythonTree node) throws Exception {
+            throw new IllegalArgumentException("Cannot assign to "
+                    + node.getClass().getName());
+        }
+
+        @Override
+        public Void visitName(Name node) throws Exception {
+            block.voidOperation(VoidOperation.STORE,
+                    graph.getVariable(node.id), value);
+            return null;
+        }
+
+        private void tupleAssign(exprType[] elts) throws Exception {
+            int before = -1, after = 0;
+            for (exprType elt : elts) {
+                if (false /*elt instanceof Starred*/) {
+                    // For 3.0 compatibility, where one can have starred targets
+                    if (before == -1) {
+                        before = after;
+                        after = 0;
+                    } else {
+                        throw new ParseException(
+                                "more than one starred expressions in assignment",
+                                elt);
+                    }
+                } else {
+                    after++;
+                }
+            }
+            Variable unpacked;
+            if (before != -1) {
+                unpacked = block.valueOperation(ValueOperation.UNPACK, value,
+                        graph.getConstant(before), graph.getConstant(after));
+            } else {
+                unpacked = block.valueOperation(ValueOperation.UNPACK, value,
+                        graph.getConstant(after));
+            }
+            int i = 0;
+            if (before != -1) {
+                for (; i < before; i++) {
+                    Variable value = block.valueOperation(
+                            ValueOperation.ARRAY_GET, unpacked,
+                            graph.getConstant(i));
+                    elts[i].accept(new Assignment(value));
+                }
+                Variable value = block.valueOperation(ValueOperation.ARRAY_GET,
+                        unpacked, graph.getConstant(i));
+                elts[i++]/* TODO: cast elts[i++] to Starred */
+                /* and pick out the value element */.accept(new Assignment(
+                        value));
+            }
+            before += 1; // either this was -1, or we consumed the extra starred
+            for (; i < before + after; i++) {
+                Variable value = block.valueOperation(ValueOperation.ARRAY_GET,
+                        unpacked, graph.getConstant(i));
+                elts[i].accept(new Assignment(value));
+            }
+        }
+
+        @Override
+        public Void visitTuple(Tuple node) throws Exception {
+            tupleAssign(node.elts);
+            return null;
+        }
+
+        @Override
+        public Void visitList(List node) throws Exception {
+            tupleAssign(node.elts);
+            return null;
+        }
+
+        @Override
+        public Void visitDict(Dict node) throws Exception {
+            // Dictionary assignment is easy enough, once the syntax supports it
+            for (int i = 0; i < node.keys.length; i++) {
+                Variable key = eval(node.keys[i]);
+                Variable value = block.valueOperation(
+                        ValueOperation.LOAD_ATTRIBUTE, this.value, key);
+                node.values[i].accept(new Assignment(value));
+            }
+            return null;
+        }
+
+        @Override
+        public Void visitAttribute(Attribute node) throws Exception {
+            Variable target = eval(node.value);
+            block.voidOperation(VoidOperation.STORE_ATTRIBUTE, target,
+                    graph.getConstant(node.attr), value);
+            return null;
+        }
+
+        @Override
+        public Void visitSubscript(Subscript node) throws Exception {
+            Variable target = eval(node.value);
+            Variable slice = eval(node.slice);
+            block.voidOperation(VoidOperation.STORE_ITEM, target, slice, value);
+            return null;
+        }
+    }
+
+    private final VisitorIF<Void> delete = new VisitorBase<Void>() {
+
+        @Override
+        public void traverse(PythonTree node) throws Exception {
+        }
+
+        @Override
+        protected Void unhandled_node(PythonTree node) throws Exception {
+            throw new IllegalArgumentException("Cannot delete "
+                    + node.getClass().getName());
+        }
+
+        @Override
+        public Void visitName(Name node) throws Exception {
+            block.voidOperation(VoidOperation.DELETE,
+                    graph.getConstant(node.id));
+            return null;
+        }
+
+        @Override
+        public Void visitAttribute(Attribute node) throws Exception {
+            Variable target = eval(node.value);
+            block.voidOperation(VoidOperation.DELETE_ATTRIBUTE, target,
+                    graph.getConstant(node.attr));
+            return null;
+        }
+
+        @Override
+        public Void visitSubscript(Subscript node) throws Exception {
+            Variable target = eval(node.value);
+            Variable slice = eval(node.slice);
+            block.voidOperation(VoidOperation.DELETE_ITEM, target, slice);
+            return null;
+        }
+    };
+
+    // -- Entry points --
+
+    // Class
+
+    public GeneratedCodeState beforeClassDef(ClassDef node,
+            ScopeInformation scope) throws Exception {
+        Variable name = graph.getConstant(node.name);
+        Variable[] baseVars;
+        if (node.bases != null) {
+            baseVars = new Variable[node.bases.length];
+            int i = 0;
+            for (exprType base : node.bases) {
+                baseVars[i] = base.accept(this);
+            }
+        } else {
+            baseVars = new Variable[0];
+        }
+        Variable bases = block.valueOperation(ValueOperation.MAKE_ARRAY,
+                baseVars);
+        return new GeneratedCodeState(name, bases, loadClosure(scope),
+                new Variable[] {});
+    }
+
+    @Override
+    public Variable visitClassDef(ClassDef node) throws Exception {
+        block = graph.entryBlock();
+        for (stmtType stmt : node.body) {
+            stmt.accept(compiler);
+        }
+        return graph.handle();
+    }
+
+    public void afterClassDef(ClassDef node, GeneratedCodeState state,
+            Variable result) throws Exception {
+        Variable dict = block.valueOperation(ValueOperation.CALL_CODE, result,
+                state.classClosure());
+        Variable klass = block.valueOperation(ValueOperation.DEFINE_CLASS,
+                state.name(), state.classBases(), dict);
+        klass = applyDecorators(klass, state.decorators());
+        block.voidOperation(VoidOperation.STORE, state.name(), klass);
+    }
+
+    // Function
+
+    public GeneratedCodeState beforeFunctionDef(FunctionDef node,
+            ScopeInformation scope) throws Exception {
+        Variable name = graph.getConstant(node.name);
+        Variable[] decorators = loadDecorators(node.decorators);
+        return new GeneratedCodeState(name, loadClosure(scope),
+                loadDefaults(node.args.defaults), decorators);
+    }
+
+    @Override
+    public Variable visitFunctionDef(FunctionDef node) throws Exception {
+        block = graph.entryBlock();
+        for (stmtType stmt : node.body) {
+            stmt.accept(compiler);
+        }
+        return graph.handle();
+    }
+
+    public void afterFunctionDef(FunctionDef node, GeneratedCodeState state,
+            Variable result) throws Exception {
+        Variable function = block.valueOperation(ValueOperation.MAKE_FUNCTION,
+                result, state.funcClosure(), state.funcDefaults());
+        function = applyDecorators(function, state.decorators());
+        block.voidOperation(VoidOperation.STORE, state.name(), function);
+    }
+
+    // Lambda
+
+    public GeneratedCodeState beforeLambda(Lambda node, ScopeInformation scope)
+            throws Exception {
+        return new GeneratedCodeState(null, loadClosure(scope),
+                loadDefaults(node.args.defaults));
+    }
+
+    @Override
+    public Variable visitLambda(Lambda node) throws Exception {
+        block = graph.entryBlock();
+        block.returnOperation(node.body.accept(compiler));
+        return graph.handle();
+    }
+
+    public Variable afterLambda(Lambda node, GeneratedCodeState state,
+            Variable result) throws Exception {
+        return block.valueOperation(ValueOperation.MAKE_FUNCTION, result,
+                state.funcClosure(), state.funcDefaults());
+    }
+
+    // Generator
+
+    public GeneratedCodeState beforeGeneratorExp(GeneratorExp node,
+            ScopeInformation scope) throws Exception {
+        Variable startArg = eval(node.generators[0].iter);
+        startArg = block.valueOperation(ValueOperation.GET_ITERATOR, startArg);
+        return new GeneratedCodeState(null, loadClosure(scope), startArg);
+    }
+
+    @Override
+    public Variable visitGeneratorExp(GeneratorExp node) throws Exception {
+        block = graph.entryBlock();
+        Variable iter = block.valueOperation(ValueOperation.LOAD,
+                graph.getArgument(0));
+        unrollComprehension(node.elt, node.generators, block, iter,
+                graph.exitBlock(), new ComprehensionAction() {
+                    @Override
+                    Link perform(Variable var, Block current, Block next,
+                            Variable... variables) {
+                        return current.yieldOperation(var, next, variables);
+                    }
+                });
+        return graph.handle();
+    }
+
+    public Variable afterGeneratorExp(GeneratorExp node,
+            GeneratedCodeState state, Variable result) throws Exception {
+        return block.valueOperation(ValueOperation.CALL_CODE, result,
+                state.funcClosure(), state.generatorStartArg());
+    }
+
+    // Suite
+
+    @Override
+    public Variable visitSuite(Suite node) throws Exception {
+        throw new ParseException("Cannot compile Suite without context.", node);
+    }
+
+    // Loops
+
+    private static abstract class ComprehensionAction {
+        /**
+         * @param var The variable that was the result of the comprehension
+         *            expression.
+         * @param current The block where we enter.
+         * @param next The block we are supposed to continue to.
+         * @param variables The variables that should be carried onto the next
+         *            block.
+         * @return the link from the current block to the next.
+         */
+        abstract Link perform(Variable var, Block current, Block next,
+                Variable... variables);
+    }
+
+    // FIXME: this is not completed... take inspiration from visitFor
+    private void unrollComprehension(exprType elt,
+            comprehensionType[] generators, Block start, Variable iter,
+            Block end, ComprehensionAction action) throws Exception {
+        Link fail = start.nextBlock(graph.exitBlock(), graph.None());
+        Variable[] iterators = null;
+        for (comprehensionType comprehension : generators) {
+            if (iterators == null) {
+                iterators = new Variable[] { iter };
+            } else {
+                Variable[] newIterators = new Variable[iterators.length + 1];
+                System.arraycopy(iterators, 0, iterators, 0, iterators.length);
+                iterators = newIterators;
+                Variable iterVar = eval(comprehension.iter);
+                iter = block.valueOperation(ValueOperation.GET_ITERATOR,
+                        iterVar);
+                iterators[iterators.length - 1] = iter;
+            }
+            block.iterNext(iter, fail, block = graph.newBlock(), iterators);
+            if (comprehension.ifs != null && comprehension.ifs.length != 0) {
+                for (exprType condition : comprehension.ifs) {
+
+                }
+            }
+        }
+    }
+
+    public Variable visitFor(For node) throws Exception {
+        LinkGroup loop = graph.newLoop();
+        Variable iterator = eval(node.iter);
+        Variable iter = block.valueOperation(ValueOperation.GET_ITERATOR,
+                iterator);
+        Block head = newBlock(block, iter);
+        Block orelse;
+        Block done = newBlock(block);
+        if (node.orelse != null) {
+            orelse = newBlock(block);
+        } else {
+            orelse = done;
+        }
+        Block body = newBlock(block, iter);
+        Link entry = block.nextBlock(head, iter);
+        Link on_exhausted = head.nextBlock(orelse);
+        Link on_hasnext = head.iterNext(iter, on_exhausted, body,
+                new Variable[] {});
+        // TODO: this should be the other way around, by utilizing SuperBlocks
+        loop.add(entry);
+        loop.add(on_exhausted);
+        loop.add(on_hasnext);
+        block = body;
+        breakStack.push(new BreakAction(loop, done, head));
+        compile(node.body);
+        if (node.orelse != null) {
+            block = orelse;
+            compile(node.orelse);
+        }
+        block = done;
+        return null;
+    }
+
+    private Block newBlock(Block template, Variable... variables) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public Variable visitWhile(While node) throws Exception {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public Variable visitBreak(Break node) throws Exception {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public Variable visitContinue(Continue node) throws Exception {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Variable visitListComp(ListComp node) throws Exception {
+        final Variable list = block.valueOperation(ValueOperation.MAKE_LIST);
+        Variable iterVal = eval(node.generators[0].iter);
+        Variable iter = block.valueOperation(ValueOperation.GET_ITERATOR,
+                iterVal);
+        Block after = graph.newBlock();
+        unrollComprehension(node.elt, node.generators, block, iter, after,
+                new ComprehensionAction() {
+                    @Override
+                    Link perform(Variable var, Block location, Block next,
+                            Variable... variables) {
+                        location.voidOperation(VoidOperation.LIST_APPEND, list,
+                                var);
+                        return location.nextBlock(next, variables);
+                    }
+                });
+        block = after;
+        return list;
+    }
+
+    // Selection
+
+    public Variable visitIf(If node) throws Exception {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public Variable visitIfExp(IfExp node) throws Exception {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    public Variable visitBoolOp(BoolOp node) throws Exception {
+        switch (node.op) {
+        case And:
+            // TODO
+            break;
+        case Or:
+            // TODO
+            break;
+        default:
+            throw new ParseException("Unknown boolean operation "
+                    + node.op.name(), node);
+        }
+        return null;
+    }
+
+    public Variable visitCompare(Compare node) throws Exception {
+        if (node.comparators.length == 1) {
+            Variable left = eval(node.left);
+            Variable comparator = eval(node.comparators[0]);
+            return block.valueOperation(comparators.get(node.ops[0]), left,
+                    comparator);
+        }
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    // Assignment
+
+    @Override
+    public Variable visitAssign(Assign node) throws Exception {
+        Variable value = eval(node.value);
+        for (exprType target : node.targets) {
+            target.accept(new Assignment(value));
+        }
+        return null;
+    }
+
+    @Override
+    public Variable visitAugAssign(AugAssign node) throws Exception {
+        Variable target = eval(node.target);
+        Variable rhs = eval(node.value);
+        Variable value = block.valueOperation(augOperators.get(node.op),
+                target, rhs);
+        node.target.accept(new Assignment(value));
+        return null;
+    }
+
+    // Deletion
+
+    @Override
+    public Variable visitDelete(Delete node) throws Exception {
+        node.traverse(delete);
+        return null;
+    }
+
+    // Call
+
+    @Override
+    public Variable visitCall(Call node) throws Exception {
+        Variable func = eval(node.func);
+  ...
 
[truncated message content]

Thread: [Jython-checkins] SF.net SVN: jython:[5272] branches/advanced

jython-checkins