Menu
▾
▴
[Jython-checkins] SF.net SVN: jython:[6990] trunk/jython
|
From: <otm...@us...> - 2010-03-28 08:21:56
|
Revision: 6990
http://jython.svn.sourceforge.net/jython/?rev=6990&view=rev
Author: otmarhumbel
Date: 2010-03-28 08:21:48 +0000 (Sun, 28 Mar 2010)
Log Message:
-----------
add google indexer (by Yin Wang and Steve Yegge)
Modified Paths:
--------------
trunk/jython/ACKNOWLEDGMENTS
trunk/jython/NEWS
trunk/jython/build.xml
trunk/jython/grammar/Python.g
trunk/jython/src/org/python/antlr/BaseParser.java
trunk/jython/src/org/python/antlr/GrammarActions.java
trunk/jython/src/org/python/antlr/PythonTree.java
trunk/jython/src/org/python/antlr/ast/Attribute.java
trunk/jython/src/org/python/antlr/ast/ClassDef.java
trunk/jython/src/org/python/antlr/ast/FunctionDef.java
trunk/jython/src/org/python/antlr/ast/Global.java
trunk/jython/src/org/python/antlr/ast/ImportFrom.java
trunk/jython/src/org/python/antlr/ast/Name.java
trunk/jython/src/org/python/antlr/ast/alias.java
trunk/jython/src/org/python/antlr/ast/arguments.java
Added Paths:
-----------
trunk/jython/src/org/python/antlr/AnalyzingParser.java
trunk/jython/src/org/python/antlr/RecordingErrorHandler.java
trunk/jython/src/org/python/indexer/
trunk/jython/src/org/python/indexer/AstCache.java
trunk/jython/src/org/python/indexer/AstConverter.java
trunk/jython/src/org/python/indexer/Builtins.java
trunk/jython/src/org/python/indexer/Def.java
trunk/jython/src/org/python/indexer/Diagnostic.java
trunk/jython/src/org/python/indexer/Indexer.java
trunk/jython/src/org/python/indexer/IndexingException.java
trunk/jython/src/org/python/indexer/NBinding.java
trunk/jython/src/org/python/indexer/Outliner.java
trunk/jython/src/org/python/indexer/Ref.java
trunk/jython/src/org/python/indexer/Scope.java
trunk/jython/src/org/python/indexer/StyleRun.java
trunk/jython/src/org/python/indexer/Util.java
trunk/jython/src/org/python/indexer/ast/
trunk/jython/src/org/python/indexer/ast/BindingFinder.java
trunk/jython/src/org/python/indexer/ast/DefaultNodeVisitor.java
trunk/jython/src/org/python/indexer/ast/GenericNodeVisitor.java
trunk/jython/src/org/python/indexer/ast/NAlias.java
trunk/jython/src/org/python/indexer/ast/NAssert.java
trunk/jython/src/org/python/indexer/ast/NAssign.java
trunk/jython/src/org/python/indexer/ast/NAttribute.java
trunk/jython/src/org/python/indexer/ast/NAugAssign.java
trunk/jython/src/org/python/indexer/ast/NBinOp.java
trunk/jython/src/org/python/indexer/ast/NBlock.java
trunk/jython/src/org/python/indexer/ast/NBody.java
trunk/jython/src/org/python/indexer/ast/NBoolOp.java
trunk/jython/src/org/python/indexer/ast/NBreak.java
trunk/jython/src/org/python/indexer/ast/NCall.java
trunk/jython/src/org/python/indexer/ast/NClassDef.java
trunk/jython/src/org/python/indexer/ast/NCompare.java
trunk/jython/src/org/python/indexer/ast/NComprehension.java
trunk/jython/src/org/python/indexer/ast/NContinue.java
trunk/jython/src/org/python/indexer/ast/NDelete.java
trunk/jython/src/org/python/indexer/ast/NDict.java
trunk/jython/src/org/python/indexer/ast/NEllipsis.java
trunk/jython/src/org/python/indexer/ast/NExceptHandler.java
trunk/jython/src/org/python/indexer/ast/NExec.java
trunk/jython/src/org/python/indexer/ast/NExprStmt.java
trunk/jython/src/org/python/indexer/ast/NFor.java
trunk/jython/src/org/python/indexer/ast/NFunctionDef.java
trunk/jython/src/org/python/indexer/ast/NGeneratorExp.java
trunk/jython/src/org/python/indexer/ast/NGlobal.java
trunk/jython/src/org/python/indexer/ast/NIf.java
trunk/jython/src/org/python/indexer/ast/NIfExp.java
trunk/jython/src/org/python/indexer/ast/NImport.java
trunk/jython/src/org/python/indexer/ast/NImportFrom.java
trunk/jython/src/org/python/indexer/ast/NIndex.java
trunk/jython/src/org/python/indexer/ast/NKeyword.java
trunk/jython/src/org/python/indexer/ast/NLambda.java
trunk/jython/src/org/python/indexer/ast/NList.java
trunk/jython/src/org/python/indexer/ast/NListComp.java
trunk/jython/src/org/python/indexer/ast/NModule.java
trunk/jython/src/org/python/indexer/ast/NName.java
trunk/jython/src/org/python/indexer/ast/NNode.java
trunk/jython/src/org/python/indexer/ast/NNodeVisitor.java
trunk/jython/src/org/python/indexer/ast/NNum.java
trunk/jython/src/org/python/indexer/ast/NPass.java
trunk/jython/src/org/python/indexer/ast/NPlaceHolder.java
trunk/jython/src/org/python/indexer/ast/NPrint.java
trunk/jython/src/org/python/indexer/ast/NQname.java
trunk/jython/src/org/python/indexer/ast/NRaise.java
trunk/jython/src/org/python/indexer/ast/NRepr.java
trunk/jython/src/org/python/indexer/ast/NReturn.java
trunk/jython/src/org/python/indexer/ast/NSequence.java
trunk/jython/src/org/python/indexer/ast/NSlice.java
trunk/jython/src/org/python/indexer/ast/NStr.java
trunk/jython/src/org/python/indexer/ast/NSubscript.java
trunk/jython/src/org/python/indexer/ast/NTryExcept.java
trunk/jython/src/org/python/indexer/ast/NTryFinally.java
trunk/jython/src/org/python/indexer/ast/NTuple.java
trunk/jython/src/org/python/indexer/ast/NUnaryOp.java
trunk/jython/src/org/python/indexer/ast/NUrl.java
trunk/jython/src/org/python/indexer/ast/NWhile.java
trunk/jython/src/org/python/indexer/ast/NWith.java
trunk/jython/src/org/python/indexer/ast/NYield.java
trunk/jython/src/org/python/indexer/ast/NameBinder.java
trunk/jython/src/org/python/indexer/demos/
trunk/jython/src/org/python/indexer/demos/DocStringParser.java
trunk/jython/src/org/python/indexer/demos/HtmlDemo.java
trunk/jython/src/org/python/indexer/demos/HtmlOutline.java
trunk/jython/src/org/python/indexer/demos/Linker.java
trunk/jython/src/org/python/indexer/demos/StyleApplier.java
trunk/jython/src/org/python/indexer/demos/Styler.java
trunk/jython/src/org/python/indexer/types/
trunk/jython/src/org/python/indexer/types/NClassType.java
trunk/jython/src/org/python/indexer/types/NDictType.java
trunk/jython/src/org/python/indexer/types/NFuncType.java
trunk/jython/src/org/python/indexer/types/NInstanceType.java
trunk/jython/src/org/python/indexer/types/NListType.java
trunk/jython/src/org/python/indexer/types/NModuleType.java
trunk/jython/src/org/python/indexer/types/NTupleType.java
trunk/jython/src/org/python/indexer/types/NType.java
trunk/jython/src/org/python/indexer/types/NUnionType.java
trunk/jython/src/org/python/indexer/types/NUnknownType.java
trunk/jython/tests/java/org/python/indexer/
trunk/jython/tests/java/org/python/indexer/IndexerTest.java
trunk/jython/tests/java/org/python/indexer/TestBase.java
trunk/jython/tests/java/org/python/indexer/data/
trunk/jython/tests/java/org/python/indexer/data/__init__.py
trunk/jython/tests/java/org/python/indexer/data/attr_infer.py
trunk/jython/tests/java/org/python/indexer/data/callnewref.py
trunk/jython/tests/java/org/python/indexer/data/class1.py
trunk/jython/tests/java/org/python/indexer/data/class2.py
trunk/jython/tests/java/org/python/indexer/data/classtype_builtins.py
trunk/jython/tests/java/org/python/indexer/data/empty_file.py
trunk/jython/tests/java/org/python/indexer/data/foo.py
trunk/jython/tests/java/org/python/indexer/data/hello.py
trunk/jython/tests/java/org/python/indexer/data/mod1.py
trunk/jython/tests/java/org/python/indexer/data/mod2.py
trunk/jython/tests/java/org/python/indexer/data/mod3.py
trunk/jython/tests/java/org/python/indexer/data/pkg/
trunk/jython/tests/java/org/python/indexer/data/pkg/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/animaltest.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/mammal/
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/mammal/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/mammal/ape.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/mammal/cat.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/mammal/dog.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/reptile/
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/reptile/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/reptile/croc.py
trunk/jython/tests/java/org/python/indexer/data/pkg/animal/reptile/snake.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/metal/
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/metal/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/metal/gold.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/metal/iron.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/metal/lead.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/stone/
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/stone/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/stone/lapis.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/stone/limestone.py
trunk/jython/tests/java/org/python/indexer/data/pkg/mineral/stone/obsidian.py
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/m1.py
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/m2.py
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/m3.py
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/moduleA.py
trunk/jython/tests/java/org/python/indexer/data/pkg/misc/moduleB.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/
trunk/jython/tests/java/org/python/indexer/data/pkg/other/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/bleu.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/blue.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/crimson.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/green.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/red.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/color/white.py
trunk/jython/tests/java/org/python/indexer/data/pkg/other/force/
trunk/jython/tests/java/org/python/indexer/data/pkg/other/force/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/food/
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/food/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/food/peach.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/food/tofu.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/garden/
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/garden/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/garden/catnip.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/garden/rose.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/garden/weed.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/poison/
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/poison/__init__.py
trunk/jython/tests/java/org/python/indexer/data/pkg/plant/poison/eggplant.py
trunk/jython/tests/java/org/python/indexer/data/pkg/test.py
trunk/jython/tests/java/org/python/indexer/data/pkgload.py
trunk/jython/tests/java/org/python/indexer/data/refs.py
trunk/jython/tests/java/org/python/indexer/data/refs2.py
trunk/jython/tests/java/org/python/indexer/data/test-load.txt
trunk/jython/tests/java/org/python/indexer/data/test.py
trunk/jython/tests/java/org/python/indexer/data/testfileload.py
trunk/jython/tests/java/org/python/indexer/data/testload.py
trunk/jython/tests/java/org/python/indexer/data/testsrc.txt
trunk/jython/tests/java/org/python/indexer/data/yinw/
Modified: trunk/jython/ACKNOWLEDGMENTS
===================================================================
--- trunk/jython/ACKNOWLEDGMENTS 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/ACKNOWLEDGMENTS 2010-03-28 08:21:48 UTC (rev 6990)
@@ -34,6 +34,8 @@
Alan Kennedy contributed modjy, which bridges WSGI to the Servlet API
Chris Gokey, David Syer and Finn Bock added PyServlet.
+
+ Yin Wang and Steve Yegge contributed the static analyzer from Google (also called indexer).
A huge thanks goes to all the members of the jpython/jython
mailing lists. Other folks who have contributed to JPython and
Modified: trunk/jython/NEWS
===================================================================
--- trunk/jython/NEWS 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/NEWS 2010-03-28 08:21:48 UTC (rev 6990)
@@ -22,6 +22,7 @@
during Django or Pylons development mode reloading)
- Fix pickling of collections.defaultdict objects
- Fix the cmath module to accept objects implementing the __float__ method
+ - Add google indexer (by Yin Wang and Steve Yegge)
Jython 2.5.1rc3
Bugs Fixed
Modified: trunk/jython/build.xml
===================================================================
--- trunk/jython/build.xml 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/build.xml 2010-03-28 08:21:48 UTC (rev 6990)
@@ -883,6 +883,7 @@
<junit fork="true" printsummary="true">
<formatter type="xml"/>
<sysproperty key="python.home" value="${dist.dir}"/>
+ <sysproperty key="python.test.source.dir" value="${test.source.dir}"/>
<classpath refid="test.classpath"/>
<batchtest todir="${junit.reports}">
<fileset dir="${test.source.dir}" includes="**/*Test*.java">
@@ -893,6 +894,50 @@
</batchtest>
</junit>
</target>
+ <target name="idxtest" depends="developer-build">
+ <mkdir dir="${junit.reports}"/>
+ <junit fork="true" printsummary="true" showoutput="true">
+ <formatter type="xml"/>
+ <sysproperty key="python.home" value="${dist.dir}"/>
+ <sysproperty key="python.test.source.dir" value="${test.source.dir}"/>
+ <classpath refid="test.classpath"/>
+ <batchtest todir="${junit.reports}">
+ <fileset dir="${test.source.dir}" includes="**/*Test*.java">
+ <exclude name="javatests/**/*" />
+ <exclude name="**/InterpTestCase.java" />
+ <exclude name="org/python/antlr/**" />
+ <exclude name="org/python/core/**" />
+ <exclude name="org/python/expose/**" />
+ <exclude name="org/python/jsr223/**" />
+ <exclude name="org/python/tests/**" />
+ <exclude name="org/python/util/**" />
+ </fileset>
+ </batchtest>
+ </junit>
+ </target>
+ <!-- XXX: how do I share common stuff with "idxtest" target? -->
+ <target name="idxtest-debug" depends="developer-build">
+ <mkdir dir="${junit.reports}"/>
+ <junit fork="true" printsummary="true">
+ <formatter type="xml"/>
+ <sysproperty key="python.home" value="${dist.dir}"/>
+ <sysproperty key="python.test.source.dir" value="${test.source.dir}"/>
+ <classpath refid="test.classpath"/>
+ <jvmarg value="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5000"/>
+ <batchtest todir="${junit.reports}">
+ <fileset dir="${test.source.dir}" includes="**/*Test*.java">
+ <exclude name="javatests/**/*" />
+ <exclude name="**/InterpTestCase.java" />
+ <exclude name="org/python/antlr/**" />
+ <exclude name="org/python/core/**" />
+ <exclude name="org/python/expose/**" />
+ <exclude name="org/python/jsr223/**" />
+ <exclude name="org/python/tests/**" />
+ <exclude name="org/python/util/**" />
+ </fileset>
+ </batchtest>
+ </junit>
+ </target>
<target name="modjytest" depends="developer-build">
<ant dir="tests/modjy">
<property name="jython_home" value="${dist.dir}"/>
Modified: trunk/jython/grammar/Python.g
===================================================================
--- trunk/jython/grammar/Python.g 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/grammar/Python.g 2010-03-28 08:21:48 UTC (rev 6990)
@@ -367,7 +367,7 @@
}
|
{
- $etype = new Name($n1, $n1.text, expr_contextType.Load);
+ $etype = actions.makeNameNode($n1);
}
)
;
@@ -783,13 +783,16 @@
import_from
: FROM (d+=DOT* dotted_name | d+=DOT+) IMPORT
(STAR
- -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.name),
+ -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.names),
+ actions.makeModuleNameNode($d, $dotted_name.names),
actions.makeStarAlias($STAR), actions.makeLevel($d)])
| i1=import_as_names
- -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.name),
+ -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.names),
+ actions.makeModuleNameNode($d, $dotted_name.names),
actions.makeAliases($i1.atypes), actions.makeLevel($d)])
| LPAREN i2=import_as_names COMMA? RPAREN
- -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.name),
+ -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.names),
+ actions.makeModuleNameNode($d, $dotted_name.names),
actions.makeAliases($i2.atypes), actions.makeLevel($d)])
)
;
@@ -811,7 +814,7 @@
}
: name=NAME (AS asname=NAME)?
{
- $atype = new alias($name, $name.text, $asname.text);
+ $atype = new alias(actions.makeNameNode($name), actions.makeNameNode($asname));
}
;
@@ -822,10 +825,9 @@
@after {
$dotted_as_name.tree = $atype;
}
-
- : dotted_name (AS NAME)?
+ : dotted_name (AS asname=NAME)?
{
- $atype = new alias($NAME, $dotted_name.name, $NAME.text);
+ $atype = new alias($dotted_name.names, actions.makeNameNode($asname));
}
;
@@ -840,17 +842,17 @@
//dotted_name: NAME ('.' NAME)*
dotted_name
- returns [String name]
+ returns [List<Name> names]
: NAME (DOT dn+=attr)*
{
- $name = actions.makeDottedText($NAME, $dn);
+ $names = actions.makeDottedName($NAME, $dn);
}
;
//global_stmt: 'global' NAME (',' NAME)*
global_stmt
: GLOBAL n+=NAME (COMMA n+=NAME)*
- -> ^(GLOBAL<Global>[$GLOBAL, actions.makeNames($n)])
+ -> ^(GLOBAL<Global>[$GLOBAL, actions.makeNames($n), actions.makeNameNodes($n)])
;
//exec_stmt: 'exec' expr ['in' test [',' test]]
@@ -1522,7 +1524,7 @@
| LBRACK subscriptlist[$begin] RBRACK
-> ^(LBRACK<Subscript>[$begin, actions.castExpr($tree), actions.castSlice($subscriptlist.tree), $expr::ctype])
| DOT attr
- -> ^(DOT<Attribute>[$begin, actions.castExpr($tree), $attr.text, $expr::ctype])
+ -> ^(DOT<Attribute>[$begin, actions.castExpr($tree), new Name($attr.tree, $attr.text, expr_contextType.Load), $expr::ctype])
;
//subscriptlist: subscript (',' subscript)* [',']
@@ -1630,7 +1632,7 @@
if ($decorators.start != null) {
t = $decorators.start;
}
- stype = new ClassDef(t, actions.cantBeNone($NAME),
+ stype = new ClassDef(t, actions.cantBeNoneName($NAME),
actions.makeBases(actions.castExpr($testlist.tree)),
actions.castStmts($suite.stypes),
actions.castExprs($decorators.etypes));
Added: trunk/jython/src/org/python/antlr/AnalyzingParser.java
===================================================================
--- trunk/jython/src/org/python/antlr/AnalyzingParser.java (rev 0)
+++ trunk/jython/src/org/python/antlr/AnalyzingParser.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -0,0 +1,71 @@
+/**
+ * Copyright 2009, Google Inc. All rights reserved.
+ * Licensed to PSF under a Contributor Agreement.
+ */
+package org.python.antlr;
+
+import org.antlr.runtime.ANTLRFileStream;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.python.antlr.ast.Name;
+import org.python.antlr.base.mod;
+
+import java.util.List;
+
+/**
+ * Parser used by the indexer.
+ */
+public class AnalyzingParser extends BaseParser {
+
+ public static class AnalyzerTreeAdaptor extends PythonTreeAdaptor {
+ /**
+ * Make sure a parenthesized {@link Name} expr has its start/stop bounds
+ * set to the bounds of the identifier.
+ */
+ @Override
+ public void setTokenBoundaries(Object t, Token startToken, Token stopToken) {
+ //XXX: should do this for all expr types, and have a prop list on Expr
+ //that records all enclosing paren locations for IDE use cases.
+ if (!(t instanceof Name)
+ || startToken == null
+ || stopToken == null
+ || startToken.getType() != PythonParser.LPAREN
+ || stopToken.getType() != PythonParser.RPAREN) {
+ super.setTokenBoundaries(t, startToken, stopToken);
+ }
+ }
+ }
+
+ public AnalyzingParser(CharStream stream, String filename, String encoding) {
+ super(stream, filename, encoding);
+ errorHandler = new RecordingErrorHandler();
+ }
+
+ public List<RecognitionException> getRecognitionErrors() {
+ return ((RecordingErrorHandler)errorHandler).errs;
+ }
+
+ @Override
+ protected PythonParser setupParser(boolean single) {
+ PythonParser parser = super.setupParser(single);
+ parser.setTreeAdaptor(new AnalyzerTreeAdaptor());
+ return parser;
+ }
+
+ public static void main(String[] args) {
+ CharStream in = null;
+ try {
+ in = new ANTLRFileStream(args[0]);
+ } catch (Exception x) {
+ x.printStackTrace();
+ }
+ AnalyzingParser p = new AnalyzingParser(in, args[0], "ascii");
+ mod ast = p.parseModule();
+ if (ast != null) {
+ System.out.println("parse result: \n" + ast.toStringTree());
+ } else {
+ System.out.println("failure: \n" + p.getRecognitionErrors());
+ }
+ }
+}
Modified: trunk/jython/src/org/python/antlr/BaseParser.java
===================================================================
--- trunk/jython/src/org/python/antlr/BaseParser.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/BaseParser.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -31,7 +31,7 @@
this.errorHandler = eh;
}
- private PythonParser setupParser(boolean single) {
+ protected PythonParser setupParser(boolean single) {
PythonLexer lexer = new PythonLexer(charStream);
lexer.setErrorHandler(errorHandler);
lexer.single = single;
Modified: trunk/jython/src/org/python/antlr/GrammarActions.java
===================================================================
--- trunk/jython/src/org/python/antlr/GrammarActions.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/GrammarActions.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -61,19 +61,41 @@
this.errorHandler = eh;
}
- String makeFromText(List dots, String name) {
- StringBuffer d = new StringBuffer();
+ String makeFromText(List dots, List<Name> names) {
+ StringBuilder d = new StringBuilder();
if (dots != null) {
for (int i=0;i<dots.size();i++) {
d.append(".");
}
}
- if (name != null) {
- d.append(name);
- }
+ d.append(PythonTree.dottedNameListToString(names));
return d.toString();
}
+ List<Name> makeModuleNameNode(List dots, List<Name> names) {
+ List<Name> result = new ArrayList<Name>();
+ if (dots != null) {
+ for (Object o : dots) {
+ Token tok = (Token)o;
+ result.add(new Name(tok, tok.getText(), expr_contextType.Load));
+ }
+ }
+ result.addAll(names);
+ return result;
+ }
+
+ List<Name> makeDottedName(Token top, List<PythonTree> attrs) {
+ List<Name> result = new ArrayList<Name>();
+ result.add(new Name(top, top.getText(), expr_contextType.Load));
+ if (attrs != null) {
+ for (PythonTree attr : attrs) {
+ Token token = attr.getToken();
+ result.add(new Name(token, token.getText(), expr_contextType.Load));
+ }
+ }
+ return result;
+ }
+
int makeLevel(List lev) {
if (lev == null) {
return 0;
@@ -113,6 +135,21 @@
return s;
}
+ Name makeNameNode(Token t) {
+ if (t == null) {
+ return null;
+ }
+ return new Name(t, t.getText(), expr_contextType.Load);
+ }
+
+ List<Name> makeNameNodes(List<Token> names) {
+ List<Name> s = new ArrayList<Name>();
+ for (int i=0; i<names.size(); i++) {
+ s.add(makeNameNode(names.get(i)));
+ }
+ return s;
+ }
+
void errorGenExpNotSoleArg(PythonTree t) {
errorHandler.error("Generator expression must be parenthesized if not sole argument", t);
}
@@ -191,7 +228,7 @@
expr current = new Name(nameToken, nameToken.getText(), expr_contextType.Load);
for (Object o: attrs) {
Token t = (Token)o;
- current = new Attribute(t, current, t.getText(),
+ current = new Attribute(t, current, cantBeNoneName(t),
expr_contextType.Load);
}
return current;
@@ -236,21 +273,21 @@
List<stmt> f = castStmts(finBody);
return new TryFinally(t, b, f);
}
-
+
stmt makeFuncdef(Token t, Token nameToken, arguments args, List funcStatements, List decorators) {
if (nameToken == null) {
return errorHandler.errorStmt(new PythonTree(t));
}
- cantBeNone(nameToken);
+ Name n = cantBeNoneName(nameToken);
arguments a;
if (args != null) {
a = args;
} else {
- a = new arguments(t, new ArrayList<expr>(), null, null, new ArrayList<expr>());
+ a = new arguments(t, new ArrayList<expr>(), (Name)null, null, new ArrayList<expr>());
}
List<stmt> s = castStmts(funcStatements);
List<expr> d = castExprs(decorators);
- return new FunctionDef(t, nameToken.getText(), a, s, d);
+ return new FunctionDef(t, n, a, s, d);
}
List<expr> makeAssignTargets(expr lhs, List rhs) {
@@ -293,17 +330,17 @@
List<expr> p = castExprs(params);
List<expr> d = castExprs(defaults);
- String s;
- String k;
+ Name s;
+ Name k;
if (snameToken == null) {
s = null;
} else {
- s = cantBeNone(snameToken);
+ s = cantBeNoneName(snameToken);
}
if (knameToken == null) {
k = null;
} else {
- k = cantBeNone(knameToken);
+ k = cantBeNoneName(knameToken);
}
return new arguments(t, p, s, k, d);
}
@@ -516,6 +553,13 @@
return t.getText();
}
+ Name cantBeNoneName(Token t) {
+ if (t == null || t.getText().equals("None")) {
+ errorHandler.error("can't be None", new PythonTree(t));
+ }
+ return new Name(t, t.getText(), expr_contextType.Load);
+ }
+
void cantBeNone(PythonTree e) {
if (e.getText().equals("None")) {
errorHandler.error("can't be None", e);
@@ -722,18 +766,4 @@
}
return s;
}
-
- public String makeDottedText(Token name, List<PythonTree> c) {
- final String dot = ".";
- if (c == null || c.isEmpty()) {
- return name.getText();
- }
- StringBuilder b = new StringBuilder(name.getText());
- for (PythonTree t : c) {
- b.append(dot);
- b.append(t.getToken().getText());
- }
- return b.toString();
- }
-
}
Modified: trunk/jython/src/org/python/antlr/PythonTree.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTree.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/PythonTree.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -5,6 +5,7 @@
import org.antlr.runtime.tree.CommonTree;
import org.python.core.PyType;
+import org.python.antlr.ast.Name;
import org.python.antlr.ast.VisitorIF;
import java.util.ArrayList;
@@ -165,6 +166,35 @@
node.setChildIndex(index);
}
+ /**
+ * Converts a list of Name to a dotted-name string.
+ * Because leading dots are indexable identifiers (referring
+ * to parent directories in relative imports), a Name list
+ * may include leading dots, but not dots between names.
+ */
+ public static String dottedNameListToString(List<Name> names) {
+ if (names == null) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ boolean leadingDot = true;
+ for (int i = 0, len = names.size(); i < len; i++) {
+ Name name = names.get(i);
+ String id = name.getInternalId();
+ if (id == null) {
+ continue;
+ }
+ if (!".".equals(id)) {
+ leadingDot = false;
+ }
+ sb.append(id);
+ if (i < len - 1 && !leadingDot) {
+ sb.append(".");
+ }
+ }
+ return sb.toString();
+ }
+
@Override
public String toString() {
if (isNil()) {
Added: trunk/jython/src/org/python/antlr/RecordingErrorHandler.java
===================================================================
--- trunk/jython/src/org/python/antlr/RecordingErrorHandler.java (rev 0)
+++ trunk/jython/src/org/python/antlr/RecordingErrorHandler.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2009, Google Inc. All rights reserved.
+ * Licensed to PSF under a Contributor Agreement.
+ */
+package org.python.antlr;
+
+import org.antlr.runtime.BaseRecognizer;
+import org.antlr.runtime.BitSet;
+import org.antlr.runtime.IntStream;
+import org.antlr.runtime.Lexer;
+import org.antlr.runtime.RecognitionException;
+import org.python.antlr.ast.ErrorExpr;
+import org.python.antlr.ast.ErrorMod;
+import org.python.antlr.ast.ErrorSlice;
+import org.python.antlr.ast.ErrorStmt;
+import org.python.antlr.base.expr;
+import org.python.antlr.base.mod;
+import org.python.antlr.base.slice;
+import org.python.antlr.base.stmt;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RecordingErrorHandler implements ErrorHandler {
+
+ public List<RecognitionException> errs = new ArrayList<RecognitionException>();
+
+ public void reportError(BaseRecognizer br, RecognitionException re) {
+ br.reportError(re);
+ errs.add(re);
+ }
+
+ public void recover(Lexer lex, RecognitionException re) {
+ lex.recover(re);
+ }
+
+ public void recover(BaseRecognizer br, IntStream input, RecognitionException re) {
+ br.recover(input, re);
+ }
+
+ public boolean mismatch(BaseRecognizer br, IntStream input, int ttype, BitSet follow) {
+ return true;
+ }
+
+ public Object recoverFromMismatchedToken(BaseRecognizer br, IntStream input,
+ int ttype, BitSet follow) {
+ return null;
+ }
+
+ public expr errorExpr(PythonTree t) {
+ return new ErrorExpr(t);
+ }
+
+ public mod errorMod(PythonTree t) {
+ return new ErrorMod(t);
+ }
+
+ public slice errorSlice(PythonTree t) {
+ return new ErrorSlice(t);
+ }
+
+ public stmt errorStmt(PythonTree t) {
+ return new ErrorStmt(t);
+ }
+
+ public void error(String message, PythonTree t) {
+ System.err.println(message);
+ }
+}
Modified: trunk/jython/src/org/python/antlr/ast/Attribute.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/Attribute.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/Attribute.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -45,6 +45,10 @@
public String getInternalAttr() {
return attr;
}
+ private Name attrName;
+ public Name getInternalAttrName() {
+ return attrName;
+ }
@ExposedGet(name = "attr")
public PyObject getAttr() {
if (attr == null) return Py.None;
@@ -119,6 +123,24 @@
this.ctx = ctx;
}
+ public Attribute(Token token, expr value, Name attr, expr_contextType ctx) {
+ super(token);
+ this.value = value;
+ addChild(value);
+ this.attr = attr.getText();
+ this.attrName = attr;
+ this.ctx = ctx;
+ }
+
+ public Attribute(Integer ttype, Token token, expr value, Name attr, expr_contextType ctx) {
+ super(ttype, token);
+ this.value = value;
+ addChild(value);
+ this.attr = attr.getText();
+ this.attrName = attr;
+ this.ctx = ctx;
+ }
+
public Attribute(Integer ttype, Token token, expr value, String attr, expr_contextType ctx) {
super(ttype, token);
this.value = value;
Modified: trunk/jython/src/org/python/antlr/ast/ClassDef.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/ClassDef.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/ClassDef.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -32,6 +32,10 @@
public String getInternalName() {
return name;
}
+ private Name nameNode;
+ public Name getInternalNameNode() {
+ return nameNode;
+ }
@ExposedGet(name = "name")
public PyObject getName() {
if (name == null) return Py.None;
@@ -154,6 +158,34 @@
}
}
+ public ClassDef(Token token, Name name, java.util.List<expr> bases, java.util.List<stmt>
+ body, java.util.List<expr> decorator_list) {
+ super(token);
+ this.name = name.getText();
+ this.nameNode = name;
+ this.bases = bases;
+ if (bases == null) {
+ this.bases = new ArrayList<expr>();
+ }
+ for(PythonTree t : this.bases) {
+ addChild(t);
+ }
+ this.body = body;
+ if (body == null) {
+ this.body = new ArrayList<stmt>();
+ }
+ for(PythonTree t : this.body) {
+ addChild(t);
+ }
+ this.decorator_list = decorator_list;
+ if (decorator_list == null) {
+ this.decorator_list = new ArrayList<expr>();
+ }
+ for(PythonTree t : this.decorator_list) {
+ addChild(t);
+ }
+ }
+
public ClassDef(Integer ttype, Token token, String name, java.util.List<expr> bases,
java.util.List<stmt> body, java.util.List<expr> decorator_list) {
super(ttype, token);
Modified: trunk/jython/src/org/python/antlr/ast/FunctionDef.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/FunctionDef.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/FunctionDef.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -32,6 +32,10 @@
public String getInternalName() {
return name;
}
+ private Name nameNode;
+ public Name getInternalNameNode() {
+ return nameNode;
+ }
@ExposedGet(name = "name")
public PyObject getName() {
if (name == null) return Py.None;
@@ -148,6 +152,28 @@
}
}
+ public FunctionDef(Token token, Name name, arguments args, java.util.List<stmt> body,
+ java.util.List<expr> decorator_list) {
+ super(token);
+ this.name = name.getText();
+ this.nameNode = name;
+ this.args = args;
+ this.body = body;
+ if (body == null) {
+ this.body = new ArrayList<stmt>();
+ }
+ for(PythonTree t : this.body) {
+ addChild(t);
+ }
+ this.decorator_list = decorator_list;
+ if (decorator_list == null) {
+ this.decorator_list = new ArrayList<expr>();
+ }
+ for(PythonTree t : this.decorator_list) {
+ addChild(t);
+ }
+ }
+
public FunctionDef(Integer ttype, Token token, String name, arguments args,
java.util.List<stmt> body, java.util.List<expr> decorator_list) {
super(ttype, token);
Modified: trunk/jython/src/org/python/antlr/ast/Global.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/Global.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/Global.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -40,8 +40,11 @@
public void setNames(PyObject names) {
this.names = AstAdapters.py2identifierList(names);
}
+ private java.util.List<Name> nameNodes;
+ public java.util.List<Name> getInternalNameNodes() {
+ return nameNodes;
+ }
-
private final static PyString[] fields =
new PyString[] {new PyString("names")};
@ExposedGet(name = "_fields")
@@ -85,6 +88,12 @@
this.names = names;
}
+ public Global(Integer ttype, Token token, java.util.List<String> names, java.util.List<Name> nameNodes) {
+ super(ttype, token);
+ this.names = names;
+ this.nameNodes = nameNodes;
+ }
+
public Global(Integer ttype, Token token, java.util.List<String> names) {
super(ttype, token);
this.names = names;
Modified: trunk/jython/src/org/python/antlr/ast/ImportFrom.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/ImportFrom.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/ImportFrom.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -1,4 +1,4 @@
-// Autogenerated AST node
+// Autogenerated AST node -*- c-basic-offset:4 -*-
package org.python.antlr.ast;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Token;
@@ -24,6 +24,7 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
@ExposedType(name = "_ast.ImportFrom", base = AST.class)
public class ImportFrom extends stmt {
@@ -42,8 +43,8 @@
this.module = AstAdapters.py2identifier(module);
}
- private java.util.List<alias> names;
- public java.util.List<alias> getInternalNames() {
+ private List<alias> names;
+ public List<alias> getInternalNames() {
return names;
}
@ExposedGet(name = "names")
@@ -55,6 +56,11 @@
this.names = AstAdapters.py2aliasList(names);
}
+ private List<Name> moduleNames;
+ public List<Name> getInternalModuleNames() {
+ return moduleNames;
+ }
+
private Integer level;
public Integer getInternalLevel() {
return level;
@@ -111,8 +117,10 @@
setLevel(level);
}
- public ImportFrom(Token token, String module, java.util.List<alias> names, Integer level) {
- super(token);
+ public ImportFrom(int ttype, Token token,
+ String module, List<Name> moduleNames,
+ List<alias> names, Integer level) {
+ super(ttype, token);
this.module = module;
this.names = names;
if (names == null) {
@@ -121,10 +129,17 @@
for(PythonTree t : this.names) {
addChild(t);
}
+ this.moduleNames = moduleNames;
+ if (moduleNames == null) {
+ this.moduleNames = new ArrayList<Name>();
+ }
+ for(PythonTree t : this.moduleNames) {
+ addChild(t);
+ }
this.level = level;
}
- public ImportFrom(Integer ttype, Token token, String module, java.util.List<alias> names,
+ public ImportFrom(Integer ttype, Token token, String module, List<alias> names,
Integer level) {
super(ttype, token);
this.module = module;
@@ -138,7 +153,7 @@
this.level = level;
}
- public ImportFrom(PythonTree tree, String module, java.util.List<alias> names, Integer level) {
+ public ImportFrom(PythonTree tree, String module, List<alias> names, Integer level) {
super(tree);
this.module = module;
this.names = names;
Modified: trunk/jython/src/org/python/antlr/ast/Name.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/Name.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/Name.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -1,4 +1,4 @@
-// Autogenerated AST node
+// Autogenerated AST node -*- c-basic-offset:4 -*-
package org.python.antlr.ast;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Token;
@@ -55,7 +55,6 @@
this.ctx = AstAdapters.py2expr_context(ctx);
}
-
private final static PyString[] fields =
new PyString[] {new PyString("id"), new PyString("ctx")};
@ExposedGet(name = "_fields")
Modified: trunk/jython/src/org/python/antlr/ast/alias.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/alias.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/alias.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -42,6 +42,10 @@
this.name = AstAdapters.py2identifier(name);
}
+ private java.util.List<Name> nameNodes;
+ public java.util.List<Name> getInternalNameNodes() {
+ return nameNodes;
+ }
private String asname;
public String getInternalAsname() {
return asname;
@@ -56,6 +60,10 @@
this.asname = AstAdapters.py2identifier(asname);
}
+ private Name asnameNode;
+ public Name getInternalAsnameNode() {
+ return asnameNode;
+ }
private final static PyString[] fields =
new PyString[] {new PyString("name"), new PyString("asname")};
@@ -92,6 +100,21 @@
this.asname = asname;
}
+ // [import] name [as asname]
+ public alias(Name name, Name asname) {
+ this(java.util.Arrays.asList(new Name[]{name}), asname);
+ }
+
+ // [import] ...foo.bar.baz [as asname]
+ public alias(java.util.List<Name> nameNodes, Name asname) {
+ this.nameNodes = nameNodes;
+ this.name = dottedNameListToString(nameNodes);
+ if (asname != null) {
+ this.asnameNode = asname;
+ this.asname = asname.getInternalId();
+ }
+ }
+
public alias(Integer ttype, Token token, String name, String asname) {
super(ttype, token);
this.name = name;
Modified: trunk/jython/src/org/python/antlr/ast/arguments.java
===================================================================
--- trunk/jython/src/org/python/antlr/ast/arguments.java 2010-03-28 04:12:27 UTC (rev 6989)
+++ trunk/jython/src/org/python/antlr/ast/arguments.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -45,6 +45,12 @@
public String getInternalVararg() {
return vararg;
}
+
+ private Name varargName;
+ public Name getInternalVarargName() {
+ return varargName;
+ }
+
@ExposedGet(name = "vararg")
public PyObject getVararg() {
if (vararg == null) return Py.None;
@@ -59,6 +65,12 @@
public String getInternalKwarg() {
return kwarg;
}
+
+ private Name kwargName;
+ public Name getInternalKwargName() {
+ return kwargName;
+ }
+
@ExposedGet(name = "kwarg")
public PyObject getKwarg() {
if (kwarg == null) return Py.None;
@@ -117,8 +129,29 @@
setDefaults(defaults);
}
- public arguments(Token token, java.util.List<expr> args, String vararg, String kwarg,
- java.util.List<expr> defaults) {
+ // public arguments(Token token, java.util.List<expr> args, String vararg, String kwarg,
+ // java.util.List<expr> defaults) {
+ // super(token);
+ // this.args = args;
+ // if (args == null) {
+ // this.args = new ArrayList<expr>();
+ // }
+ // for(PythonTree t : this.args) {
+ // addChild(t);
+ // }
+ // this.vararg = vararg;
+ // this.kwarg = kwarg;
+ // this.defaults = defaults;
+ // if (defaults == null) {
+ // this.defaults = new ArrayList<expr>();
+ // }
+ // for(PythonTree t : this.defaults) {
+ // addChild(t);
+ // }
+ // }
+
+ public arguments(Token token, java.util.List<expr> args, Name vararg, Name kwarg,
+ java.util.List<expr> defaults) {
super(token);
this.args = args;
if (args == null) {
@@ -127,8 +160,10 @@
for(PythonTree t : this.args) {
addChild(t);
}
- this.vararg = vararg;
- this.kwarg = kwarg;
+ this.vararg = vararg == null ? null : vararg.getText();
+ this.varargName = vararg;
+ this.kwarg = kwarg == null ? null : kwarg.getText();
+ this.kwargName = kwarg;
this.defaults = defaults;
if (defaults == null) {
this.defaults = new ArrayList<expr>();
Added: trunk/jython/src/org/python/indexer/AstCache.java
===================================================================
--- trunk/jython/src/org/python/indexer/AstCache.java (rev 0)
+++ trunk/jython/src/org/python/indexer/AstCache.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -0,0 +1,325 @@
+/**
+ * Copyright 2009, Google Inc. All rights reserved.
+ * Licensed to PSF under a Contributor Agreement.
+ */
+package org.python.indexer;
+
+import org.antlr.runtime.ANTLRFileStream;
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.RecognitionException;
+import org.python.antlr.AnalyzingParser;
+import org.python.antlr.base.mod;
+import org.python.indexer.ast.NModule;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Provides a factory for python source ASTs. Maintains configurable on-disk and
+ * in-memory caches to avoid re-parsing files during analysis.
+ */
+public class AstCache {
+
+ public static final String CACHE_DIR = Util.getSystemTempDir() + "jython/ast_cache/";
+
+ private static final Logger LOG = Logger.getLogger(AstCache.class.getCanonicalName());
+
+ private Map<String, NModule> cache = new HashMap<String, NModule>();
+
+ private static AstCache INSTANCE;
+
+ private AstCache() throws Exception {
+ File f = new File(CACHE_DIR);
+ if (!f.exists()) {
+ f.mkdirs();
+ }
+ }
+
+ public static AstCache get() throws Exception {
+ if (INSTANCE == null) {
+ INSTANCE = new AstCache();
+ }
+ return INSTANCE;
+ }
+
+ /**
+ * Clears the memory cache.
+ */
+ public void clear() {
+ cache.clear();
+ }
+
+ /**
+ * Removes all serialized ASTs from the on-disk cache.
+ * @return {@code true} if all cached AST files were removed
+ */
+ public boolean clearDiskCache() {
+ try {
+ File dir = new File(CACHE_DIR);
+ for (File f : dir.listFiles()) {
+ if (f.isFile()) {
+ f.delete();
+ }
+ }
+ return true;
+ } catch (Exception x) {
+ severe("Failed to clear disk cache: " + x);
+ return false;
+ }
+ }
+
+ /**
+ * Returns the syntax tree for {@code path}. May find and/or create a
+ * cached copy in the mem cache or the disk cache.
+ * @param path absolute path to a source file
+ * @return the AST, or {@code null} if the parse failed for any reason
+ * @throws Exception if anything unexpected occurs
+ */
+ public NModule getAST(String path) throws Exception {
+ if (path == null) throw new IllegalArgumentException("null path");
+ return fetch(path);
+ }
+
+ /**
+ * Returns the syntax tree for {@code path} with {@code contents}.
+ * Uses the memory cache but not the disk cache.
+ * This method exists primarily for unit testing.
+ * @param path a name for the file. Can be relative.
+ * @param contents the source to parse
+ */
+ public NModule getAST(String path, String contents) throws Exception {
+ if (path == null) throw new IllegalArgumentException("null path");
+ if (contents == null) throw new IllegalArgumentException("null contents");
+
+ // Cache stores null value if the parse failed.
+ if (cache.containsKey(path)) {
+ return cache.get(path);
+ }
+
+ NModule mod = null;
+ try {
+ mod = parse(path, contents);
+ if (mod != null) {
+ mod.setFileAndMD5(path, Util.getMD5(contents.getBytes("UTF-8")));
+ }
+ } finally {
+ cache.put(path, mod); // may be null
+ }
+ return mod;
+ }
+
+ /**
+ * Get or create an AST for {@code path}, checking and if necessary updating
+ * the disk and memory caches.
+ * @param path absolute source path
+ */
+ private NModule fetch(String path) throws Exception {
+ // Cache stores null value if the parse failed.
+ if (cache.containsKey(path)) {
+ return cache.get(path);
+ }
+
+ // Might be cached on disk but not in memory.
+ NModule mod = getSerializedModule(path);
+ if (mod != null) {
+ fine("reusing " + path);
+ cache.put(path, mod);
+ return mod;
+ }
+
+ try {
+ mod = parse(path);
+ } finally {
+ cache.put(path, mod); // may be null
+ }
+
+ if (mod != null) {
+ serialize(mod);
+ }
+
+ return mod;
+ }
+
+ /**
+ * Parse a file. Does not look in the cache or cache the result.
+ */
+ private NModule parse(String path) throws Exception {
+ fine("parsing " + path);
+ mod ast = invokeANTLR(path);
+ return generateAST(ast, path);
+ }
+
+ /**
+ * Parse a string. Does not look in the cache or cache the result.
+ */
+ private NModule parse(String path, String contents) throws Exception {
+ fine("parsing " + path);
+ mod ast = invokeANTLR(path, contents);
+ return generateAST(ast, path);
+ }
+
+ private NModule generateAST(mod ast, String path) throws Exception {
+ if (ast == null) {
+ Indexer.idx.reportFailedAssertion("ANTLR returned NULL for " + path);
+ return null;
+ }
+
+ // Convert to indexer's AST. Type conversion warnings are harmless here.
+ Object obj = ast.accept(new AstConverter());
+ if (!(obj instanceof NModule)) {
+ warn("\n[warning] converted AST is not a module: " + obj);
+ return null;
+ }
+
+ NModule module = (NModule)obj;
+ if (new File(path).canRead()) {
+ module.setFile(path);
+ }
+ return module;
+ }
+
+ private mod invokeANTLR(String filename) {
+ CharStream text = null;
+ try {
+ text = new ANTLRFileStream(filename);
+ } catch (IOException iox) {
+ fine(filename + ": " + iox);
+ return null;
+ }
+ return invokeANTLR(text, filename);
+ }
+
+ private mod invokeANTLR(String filename, String contents) {
+ CharStream text = new ANTLRStringStream(contents);
+ return invokeANTLR(text, filename);
+ }
+
+ private mod invokeANTLR(CharStream text, String filename) {
+ AnalyzingParser p = new AnalyzingParser(text, filename, null);
+ mod ast = null;
+ try {
+ ast = p.parseModule();
+ } catch (Exception x) {
+ fine("parse for " + filename + " failed: " + x);
+ }
+ recordParseErrors(filename, p.getRecognitionErrors());
+ return ast;
+ }
+
+ private void recordParseErrors(String path, List<RecognitionException> errs) {
+ if (errs.isEmpty()) {
+ return;
+ }
+ List<Diagnostic> diags = Indexer.idx.getParseErrs(path);
+ for (RecognitionException rx : errs) {
+ String msg = rx.line + ":" + rx.charPositionInLine + ":" + rx;
+ diags.add(new Diagnostic(path, Diagnostic.Type.ERROR, -1, -1, msg));
+ }
+ }
+
+ /**
+ * Each source file's AST is saved in an object file named for the MD5
+ * checksum of the source file. All that is needed is the MD5, but the
+ * file's base name is included for ease of debugging.
+ */
+ public String getCachePath(File sourcePath) throws Exception {
+ return getCachePath(Util.getMD5(sourcePath), sourcePath.getName());
+ }
+
+ public String getCachePath(String md5, String name) {
+ return CACHE_DIR + name + md5 + ".ast";
+ }
+
+ // package-private for testing
+ void serialize(NModule ast) throws Exception {
+ String path = getCachePath(ast.getMD5(), new File(ast.getFile()).getName());
+ ObjectOutputStream oos = null;
+ FileOutputStream fos = null;
+ try {
+ fos = new FileOutputStream(path);
+ oos = new ObjectOutputStream(fos);
+ oos.writeObject(ast);
+ } finally {
+ if (oos != null) {
+ oos.close();
+ } else if (fos != null) {
+ fos.close();
+ }
+ }
+ }
+
+ // package-private for testing
+ NModule getSerializedModule(String sourcePath) {
+ try {
+ File sourceFile = new File(sourcePath);
+ if (sourceFile == null || !sourceFile.canRead()) {
+ return null;
+ }
+ File cached = new File(getCachePath(sourceFile));
+ if (!cached.canRead()) {
+ return null;
+ }
+ return deserialize(sourceFile);
+ } catch (Exception x) {
+ severe("Failed to deserialize " + sourcePath + ": " + x);
+ return null;
+ }
+ }
+
+ // package-private for testing
+ NModule deserialize(File sourcePath) throws Exception {
+ String cachePath = getCachePath(sourcePath);
+ FileInputStream fis = null;
+ ObjectInputStream ois = null;
+ try {
+ fis = new FileInputStream(cachePath);
+ ois = new ObjectInputStream(fis);
+ NModule mod = (NModule)ois.readObject();
+ // Files in different dirs may have the same base name and contents.
+ mod.setFile(sourcePath);
+ return mod;
+ } finally {
+ if (ois != null) {
+ ois.close();
+ } else if (fis != null) {
+ fis.close();
+ }
+ }
+ }
+
+ private void log(Level level, String msg) {
+ if (LOG.isLoggable(level)) {
+ LOG.log(level, msg);
+ }
+ }
+
+ private void severe(String msg) {
+ log(Level.SEVERE, msg);
+ }
+
+ private void warn(String msg) {
+ log(Level.WARNING, msg);
+ }
+
+ private void info(String msg) {
+ log(Level.INFO, msg);
+ }
+
+ private void fine(String msg) {
+ log(Level.FINE, msg);
+ }
+
+ private void finer(String msg) {
+ log(Level.FINER, msg);
+ }
+}
Added: trunk/jython/src/org/python/indexer/AstConverter.java
===================================================================
--- trunk/jython/src/org/python/indexer/AstConverter.java (rev 0)
+++ trunk/jython/src/org/python/indexer/AstConverter.java 2010-03-28 08:21:48 UTC (rev 6990)
@@ -0,0 +1,676 @@
+/**
+ * Copyright 2009, Google Inc. All rights reserved.
+ * Licensed to PSF under a Contributor Agreement.
+ */
+package org.python.indexer;
+
+import org.python.antlr.PythonTree;
+import org.python.antlr.Visitor;
+import org.python.antlr.ast.Assert;
+import org.python.antlr.ast.Assign;
+import org.python.antlr.ast.Attribute;
+import org.python.antlr.ast.AugAssign;
+import org.python.antlr.ast.BinOp;
+import org.python.antlr.ast.BoolOp;
+import org.python.antlr.ast.Break;
+import org.python.antlr.ast.Call;
+import org.python.antlr.ast.ClassDef;
+import org.python.antlr.ast.Compare;
+import org.python.antlr.ast.Continue;
+import org.python.antlr.ast.Delete;
+import org.python.antlr.ast.Dict;
+import org.python.antlr.ast.Ellipsis;
+import org.python.antlr.ast.ExceptHandler;
+import org.python.antlr.ast.Exec;
+import org.python.antlr.ast.Expr;
+import org.python.antlr.ast.For;
+import org.python.antlr.ast.FunctionDef;
+import org.python.antlr.ast.GeneratorExp;
+import org.python.antlr.ast.Global;
+import org.python.antlr.ast.If;
+import org.python.antlr.ast.IfExp;
+import org.python.antlr.ast.Import;
+import org.python.antlr.ast.ImportFrom;
+import org.python.antlr.ast.Index;
+import org.python.antlr.ast.Lambda;
+import org.python.antlr.ast.ListComp;
+import org.python.antlr.ast.Module;
+import org.python.antlr.ast.Name;
+import org.python.antlr.ast.Num;
+import org.python.antlr.ast.Pass;
+import org.python.antlr.ast.Print;
+import org.python.antlr.ast.Raise;
+import org.python.antlr.ast.Repr;
+import org.python.antlr.ast.Return;
+import org.python.antlr.ast.Slice;
+import org.python.antlr.ast.Str;
+import org.python.antlr.ast.Subscript;
+import org.python.antlr.ast.TryExcept;
+import org.python.antlr.ast.TryFinally;
+import org.python.antlr.ast.Tuple;
+import org.python.antlr.ast.UnaryOp;
+import org.python.antlr.ast.While;
+import org.python.antlr.ast.With;
+import org.python.antlr.ast.Yield;
+import org.python.antlr.ast.alias;
+import org.python.antlr.ast.arguments;
+import org.python.antlr.ast.boolopType;
+import org.python.antlr.ast.cmpopType;
+import org.python.antlr.ast.comprehension;
+import org.python.antlr.ast.keyword;
+import org.python.antlr.ast.operatorType;
+import org.python.antlr.ast.unaryopType;
+import org.python.antlr.base.excepthandler;
+import org.python.antlr.base.expr;
+import org.python.antlr.base.stmt;
+import org.python.indexer.ast.NAlias;
+import org.python.indexer.ast.NAssert;
+import org.python.indexer.ast.NAssign;
+import org.python.indexer.ast.NAttribute;
+import org.python.indexer.ast.NAugAssign;
+import org.python.indexer.ast.NBinOp;
+import org.python.indexer.ast.NBlock;
+import org.python.indexer.ast.NBoolOp;
+import org.python.indexer.ast.NBreak;
+import org.python.indexer.ast.NCall;
+import org.python.indexer.ast.NClassDef;
+import org.python.indexer.ast.NCompare;
+import org.python.indexer.ast.NComprehension;
+import org.python.indexer.ast.NContinue;
+import org.python.indexer.ast.NDelete;
+import org.python.indexer.ast.NDict;
+import org.python.indexer.ast.NEllipsis;
+import org.python.indexer.ast.NExceptHandler;
+import org.python.indexer.ast.NExec;
+import org.python.indexer.ast.NFor;
+import org.python.indexer.ast.NFunctionDef;
+import org.python.indexer.ast.NGeneratorExp;
+import org.python.indexer.ast.NGlobal;
+import org.python.indexer.ast.NIf;
+import org.python.indexer...
[truncated message content] |
