You can subscribe to this list here.
2003 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(15) |
Oct
(31) |
Nov
(38) |
Dec
(21) |
---|---|---|---|---|---|---|---|---|---|---|---|---|
2004 |
Jan
(46) |
Feb
(2) |
Mar
(10) |
Apr
(13) |
May
(2) |
Jun
(14) |
Jul
(28) |
Aug
(3) |
Sep
(7) |
Oct
(8) |
Nov
(8) |
Dec
(11) |
2005 |
Jan
|
Feb
|
Mar
|
Apr
(42) |
May
(1) |
Jun
(8) |
Jul
(4) |
Aug
(18) |
Sep
(38) |
Oct
(10) |
Nov
|
Dec
(7) |
2006 |
Jan
|
Feb
(8) |
Mar
(9) |
Apr
|
May
|
Jun
(6) |
Jul
(6) |
Aug
|
Sep
|
Oct
|
Nov
(12) |
Dec
(8) |
2007 |
Jan
(3) |
Feb
|
Mar
(9) |
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(5) |
Oct
|
Nov
|
Dec
|
2008 |
Jan
|
Feb
(4) |
Mar
(2) |
Apr
(17) |
May
(16) |
Jun
(2) |
Jul
|
Aug
(3) |
Sep
(1) |
Oct
|
Nov
(11) |
Dec
(1) |
2009 |
Jan
|
Feb
|
Mar
|
Apr
(2) |
May
|
Jun
|
Jul
|
Aug
(1) |
Sep
|
Oct
|
Nov
|
Dec
|
2010 |
Jan
|
Feb
|
Mar
|
Apr
(1) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
2011 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
(1) |
Nov
|
Dec
|
2012 |
Jan
(1) |
Feb
(2) |
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
(2) |
2013 |
Jan
(14) |
Feb
(14) |
Mar
|
Apr
|
May
(3) |
Jun
(10) |
Jul
(2) |
Aug
|
Sep
(1) |
Oct
|
Nov
|
Dec
|
2014 |
Jan
|
Feb
|
Mar
|
Apr
|
May
(10) |
Jun
|
Jul
|
Aug
|
Sep
(9) |
Oct
(8) |
Nov
|
Dec
|
2017 |
Jan
(3) |
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
(4) |
Sep
|
Oct
|
Nov
|
Dec
|
From: <th...@us...> - 2013-02-04 08:00:35
|
Revision: 722 http://py2exe.svn.sourceforge.net/py2exe/?rev=722&view=rev Author: theller Date: 2013-02-04 08:00:26 +0000 (Mon, 04 Feb 2013) Log Message: ----------- Remove unneeded argument, add a todo comment. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-03 11:16:48 UTC (rev 721) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-04 08:00:26 UTC (rev 722) @@ -7,9 +7,18 @@ It requires Python 3.3 or later because it uses importlib. -Currently is suffers from bugs in Python 3.3.0. +Contains workaround for a bug in Python 3.3.0. """ +## TODO/Think about: +## pyexpat/xml.parsers.expat create their errors and model modules from +## scratch. This means they do not set __loader__ by default. This is +## acceptable under importlib/PEP 302 definitions. +## +## XXX are there more modules doing something similar? +## Is this a use-case for hooks? + + import dis import importlib import importlib.machinery @@ -117,7 +126,7 @@ def get_source(self): return self._imp_loader.get_source(self.name) -def adapt_loader(name, path, loader=None): +def adapt_loader(name, path): """Wrap the passed loader, or the loader returned from importlib.find_loader(), into a LoaderAdapter instance, or return This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-03 11:16:55
|
Revision: 721 http://py2exe.svn.sourceforge.net/py2exe/?rev=721&view=rev Author: theller Date: 2013-02-03 11:16:48 +0000 (Sun, 03 Feb 2013) Log Message: ----------- Simplify some code. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-02 18:37:32 UTC (rev 720) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-03 11:16:48 UTC (rev 721) @@ -11,6 +11,7 @@ """ import dis +import importlib import importlib.machinery import marshal import os @@ -166,8 +167,7 @@ self.msg(2, "run_script", pathname) dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) - loader = adapt_loader(name, pathname, - importlib.machinery.SourceFileLoader(name, pathname)) + loader = adapt_loader(name, [dir]) if loader is None: raise ImportError("could run script {!r}".format(pathname)) self.load_module('__main__', loader) @@ -176,8 +176,7 @@ dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) self.msg(2, "load_file", pathname) - loader = adapt_loader(name, pathname, - importlib.machinery.SourceFileLoader(name, pathname)) + loader = adapt_loader(name, [dir]) if loader is None: raise ImportError("could not load file {!r}".format(pathname)) self.load_module(name, loader) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-02 18:37:38
|
Revision: 720 http://py2exe.svn.sourceforge.net/py2exe/?rev=720&view=rev Author: theller Date: 2013-02-02 18:37:32 +0000 (Sat, 02 Feb 2013) Log Message: ----------- Refactoring. Important bug fix: Modules in packages were not found correctly. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 19:57:20 UTC (rev 719) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-02 18:37:32 UTC (rev 720) @@ -74,6 +74,7 @@ else: self.__path__ = None self.__code__ = loader.get_code() + self.__loader__ = loader # The set of global names that are assigned to in the module. # This includes those names imported through starimports of @@ -92,8 +93,11 @@ s = s + ")" return s -class Loader: +class LoaderAdapter: + """This class adapts the PEP 302 loaders to a convenient and + extended interface. """ + def __init__(self, imp_loader, name, path): self._imp_loader = imp_loader self.name = name @@ -108,12 +112,20 @@ def get_code(self): return self._imp_loader.get_code(self.name) - -def wrap_loader(name, path): + + def get_source(self): + return self._imp_loader.get_source(self.name) + +def adapt_loader(name, path, loader=None): + + """Wrap the passed loader, or the loader returned from + importlib.find_loader(), into a LoaderAdapter instance, or return + None if no loader is found. """ + ldr = importlib.find_loader(name, path) if ldr is None: return None - return Loader(ldr, name, path) + return LoaderAdapter(ldr, name, path) class ModuleFinder: @@ -152,16 +164,24 @@ def run_script(self, pathname): self.msg(2, "run_script", pathname) - self.load_module('__main__', - importlib.machinery.SourceFileLoader('__main__', pathname)) + dir, name = os.path.split(pathname) + name, ext = os.path.splitext(name) + loader = adapt_loader(name, pathname, + importlib.machinery.SourceFileLoader(name, pathname)) + if loader is None: + raise ImportError("could run script {!r}".format(pathname)) + self.load_module('__main__', loader) def load_file(self, pathname): dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) self.msg(2, "load_file", pathname) - self.load_module('__main__', - importlib.machinery.SourceFileLoader(name, pathname)) - + loader = adapt_loader(name, pathname, + importlib.machinery.SourceFileLoader(name, pathname)) + if loader is None: + raise ImportError("could not load file {!r}".format(pathname)) + self.load_module(name, loader) + def import_hook(self, name, caller=None, fromlist=None, level=-1): self.msg(3, "import_hook", name, caller, fromlist, level) parent = self.determine_parent(caller, level=level) @@ -316,6 +336,7 @@ return m def load_module(self, fqname, loader): + assert loader is not None is_pkg = loader.is_package() self.msgin(2, "load_package" if is_pkg else "load_module", fqname, loader) @@ -480,11 +501,13 @@ raise ImportError(name) if path is None: - if name in sys.builtin_module_names: - return wrap_loader(name, []) - + # look for builtin modules on an empty path + ldr = adapt_loader(fullname, []) + if ldr: + return ldr + # no builtin, search again with self.path (which is our sys.path) path = self.path - return wrap_loader(name, path) + return adapt_loader(fullname, path) def report(self): """Print a report to stdout, listing the found modules with their @@ -671,7 +694,7 @@ mf.import_hook(arg) else: mf.load_file(arg) -# mf.run_script(script) + mf.run_script(script) mf.report() return mf # for -i debugging This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 19:57:28
|
Revision: 719 http://py2exe.svn.sourceforge.net/py2exe/?rev=719&view=rev Author: theller Date: 2013-02-01 19:57:20 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Work in progress. Does now find modules in (possibly zipped) eggs. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:53:32 UTC (rev 718) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 19:57:20 UTC (rev 719) @@ -68,15 +68,12 @@ def __init__(self, name, loader): self.__name__ = name self.__file__ = loader.path - if loader.is_package(loader.name): - # XXX zipimporter loaders doen't have .path attribute. - # XXX They have get_filename(fqname) and prefix - # which should be used to simulate these: + if loader.is_package(): # As per comment at top of file, simulate runtime __path__ additions. self.__path__ = [os.path.dirname(loader.path)] + packagePathMap.get(name, []) else: self.__path__ = None - self.__code__ = loader.get_code(loader.name) + self.__code__ = loader.get_code() # The set of global names that are assigned to in the module. # This includes those names imported through starimports of @@ -95,6 +92,29 @@ s = s + ")" return s +class Loader: + + def __init__(self, imp_loader, name, path): + self._imp_loader = imp_loader + self.name = name + self.path = getattr(imp_loader, "path", None) + + import zipimport + if isinstance(imp_loader, zipimport.zipimporter): + self.path = imp_loader.get_filename(self.name) + + def is_package(self): + return self._imp_loader.is_package(self.name) + + def get_code(self): + return self._imp_loader.get_code(self.name) + +def wrap_loader(name, path): + ldr = importlib.find_loader(name, path) + if ldr is None: + return None + return Loader(ldr, name, path) + class ModuleFinder: def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): @@ -296,7 +316,7 @@ return m def load_module(self, fqname, loader): - is_pkg = loader.is_package(loader.name) + is_pkg = loader.is_package() self.msgin(2, "load_package" if is_pkg else "load_module", fqname, loader) m = self.add_module(fqname, loader) @@ -443,7 +463,6 @@ if fqname in self.modules: return self.modules[fqname] self.modules[fqname] = m = Module(fqname, loader) - if m.__code__: if self.replace_paths: m.__code__ = self.replace_paths_in_code(m.__code__) @@ -462,15 +481,11 @@ if path is None: if name in sys.builtin_module_names: - ldr = importlib.find_loader(name) - ldr.name = name - ldr.path = None - return ldr + return wrap_loader(name, []) path = self.path - return importlib.find_loader(name, path) + return wrap_loader(name, path) - def report(self): """Print a report to stdout, listing the found modules with their paths, as well as modules that are missing, or seem to be missing. @@ -656,7 +671,7 @@ mf.import_hook(arg) else: mf.load_file(arg) - mf.run_script(script) +# mf.run_script(script) mf.report() return mf # for -i debugging This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 16:53:38
|
Revision: 718 http://py2exe.svn.sourceforge.net/py2exe/?rev=718&view=rev Author: theller Date: 2013-02-01 16:53:32 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Implement run_script() and load_file() methods. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:46:02 UTC (rev 717) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:53:32 UTC (rev 718) @@ -130,19 +130,18 @@ self.indent = self.indent - 1 self.msg(*args) - ## def run_script(self, pathname): - ## self.msg(2, "run_script", pathname) - ## with open(pathname) as fp: - ## stuff = ("", "r", imp.PY_SOURCE) - ## self.load_module('__main__', fp, pathname, stuff) + def run_script(self, pathname): + self.msg(2, "run_script", pathname) + self.load_module('__main__', + importlib.machinery.SourceFileLoader('__main__', pathname)) - ## def load_file(self, pathname): - ## dir, name = os.path.split(pathname) - ## name, ext = os.path.splitext(name) - ## with open(pathname) as fp: - ## stuff = (ext, "r", imp.PY_SOURCE) - ## self.load_module(name, fp, pathname, stuff) - + def load_file(self, pathname): + dir, name = os.path.split(pathname) + name, ext = os.path.splitext(name) + self.msg(2, "load_file", pathname) + self.load_module('__main__', + importlib.machinery.SourceFileLoader(name, pathname)) + def import_hook(self, name, caller=None, fromlist=None, level=-1): self.msg(3, "import_hook", name, caller, fromlist, level) parent = self.determine_parent(caller, level=level) @@ -657,10 +656,8 @@ mf.import_hook(arg) else: mf.load_file(arg) -# mf.run_script(script) + mf.run_script(script) mf.report() - ## for m in mf.modules.values(): - ## print(m, m.globalnames.keys()) return mf # for -i debugging This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 16:46:09
|
Revision: 717 http://py2exe.svn.sourceforge.net/py2exe/?rev=717&view=rev Author: theller Date: 2013-02-01 16:46:02 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Fix finding modules in packages. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:33:47 UTC (rev 716) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:46:02 UTC (rev 717) @@ -73,7 +73,7 @@ # XXX They have get_filename(fqname) and prefix # which should be used to simulate these: # As per comment at top of file, simulate runtime __path__ additions. - self.__path__ = [loader.path] + packagePathMap.get(name, []) + self.__path__ = [os.path.dirname(loader.path)] + packagePathMap.get(name, []) else: self.__path__ = None self.__code__ = loader.get_code(loader.name) @@ -659,6 +659,8 @@ mf.load_file(arg) # mf.run_script(script) mf.report() + ## for m in mf.modules.values(): + ## print(m, m.globalnames.keys()) return mf # for -i debugging This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 16:33:54
|
Revision: 716 http://py2exe.svn.sourceforge.net/py2exe/?rev=716&view=rev Author: theller Date: 2013-02-01 16:33:47 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Refactor. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:27:27 UTC (rev 715) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:33:47 UTC (rev 716) @@ -65,11 +65,19 @@ class Module: - def __init__(self, name, file=None, path=None): + def __init__(self, name, loader): self.__name__ = name - self.__file__ = file - self.__path__ = path - self.__code__ = None + self.__file__ = loader.path + if loader.is_package(loader.name): + # XXX zipimporter loaders doen't have .path attribute. + # XXX They have get_filename(fqname) and prefix + # which should be used to simulate these: + # As per comment at top of file, simulate runtime __path__ additions. + self.__path__ = [loader.path] + packagePathMap.get(name, []) + else: + self.__path__ = None + self.__code__ = loader.get_code(loader.name) + # The set of global names that are assigned to in the module. # This includes those names imported through starimports of # Python modules. @@ -435,23 +443,12 @@ def add_module(self, fqname, loader): if fqname in self.modules: return self.modules[fqname] - self.modules[fqname] = m = Module(fqname) - m.__file__ = loader.path + self.modules[fqname] = m = Module(fqname, loader) - if loader.is_package(loader.name): - # XXX zipimporter loaders doen't have .path attribute. - # XXX They have get_filename(fqname) and prefix - # which should be used to simulate these: - m.__path__ = [loader.path] - # As per comment at top of file, simulate runtime __path__ additions. - m.__path__ = m.__path__ + packagePathMap.get(fqname, []) - - co = loader.get_code(loader.name) - if co: + if m.__code__: if self.replace_paths: - co = self.replace_paths_in_code(co) - m.__code__ = co - self.scan_code(co, m) + m.__code__ = self.replace_paths_in_code(m.__code__) + self.scan_code(m.__code__, m) return m def find_module(self, name, path, parent=None): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 16:27:33
|
Revision: 715 http://py2exe.svn.sourceforge.net/py2exe/?rev=715&view=rev Author: theller Date: 2013-02-01 16:27:27 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Fix the workaround. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:07:36 UTC (rev 714) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:27:27 UTC (rev 715) @@ -54,7 +54,8 @@ if sys.version_info[:3] == (3, 3, 0): # Work around Python bug #17098: # Set __loader__ on modules imported by the C level - for m in sys.modules.values(): + for name in sys.builtin_module_names + ("_frozen_importlib",): + m = __import__(name) try: m.__loader__ except AttributeError: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 16:07:43
|
Revision: 714 http://py2exe.svn.sourceforge.net/py2exe/?rev=714&view=rev Author: theller Date: 2013-02-01 16:07:36 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Workaround for bug #17098. Cleanup a bit. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 15:14:11 UTC (rev 713) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 16:07:36 UTC (rev 714) @@ -50,6 +50,18 @@ replacePackageMap[oldname] = newname +################################################################ +if sys.version_info[:3] == (3, 3, 0): + # Work around Python bug #17098: + # Set __loader__ on modules imported by the C level + for m in sys.modules.values(): + try: + m.__loader__ + except AttributeError: + m.__loader__ = importlib.machinery.BuiltinImporter +################################################################ + + class Module: def __init__(self, name, file=None, path=None): @@ -57,7 +69,6 @@ self.__file__ = file self.__path__ = path self.__code__ = None - self.__loader__ = None # The set of global names that are assigned to in the module. # This includes those names imported through starimports of # Python modules. @@ -75,20 +86,6 @@ s = s + ")" return s -class FakeBuiltinLoader: - path = None - def __init__(self, name): - self.name = name - - def is_package(self, fqname): - return False - - def get_code(self, fqname): - return None - - def get_source(self, fqname): - return None - class ModuleFinder: def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): @@ -269,11 +266,8 @@ def import_module(self, partname, fqname, parent): self.msgin(3, "import_module", partname, fqname, parent) - try: + if fqname in self.modules: m = self.modules[fqname] - except KeyError: - pass - else: self.msgout(3, "import_module ->", m) return m if fqname in self.badmodules: @@ -441,7 +435,6 @@ if fqname in self.modules: return self.modules[fqname] self.modules[fqname] = m = Module(fqname) - m.__loader__ = loader m.__file__ = loader.path if loader.is_package(loader.name): @@ -471,17 +464,14 @@ raise ImportError(name) if path is None: - # XXX importlib.find_loader should be able to handle - # builtin modules, but in Python 3.3.0 it isn't because of - # a bug. if name in sys.builtin_module_names: - return FakeBuiltinLoader(name) + ldr = importlib.find_loader(name) + ldr.name = name + ldr.path = None + return ldr path = self.path - try: - return importlib.find_loader(name, path) - except (ValueError, AttributeError): - return None # XXX Is this correct? + return importlib.find_loader(name, path) def report(self): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 15:14:18
|
Revision: 713 http://py2exe.svn.sourceforge.net/py2exe/?rev=713&view=rev Author: theller Date: 2013-02-01 15:14:11 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Refactor: Merge load_module() and load_package() into one method. Move code into add_module(). Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 15:09:40 UTC (rev 712) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 15:14:11 UTC (rev 713) @@ -294,18 +294,11 @@ return m def load_module(self, fqname, loader): - self.msgin(2, "load_module", fqname, loader) - if loader.is_package(loader.name): - m = self.load_package(fqname, loader) - return m + is_pkg = loader.is_package(loader.name) + self.msgin(2, "load_package" if is_pkg else "load_module", + fqname, loader) m = self.add_module(fqname, loader) - co = loader.get_code(loader.name) - if co: - if self.replace_paths: - co = self.replace_paths_in_code(co) - m.__code__ = co - self.scan_code(co, m) - self.msgout(2, "load_module ->", m) + self.msgout(2, "load_package" if is_pkg else "load_module", m) return m def _add_badmodule(self, name, caller): @@ -444,21 +437,6 @@ if isinstance(c, type(co)): self.scan_code(c, m) - def load_package(self, fqname, loader): - self.msgin(2, "load_package", fqname, loader) - newname = replacePackageMap.get(fqname) - if newname: - fqname = newname - m = self.add_module(fqname, loader) - co = loader.get_code(loader.name) - if co: - if self.replace_paths: - co = self.replace_paths_in_code(co) - m.__code__ = co - self.scan_code(co, m) - self.msgout(2, "load_package ->", m) - return m - def add_module(self, fqname, loader): if fqname in self.modules: return self.modules[fqname] @@ -474,6 +452,12 @@ # As per comment at top of file, simulate runtime __path__ additions. m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + co = loader.get_code(loader.name) + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) return m def find_module(self, name, path, parent=None): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 15:09:50
|
Revision: 712 http://py2exe.svn.sourceforge.net/py2exe/?rev=712&view=rev Author: theller Date: 2013-02-01 15:09:40 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Refactor. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 12:22:56 UTC (rev 711) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 15:09:40 UTC (rev 712) @@ -76,6 +76,7 @@ return s class FakeBuiltinLoader: + path = None def __init__(self, name): self.name = name @@ -297,10 +298,8 @@ if loader.is_package(loader.name): m = self.load_package(fqname, loader) return m + m = self.add_module(fqname, loader) co = loader.get_code(loader.name) - m = self.add_module(fqname, loader) - if hasattr(loader, "path"): - m.__file__ = loader.path if co: if self.replace_paths: co = self.replace_paths_in_code(co) @@ -451,14 +450,6 @@ if newname: fqname = newname m = self.add_module(fqname, loader) - # XXX zipimporter loaders doen't have .path attribute. - # XXX They have get_filename(fqname) and prefix - # which should be used to simulate these: - m.__file__ = loader.path - m.__path__ = [loader.path] - # As per comment at top of file, simulate runtime __path__ additions. - m.__path__ = m.__path__ + packagePathMap.get(fqname, []) - co = loader.get_code(loader.name) if co: if self.replace_paths: @@ -473,6 +464,16 @@ return self.modules[fqname] self.modules[fqname] = m = Module(fqname) m.__loader__ = loader + m.__file__ = loader.path + + if loader.is_package(loader.name): + # XXX zipimporter loaders doen't have .path attribute. + # XXX They have get_filename(fqname) and prefix + # which should be used to simulate these: + m.__path__ = [loader.path] + # As per comment at top of file, simulate runtime __path__ additions. + m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + return m def find_module(self, name, path, parent=None): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 12:23:03
|
Revision: 711 http://py2exe.svn.sourceforge.net/py2exe/?rev=711&view=rev Author: theller Date: 2013-02-01 12:22:56 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Add missing return. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 11:48:10 UTC (rev 710) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 12:22:56 UTC (rev 711) @@ -296,6 +296,7 @@ self.msgin(2, "load_module", fqname, loader) if loader.is_package(loader.name): m = self.load_package(fqname, loader) + return m co = loader.get_code(loader.name) m = self.add_module(fqname, loader) if hasattr(loader, "path"): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 11:48:17
|
Revision: 710 http://py2exe.svn.sourceforge.net/py2exe/?rev=710&view=rev Author: theller Date: 2013-02-01 11:48:10 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Call the loader methods with loader.name as argument. Modified Paths: -------------- trunk/py3exe/py3exe/mf_metapath.py Modified: trunk/py3exe/py3exe/mf_metapath.py =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 08:12:09 UTC (rev 709) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 11:48:10 UTC (rev 710) @@ -1,6 +1,15 @@ #!/usr/bin/python3.3 -"""Find modules used by a script, using introspection.""" +"""Find modules used by a script, using introspection. +This is a modulefinder that finds modules using sys.meta_path, so it +should (eventually) be able to handle modules imported even from +zipped eggs, or other 'strange' mechanisms. + +It requires Python 3.3 or later because it uses importlib. + +Currently is suffers from bugs in Python 3.3.0. +""" + import dis import importlib.machinery import marshal @@ -285,9 +294,9 @@ def load_module(self, fqname, loader): self.msgin(2, "load_module", fqname, loader) - if loader.is_package(fqname): + if loader.is_package(loader.name): m = self.load_package(fqname, loader) - co = loader.get_code(fqname) + co = loader.get_code(loader.name) m = self.add_module(fqname, loader) if hasattr(loader, "path"): m.__file__ = loader.path @@ -449,7 +458,7 @@ # As per comment at top of file, simulate runtime __path__ additions. m.__path__ = m.__path__ + packagePathMap.get(fqname, []) - co = loader.get_code(fqname) + co = loader.get_code(loader.name) if co: if self.replace_paths: co = self.replace_paths_in_code(co) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-02-01 08:12:16
|
Revision: 709 http://py2exe.svn.sourceforge.net/py2exe/?rev=709&view=rev Author: theller Date: 2013-02-01 08:12:09 +0000 (Fri, 01 Feb 2013) Log Message: ----------- Work in progress: Modulefinder that uses importlib, which uses sys.meta_path to find modules. Added Paths: ----------- trunk/py3exe/py3exe/mf_metapath.py Copied: trunk/py3exe/py3exe/mf_metapath.py (from rev 708, trunk/py3exe/py3exe/mf34.py) =================================================================== --- trunk/py3exe/py3exe/mf_metapath.py (rev 0) +++ trunk/py3exe/py3exe/mf_metapath.py 2013-02-01 08:12:09 UTC (rev 709) @@ -0,0 +1,686 @@ +#!/usr/bin/python3.3 +"""Find modules used by a script, using introspection.""" + +import dis +import importlib.machinery +import marshal +import os +import sys +import types +import struct + +# XXX Clean up once str8's cstor matches bytes. +LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) +IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')]) +STORE_NAME = bytes([dis.opname.index('STORE_NAME')]) +STORE_GLOBAL = bytes([dis.opname.index('STORE_GLOBAL')]) +STORE_OPS = [STORE_NAME, STORE_GLOBAL] +HAVE_ARGUMENT = bytes([dis.HAVE_ARGUMENT]) + +# Modulefinder does a good job at simulating Python's, but it can not +# handle __path__ modifications packages make at runtime. Therefore there +# is a mechanism whereby you can register extra paths in this map for a +# package, and it will be honored. + +# Note this is a mapping is lists of paths. +packagePathMap = {} + +# A Public interface +def AddPackagePath(packagename, path): + packagePathMap.setdefault(packagename, []).append(path) + +replacePackageMap = {} + +# This ReplacePackage mechanism allows modulefinder to work around +# situations in which a package injects itself under the name +# of another package into sys.modules at runtime by calling +# ReplacePackage("real_package_name", "faked_package_name") +# before running ModuleFinder. + +def ReplacePackage(oldname, newname): + replacePackageMap[oldname] = newname + + +class Module: + + def __init__(self, name, file=None, path=None): + self.__name__ = name + self.__file__ = file + self.__path__ = path + self.__code__ = None + self.__loader__ = None + # The set of global names that are assigned to in the module. + # This includes those names imported through starimports of + # Python modules. + self.globalnames = {} + # The set of starimports this module did that could not be + # resolved, ie. a starimport from a non-Python module. + self.starimports = {} + + def __repr__(self): + s = "Module(%r" % (self.__name__,) + if self.__file__ is not None: + s = s + ", %r" % (self.__file__,) + if self.__path__ is not None: + s = s + ", %r" % (self.__path__,) + s = s + ")" + return s + +class FakeBuiltinLoader: + def __init__(self, name): + self.name = name + + def is_package(self, fqname): + return False + + def get_code(self, fqname): + return None + + def get_source(self, fqname): + return None + +class ModuleFinder: + + def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): + if path is None: + path = sys.path + self.path = path + self.modules = {} + self.badmodules = {} + self.debug = debug + self.indent = 0 + self.excludes = excludes + self.replace_paths = replace_paths + self.processed_paths = [] # Used in debugging only + + def msg(self, level, str, *args): + if level <= self.debug: + for i in range(self.indent): + print(" ", end=' ') + print(str, end=' ') + for arg in args: + print(repr(arg), end=' ') + print() + + def msgin(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent + 1 + self.msg(*args) + + def msgout(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent - 1 + self.msg(*args) + + ## def run_script(self, pathname): + ## self.msg(2, "run_script", pathname) + ## with open(pathname) as fp: + ## stuff = ("", "r", imp.PY_SOURCE) + ## self.load_module('__main__', fp, pathname, stuff) + + ## def load_file(self, pathname): + ## dir, name = os.path.split(pathname) + ## name, ext = os.path.splitext(name) + ## with open(pathname) as fp: + ## stuff = (ext, "r", imp.PY_SOURCE) + ## self.load_module(name, fp, pathname, stuff) + + def import_hook(self, name, caller=None, fromlist=None, level=-1): + self.msg(3, "import_hook", name, caller, fromlist, level) + parent = self.determine_parent(caller, level=level) + q, tail = self.find_head_package(parent, name) + m = self.load_tail(q, tail) + if not fromlist: + return q + if m.__path__: + self.ensure_fromlist(m, fromlist) + return None + + def determine_parent(self, caller, level=-1): + self.msgin(4, "determine_parent", caller, level) + if not caller or level == 0: + self.msgout(4, "determine_parent -> None") + return None + pname = caller.__name__ + if level >= 1: # relative import + if caller.__path__: + level -= 1 + if level == 0: + parent = self.modules[pname] + assert parent is caller + self.msgout(4, "determine_parent ->", parent) + return parent + if pname.count(".") < level: + raise ImportError("relative importpath too deep") + pname = ".".join(pname.split(".")[:-level]) + parent = self.modules[pname] + self.msgout(4, "determine_parent ->", parent) + return parent + if caller.__path__: + parent = self.modules[pname] + assert caller is parent + self.msgout(4, "determine_parent ->", parent) + return parent + if '.' in pname: + i = pname.rfind('.') + pname = pname[:i] + parent = self.modules[pname] + assert parent.__name__ == pname + self.msgout(4, "determine_parent ->", parent) + return parent + self.msgout(4, "determine_parent -> None") + return None + + def find_head_package(self, parent, name): + self.msgin(4, "find_head_package", parent, name) + if '.' in name: + i = name.find('.') + head = name[:i] + tail = name[i+1:] + else: + head = name + tail = "" + if parent: + qname = "%s.%s" % (parent.__name__, head) + else: + qname = head + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + if parent: + qname = head + parent = None + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + self.msgout(4, "raise ImportError: No module named", qname) + raise ImportError("No module named " + qname) + + def load_tail(self, q, tail): + self.msgin(4, "load_tail", q, tail) + m = q + while tail: + i = tail.find('.') + if i < 0: i = len(tail) + head, tail = tail[:i], tail[i+1:] + mname = "%s.%s" % (m.__name__, head) + m = self.import_module(head, mname, m) + if not m: + self.msgout(4, "raise ImportError: No module named", mname) + raise ImportError("No module named " + mname) + self.msgout(4, "load_tail ->", m) + return m + + def ensure_fromlist(self, m, fromlist, recursive=0): + self.msg(4, "ensure_fromlist", m, fromlist, recursive) + for sub in fromlist: + if sub == "*": + if not recursive: + all = self.find_all_submodules(m) + if all: + self.ensure_fromlist(m, all, 1) + elif not hasattr(m, sub): + subname = "%s.%s" % (m.__name__, sub) + submod = self.import_module(sub, subname, m) + if not submod: + raise ImportError("No module named " + subname) + + def find_all_submodules(self, m): + if not m.__path__: + return + modules = {} + # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. + # But we must also collect Python extension modules - although + # we cannot separate normal dlls from Python extensions. + suffixes = [] + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] + for dir in m.__path__: + try: + names = os.listdir(dir) + except OSError: + self.msg(2, "can't list directory", dir) + continue + for name in names: + mod = None + for suff in suffixes: + n = len(suff) + if name[-n:] == suff: + mod = name[:-n] + break + if mod and mod != "__init__": + modules[mod] = mod + return modules.keys() + + def import_module(self, partname, fqname, parent): + self.msgin(3, "import_module", partname, fqname, parent) + try: + m = self.modules[fqname] + except KeyError: + pass + else: + self.msgout(3, "import_module ->", m) + return m + if fqname in self.badmodules: + self.msgout(3, "import_module -> None") + return None + if parent and parent.__path__ is None: + self.msgout(3, "import_module -> None") + return None + loader = self.find_module(partname, + parent and parent.__path__, parent) + if loader is None: + self.msgout(3, "import_module ->", None) + return None + m = self.load_module(fqname, loader) + if parent: + setattr(parent, partname, m) + self.msgout(3, "import_module ->", m) + return m + + def load_module(self, fqname, loader): + self.msgin(2, "load_module", fqname, loader) + if loader.is_package(fqname): + m = self.load_package(fqname, loader) + co = loader.get_code(fqname) + m = self.add_module(fqname, loader) + if hasattr(loader, "path"): + m.__file__ = loader.path + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_module ->", m) + return m + + def _add_badmodule(self, name, caller): + if name not in self.badmodules: + self.badmodules[name] = {} + if caller: + self.badmodules[name][caller.__name__] = 1 + else: + self.badmodules[name]["-"] = 1 + + def _safe_import_hook(self, name, caller, fromlist, level=-1): + # wrapper for self.import_hook() that won't raise ImportError + if name in self.badmodules: + self._add_badmodule(name, caller) + return + try: + self.import_hook(name, caller, level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(name, caller) + else: + if fromlist: + for sub in fromlist: + if sub in self.badmodules: + self._add_badmodule(sub, caller) + continue + try: + self.import_hook(name, caller, [sub], level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + fullname = name + "." + sub + self._add_badmodule(fullname, caller) + + def scan_opcodes(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Version for Python 2.4 and older + code = co.co_code + names = co.co_names + consts = co.co_consts + while code: + c = code[0] + if c in STORE_OPS: + oparg, = unpack('<H', code[1:3]) + yield "store", (names[oparg],) + code = code[3:] + continue + if c == LOAD_CONST and code[3] == IMPORT_NAME: + oparg_1, oparg_2 = unpack('<xHxH', code[:6]) + yield "import", (consts[oparg_1], names[oparg_2]) + code = code[6:] + continue + if c >= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + + def scan_opcodes_25(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Python 2.5 version (has absolute and relative imports) + code = co.co_code + names = co.co_names + consts = co.co_consts + LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME + while code: + c = bytes([code[0]]) + if c in STORE_OPS: + oparg, = unpack('<H', code[1:3]) + yield "store", (names[oparg],) + code = code[3:] + continue + if code[:9:3] == LOAD_LOAD_AND_IMPORT: + oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9]) + level = consts[oparg_1] + if level == 0: # absolute import + yield "absolute_import", (consts[oparg_2], names[oparg_3]) + else: # relative import + yield "relative_import", (level, consts[oparg_2], names[oparg_3]) + code = code[9:] + continue + if c >= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + + def scan_code(self, co, m): + code = co.co_code + if sys.version_info >= (2, 5): + scanner = self.scan_opcodes_25 + else: + scanner = self.scan_opcodes + for what, args in scanner(co): + if what == "store": + name, = args + m.globalnames[name] = 1 + elif what == "absolute_import": + fromlist, name = args + have_star = 0 + if fromlist is not None: + if "*" in fromlist: + have_star = 1 + fromlist = [f for f in fromlist if f != "*"] + self._safe_import_hook(name, m, fromlist, level=0) + if have_star: + # We've encountered an "import *". If it is a Python module, + # the code has already been parsed and we can suck out the + # global names. + mm = None + if m.__path__: + # At this point we don't know whether 'name' is a + # submodule of 'm' or a global module. Let's just try + # the full name first. + mm = self.modules.get(m.__name__ + "." + name) + if mm is None: + mm = self.modules.get(name) + if mm is not None: + m.globalnames.update(mm.globalnames) + m.starimports.update(mm.starimports) + if mm.__code__ is None: + m.starimports[name] = 1 + else: + m.starimports[name] = 1 + elif what == "relative_import": + level, fromlist, name = args + if name: + self._safe_import_hook(name, m, fromlist, level=level) + else: + parent = self.determine_parent(m, level=level) + self._safe_import_hook(parent.__name__, None, fromlist, level=0) + else: + # We don't expect anything else from the generator. + raise RuntimeError(what) + + for c in co.co_consts: + if isinstance(c, type(co)): + self.scan_code(c, m) + + def load_package(self, fqname, loader): + self.msgin(2, "load_package", fqname, loader) + newname = replacePackageMap.get(fqname) + if newname: + fqname = newname + m = self.add_module(fqname, loader) + # XXX zipimporter loaders doen't have .path attribute. + # XXX They have get_filename(fqname) and prefix + # which should be used to simulate these: + m.__file__ = loader.path + m.__path__ = [loader.path] + # As per comment at top of file, simulate runtime __path__ additions. + m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + + co = loader.get_code(fqname) + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_package ->", m) + return m + + def add_module(self, fqname, loader): + if fqname in self.modules: + return self.modules[fqname] + self.modules[fqname] = m = Module(fqname) + m.__loader__ = loader + return m + + def find_module(self, name, path, parent=None): + if parent is not None: + # assert path is not None + fullname = parent.__name__+'.'+name + else: + fullname = name + if fullname in self.excludes: + self.msgout(3, "find_module -> Excluded", fullname) + raise ImportError(name) + + if path is None: + # XXX importlib.find_loader should be able to handle + # builtin modules, but in Python 3.3.0 it isn't because of + # a bug. + if name in sys.builtin_module_names: + return FakeBuiltinLoader(name) + + path = self.path + try: + return importlib.find_loader(name, path) + except (ValueError, AttributeError): + return None # XXX Is this correct? + + + def report(self): + """Print a report to stdout, listing the found modules with their + paths, as well as modules that are missing, or seem to be missing. + """ + print() + print(" %-25s %s" % ("Name", "File")) + print(" %-25s %s" % ("----", "----")) + # Print modules found + keys = sorted(self.modules.keys()) + for key in keys: + m = self.modules[key] + if m.__path__: + print("P", end=' ') + else: + print("m", end=' ') + print("%-25s" % key, m.__file__ or "") + + # Print missing modules + missing, maybe = self.any_missing_maybe() + if missing: + print() + print("Missing modules:") + for name in missing: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + # Print modules that may be missing, but then again, maybe not... + if maybe: + print() + print("Submodules thay appear to be missing, but could also be", end=' ') + print("global names in the parent package:") + for name in maybe: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + + def any_missing(self): + """Return a list of modules that appear to be missing. Use + any_missing_maybe() if you want to know which modules are + certain to be missing, and which *may* be missing. + """ + missing, maybe = self.any_missing_maybe() + return missing + maybe + + def any_missing_maybe(self): + """Return two lists, one with modules that are certainly missing + and one with modules that *may* be missing. The latter names could + either be submodules *or* just global names in the package. + + The reason it can't always be determined is that it's impossible to + tell which names are imported when "from module import *" is done + with an extension module, short of actually importing it. + """ + missing = [] + maybe = [] + for name in self.badmodules: + if name in self.excludes: + continue + i = name.rfind(".") + if i < 0: + missing.append(name) + continue + subname = name[i+1:] + pkgname = name[:i] + pkg = self.modules.get(pkgname) + if pkg is not None: + if pkgname in self.badmodules[name]: + # The package tried to import this module itself and + # failed. It's definitely missing. + missing.append(name) + elif subname in pkg.globalnames: + # It's a global in the package: definitely not missing. + pass + elif pkg.starimports: + # It could be missing, but the package did an "import *" + # from a non-Python module, so we simply can't be sure. + maybe.append(name) + else: + # It's not a global in the package, the package didn't + # do funny star imports, it's very likely to be missing. + # The symbol could be inserted into the package from the + # outside, but since that's not good style we simply list + # it missing. + missing.append(name) + else: + missing.append(name) + missing.sort() + maybe.sort() + return missing, maybe + + def replace_paths_in_code(self, co): + new_filename = original_filename = os.path.normpath(co.co_filename) + for f, r in self.replace_paths: + if original_filename.startswith(f): + new_filename = r + original_filename[len(f):] + break + + if self.debug and original_filename not in self.processed_paths: + if new_filename != original_filename: + self.msgout(2, "co_filename %r changed to %r" \ + % (original_filename,new_filename,)) + else: + self.msgout(2, "co_filename %r remains unchanged" \ + % (original_filename,)) + self.processed_paths.append(original_filename) + + consts = list(co.co_consts) + for i in range(len(consts)): + if isinstance(consts[i], type(co)): + consts[i] = self.replace_paths_in_code(consts[i]) + + return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, + co.co_flags, co.co_code, tuple(consts), co.co_names, + co.co_varnames, new_filename, co.co_name, + co.co_firstlineno, co.co_lnotab, + co.co_freevars, co.co_cellvars) + + +def test(): + """This test function has a somwhat unusual command line. + mf.py [-d] [-m] [-p path] [-q] [-x exclude] script modules... + + -d increase debug level + -m script name is followed by module or package names + -p extend searchpath + -q reset debug level + -x exclude module/package + """ + # There also was a bug in the original function in modulefinder, + # which is fixed in this version + + + # Parse command line + import sys + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + except getopt.error as msg: + print(msg) + return + + # Process options + debug = 1 + domods = 0 + addpath = [] + exclude = [] + for o, a in opts: + if o == '-d': + debug = debug + 1 + if o == '-m': + domods = 1 + if o == '-p': + addpath = addpath + a.split(os.pathsep) + if o == '-q': + debug = 0 + if o == '-x': + exclude.append(a) + + # Provide default arguments + if not args: + script = "hello.py" + else: + script = args[0] + args = args[1:] # BUGFIX: This line was missing in the original + + # Set the path based on sys.path and the script directory + path = sys.path[:] + path[0] = os.path.dirname(script) + path = addpath + path + if debug > 1: + print("path:") + for item in path: + print(" ", repr(item)) + + # Create the module finder and turn its crank + mf = ModuleFinder(path, debug, exclude) + for arg in args[:]: # BUGFIX: the original used 'for arg in args[1:]' + if arg == '-m': + domods = 1 + continue + if domods: + if arg[-2:] == '.*': + mf.import_hook(arg[:-2], None, ["*"]) + else: + mf.import_hook(arg) + else: + mf.load_file(arg) +# mf.run_script(script) + mf.report() + return mf # for -i debugging + + +if __name__ == '__main__': + try: + mf = test() + except KeyboardInterrupt: + print("\n[interrupted]") This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-31 21:08:57
|
Revision: 708 http://py2exe.svn.sourceforge.net/py2exe/?rev=708&view=rev Author: theller Date: 2013-01-31 21:08:46 +0000 (Thu, 31 Jan 2013) Log Message: ----------- Set eol-style to native. Property Changed: ---------------- trunk/py3exe/py3exe/mf34.py Property changes on: trunk/py3exe/py3exe/mf34.py ___________________________________________________________________ Added: svn:eol-style + native This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-31 19:34:11
|
Revision: 707 http://py2exe.svn.sourceforge.net/py2exe/?rev=707&view=rev Author: theller Date: 2013-01-31 19:34:04 +0000 (Thu, 31 Jan 2013) Log Message: ----------- Replace the tst() function with the fixed one. Modified Paths: -------------- trunk/py3exe/py3exe/mf34.py Modified: trunk/py3exe/py3exe/mf34.py =================================================================== --- trunk/py3exe/py3exe/mf34.py 2013-01-31 19:32:55 UTC (rev 706) +++ trunk/py3exe/py3exe/mf34.py 2013-01-31 19:34:04 UTC (rev 707) @@ -598,7 +598,21 @@ def test(): + """This test function has a somwhat unusual command line. + mf.py [-d] [-m] [-p path] [-q] [-x exclude] script modules... + + -d increase debug level + -m script name is followed by module or package names + -p extend searchpath + -q reset debug level + -x exclude module/package + """ + # There also was a bug in the original function in modulefinder, + # which is fixed in this version + + # Parse command line + import sys import getopt try: opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") @@ -628,6 +642,7 @@ script = "hello.py" else: script = args[0] + args = args[1:] # BUGFIX: This line was missing in the original # Set the path based on sys.path and the script directory path = sys.path[:] @@ -640,7 +655,7 @@ # Create the module finder and turn its crank mf = ModuleFinder(path, debug, exclude) - for arg in args[1:]: + for arg in args[:]: # BUGFIX: the original used 'for arg in args[1:]' if arg == '-m': domods = 1 continue This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-31 19:33:03
|
Revision: 706 http://py2exe.svn.sourceforge.net/py2exe/?rev=706&view=rev Author: theller Date: 2013-01-31 19:32:55 +0000 (Thu, 31 Jan 2013) Log Message: ----------- The modulefinder from python3.4. Added Paths: ----------- trunk/py3exe/py3exe/mf34.py Added: trunk/py3exe/py3exe/mf34.py =================================================================== --- trunk/py3exe/py3exe/mf34.py (rev 0) +++ trunk/py3exe/py3exe/mf34.py 2013-01-31 19:32:55 UTC (rev 706) @@ -0,0 +1,663 @@ +"""Find modules used by a script, using introspection.""" + +import dis +import imp +import importlib.machinery +import marshal +import os +import sys +import types +import struct + +# XXX Clean up once str8's cstor matches bytes. +LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) +IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')]) +STORE_NAME = bytes([dis.opname.index('STORE_NAME')]) +STORE_GLOBAL = bytes([dis.opname.index('STORE_GLOBAL')]) +STORE_OPS = [STORE_NAME, STORE_GLOBAL] +HAVE_ARGUMENT = bytes([dis.HAVE_ARGUMENT]) + +# Modulefinder does a good job at simulating Python's, but it can not +# handle __path__ modifications packages make at runtime. Therefore there +# is a mechanism whereby you can register extra paths in this map for a +# package, and it will be honored. + +# Note this is a mapping is lists of paths. +packagePathMap = {} + +# A Public interface +def AddPackagePath(packagename, path): + packagePathMap.setdefault(packagename, []).append(path) + +replacePackageMap = {} + +# This ReplacePackage mechanism allows modulefinder to work around +# situations in which a package injects itself under the name +# of another package into sys.modules at runtime by calling +# ReplacePackage("real_package_name", "faked_package_name") +# before running ModuleFinder. + +def ReplacePackage(oldname, newname): + replacePackageMap[oldname] = newname + + +class Module: + + def __init__(self, name, file=None, path=None): + self.__name__ = name + self.__file__ = file + self.__path__ = path + self.__code__ = None + # The set of global names that are assigned to in the module. + # This includes those names imported through starimports of + # Python modules. + self.globalnames = {} + # The set of starimports this module did that could not be + # resolved, ie. a starimport from a non-Python module. + self.starimports = {} + + def __repr__(self): + s = "Module(%r" % (self.__name__,) + if self.__file__ is not None: + s = s + ", %r" % (self.__file__,) + if self.__path__ is not None: + s = s + ", %r" % (self.__path__,) + s = s + ")" + return s + +class ModuleFinder: + + def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): + if path is None: + path = sys.path + self.path = path + self.modules = {} + self.badmodules = {} + self.debug = debug + self.indent = 0 + self.excludes = excludes + self.replace_paths = replace_paths + self.processed_paths = [] # Used in debugging only + + def msg(self, level, str, *args): + if level <= self.debug: + for i in range(self.indent): + print(" ", end=' ') + print(str, end=' ') + for arg in args: + print(repr(arg), end=' ') + print() + + def msgin(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent + 1 + self.msg(*args) + + def msgout(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent - 1 + self.msg(*args) + + def run_script(self, pathname): + self.msg(2, "run_script", pathname) + with open(pathname) as fp: + stuff = ("", "r", imp.PY_SOURCE) + self.load_module('__main__', fp, pathname, stuff) + + def load_file(self, pathname): + dir, name = os.path.split(pathname) + name, ext = os.path.splitext(name) + with open(pathname) as fp: + stuff = (ext, "r", imp.PY_SOURCE) + self.load_module(name, fp, pathname, stuff) + + def import_hook(self, name, caller=None, fromlist=None, level=-1): + self.msg(3, "import_hook", name, caller, fromlist, level) + parent = self.determine_parent(caller, level=level) + q, tail = self.find_head_package(parent, name) + m = self.load_tail(q, tail) + if not fromlist: + return q + if m.__path__: + self.ensure_fromlist(m, fromlist) + return None + + def determine_parent(self, caller, level=-1): + self.msgin(4, "determine_parent", caller, level) + if not caller or level == 0: + self.msgout(4, "determine_parent -> None") + return None + pname = caller.__name__ + if level >= 1: # relative import + if caller.__path__: + level -= 1 + if level == 0: + parent = self.modules[pname] + assert parent is caller + self.msgout(4, "determine_parent ->", parent) + return parent + if pname.count(".") < level: + raise ImportError("relative importpath too deep") + pname = ".".join(pname.split(".")[:-level]) + parent = self.modules[pname] + self.msgout(4, "determine_parent ->", parent) + return parent + if caller.__path__: + parent = self.modules[pname] + assert caller is parent + self.msgout(4, "determine_parent ->", parent) + return parent + if '.' in pname: + i = pname.rfind('.') + pname = pname[:i] + parent = self.modules[pname] + assert parent.__name__ == pname + self.msgout(4, "determine_parent ->", parent) + return parent + self.msgout(4, "determine_parent -> None") + return None + + def find_head_package(self, parent, name): + self.msgin(4, "find_head_package", parent, name) + if '.' in name: + i = name.find('.') + head = name[:i] + tail = name[i+1:] + else: + head = name + tail = "" + if parent: + qname = "%s.%s" % (parent.__name__, head) + else: + qname = head + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + if parent: + qname = head + parent = None + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + self.msgout(4, "raise ImportError: No module named", qname) + raise ImportError("No module named " + qname) + + def load_tail(self, q, tail): + self.msgin(4, "load_tail", q, tail) + m = q + while tail: + i = tail.find('.') + if i < 0: i = len(tail) + head, tail = tail[:i], tail[i+1:] + mname = "%s.%s" % (m.__name__, head) + m = self.import_module(head, mname, m) + if not m: + self.msgout(4, "raise ImportError: No module named", mname) + raise ImportError("No module named " + mname) + self.msgout(4, "load_tail ->", m) + return m + + def ensure_fromlist(self, m, fromlist, recursive=0): + self.msg(4, "ensure_fromlist", m, fromlist, recursive) + for sub in fromlist: + if sub == "*": + if not recursive: + all = self.find_all_submodules(m) + if all: + self.ensure_fromlist(m, all, 1) + elif not hasattr(m, sub): + subname = "%s.%s" % (m.__name__, sub) + submod = self.import_module(sub, subname, m) + if not submod: + raise ImportError("No module named " + subname) + + def find_all_submodules(self, m): + if not m.__path__: + return + modules = {} + # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. + # But we must also collect Python extension modules - although + # we cannot separate normal dlls from Python extensions. + suffixes = [] + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] + for dir in m.__path__: + try: + names = os.listdir(dir) + except OSError: + self.msg(2, "can't list directory", dir) + continue + for name in names: + mod = None + for suff in suffixes: + n = len(suff) + if name[-n:] == suff: + mod = name[:-n] + break + if mod and mod != "__init__": + modules[mod] = mod + return modules.keys() + + def import_module(self, partname, fqname, parent): + self.msgin(3, "import_module", partname, fqname, parent) + try: + m = self.modules[fqname] + except KeyError: + pass + else: + self.msgout(3, "import_module ->", m) + return m + if fqname in self.badmodules: + self.msgout(3, "import_module -> None") + return None + if parent and parent.__path__ is None: + self.msgout(3, "import_module -> None") + return None + try: + fp, pathname, stuff = self.find_module(partname, + parent and parent.__path__, parent) + except ImportError: + self.msgout(3, "import_module ->", None) + return None + try: + m = self.load_module(fqname, fp, pathname, stuff) + finally: + if fp: + fp.close() + if parent: + setattr(parent, partname, m) + self.msgout(3, "import_module ->", m) + return m + + def load_module(self, fqname, fp, pathname, file_info): + suffix, mode, type = file_info + self.msgin(2, "load_module", fqname, fp and "fp", pathname) + if type == imp.PKG_DIRECTORY: + m = self.load_package(fqname, pathname) + self.msgout(2, "load_module ->", m) + return m + if type == imp.PY_SOURCE: + co = compile(fp.read()+'\n', pathname, 'exec') + elif type == imp.PY_COMPILED: + if fp.read(4) != imp.get_magic(): + self.msgout(2, "raise ImportError: Bad magic number", pathname) + raise ImportError("Bad magic number in %s" % pathname) + fp.read(4) + co = marshal.load(fp) + else: + co = None + m = self.add_module(fqname) + m.__file__ = pathname + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_module ->", m) + return m + + def _add_badmodule(self, name, caller): + if name not in self.badmodules: + self.badmodules[name] = {} + if caller: + self.badmodules[name][caller.__name__] = 1 + else: + self.badmodules[name]["-"] = 1 + + def _safe_import_hook(self, name, caller, fromlist, level=-1): + # wrapper for self.import_hook() that won't raise ImportError + if name in self.badmodules: + self._add_badmodule(name, caller) + return + try: + self.import_hook(name, caller, level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(name, caller) + else: + if fromlist: + for sub in fromlist: + if sub in self.badmodules: + self._add_badmodule(sub, caller) + continue + try: + self.import_hook(name, caller, [sub], level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + fullname = name + "." + sub + self._add_badmodule(fullname, caller) + + def scan_opcodes(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Version for Python 2.4 and older + code = co.co_code + names = co.co_names + consts = co.co_consts + while code: + c = code[0] + if c in STORE_OPS: + oparg, = unpack('<H', code[1:3]) + yield "store", (names[oparg],) + code = code[3:] + continue + if c == LOAD_CONST and code[3] == IMPORT_NAME: + oparg_1, oparg_2 = unpack('<xHxH', code[:6]) + yield "import", (consts[oparg_1], names[oparg_2]) + code = code[6:] + continue + if c >= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + + def scan_opcodes_25(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Python 2.5 version (has absolute and relative imports) + code = co.co_code + names = co.co_names + consts = co.co_consts + LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME + while code: + c = bytes([code[0]]) + if c in STORE_OPS: + oparg, = unpack('<H', code[1:3]) + yield "store", (names[oparg],) + code = code[3:] + continue + if code[:9:3] == LOAD_LOAD_AND_IMPORT: + oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9]) + level = consts[oparg_1] + if level == 0: # absolute import + yield "absolute_import", (consts[oparg_2], names[oparg_3]) + else: # relative import + yield "relative_import", (level, consts[oparg_2], names[oparg_3]) + code = code[9:] + continue + if c >= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + + def scan_code(self, co, m): + code = co.co_code + if sys.version_info >= (2, 5): + scanner = self.scan_opcodes_25 + else: + scanner = self.scan_opcodes + for what, args in scanner(co): + if what == "store": + name, = args + m.globalnames[name] = 1 + elif what == "absolute_import": + fromlist, name = args + have_star = 0 + if fromlist is not None: + if "*" in fromlist: + have_star = 1 + fromlist = [f for f in fromlist if f != "*"] + self._safe_import_hook(name, m, fromlist, level=0) + if have_star: + # We've encountered an "import *". If it is a Python module, + # the code has already been parsed and we can suck out the + # global names. + mm = None + if m.__path__: + # At this point we don't know whether 'name' is a + # submodule of 'm' or a global module. Let's just try + # the full name first. + mm = self.modules.get(m.__name__ + "." + name) + if mm is None: + mm = self.modules.get(name) + if mm is not None: + m.globalnames.update(mm.globalnames) + m.starimports.update(mm.starimports) + if mm.__code__ is None: + m.starimports[name] = 1 + else: + m.starimports[name] = 1 + elif what == "relative_import": + level, fromlist, name = args + if name: + self._safe_import_hook(name, m, fromlist, level=level) + else: + parent = self.determine_parent(m, level=level) + self._safe_import_hook(parent.__name__, None, fromlist, level=0) + else: + # We don't expect anything else from the generator. + raise RuntimeError(what) + + for c in co.co_consts: + if isinstance(c, type(co)): + self.scan_code(c, m) + + def load_package(self, fqname, pathname): + self.msgin(2, "load_package", fqname, pathname) + newname = replacePackageMap.get(fqname) + if newname: + fqname = newname + m = self.add_module(fqname) + m.__file__ = pathname + m.__path__ = [pathname] + + # As per comment at top of file, simulate runtime __path__ additions. + m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + + fp, buf, stuff = self.find_module("__init__", m.__path__) + try: + self.load_module(fqname, fp, buf, stuff) + self.msgout(2, "load_package ->", m) + return m + finally: + if fp: + fp.close() + + def add_module(self, fqname): + if fqname in self.modules: + return self.modules[fqname] + self.modules[fqname] = m = Module(fqname) + return m + + def find_module(self, name, path, parent=None): + if parent is not None: + # assert path is not None + fullname = parent.__name__+'.'+name + else: + fullname = name + if fullname in self.excludes: + self.msgout(3, "find_module -> Excluded", fullname) + raise ImportError(name) + + if path is None: + if name in sys.builtin_module_names: + return (None, None, ("", "", imp.C_BUILTIN)) + + path = self.path + return imp.find_module(name, path) + + def report(self): + """Print a report to stdout, listing the found modules with their + paths, as well as modules that are missing, or seem to be missing. + """ + print() + print(" %-25s %s" % ("Name", "File")) + print(" %-25s %s" % ("----", "----")) + # Print modules found + keys = sorted(self.modules.keys()) + for key in keys: + m = self.modules[key] + if m.__path__: + print("P", end=' ') + else: + print("m", end=' ') + print("%-25s" % key, m.__file__ or "") + + # Print missing modules + missing, maybe = self.any_missing_maybe() + if missing: + print() + print("Missing modules:") + for name in missing: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + # Print modules that may be missing, but then again, maybe not... + if maybe: + print() + print("Submodules thay appear to be missing, but could also be", end=' ') + print("global names in the parent package:") + for name in maybe: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + + def any_missing(self): + """Return a list of modules that appear to be missing. Use + any_missing_maybe() if you want to know which modules are + certain to be missing, and which *may* be missing. + """ + missing, maybe = self.any_missing_maybe() + return missing + maybe + + def any_missing_maybe(self): + """Return two lists, one with modules that are certainly missing + and one with modules that *may* be missing. The latter names could + either be submodules *or* just global names in the package. + + The reason it can't always be determined is that it's impossible to + tell which names are imported when "from module import *" is done + with an extension module, short of actually importing it. + """ + missing = [] + maybe = [] + for name in self.badmodules: + if name in self.excludes: + continue + i = name.rfind(".") + if i < 0: + missing.append(name) + continue + subname = name[i+1:] + pkgname = name[:i] + pkg = self.modules.get(pkgname) + if pkg is not None: + if pkgname in self.badmodules[name]: + # The package tried to import this module itself and + # failed. It's definitely missing. + missing.append(name) + elif subname in pkg.globalnames: + # It's a global in the package: definitely not missing. + pass + elif pkg.starimports: + # It could be missing, but the package did an "import *" + # from a non-Python module, so we simply can't be sure. + maybe.append(name) + else: + # It's not a global in the package, the package didn't + # do funny star imports, it's very likely to be missing. + # The symbol could be inserted into the package from the + # outside, but since that's not good style we simply list + # it missing. + missing.append(name) + else: + missing.append(name) + missing.sort() + maybe.sort() + return missing, maybe + + def replace_paths_in_code(self, co): + new_filename = original_filename = os.path.normpath(co.co_filename) + for f, r in self.replace_paths: + if original_filename.startswith(f): + new_filename = r + original_filename[len(f):] + break + + if self.debug and original_filename not in self.processed_paths: + if new_filename != original_filename: + self.msgout(2, "co_filename %r changed to %r" \ + % (original_filename,new_filename,)) + else: + self.msgout(2, "co_filename %r remains unchanged" \ + % (original_filename,)) + self.processed_paths.append(original_filename) + + consts = list(co.co_consts) + for i in range(len(consts)): + if isinstance(consts[i], type(co)): + consts[i] = self.replace_paths_in_code(consts[i]) + + return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, + co.co_flags, co.co_code, tuple(consts), co.co_names, + co.co_varnames, new_filename, co.co_name, + co.co_firstlineno, co.co_lnotab, + co.co_freevars, co.co_cellvars) + + +def test(): + # Parse command line + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + except getopt.error as msg: + print(msg) + return + + # Process options + debug = 1 + domods = 0 + addpath = [] + exclude = [] + for o, a in opts: + if o == '-d': + debug = debug + 1 + if o == '-m': + domods = 1 + if o == '-p': + addpath = addpath + a.split(os.pathsep) + if o == '-q': + debug = 0 + if o == '-x': + exclude.append(a) + + # Provide default arguments + if not args: + script = "hello.py" + else: + script = args[0] + + # Set the path based on sys.path and the script directory + path = sys.path[:] + path[0] = os.path.dirname(script) + path = addpath + path + if debug > 1: + print("path:") + for item in path: + print(" ", repr(item)) + + # Create the module finder and turn its crank + mf = ModuleFinder(path, debug, exclude) + for arg in args[1:]: + if arg == '-m': + domods = 1 + continue + if domods: + if arg[-2:] == '.*': + mf.import_hook(arg[:-2], None, ["*"]) + else: + mf.import_hook(arg) + else: + mf.load_file(arg) + mf.run_script(script) + mf.report() + return mf # for -i debugging + + +if __name__ == '__main__': + try: + mf = test() + except KeyboardInterrupt: + print("\n[interrupted]") This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-11 21:25:58
|
Revision: 705 http://py2exe.svn.sourceforge.net/py2exe/?rev=705&view=rev Author: theller Date: 2013-01-11 21:25:50 +0000 (Fri, 11 Jan 2013) Log Message: ----------- Fix wrong code. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-11 20:17:22 UTC (rev 704) +++ trunk/py3exe/py3exe/mf.py 2013-01-11 21:25:50 UTC (rev 705) @@ -114,16 +114,17 @@ if reason == _wapi.BindImportModule: dllname = dllname.decode("mbcs").lower() dependends.add(dllname) - ## if imagename.lower().endswith("python%d%d.dll" % sys.version_info[:2]): - ## # This image binds to the pythondll, canot be a systemdll - ## print("***PYDLL", imagename, dllname) + ## print(imagename, "needs", dllname) + if dllname.lower().endswith("python%d%d.dll" % sys.version_info[:2]): + # This image binds to the pythondll + print("***PYDLL", imagename) elif reason == _wapi.BindImportProcedure: dllname = dllname.decode("mbcs").lower() procname = _wapi.STRING(parameter).value.decode("mbcs") imagename = imagename.decode("mbcs").lower() - ## if procname == "PyImport_ImportModule": - ## # This image imports other modules, needs a hint - ## print("***NEED HINT", imagename, procname) + if procname == "PyImport_ImportModule": + # This image imports other modules, needs a hint + print("***NEED HINT", imagename, procname) return True self._bound_images.add(pathname.lower()) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-11 20:17:32
|
Revision: 704 http://py2exe.svn.sourceforge.net/py2exe/?rev=704&view=rev Author: theller Date: 2013-01-11 20:17:22 +0000 (Fri, 11 Jan 2013) Log Message: ----------- Update the plan. Modified Paths: -------------- trunk/py3exe/README.txt Modified: trunk/py3exe/README.txt =================================================================== --- trunk/py3exe/README.txt 2013-01-11 20:14:11 UTC (rev 703) +++ trunk/py3exe/README.txt 2013-01-11 20:17:22 UTC (rev 704) @@ -5,6 +5,15 @@ Planned features: -Compatible with Python 2.7, and Python 3.x with the same codebase. -Should work with 32-bit and 64-bit builds. -MemoryLoadLibrary will no longer be used. +- Compatible with Python 2.7, and Python 3.x with the same codebase. + +- A single 32-bit installer for all Python versions, a single 64-bit + installer for all Python versions. + +- Should work with 32-bit and 64-bit builds. + +- MemoryLoadLibrary will no longer be used. + +- Implement a 'hint' mechanism to find hidden imports and imports done + by extension modules, runtime __path__ additions, and more; somewhat + similar to PyInstaller hooks. What does cx_Freeze do? This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-11 20:14:18
|
Revision: 703 http://py2exe.svn.sourceforge.net/py2exe/?rev=703&view=rev Author: theller Date: 2013-01-11 20:14:11 +0000 (Fri, 11 Jan 2013) Log Message: ----------- Recursively bind images, collect dlls needed. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-11 13:11:15 UTC (rev 702) +++ trunk/py3exe/py3exe/mf.py 2013-01-11 20:14:11 UTC (rev 703) @@ -36,6 +36,7 @@ self._types = {} self._last_caller = None self._scripts = set() + self._dlls = set() self._bound_images = set() self.__sysdir = _wapi.GetSystemDirectory() ModuleFinder.__init__(self, *args, **kw) @@ -88,10 +89,10 @@ self.bind_image(pathname) return mod - ################################ + def msg(self, level, str, *args): # overridden for smaller indent - if level <= self.debug-1: + if level <= self.debug: for i in range(self.indent): print(" ", end=' ') print(str, end=' ') @@ -113,18 +114,16 @@ if reason == _wapi.BindImportModule: dllname = dllname.decode("mbcs").lower() dependends.add(dllname) - ## imagepath = self.find_image(dllname, searchpath) - ## if imagepath: - ## dependends.add(imagepath) - ## else: - ## print(" NOT FOUND:", pathname) - ## elif reason == _wapi.BindImportProcedure: - ## dllname = dllname.decode("mbcs").lower() - ## procname = _wapi.STRING(parameter).value.decode("mbcs") - ## imagename = imagename.decode("mbcs").lower() - ## if procname == "PyImport_ImportModule": - ## dependends[dllname] = [1, self.find_image(dllname, searchpath)] - ## print(imagename, procname) + ## if imagename.lower().endswith("python%d%d.dll" % sys.version_info[:2]): + ## # This image binds to the pythondll, canot be a systemdll + ## print("***PYDLL", imagename, dllname) + elif reason == _wapi.BindImportProcedure: + dllname = dllname.decode("mbcs").lower() + procname = _wapi.STRING(parameter).value.decode("mbcs") + imagename = imagename.decode("mbcs").lower() + ## if procname == "PyImport_ImportModule": + ## # This image imports other modules, needs a hint + ## print("***NEED HINT", imagename, procname) return True self._bound_images.add(pathname.lower()) @@ -137,6 +136,7 @@ None, # symbolpath _wapi.PIMAGEHLP_STATUS_ROUTINE(StatusRoutine)) except WindowsError as details: + # XXX Error? Missing Module? print("Error binding %s in %s: %s" % (pathname, searchpath, details)) self.msgout(1, "bind_image ->", pathname, ()) return @@ -148,25 +148,44 @@ elif self.is_system_dll(path): self._bound_images.add(path.lower()) else: - result.add(self.bind_image(path)) + r = self.add_image(path.lower()) + result.add(r) self.msgout(1, "bind_image ->", pathname, result or None) + + + def add_image(self, pathname): + self.bind_image(pathname) + # Should add a special kind of Module()? A DLL Module()? + # How can that be integrated into the report() method? + self._dlls.add(pathname) return pathname + def report_dlls(self): + print("\n") + print(" %-25s %s" % ("Name", "Dll")) + print(" %-25s %s" % ("----", "----")) + dlls = sorted(self._dlls) + for dll in dlls: + print("D", end=' ') + print("%-25s" % os.path.basename(dll), dll) + def is_system_dll(self, path): + basename = os.path.basename(path) dirname = os.path.dirname(path).lower() - if dirname.startswith(self.__sysdir.lower()): + # XXX replace 'not "py" in basename' with a better mechanism... + if dirname.startswith(self.__sysdir.lower()) and not "py" in basename: return True return False def find_image(self, imagename, searchpath): - self.msgin(1, "find_image", imagename) + self.msgin(2, "find_image", imagename) for p in searchpath.split(";"): if os.path.isfile(os.path.join(p, imagename)): result = os.path.join(p, imagename) - self.msgout(1, "find_image ->", result) + self.msgout(2, "find_image ->", result) return result - self.msgout(1, "find_image ->", None) + self.msgout(2, "find_image ->", None) return None ################################ @@ -325,6 +344,7 @@ mf.run_script(script) if doreport: mf.report() + mf.report_dlls() return mf # for -i debugging This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-11 13:11:23
|
Revision: 702 http://py2exe.svn.sourceforge.net/py2exe/?rev=702&view=rev Author: theller Date: 2013-01-11 13:11:15 +0000 (Fri, 11 Jan 2013) Log Message: ----------- work in progress. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Added Paths: ----------- trunk/py3exe/py3exe/_wapi.py Added: trunk/py3exe/py3exe/_wapi.py =================================================================== --- trunk/py3exe/py3exe/_wapi.py (rev 0) +++ trunk/py3exe/py3exe/_wapi.py 2013-01-11 13:11:15 UTC (rev 702) @@ -0,0 +1,105 @@ +"""Windows constants, structure definitions, function prototypes, and +some pythonic wrappings of windows functions. +""" +from ctypes import * +from ctypes.wintypes import * + +_kernel32 = WinDLL("kernel32") +_imagehlp = WinDLL("imagehlp") + +def nonnull(result, func, args): + if result: + return result + raise WinError() + +BOOL_errcheck = nonnull +HDC_errcheck = nonnull +HENHMETAFILE_errcheck = nonnull +HFONT_errcheck = nonnull + +################################################################ +## _gdi32 = WinDLL("gdi32") +## _user32 = WinDLL("user32") +## _comdlg32 = WinDLL("comdlg32") +## _winspool = WinDLL("winspool.drv") +## _ole32 = OleDLL("ole32") +## _commctrl = WinDLL("comctl32") +## _shlwapi = WinDLL("shlwapi") +## _shell32 = WinDLL("shell32") +## _psapi = WinDLL("psapi") +## _uxtheme = WinDLL("uxtheme") +## _setupapi = WinDLL("setupapi") +##_riched20 = WinDLL("riched20") +## +## if __debug__: +## import sys +## if sys.version_info < (3, 0): +## from ctypeslib.dynamic_module import include +## include("""\ +## #define UNICODE +## #define NO_STRICT +## #define WINVER 0x501 +## #define _WIN32_WINNT 0x501 +## #include <windows.h> +## #include <winspool.h> +## #include <commctrl.h> +## #include <shlwapi.h> +## #include <shlobj.h> +## #include <psapi.h> +## #include <richedit.h> +## #include <tmschema.h> +## #include <uxtheme.h> +## #include <setupapi.h> +## #include <imagehlp.h> +## """, +## persist=True) + +################################################################ +WSTRING = c_wchar_p +_WIN64 = (sizeof(c_void_p) == 8) + +STRING = c_char_p +CHAR = c_char +PSTR = STRING + +# values for enumeration '_IMAGEHLP_STATUS_REASON' +BindOutOfMemory = 0 +BindRvaToVaFailed = 1 +BindNoRoomInImage = 2 +BindImportModuleFailed = 3 +BindImportProcedureFailed = 4 +BindImportModule = 5 +BindImportProcedure = 6 +BindForwarder = 7 +BindForwarderNOT = 8 +BindImageModified = 9 +BindExpandFileHeaders = 10 +BindImageComplete = 11 +BindMismatchedSymbols = 12 +BindSymbolsNotUpdated = 13 +BindImportProcedure32 = 14 +BindImportProcedure64 = 15 +BindForwarder32 = 16 +BindForwarder64 = 17 +BindForwarderNOT32 = 18 +BindForwarderNOT64 = 19 +_IMAGEHLP_STATUS_REASON = c_int # enum +PIMAGEHLP_STATUS_ROUTINE = WINFUNCTYPE(BOOL, _IMAGEHLP_STATUS_REASON, STRING, STRING, c_ulong, c_ulong) +BindImageEx = _imagehlp.BindImageEx +BindImageEx.restype = BOOL +BindImageEx.argtypes = [DWORD, PSTR, PSTR, PSTR, PIMAGEHLP_STATUS_ROUTINE] +BindImageEx.errcheck = BOOL_errcheck +BIND_NO_BOUND_IMPORTS = 1 # Variable c_int +BIND_NO_UPDATE = 2 # Variable c_int +BIND_ALL_IMAGES = 4 # Variable c_int +_GetSystemDirectoryW = _kernel32.GetSystemDirectoryW +_GetSystemDirectoryW.restype = UINT +_GetSystemDirectoryW.argtypes = [LPWSTR, UINT] +_GetSystemDirectory = _GetSystemDirectoryW # alias +_MAX_PATH = 260 # Variable c_int +def GetSystemDirectory(): + buf = create_unicode_buffer(_MAX_PATH) + res = _GetSystemDirectory(buf, len(buf)) + if res: + return buf[:res] + raise WinError() Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-11 09:37:34 UTC (rev 701) +++ trunk/py3exe/py3exe/mf.py 2013-01-11 13:11:15 UTC (rev 702) @@ -37,7 +37,6 @@ self._last_caller = None self._scripts = set() self._bound_images = set() - self.__windir = _wapi.GetWindowsDirectory() self.__sysdir = _wapi.GetSystemDirectory() ModuleFinder.__init__(self, *args, **kw) @@ -130,7 +129,9 @@ self._bound_images.add(pathname.lower()) try: - res = _wapi.BindImageEx(_wapi.BIND_NO_BOUND_IMPORTS | _wapi.BIND_NO_UPDATE | _wapi.BIND_ALL_IMAGES, + res = _wapi.BindImageEx(_wapi.BIND_NO_BOUND_IMPORTS + | _wapi.BIND_NO_UPDATE + | _wapi.BIND_ALL_IMAGES, pathname.encode("mbcs"), # imagename searchpath.encode("mbcs"), # dllpath None, # symbolpath @@ -154,7 +155,7 @@ def is_system_dll(self, path): dirname = os.path.dirname(path).lower() - if dirname.startswith(self.__windir) or dirname.startswith(self.__sysdir): + if dirname.startswith(self.__sysdir.lower()): return True return False @@ -296,20 +297,21 @@ script = "hello.py" else: script = args[0] - args = args[1:] # BUGFIX: This line was missing in the original + args = args[1:] # BUGFIX (theller): This line was missing in the original # Set the path based on sys.path and the script directory path = sys.path[:] path[0] = os.path.dirname(script) path = addpath + path if debug > 1: + print("version:", sys.version) print("path:") for item in path: print(" ", repr(item)) # Create the module finder and turn its crank mf = ModuleFinderEx(path, debug, exclude) - for arg in args[:]: # BUGFIX: the original used 'for arg in args[1:]' + for arg in args[:]: # BUGFIX (theller): the original used 'for arg in args[1:]' if arg == '-m': domods = 1 continue This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-11 09:37:42
|
Revision: 701 http://py2exe.svn.sourceforge.net/py2exe/?rev=701&view=rev Author: theller Date: 2013-01-11 09:37:34 +0000 (Fri, 11 Jan 2013) Log Message: ----------- Commit work in progress. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-10 20:30:30 UTC (rev 700) +++ trunk/py3exe/py3exe/mf.py 2013-01-11 09:37:34 UTC (rev 701) @@ -5,6 +5,7 @@ """ from modulefinder import ModuleFinder +import _wapi def _monkeypatch_330(): # Fix a bug in Python 3.3.0 by monkeypatching @@ -35,6 +36,9 @@ self._types = {} self._last_caller = None self._scripts = set() + self._bound_images = set() + self.__windir = _wapi.GetWindowsDirectory() + self.__sysdir = _wapi.GetSystemDirectory() ModuleFinder.__init__(self, *args, **kw) def run_script(self, pathname): @@ -78,11 +82,94 @@ Returns a modulefinder.Module instance. """ (suffix, mode, typ) = info - r = ModuleFinder.load_module(self, fqname, fp, pathname, (suffix, mode, typ)) - if r is not None: - self._types[r.__name__] = typ - return r + mod = ModuleFinder.load_module(self, fqname, fp, pathname, (suffix, mode, typ)) + if mod is not None: + self._types[mod.__name__] = typ + if typ == imp.C_EXTENSION: + self.bind_image(pathname) + return mod + ################################ + def msg(self, level, str, *args): + # overridden for smaller indent + if level <= self.debug-1: + for i in range(self.indent): + print(" ", end=' ') + print(str, end=' ') + for arg in args: + print(repr(arg), end=' ') + print() + + def bind_image(self, pathname): + if pathname.lower() in self._bound_images: + return + + # c:/python33/lib/modulefinder.py + self.msgin(1, "bind_image", pathname) + # XXX Check LoadLibrary search algo... + searchpath = ";".join(sys.path + [os.path.dirname(pathname)]) + ";" + os.environ["PATH"] + dependends = set() + + def StatusRoutine(reason, imagename, dllname, va, parameter): + if reason == _wapi.BindImportModule: + dllname = dllname.decode("mbcs").lower() + dependends.add(dllname) + ## imagepath = self.find_image(dllname, searchpath) + ## if imagepath: + ## dependends.add(imagepath) + ## else: + ## print(" NOT FOUND:", pathname) + ## elif reason == _wapi.BindImportProcedure: + ## dllname = dllname.decode("mbcs").lower() + ## procname = _wapi.STRING(parameter).value.decode("mbcs") + ## imagename = imagename.decode("mbcs").lower() + ## if procname == "PyImport_ImportModule": + ## dependends[dllname] = [1, self.find_image(dllname, searchpath)] + ## print(imagename, procname) + return True + + self._bound_images.add(pathname.lower()) + try: + res = _wapi.BindImageEx(_wapi.BIND_NO_BOUND_IMPORTS | _wapi.BIND_NO_UPDATE | _wapi.BIND_ALL_IMAGES, + pathname.encode("mbcs"), # imagename + searchpath.encode("mbcs"), # dllpath + None, # symbolpath + _wapi.PIMAGEHLP_STATUS_ROUTINE(StatusRoutine)) + except WindowsError as details: + print("Error binding %s in %s: %s" % (pathname, searchpath, details)) + self.msgout(1, "bind_image ->", pathname, ()) + return + result = set() + for dll in dependends.copy(): + path = self.find_image(dll, searchpath) + if path is None: + self._bound_images.add(dll.lower()) + elif self.is_system_dll(path): + self._bound_images.add(path.lower()) + else: + result.add(self.bind_image(path)) + + self.msgout(1, "bind_image ->", pathname, result or None) + return pathname + + def is_system_dll(self, path): + dirname = os.path.dirname(path).lower() + if dirname.startswith(self.__windir) or dirname.startswith(self.__sysdir): + return True + return False + + def find_image(self, imagename, searchpath): + self.msgin(1, "find_image", imagename) + for p in searchpath.split(";"): + if os.path.isfile(os.path.join(p, imagename)): + result = os.path.join(p, imagename) + self.msgout(1, "find_image ->", result) + return result + self.msgout(1, "find_image ->", None) + return None + + ################################ + def create_xref(self): # this code probably needs cleanup depgraph = {} @@ -168,6 +255,7 @@ -m script name is followed by module or package names -p extend searchpath -q reset debug level + -r report -x exclude module/package """ # There also was a bug in the original function in modulefinder, @@ -178,7 +266,7 @@ import sys import getopt try: - opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + opts, args = getopt.getopt(sys.argv[1:], "dmp:qrx:") except getopt.error as msg: print(msg) return @@ -186,6 +274,7 @@ # Process options debug = 1 domods = 0 + doreport = 0 addpath = [] exclude = [] for o, a in opts: @@ -199,6 +288,8 @@ debug = 0 if o == '-x': exclude.append(a) + if o == '-r': + doreport = 1 # Provide default arguments if not args: @@ -230,7 +321,8 @@ else: mf.load_file(arg) mf.run_script(script) - mf.report() + if doreport: + mf.report() return mf # for -i debugging This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-10 20:30:38
|
Revision: 700 http://py2exe.svn.sourceforge.net/py2exe/?rev=700&view=rev Author: theller Date: 2013-01-10 20:30:30 +0000 (Thu, 10 Jan 2013) Log Message: ----------- Fix some bugs, monkeypatch a Python bug away, add our own copy of the test function. Some explaining comments as well. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-10 19:30:39 UTC (rev 699) +++ trunk/py3exe/py3exe/mf.py 2013-01-10 20:30:30 UTC (rev 700) @@ -6,9 +6,20 @@ from modulefinder import ModuleFinder +def _monkeypatch_330(): + # Fix a bug in Python 3.3.0 by monkeypatching + import modulefinder + import importlib + modulefinder.importlib = importlib + import imp +import os +import sys import tempfile +if sys.version_info[:3] == (3, 3, 0): + _monkeypatch_330() + try: # Python3.x from urllib.request import pathname2url @@ -31,25 +42,43 @@ # have more than one script in py2exe, so we want to keep # *all* the pathnames. self._scripts.add(pathname) - Base.run_script(self, pathname) + ModuleFinder.run_script(self, pathname) def import_hook(self, name, caller=None, fromlist=None, level=-1): old_last_caller = self._last_caller try: self._last_caller = caller - return Base.import_hook(self,name,caller,fromlist,level) + return ModuleFinder.import_hook(self, name, caller, fromlist, level) finally: self._last_caller = old_last_caller - def import_module(self,partnam,fqname,parent): - r = Base.import_module(self,partnam,fqname,parent) + def import_module(self, partnam, fqname, parent): + """ModuleFinder.import_module() calls find_module() then load_module() + unless the module is already loaded. + """ + r = ModuleFinder.import_module(self, partnam, fqname, parent) if r is not None and self._last_caller: self._depgraph.setdefault(self._last_caller.__name__, set()).add(r.__name__) return r def load_module(self, fqname, fp, pathname, info): + """Base.load_module() determines the module type. + + For a package, it calls load_package(). + + Calls add_module(fqname). + + For PY_COMPILED or PY_SOURCE it compiles or loads the bytecode + and calls scan_code to determine imports. + + For extension modules (C_EXTENSION: .pyd, .dll) it should scan + for binary dependencies, also determine whether the DLL calls + PyImport_ImportModule, call the hook (if present), ... + + Returns a modulefinder.Module instance. + """ (suffix, mode, typ) = info - r = Base.load_module(self, fqname, fp, pathname, (suffix, mode, typ)) + r = ModuleFinder.load_module(self, fqname, fp, pathname, (suffix, mode, typ)) if r is not None: self._types[r.__name__] = typ return r @@ -130,6 +159,80 @@ imp.SEARCH_ERROR: "SEARCH_ERROR" } +################################################################ +def test(): + """This test function has a somwhat unusual command line. + mf.py [-d] [-m] [-p path] [-q] [-x exclude] script modules... + + -d increase debug level + -m script name is followed by module or package names + -p extend searchpath + -q reset debug level + -x exclude module/package + """ + # There also was a bug in the original function in modulefinder, + # which is fixed in this version + + + # Parse command line + import sys + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + except getopt.error as msg: + print(msg) + return + + # Process options + debug = 1 + domods = 0 + addpath = [] + exclude = [] + for o, a in opts: + if o == '-d': + debug = debug + 1 + if o == '-m': + domods = 1 + if o == '-p': + addpath = addpath + a.split(os.pathsep) + if o == '-q': + debug = 0 + if o == '-x': + exclude.append(a) + + # Provide default arguments + if not args: + script = "hello.py" + else: + script = args[0] + args = args[1:] # BUGFIX: This line was missing in the original + + # Set the path based on sys.path and the script directory + path = sys.path[:] + path[0] = os.path.dirname(script) + path = addpath + path + if debug > 1: + print("path:") + for item in path: + print(" ", repr(item)) + + # Create the module finder and turn its crank + mf = ModuleFinderEx(path, debug, exclude) + for arg in args[:]: # BUGFIX: the original used 'for arg in args[1:]' + if arg == '-m': + domods = 1 + continue + if domods: + if arg[-2:] == '.*': + mf.import_hook(arg[:-2], None, ["*"]) + else: + mf.import_hook(arg) + else: + mf.load_file(arg) + mf.run_script(script) + mf.report() + return mf # for -i debugging + + if __name__ == "__main__": - from modulefinder import test - test() + mf = test() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-10 19:30:50
|
Revision: 699 http://py2exe.svn.sourceforge.net/py2exe/?rev=699&view=rev Author: theller Date: 2013-01-10 19:30:39 +0000 (Thu, 10 Jan 2013) Log Message: ----------- Add standard header, ran 2to3 on the code, make it work with 2.7 again. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-10 19:15:32 UTC (rev 698) +++ trunk/py3exe/py3exe/mf.py 2013-01-10 19:30:39 UTC (rev 699) @@ -1,3 +1,9 @@ +#!/usr/bin/python3.3 +# -*- coding: utf-8 -*- +from __future__ import division, with_statement, absolute_import, print_function +"""The py2exe modulefinder. +""" + from modulefinder import ModuleFinder import imp @@ -2,4 +8,10 @@ import tempfile -import urllib +try: + # Python3.x + from urllib.request import pathname2url +except ImportError: + # Python2.x + from urllib import pathname2url + # Much inspired by Toby Dickenson's code: @@ -35,7 +47,8 @@ self._depgraph.setdefault(self._last_caller.__name__, set()).add(r.__name__) return r - def load_module(self, fqname, fp, pathname, (suffix, mode, typ)): + def load_module(self, fqname, fp, pathname, info): + (suffix, mode, typ) = info r = Base.load_module(self, fqname, fp, pathname, (suffix, mode, typ)) if r is not None: self._types[r.__name__] = typ @@ -45,59 +58,59 @@ # this code probably needs cleanup depgraph = {} importedby = {} - for name, value in self._depgraph.items(): + for name, value in list(self._depgraph.items()): depgraph[name] = list(value) for needs in value: importedby.setdefault(needs, set()).add(name) - names = self._types.keys() + names = list(self._types.keys()) names.sort() fd, htmlfile = tempfile.mkstemp(".html") ofi = open(htmlfile, "w") os.close(fd) - print >> ofi, "<html><title>py2exe cross reference for %s</title><body>" % sys.argv[0] + print("<html><title>py2exe cross reference for %s</title><body>" % sys.argv[0], file=ofi) - print >> ofi, "<h1>py2exe cross reference for %s</h1>" % sys.argv[0] + print("<h1>py2exe cross reference for %s</h1>" % sys.argv[0], file=ofi) for name in names: if self._types[name] in (imp.PY_SOURCE, imp.PKG_DIRECTORY): - print >> ofi, '<a name="%s"><b><tt>%s</tt></b></a>' % (name, name) + print('<a name="%s"><b><tt>%s</tt></b></a>' % (name, name), file=ofi) if name == "__main__": for fname in self._scripts: - path = urllib.pathname2url(os.path.abspath(fname)) - print >> ofi, '<a target="code" href="%s" type="text/plain"><tt>%s</tt></a> ' \ - % (path, fname) - print >> ofi, '<br>imports:' + path = pathname2url(os.path.abspath(fname)) + print('<a target="code" href="%s" type="text/plain"><tt>%s</tt></a> ' \ + % (path, fname), file=ofi) + print('<br>imports:', file=ofi) else: - fname = urllib.pathname2url(self.modules[name].__file__) - print >> ofi, '<a target="code" href="%s" type="text/plain"><tt>%s</tt></a><br>imports:' \ - % (fname, self.modules[name].__file__) + fname = pathname2url(self.modules[name].__file__) + print('<a target="code" href="%s" type="text/plain"><tt>%s</tt></a><br>imports:' \ + % (fname, self.modules[name].__file__), file=ofi) else: fname = self.modules[name].__file__ if fname: - print >> ofi, '<a name="%s"><b><tt>%s</tt></b></a> <tt>%s</tt><br>imports:' \ - % (name, name, fname) + print('<a name="%s"><b><tt>%s</tt></b></a> <tt>%s</tt><br>imports:' \ + % (name, name, fname), file=ofi) else: - print >> ofi, '<a name="%s"><b><tt>%s</tt></b></a> <i>%s</i><br>imports:' \ - % (name, name, TYPES[self._types[name]]) + print('<a name="%s"><b><tt>%s</tt></b></a> <i>%s</i><br>imports:' \ + % (name, name, TYPES[self._types[name]]), file=ofi) if name in depgraph: needs = depgraph[name] for n in needs: - print >> ofi, '<a href="#%s"><tt>%s</tt></a> ' % (n, n) - print >> ofi, "<br>\n" + print('<a href="#%s"><tt>%s</tt></a> ' % (n, n), file=ofi) + print("<br>\n", file=ofi) - print >> ofi, 'imported by:' + print('imported by:', file=ofi) if name in importedby: for i in importedby[name]: - print >> ofi, '<a href="#%s"><tt>%s</tt></a> ' % (i, i) + print('<a href="#%s"><tt>%s</tt></a> ' % (i, i), file=ofi) - print >> ofi, "<br>\n" + print("<br>\n", file=ofi) - print >> ofi, "<br>\n" + print("<br>\n", file=ofi) - print >> ofi, "</body></html>" + print("</body></html>", file=ofi) ofi.close() os.startfile(htmlfile) # how long does it take to start the browser? This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <th...@us...> - 2013-01-10 19:15:39
|
Revision: 698 http://py2exe.svn.sourceforge.net/py2exe/?rev=698&view=rev Author: theller Date: 2013-01-10 19:15:32 +0000 (Thu, 10 Jan 2013) Log Message: ----------- Remove the ModuleFinder code (use from python library instead), and call test function when used as script. Modified Paths: -------------- trunk/py3exe/py3exe/mf.py Modified: trunk/py3exe/py3exe/mf.py =================================================================== --- trunk/py3exe/py3exe/mf.py 2013-01-10 19:00:09 UTC (rev 697) +++ trunk/py3exe/py3exe/mf.py 2013-01-10 19:15:32 UTC (rev 698) @@ -1,709 +1,18 @@ -"""Find modules used by a script, using introspection.""" -# This module should be kept compatible with Python 2.2, see PEP 291. +from modulefinder import ModuleFinder -from __future__ import generators -import dis import imp -import marshal -import os -import sys -import types -import struct - -if hasattr(sys.__stdout__, "newlines"): - READ_MODE = "U" # universal line endings -else: - # remain compatible with Python < 2.3 - READ_MODE = "r" - -LOAD_CONST = chr(dis.opname.index('LOAD_CONST')) -IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME')) -STORE_NAME = chr(dis.opname.index('STORE_NAME')) -STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL')) -STORE_OPS = [STORE_NAME, STORE_GLOBAL] -HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT) - -# !!! NOTE BEFORE INCLUDING IN PYTHON DISTRIBUTION !!! -# To clear up issues caused by the duplication of data structures between -# the real Python modulefinder and this duplicate version, packagePathMap -# and replacePackageMap are imported from the actual modulefinder. This -# should be changed back to the assigments that are commented out below. -# There are also py2exe specific pieces at the bottom of this file. - - -# Modulefinder does a good job at simulating Python's, but it can not -# handle __path__ modifications packages make at runtime. Therefore there -# is a mechanism whereby you can register extra paths in this map for a -# package, and it will be honored. - -# Note this is a mapping is lists of paths. -#~ packagePathMap = {} -from modulefinder import packagePathMap - -# A Public interface -def AddPackagePath(packagename, path): - paths = packagePathMap.get(packagename, []) - paths.append(path) - packagePathMap[packagename] = paths - -#~ replacePackageMap = {} -from modulefinder import replacePackageMap - -# This ReplacePackage mechanism allows modulefinder to work around the -# way the _xmlplus package injects itself under the name "xml" into -# sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml") -# before running ModuleFinder. - -def ReplacePackage(oldname, newname): - replacePackageMap[oldname] = newname - - -class Module: - - def __init__(self, name, file=None, path=None): - self.__name__ = name - self.__file__ = file - self.__path__ = path - self.__code__ = None - # The set of global names that are assigned to in the module. - # This includes those names imported through starimports of - # Python modules. - self.globalnames = {} - # The set of starimports this module did that could not be - # resolved, ie. a starimport from a non-Python module. - self.starimports = {} - - def __repr__(self): - s = "Module(%r" % (self.__name__,) - if self.__file__ is not None: - s = s + ", %r" % (self.__file__,) - if self.__path__ is not None: - s = s + ", %r" % (self.__path__,) - s = s + ")" - return s - -class ModuleFinder: - - def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): - if path is None: - path = sys.path - self.path = path - self.modules = {} - self.badmodules = {} - self.debug = debug - self.indent = 0 - self.excludes = excludes - self.replace_paths = replace_paths - self.processed_paths = [] # Used in debugging only - - def msg(self, level, str, *args): - if level <= self.debug: - for i in range(self.indent): - print " ", - print str, - for arg in args: - print repr(arg), - print - - def msgin(self, *args): - level = args[0] - if level <= self.debug: - self.indent = self.indent + 1 - self.msg(*args) - - def msgout(self, *args): - level = args[0] - if level <= self.debug: - self.indent = self.indent - 1 - self.msg(*args) - - def run_script(self, pathname): - self.msg(2, "run_script", pathname) - fp = open(pathname, READ_MODE) - stuff = ("", "r", imp.PY_SOURCE) - self.load_module('__main__', fp, pathname, stuff) - - def load_file(self, pathname): - dir, name = os.path.split(pathname) - name, ext = os.path.splitext(name) - fp = open(pathname, READ_MODE) - stuff = (ext, "r", imp.PY_SOURCE) - self.load_module(name, fp, pathname, stuff) - - def import_hook(self, name, caller=None, fromlist=None, level=-1): - self.msg(3, "import_hook", name, caller, fromlist, level) - parent = self.determine_parent(caller, level=level) - q, tail = self.find_head_package(parent, name) - m = self.load_tail(q, tail) - if not fromlist: - return q - if m.__path__: - self.ensure_fromlist(m, fromlist) - return None - - def determine_parent(self, caller, level=-1): - self.msgin(4, "determine_parent", caller, level) - if not caller or level == 0: - self.msgout(4, "determine_parent -> None") - return None - pname = caller.__name__ - if level >= 1: # relative import - if caller.__path__: - level -= 1 - if level == 0: - parent = self.modules[pname] - assert parent is caller - self.msgout(4, "determine_parent ->", parent) - return parent - if pname.count(".") < level: - raise ImportError, "relative importpath too deep" - pname = ".".join(pname.split(".")[:-level]) - parent = self.modules[pname] - self.msgout(4, "determine_parent ->", parent) - return parent - if caller.__path__: - parent = self.modules[pname] - assert caller is parent - self.msgout(4, "determine_parent ->", parent) - return parent - if '.' in pname: - i = pname.rfind('.') - pname = pname[:i] - parent = self.modules[pname] - assert parent.__name__ == pname - self.msgout(4, "determine_parent ->", parent) - return parent - self.msgout(4, "determine_parent -> None") - return None - - def find_head_package(self, parent, name): - self.msgin(4, "find_head_package", parent, name) - if '.' in name: - i = name.find('.') - head = name[:i] - tail = name[i+1:] - else: - head = name - tail = "" - if parent: - qname = "%s.%s" % (parent.__name__, head) - else: - qname = head - q = self.import_module(head, qname, parent) - if q: - self.msgout(4, "find_head_package ->", (q, tail)) - return q, tail - if parent: - qname = head - parent = None - q = self.import_module(head, qname, parent) - if q: - self.msgout(4, "find_head_package ->", (q, tail)) - return q, tail - self.msgout(4, "raise ImportError: No module named", qname) - raise ImportError, "No module named " + qname - - def load_tail(self, q, tail): - self.msgin(4, "load_tail", q, tail) - m = q - while tail: - i = tail.find('.') - if i < 0: i = len(tail) - head, tail = tail[:i], tail[i+1:] - mname = "%s.%s" % (m.__name__, head) - m = self.import_module(head, mname, m) - if not m: - self.msgout(4, "raise ImportError: No module named", mname) - raise ImportError, "No module named " + mname - self.msgout(4, "load_tail ->", m) - return m - - def ensure_fromlist(self, m, fromlist, recursive=0): - self.msg(4, "ensure_fromlist", m, fromlist, recursive) - for sub in fromlist: - if sub == "*": - if not recursive: - all = self.find_all_submodules(m) - if all: - self.ensure_fromlist(m, all, 1) - elif not hasattr(m, sub): - subname = "%s.%s" % (m.__name__, sub) - submod = self.import_module(sub, subname, m) - if not submod: - raise ImportError, "No module named " + subname - - def find_all_submodules(self, m): - if not m.__path__: - return - modules = {} - # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. - # But we must also collect Python extension modules - although - # we cannot separate normal dlls from Python extensions. - suffixes = [] - for triple in imp.get_suffixes(): - suffixes.append(triple[0]) - for dir in m.__path__: - try: - names = os.listdir(dir) - except os.error: - self.msg(2, "can't list directory", dir) - continue - for name in names: - mod = None - for suff in suffixes: - n = len(suff) - if name[-n:] == suff: - mod = name[:-n] - break - if mod and mod != "__init__": - modules[mod] = mod - return modules.keys() - - def import_module(self, partname, fqname, parent): - self.msgin(3, "import_module", partname, fqname, parent) - try: - m = self.modules[fqname] - except KeyError: - pass - else: - self.msgout(3, "import_module ->", m) - return m - if self.badmodules.has_key(fqname): - self.msgout(3, "import_module -> None") - return None - if parent and parent.__path__ is None: - self.msgout(3, "import_module -> None") - return None - try: - fp, pathname, stuff = self.find_module(partname, - parent and parent.__path__, parent) - except ImportError: - self.msgout(3, "import_module ->", None) - return None - try: - m = self.load_module(fqname, fp, pathname, stuff) - finally: - if fp: fp.close() - if parent: - setattr(parent, partname, m) - self.msgout(3, "import_module ->", m) - return m - - def load_module(self, fqname, fp, pathname, (suffix, mode, type)): - self.msgin(2, "load_module", fqname, fp and "fp", pathname) - if type == imp.PKG_DIRECTORY: - m = self.load_package(fqname, pathname) - self.msgout(2, "load_module ->", m) - return m - if type == imp.PY_SOURCE: - co = compile(fp.read()+'\n', pathname, 'exec') - elif type == imp.PY_COMPILED: - if fp.read(4) != imp.get_magic(): - self.msgout(2, "raise ImportError: Bad magic number", pathname) - raise ImportError, "Bad magic number in %s" % pathname - fp.read(4) - co = marshal.load(fp) - else: - co = None - m = self.add_module(fqname) - m.__file__ = pathname - if co: - if self.replace_paths: - co = self.replace_paths_in_code(co) - m.__code__ = co - self.scan_code(co, m) - self.msgout(2, "load_module ->", m) - return m - - def _add_badmodule(self, name, caller): - if name not in self.badmodules: - self.badmodules[name] = {} - if caller: - self.badmodules[name][caller.__name__] = 1 - else: - self.badmodules[name]["-"] = 1 - - def _safe_import_hook(self, name, caller, fromlist, level=-1): - # wrapper for self.import_hook() that won't raise ImportError - if name in self.badmodules: - self._add_badmodule(name, caller) - return - try: - self.import_hook(name, caller, level=level) - except ImportError, msg: - self.msg(2, "ImportError:", str(msg)) - self._add_badmodule(name, caller) - else: - if fromlist: - for sub in fromlist: - if sub in self.badmodules: - self._add_badmodule(sub, caller) - continue - try: - self.import_hook(name, caller, [sub], level=level) - except ImportError, msg: - self.msg(2, "ImportError:", str(msg)) - fullname = name + "." + sub - self._add_badmodule(fullname, caller) - - def scan_opcodes(self, co, - unpack = struct.unpack): - # Scan the code, and yield 'interesting' opcode combinations - # Version for Python 2.4 and older - code = co.co_code - names = co.co_names - consts = co.co_consts - while code: - c = code[0] - if c in STORE_OPS: - oparg, = unpack('<H', code[1:3]) - yield "store", (names[oparg],) - code = code[3:] - continue - if c == LOAD_CONST and code[3] == IMPORT_NAME: - oparg_1, oparg_2 = unpack('<xHxH', code[:6]) - yield "import", (consts[oparg_1], names[oparg_2]) - code = code[6:] - continue - if c >= HAVE_ARGUMENT: - code = code[3:] - else: - code = code[1:] - - def scan_opcodes_25(self, co, - unpack = struct.unpack): - # Scan the code, and yield 'interesting' opcode combinations - # Python 2.5 version (has absolute and relative imports) - code = co.co_code - names = co.co_names - consts = co.co_consts - LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME - while code: - c = code[0] - if c in STORE_OPS: - oparg, = unpack('<H', code[1:3]) - yield "store", (names[oparg],) - code = code[3:] - continue - if code[:9:3] == LOAD_LOAD_AND_IMPORT: - oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9]) - level = consts[oparg_1] - if level == -1: # normal import - yield "import", (consts[oparg_2], names[oparg_3]) - elif level == 0: # absolute import - yield "absolute_import", (consts[oparg_2], names[oparg_3]) - else: # relative import - yield "relative_import", (level, consts[oparg_2], names[oparg_3]) - code = code[9:] - continue - if c >= HAVE_ARGUMENT: - code = code[3:] - else: - code = code[1:] - - def scan_code(self, co, m): - code = co.co_code - if sys.version_info >= (2, 5): - scanner = self.scan_opcodes_25 - else: - scanner = self.scan_opcodes - for what, args in scanner(co): - if what == "store": - name, = args - m.globalnames[name] = 1 - elif what in ("import", "absolute_import"): - fromlist, name = args - have_star = 0 - if fromlist is not None: - if "*" in fromlist: - have_star = 1 - fromlist = [f for f in fromlist if f != "*"] - if what == "absolute_import": level = 0 - else: level = -1 - self._safe_import_hook(name, m, fromlist, level=level) - if have_star: - # We've encountered an "import *". If it is a Python module, - # the code has already been parsed and we can suck out the - # global names. - mm = None - if m.__path__: - # At this point we don't know whether 'name' is a - # submodule of 'm' or a global module. Let's just try - # the full name first. - mm = self.modules.get(m.__name__ + "." + name) - if mm is None: - mm = self.modules.get(name) - if mm is not None: - m.globalnames.update(mm.globalnames) - m.starimports.update(mm.starimports) - if mm.__code__ is None: - m.starimports[name] = 1 - else: - m.starimports[name] = 1 - elif what == "relative_import": - level, fromlist, name = args - if name: - self._safe_import_hook(name, m, fromlist, level=level) - else: - parent = self.determine_parent(m, level=level) - self._safe_import_hook(parent.__name__, None, fromlist, level=0) - else: - # We don't expect anything else from the generator. - raise RuntimeError(what) - - for c in co.co_consts: - if isinstance(c, type(co)): - self.scan_code(c, m) - - def load_package(self, fqname, pathname): - self.msgin(2, "load_package", fqname, pathname) - newname = replacePackageMap.get(fqname) - if newname: - fqname = newname - m = self.add_module(fqname) - m.__file__ = pathname - m.__path__ = [pathname] - - # As per comment at top of file, simulate runtime __path__ additions. - m.__path__ = m.__path__ + packagePathMap.get(fqname, []) - - fp, buf, stuff = self.find_module("__init__", m.__path__) - self.load_module(fqname, fp, buf, stuff) - self.msgout(2, "load_package ->", m) - return m - - def add_module(self, fqname): - if self.modules.has_key(fqname): - return self.modules[fqname] - self.modules[fqname] = m = Module(fqname) - return m - - def find_module(self, name, path, parent=None): - if parent is not None: - # assert path is not None - fullname = parent.__name__+'.'+name - else: - fullname = name - if fullname in self.excludes: - self.msgout(3, "find_module -> Excluded", fullname) - raise ImportError, name - - if path is None: - if name in sys.builtin_module_names: - return (None, None, ("", "", imp.C_BUILTIN)) - - path = self.path - return imp.find_module(name, path) - - def report(self): - """Print a report to stdout, listing the found modules with their - paths, as well as modules that are missing, or seem to be missing. - """ - print - print " %-25s %s" % ("Name", "File") - print " %-25s %s" % ("----", "----") - # Print modules found - keys = self.modules.keys() - keys.sort() - for key in keys: - m = self.modules[key] - if m.__path__: - print "P", - else: - print "m", - print "%-25s" % key, m.__file__ or "" - - # Print missing modules - missing, maybe = self.any_missing_maybe() - if missing: - print - print "Missing modules:" - for name in missing: - mods = self.badmodules[name].keys() - mods.sort() - print "?", name, "imported from", ', '.join(mods) - # Print modules that may be missing, but then again, maybe not... - if maybe: - print - print "Submodules thay appear to be missing, but could also be", - print "global names in the parent package:" - for name in maybe: - mods = self.badmodules[name].keys() - mods.sort() - print "?", name, "imported from", ', '.join(mods) - - def any_missing(self): - """Return a list of modules that appear to be missing. Use - any_missing_maybe() if you want to know which modules are - certain to be missing, and which *may* be missing. - """ - missing, maybe = self.any_missing_maybe() - return missing + maybe - - def any_missing_maybe(self): - """Return two lists, one with modules that are certainly missing - and one with modules that *may* be missing. The latter names could - either be submodules *or* just global names in the package. - - The reason it can't always be determined is that it's impossible to - tell which names are imported when "from module import *" is done - with an extension module, short of actually importing it. - """ - missing = [] - maybe = [] - for name in self.badmodules: - if name in self.excludes: - continue - i = name.rfind(".") - if i < 0: - missing.append(name) - continue - subname = name[i+1:] - pkgname = name[:i] - pkg = self.modules.get(pkgname) - if pkg is not None: - if pkgname in self.badmodules[name]: - # The package tried to import this module itself and - # failed. It's definitely missing. - missing.append(name) - elif subname in pkg.globalnames: - # It's a global in the package: definitely not missing. - pass - elif pkg.starimports: - # It could be missing, but the package did an "import *" - # from a non-Python module, so we simply can't be sure. - maybe.append(name) - else: - # It's not a global in the package, the package didn't - # do funny star imports, it's very likely to be missing. - # The symbol could be inserted into the package from the - # outside, but since that's not good style we simply list - # it missing. - missing.append(name) - else: - missing.append(name) - missing.sort() - maybe.sort() - return missing, maybe - - def replace_paths_in_code(self, co): - new_filename = original_filename = os.path.normpath(co.co_filename) - for f, r in self.replace_paths: - if original_filename.startswith(f): - new_filename = r + original_filename[len(f):] - break - - if self.debug and original_filename not in self.processed_paths: - if new_filename != original_filename: - self.msgout(2, "co_filename %r changed to %r" \ - % (original_filename,new_filename,)) - else: - self.msgout(2, "co_filename %r remains unchanged" \ - % (original_filename,)) - self.processed_paths.append(original_filename) - - consts = list(co.co_consts) - for i in range(len(consts)): - if isinstance(consts[i], type(co)): - consts[i] = self.replace_paths_in_code(consts[i]) - - return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, - co.co_flags, co.co_code, tuple(consts), co.co_names, - co.co_varnames, new_filename, co.co_name, - co.co_firstlineno, co.co_lnotab, - co.co_freevars, co.co_cellvars) - - -def test(): - # Parse command line - import getopt - try: - opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") - except getopt.error, msg: - print msg - return - - # Process options - debug = 1 - domods = 0 - addpath = [] - exclude = [] - for o, a in opts: - if o == '-d': - debug = debug + 1 - if o == '-m': - domods = 1 - if o == '-p': - addpath = addpath + a.split(os.pathsep) - if o == '-q': - debug = 0 - if o == '-x': - exclude.append(a) - - # Provide default arguments - if not args: - script = "hello.py" - else: - script = args[0] - - # Set the path based on sys.path and the script directory - path = sys.path[:] - path[0] = os.path.dirname(script) - path = addpath + path - if debug > 1: - print "path:" - for item in path: - print " ", repr(item) - - # Create the module finder and turn its crank - mf = ModuleFinder(path, debug, exclude) - for arg in args[1:]: - if arg == '-m': - domods = 1 - continue - if domods: - if arg[-2:] == '.*': - mf.import_hook(arg[:-2], None, ["*"]) - else: - mf.import_hook(arg) - else: - mf.load_file(arg) - mf.run_script(script) - mf.report() - return mf # for -i debugging - - -if __name__ == '__main__': - try: - mf = test() - except KeyboardInterrupt: - print "\n[interrupt]" - - - -# py2exe specific portion - this should be removed before inclusion in the -# Python distribution - import tempfile import urllib -try: - set -except NameError: - from sets import Set as set - -Base = ModuleFinder -del ModuleFinder - # Much inspired by Toby Dickenson's code: # http://www.tarind.com/depgraph.html -class ModuleFinder(Base): +class ModuleFinderEx(ModuleFinder): def __init__(self, *args, **kw): self._depgraph = {} self._types = {} self._last_caller = None self._scripts = set() - Base.__init__(self, *args, **kw) + ModuleFinder.__init__(self, *args, **kw) def run_script(self, pathname): # Scripts always end in the __main__ module, but we possibly @@ -807,3 +116,7 @@ imp.PY_SOURCE: "python module", imp.SEARCH_ERROR: "SEARCH_ERROR" } + +if __name__ == "__main__": + from modulefinder import test + test() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |