""" High-level module dependency finding interface See find_modules(...) Originally (loosely) based on code in py2exe's build_exe.py by Thomas Heller. """ import sys import os import imp import warnings try: set except NameError: from sets import Set as set #from modulegraph import modulegraph #from modulegraph.modulegraph import Alias #from modulegraph.util import imp_find_module import modulegraph from modulegraph import Alias from util import imp_find_module, imp_walk __all__ = [ 'find_modules', 'parse_mf_results' ] def get_implies(): result = { # imports done from builtin modules in C code (untrackable by modulegraph) "time": ["_strptime"], "datetime": ["time"], "MacOS": ["macresource"], "cPickle": ["copy_reg", "cStringIO"], "parser": ["copy_reg"], "codecs": ["encodings"], "cStringIO": ["copy_reg"], "_sre": ["copy", "string", "sre"], "zipimport": ["zlib"], # mactoolboxglue can do a bunch more of these # that are far harder to predict, these should be tracked # manually for now. # this isn't C, but it uses __import__ "anydbm": ["dbhash", "gdbm", "dbm", "dumbdbm", "whichdb"], # package aliases "wxPython.wx": Alias('wx'), } if sys.version_info[:2] >= (2, 5): result["_elementtree"] = ["pyexpat"] import xml.etree files = os.listdir(xml.etree.__path__[0]) for fn in files: if fn.endswith('.py') and fn != "__init__.py": result["_elementtree"].append("xml.etree.%s"%(fn[:-3],)) return result def parse_mf_results(mf): #for name, imports in get_hidden_imports().items(): # if name in mf.modules.keys(): # for mod in imports: # mf.import_hook(mod) # Retrieve modules from modulegraph py_files = [] extensions = [] for item in mf.flatten(): # There may be __main__ modules (from mf.run_script), but # we don't need it in the zipfile we build. if item.identifier == "__main__": continue src = item.filename if src: suffix = os.path.splitext(src)[1] if suffix in PY_SUFFIXES: py_files.append(item) elif suffix in C_SUFFIXES: extensions.append(item) else: raise TypeError("Don't know how to handle '%s'" % repr(src)) # sort on the file names, the output is nicer to read py_files.sort(lambda a,b:cmp(a.filename, b.filename)) extensions.sort(lambda a,b:cmp(a.filename, b.filename)) return py_files, extensions def plat_prepare(includes, packages, excludes): # used by Python itself includes.update(["warnings", "unicodedata", "weakref"]) if not sys.platform.startswith('irix'): excludes.update([ 'AL', 'sgi', ]) if not sys.platform in ('mac', 'darwin'): # XXX - this doesn't look nearly complete excludes.update([ 'Audio_mac', 'Carbon.File', 'Carbon.Folder', 'Carbon.Folders', 'EasyDialogs', 'MacOS', 'macfs', 'macostools', 'macpath', ]) if not sys.platform == 'mac': excludes.update([ 'mkcwproject', ]) if not sys.platform == 'win32': # only win32 excludes.update([ 'ntpath', 'nturl2path', 'win32api', 'win32con', 'win32event', 'win32evtlogutil', 'win32evtlog', 'win32file', 'win32gui', 'win32pipe', 'win32process', 'win32security', 'pywintypes', 'winsound', 'win32', '_winreg', ]) if not sys.platform == 'riscos': excludes.update([ 'riscosenviron', 'riscospath', 'rourl2path', ]) if not sys.platform == 'dos' or sys.platform.startswith('ms-dos'): excludes.update([ 'dos', ]) if not sys.platform == 'os2emx': excludes.update([ 'os2emxpath' ]) excludes.update(set(['posix', 'nt', 'os2', 'mac', 'ce', 'riscos']) - set(sys.builtin_module_names)) try: imp_find_module('poll') except ImportError: excludes.update([ 'poll', ]) def find_needed_modules(mf=None, scripts=(), includes=(), packages=(), warn=warnings.warn): if mf is None: mf = modulegraph.ModuleGraph() # feed Modulefinder with everything, and return it. for path in scripts: mf.run_script(path) for mod in includes: if mod[-2:] == '.*': mf.import_hook(mod[:-2], None, ['*']) else: mf.import_hook(mod) for f in packages: # If modulegraph has seen a reference to the package, then # we prefer to believe that (imp_find_module doesn't seem to locate # sub-packages) m = mf.findNode(f) if m is not None: path = m.packagepath[0] else: # Find path of package # TODO: use imp_find_module_or_importer try: path = imp_find_module(f)[1] except ImportError: warn("No package named %s" % f) continue # walk the path to find subdirs containing __init__.py files # scan the results (directory of __init__.py files) # first trim the path (of the head package), # then convert directory name in package name, # finally push into modulegraph. for (dirpath, dirnames, filenames) in os.walk(path): if '__init__.py' in filenames and dirpath.startswith(path): package = f + '.' + path[len(path)+1:].replace(os.sep, '.') mf.import_hook(package, None, ["*"]) return mf # # resource constants # PY_SUFFIXES = ['.py', '.pyw', '.pyo', '.pyc'] C_SUFFIXES = [ _triple[0] for _triple in imp.get_suffixes() if _triple[2] == imp.C_EXTENSION ] # # side-effects # def _replacePackages(): REPLACEPACKAGES = { '_xmlplus': 'xml', } for k,v in REPLACEPACKAGES.iteritems(): modulegraph.ReplacePackage(k, v) _replacePackages() def find_modules(scripts=(), includes=(), packages=(), excludes=(), path=None, debug=0): """ High-level interface, takes iterables for: scripts, includes, packages, excludes And returns a ModuleGraph instance, python_files, and extensions python_files is a list of pure python dependencies as modulegraph.Module objects, extensions is a list of platform-specific C extension dependencies as modulegraph.Module objects """ scripts = set(scripts) includes = set(includes) packages = set(packages) excludes = set(excludes) plat_prepare(includes, packages, excludes) mf = modulegraph.ModuleGraph( path=path, excludes=(excludes - includes), implies=get_implies(), debug=debug, ) find_needed_modules(mf, scripts, includes, packages) return mf def test(): if '-g' in sys.argv[1:]: sys.argv.remove('-g') dograph = True else: dograph = False if '-x' in sys.argv[1:]: sys.argv.remove('-x') doxref = True else: doxref= False scripts = sys.argv[1:] or [__file__] mf = find_modules(scripts=scripts) if doxref: mf.create_xref() elif dograph: mf.graphreport() else: mf.report() if __name__ == '__main__': test()