#! /usr/bin/python # -*- coding: UTF-8 -*- vim: et ts=4 sw=4 # Copyright © 2010 Piotr Ożarowski # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. from __future__ import with_statement import logging import os import re import sys from filecmp import dircmp, cmpfiles from optparse import OptionParser, SUPPRESS_HELP from os.path import isdir, islink, exists, join from shutil import rmtree, copy as fcopy from stat import ST_MODE, S_IXUSR, S_IXGRP, S_IXOTH sys.path.insert(1, '/usr/share/python/') from debpython.debhelper import DebHelper from debpython.depends import Dependencies from debpython.version import SUPPORTED, DEFAULT, \ debsorted, getver, vrepr, parse_pycentral_vrange, \ get_requested_versions, parse_vrange, vrange_str from debpython.pydist import validate as validate_pydist, \ PUBLIC_DIR_RE from debpython.tools import sitedir, relative_symlink, \ shebang2pyver from debpython.option import Option # initialize script logging.basicConfig(format='%(levelname).1s: %(module)s:%(lineno)d: ' '%(message)s') log = logging.getLogger(__name__) os.umask(022) EGGnPTH_RE = re.compile(r'(.*?)(-py\d\.\d+)?(.*?)(\.egg-info|\.pth)$') """TODO: move it to manpage Examples: dh_python2 dh_python2 -V 2.4- # public files only, Python >= 2.4 dh_python2 -p python-foo -X 'bar.*' /usr/lib/baz/ # private files in python-foo package """ # naming conventions used in the file: # * version - tuple of integers # * ver - string representation of version # * vrange - version range, pair of max and min versions # * fn - file name (without path) # * fpath - file path ### FILES ###################################################### def fix_locations(package): """Move files to the right location.""" found_versions = {} for version in SUPPORTED: ver = vrepr(version) to_check = [i % ver for i in (\ 'usr/local/lib/python%s/site-packages', 'usr/local/lib/python%s/dist-packages', 'var/lib/python-support/python%s', 'usr/lib/pymodules/python%s')] if version >= (2, 6): to_check.append("usr/lib/python%s/site-packages" % ver) dstdir = sitedir(version, package) for location in to_check: srcdir = "debian/%s/%s" % (package, location) if isdir(srcdir): if ver in found_versions: log.error('files for version %s ' 'found in two locations:\n %s\n %s', ver, location, found_versions[ver]) exit(2) log.warn('Python %s should install files in %s. ' 'Did you forget "--install-layout=deb"?', ver, sitedir(version)) if not isdir(dstdir): os.makedirs(dstdir) # TODO: what about relative symlinks? log.debug('moving files from %s to %s', srcdir, dstdir) os.renames(srcdir, dstdir) found_versions[ver] = location # do the same with debug locations dbg_to_check = ['usr/lib/debug/%s' % i for i in to_check] dbg_to_check.append("usr/lib/debug/usr/lib/pyshared/python%s" % ver) dstdir = sitedir(version, package, gdb=True) for location in to_check: srcdir = "debian/%s/%s" % (package, location) if isdir(srcdir): if not isdir(dstdir): os.makedirs(dstdir) log.debug('moving files from %s to %s', srcdir, dstdir) os.renames(srcdir, dstdir) ### SHARING FILES ############################################## def share(package, stats, options): """Move files to /usr/share/pyshared/ if possible.""" if package.endswith('-dbg'): # nothing to share in debug packages return pubvers = debsorted(i for i in stats['public_vers'] if i[0] == 2) if len(pubvers) > 1: for pos, version1 in enumerate(pubvers): dir1 = sitedir(version1, package) for version2 in pubvers[pos + 1:]: dir2 = sitedir(version2, package) dc = dircmp(dir1, dir2) share_2x(dir1, dir2, dc) elif len(pubvers) == 1: # TODO: remove this once file conflicts will not be needed anymore move_to_pyshared(sitedir(pubvers[0], package)) for version in stats['public_ext']: create_ext_links(sitedir(version, package)) if options.guess_versions and pubvers: versions = get_requested_versions(options.vrange) for version in (i for i in versions if i[0] == 2): if version not in pubvers: log.debug('guessing files for Python %s', vrepr(version)) versions_without_ext = debsorted(set(pubvers) -\ stats['public_ext']) if not versions_without_ext: log.error('you most probably have to build extension ' 'for python%s.', vrepr(version)) exit(12) srcver = versions_without_ext[0] if srcver in stats['public_vers']: stats['public_vers'].add(version) share_2x(sitedir(srcver, package), sitedir(version, package)) def move_to_pyshared(dir1): # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/ debian, package, path = dir1.split('/', 2) dstdir = join(debian, package, 'usr/share/pyshared/', \ '/'.join(dir1.split('/')[6:])) fext = lambda fname: fname.rsplit('.', 1)[-1] for i in os.listdir(dir1): fpath1 = join(dir1, i) if isdir(fpath1): if any(fn for fn in os.listdir(fpath1) if fext(fn) != 'so'): # at least one file that is not an extension move_to_pyshared(join(dir1, i)) else: if fext(i) == 'so': continue fpath2 = join(dstdir, i) if not exists(fpath2): if not exists(dstdir): os.makedirs(dstdir) os.rename(fpath1, fpath2) relative_symlink(fpath2, fpath1) def create_ext_links(dir1): """Create extension symlinks in /usr/lib/pyshared/pythonX.Y. These symlinks are used to let dpkg detect file conflicts with python-support and python-central packages. """ debian, package, path = dir1.split('/', 2) python, _, module_subpath = path[8:].split('/', 2) dstdir = join(debian, package, 'usr/lib/pyshared/', python, module_subpath) for i in os.listdir(dir1): fpath1 = join(dir1, i) if isdir(fpath1): create_ext_links(fpath1) elif i.rsplit('.', 1)[-1] == 'so': fpath2 = join(dstdir, i) if exists(fpath2): continue if not exists(dstdir): os.makedirs(dstdir) relative_symlink(fpath1, join(dstdir, i)) def share_2x(dir1, dir2, dc=None): """Move common files to pyshared and create symlinks in original locations.""" debian, package, path = dir2.split('/', 2) # dir1 starts with debian/packagename/usr/lib/pythonX.Y/*-packages/ dstdir = join(debian, package, 'usr/share/pyshared/', \ '/'.join(dir1.split('/')[6:])) if not exists(dstdir): os.makedirs(dstdir) if dc is None: # guess/copy mode if not exists(dir2): os.makedirs(dir2) common_dirs = [] common_files = [] for i in os.listdir(dir1): if isdir(join(dir1, i)): common_dirs.append([i, None]) else: # directories with .so files will be blocked earlier common_files.append(i) else: common_dirs = dc.subdirs.iteritems() common_files = dc.common_files # dircmp returns common names only, lets check files more carefully... common_files = cmpfiles(dir1, dir2, common_files, shallow=False)[0] for fn in common_files: fpath1 = join(dir1, fn) fpath2 = join(dir2, fn) fpath3 = join(dstdir, fn) # do not touch symlinks created by previous loop or other tools if dc and not islink(fpath1): # replace with a link to pyshared os.rename(fpath1, fpath3) relative_symlink(fpath3, fpath1) if dc is None: # guess/copy mode if islink(fpath1): # ralative links will work as well, it's always the same level os.symlink(os.readlink(fpath1), fpath2) else: if exists(fpath3): # cannot share it, pyshared contains another copy fcopy(fpath1, fpath2) else: # replace with a link to pyshared os.rename(fpath1, fpath3) relative_symlink(fpath3, fpath1) relative_symlink(fpath3, fpath2) else: os.remove(fpath2) relative_symlink(fpath3, fpath2) for dn, dc in common_dirs: share_2x(join(dir1, dn), join(dir2, dn), dc) ### PACKAGE DETAILS ############################################ def scan(package, dname=None): """Gather statistics about Python files in given package.""" r = {'requires.txt': set(), 'shebangs': set(), 'public_vers': set(), 'private_dirs': {}, 'compile': False, 'public_ext': set()} dbg_package = package.endswith('-dbg') if not dname: proot = "debian/%s" % package if dname is False: private_to_check = [] else: private_to_check = [i % package for i in ('usr/lib/%s', 'usr/lib/games/%s', 'usr/share/%s', 'usr/share/games/%s')] else: proot = join('debian', package, dname.strip('/')) private_to_check = [dname[1:]] for root, dirs, file_names in os.walk(proot): # ignore Python 3.X locations if '/usr/lib/python3' in root or\ '/usr/local/lib/python3' in root: # warn only once warn = root[root.find('/lib/python'):].count('/') == 2 if warn: log.warning('Python 3.x location detected, ' 'please use dh_python3: %s', root) continue bin_dir = private_dir = None public_dir = PUBLIC_DIR_RE.match(root) if public_dir: version = getver(public_dir.group(1)) if root.endswith('-packages'): r['public_vers'].add(version) else: version = False for i in private_to_check: if root.startswith(join('debian', package, i)): private_dir = '/' + i break else: # i.e. not public_dir and not private_dir if len(root.split('/', 6)) < 6 and (\ root.endswith('/sbin') or root.endswith('/bin') or\ root.endswith('/usr/games')): # /bin or /usr/bin or /usr/games bin_dir = root # handle some EGG related data (.egg-info dirs) for name in dirs: match = EGGnPTH_RE.match(name) if match: if dbg_package: rmtree(join(root, name)) dirs.pop(dirs.index(name)) continue if match.group(2) is not None: new_name = ''.join(match.group(1, 3, 4)) log.debug('renaming %s to %s', name, new_name) os.rename(join(root, name), join(root, new_name)) if root.endswith('.egg-info') and 'requires.txt' in file_names: r['requires.txt'].add(join(root, 'requires.txt')) continue # check files for fn in file_names: fext = fn.rsplit('.', 1)[-1] if fext in ('pyc', 'pyo'): os.remove(join(root, fn)) continue if public_dir: if dbg_package and fext not in ('so', 'h'): os.remove(join(root, fn)) continue elif private_dir: mode = os.stat(join(root, fn))[ST_MODE] if mode is S_IXUSR or mode is S_IXGRP or mode is S_IXOTH: res = shebang2pyver(join(root, fn)) if res: r['private_dirs'].setdefault(private_dir, {})\ .setdefault('shebangs', set()).add(res) if public_dir or private_dir: if fext == 'so': (r if public_dir else r['private_dirs'].setdefault(private_dir, {}))\ ['public_ext'].add(version) continue elif fext == 'py': (r if public_dir else r['private_dirs'].setdefault(private_dir, {}))\ ['compile'] = True continue # .egg-info files match = EGGnPTH_RE.match(fn) if match: if match.group(2) is not None: new_name = ''.join(match.group(1, 3, 4)) log.debug('renaming %s to %s', fn, new_name) os.rename(join(root, fn), join(root, new_name)) continue # search for scripts in bin dirs if bin_dir: fpath = join(root, fn) res = shebang2pyver(fpath) if res: r['shebangs'].add(res) if dbg_package: # remove empty directories in -dbg packages proot = proot + '/usr/lib' for root, dirs, file_names in os.walk(proot, topdown=False): if '-packages/' in root and not file_names: try: os.rmdir(root) except: pass log.debug("package %s details = %s", package, r) return r ################################################################ def main(): usage = '%prog -p PACKAGE [-V [X.Y][-][A.B]] DIR_OR_FILE [-X REGEXPR]\n' parser = OptionParser(usage, version='%prog 2.0~beta1', option_class=Option) parser.add_option('--no-guessing-versions', action='store_false', dest='guess_versions', default=True, help='disable guessing other supported Python versions') parser.add_option('--no-guessing-deps', action='store_false', dest='guess_deps', default=True, help='disable guessing dependencies') parser.add_option('--skip-private', action='store_true', dest='skip_private', default=False, help='don\'t check private directories') parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False, help='turn verbose mode on') # arch=False->arch:all only, arch=True->arch:any only, None->all of them parser.add_option('-i', '--indep', action='store_false', dest='arch', default=None, help='act on architecture independent packages') parser.add_option('-a', '--arch', action='store_true', dest='arch', help='act on architecture dependent packages') parser.add_option('-q', '--quiet', action='store_false', dest='verbose', help='be quiet') parser.add_option('-p', '--package', action='append', dest='package', help='act on the package named PACKAGE') parser.add_option('-N', '--no-package', action='append', dest='no_package', help='do not act on the specified package') parser.add_option('-V', type='version_range', dest='vrange', help='specify list of supported Python versions. ' +\ 'See pycompile(1) for examples') parser.add_option('-X', '--exclude', action='append', dest='regexpr', help='exclude items that match given REGEXPR. You may use this option ' 'multiple times to build up a list of things to exclude.') parser.add_option('--depends', action='append', dest='depends', help='translate given requirements into Debian dependencies ' 'and add them to ${python:Depends}. ' 'Use it for missing items in requires.txt.') parser.add_option('--recommends', action='append', dest='recommends', help='translate given requirements into Debian ' 'dependencies and add them to ${python:Recommends}') parser.add_option('--suggests', action='append', dest='suggests', help='translate given requirements into Debian ' 'dependencies and add them to ${python:Suggests}') # ignore some debhelper options: parser.add_option('-O', help=SUPPRESS_HELP) (options, args) = parser.parse_args() # regexpr option type is not used so lets check patterns here for pattern in options.regexpr or []: # fail now rather than at runtime try: pattern = re.compile(pattern) except: log.error('regular expression is not valid: %s', pattern) exit(1) if not options.vrange and exists('debian/pyversions'): log.debug('parsing version range from debian/pyversions') with open('debian/pyversions') as fp: for line in fp: line = line.strip() if line and not line.startswith('#'): options.vrange = parse_vrange(line) break # disable PyDist if dh_pydeb is used if options.guess_deps: try: fp = open('debian/rules', 'r') except IOError: log.warning('cannot open debian/rules file') else: if re.compile('\n\s*dh_pydeb').search(fp.read()): log.warning('dh_pydeb detected, PyDist feature disabled') options.guess_deps = False private_dir = None if not args else args[0] # TODO: support more than one private dir at the same time (see :meth:scan) if options.skip_private: private_dir = False if options.verbose or os.environ.get('DH_VERBOSE') == '1': log.setLevel(logging.DEBUG) log.debug('argv: %s', sys.argv) log.debug('options: %s', options) log.debug('args: %s', args) dh = DebHelper(options.package, options.no_package) if not options.vrange and dh.python_version: options.vrange = parse_pycentral_vrange(dh.python_version) for package, pdetails in dh.packages.iteritems(): if options.arch is False and pdetails['arch'] != 'all' or \ options.arch is True and pdetails['arch'] == 'all': continue log.debug('processing package %s...', package) fix_locations(package) stats = scan(package, private_dir) share(package, stats, options) dependencies = Dependencies(package, dh.packages[package]['uses_breaks']) dependencies.parse(stats, options) dependencies.export_to(dh) if stats['public_vers']: dh.addsubstvar(package, 'python:Versions', \ ', '.join(sorted(vrepr(stats['public_vers'])))) ps = package.split('-', 1) if len(ps) > 1 and ps[0] == 'python': dh.addsubstvar(package, 'python:Provides', \ ', '.join("python%s-%s" % (i, ps[1])\ for i in sorted(vrepr(stats['public_vers'])))) pyclean_added = False # invoke pyclean only once in maintainer script if stats['compile']: dh.autoscript(package, 'preinst', 'preinst-pycentral-clean', '') dh.autoscript(package, 'postinst', 'postinst-pycompile', '') dh.autoscript(package, 'prerm', 'prerm-pyclean', '') pyclean_added = True for pdir, details in stats['private_dirs'].iteritems(): if not details.get('compile'): continue if not pyclean_added: dh.autoscript(package, 'prerm', 'prerm-pyclean', '') pyclean_added = True args = pdir ext_for = details.get('public_ext') if ext_for is None: # no extension if options.vrange: args += " -V %s" % vrange_str(options.vrange) elif ext_for is False: # extension's version not detected if options.vrange and '-' not in vrange_str(options.vrange): ver = vrange_str(options.vrange) else: # try shebang or default Python version ver = (list(v for i, v in details.get('shebangs', []) if v) or [None])[0] or DEFAULT args += " -V %s" % vrepr(ver) else: args += " -V %s" % vrepr(ext_for.pop()) for pattern in options.regexpr or []: args += " -X '%s'" % pattern.replace("'", r"\'") dh.autoscript(package, 'postinst', 'postinst-pycompile', args) pydist_file = join('debian', "%s.pydist" % package) if exists(pydist_file): if not validate_pydist(pydist_file, True): log.warning("%s.pydist file is invalid", package) else: dstdir = join('debian', package, 'usr/share/python/dist/') if not exists(dstdir): os.makedirs(dstdir) fcopy(pydist_file, join(dstdir, package)) dh.save() if __name__ == '__main__': main()