Your IP : 172.28.240.42
| Current Path : /usr/sbin/ |
|
|
| Current File : //usr/sbin/update-python-modules |
#! /usr/bin/python
#
# copyright (c) 2006 Josselin Mouette <joss@debian.org>
# Licensed under the GNU Lesser General Public License, version 2.1
# See COPYING for details
# Everything prefixed by old_ is compatibility code with older versions
# Modules used to lie in /usr/{lib,share}/python-support/$package
# They now lie in /usr/{lib,share}/pyshared
import sys,os,shutil
from optparse import OptionParser
from subprocess import call
from py_compile import compile, PyCompileError
sys.path.append("/usr/share/python-support/private/")
import pysupport
from pysupport import py_supported,py_installed,py_oldversions
basepath='/usr/lib/pymodules'
sourcepath='/usr/share/python-support'
old_extensionpath='/usr/lib/python-support'
shared_path='/usr/share/pyshared'
shared_extensionpath='/usr/lib/pyshared'
parser = OptionParser(usage="usage: %prog [-v] [-c] package_directory [...]\n"+
" %prog [-v] [-c] package.dirs [...]\n"+
" %prog [-v] [-a|-f|-p]")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
help="verbose output", default=False)
parser.add_option("-c", "--clean", action="store_true", dest="clean_mode",
help="clean modules instead of compiling them",
default=False)
parser.add_option("-a", "--rebuild-all", action="store_true",
dest="rebuild_all", default=False,
help="rebuild all private modules for a new default python version")
parser.add_option("-f", "--force-rebuild-all", action="store_true",
dest="rebuild_everything", default=False,
help="rebuild all modules, including public modules for all python versions")
parser.add_option("-p", "--post-install", action="store_true", dest="post_install",
help="run post-installation operations, common to many packages",
default=False)
parser.add_option("-b", "--bytecompile", action="store_true", dest="force_private",
help="[deprecated] byte-compilation mode: only handle private modules",
default=False)
parser.add_option("-i", "--install", action="store_true", dest="force_public",
help="[deprecated] installation mode: only handle public modules",
default=False)
(options, args) = parser.parse_args()
def debug(x):
if(options.verbose):
print x
def warning(x):
sys.stderr.write("WARNING: %s\n"%x)
def isect(l1,l2):
return [i for i in l1 if i in l2]
def concat(l1,l2):
return l1 + [i for i in l2 if i not in l1]
# Abstract class implementing the methods related to public modules
class _PublicList (list):
pyversions = py_supported
def install (self, versions):
versions = isect (self.pyversions, versions)
for filename in self:
version = None
rng = versions
try:
if filename.startswith (shared_path+"/"):
# New layout, module
relname = filename[len(shared_path)+1:]
elif filename.startswith (shared_extensionpath+"/python"):
# New layout, extension
[ version, relname ] = filename[len(shared_extensionpath)+1:].split("/", 1)
elif filename.startswith (sourcepath+"/"):
[ package, relname ] = filename[len(sourcepath)+1:].split("/",1)
elif filename.startswith (old_extensionpath+"/"):
[ package, version, relname ] = filename[len(old_extensionpath)+1:].split("/",2)
else:
raise ValueError
except ValueError:
warning ("%s contains an invalid filename (%s)"%(self.name, filename))
continue
if version:
if version not in versions:
continue
rng = [version]
for pyversion in rng:
destpath = os.path.join (basepath, pyversion, relname)
try:
os.makedirs(os.path.dirname(destpath))
except OSError:
pass
if filename[-4:] not in ['.pyc', '.pyo']:
debug("link "+destpath)
# os.path.exists returns False for broken symbolic links
if os.path.exists(destpath) or os.path.islink(destpath):
if file!="__init__.py" or (os.path.exists(destpath) and os.path.getsize(destpath)):
# The file is already here, probably from the previous version.
# No need to check for conflicts, dpkg catches them earlier now
debug("overwrite "+destpath)
else:
debug("overwrite namespace "+destpath)
if os.path.isdir(destpath):
shutil.rmtree(destpath)
else:
os.remove(destpath)
os.symlink(filename,destpath)
# Abstract class implementing the methods related to private modules
class _PrivateList (list):
pyversion = None
def bytecompile (self):
if self.pyversion:
debug("Byte-compilation of whole %s with python%s..."%(self.name,self.pyversion))
call(['/usr/bin/python'+self.pyversion,
os.path.join('/usr/lib','python'+self.pyversion,'py_compile.py')]
+ self)
else:
for filename in self:
debug("compile "+filename+'c')
try:
# Note that compile doesn't raise PyCompileError by default
compile(filename, doraise=True)
except IOError, (errno, strerror):
warning("I/O error while trying to byte-compile %s (%s): %s" % (filename, errno, strerror))
except PyCompileError, inst:
warning("compile error while trying to byte-compile %s: %s" % (filename, inst.msg))
except:
warning("unexpected error while trying to byte-compile %s: %s" % (filename, sys.exc_info()[0]))
def clean(self):
for filename in self:
for ext in ['c', 'o']:
fullpath=filename+ext
if os.path.exists(fullpath):
debug("remove "+fullpath)
os.remove(fullpath)
# Abstract class for PrivateFileList and SharedFileList
class _FileList(list):
def __init__ (self, path):
self.name = path
for line in file(path):
line = line.strip()
if (not line) or line.startswith('#'):
continue
if line.startswith('/'):
self.append(line)
continue
line = [x.strip() for x in line.split('=',1)]
if len(line) != 2:
warning("Parse error in %s"%path)
continue
self.parse_option(*line)
# This class represents a file list as provided in the /usr/share/python-support/$package.public
# Useful for public modules and extensions
class SharedFileList(_FileList, _PublicList):
def parse_option (self, arg, value):
if arg=='pyversions':
self.pyversions = pysupport.version_list(value)
# Ignore unknown arguments for extensivity
# This class represents a file list as provided in the /usr/share/python-support/$package.private
# Useful for private modules
class PrivateFileList(_FileList, _PrivateList):
def parse_option (self, arg, value):
if arg=='pyversion':
self.pyversion = value
# This is a helper generator that goes through files of interest in a given directory
def allfiles(path, onlypy=False):
for root, dirs, files in os.walk(path):
for f in files:
if (onlypy and not f.endswith(".py")) or f== ".version":
continue
yield os.path.join(root,f)
if not onlypy:
for d in dirs:
d = os.path.join(root, d)
if os.path.islink(d):
yield d
# This class emulates the file listing as provided by /usr/share/python-support/$package.public
# with the deprecated layout /usr/{lib,share}/python-support/$package/
class SharedDirList(_PublicList):
def __init__ (self, path):
self.name = path
# Add all files to the file listing
self.extend(allfiles(path))
verfile=os.path.join(path,'.version')
extdir=path.replace(sourcepath,old_extensionpath,1)
if os.path.isfile(verfile):
# If we have a .version, use it
self.pyversions = pysupport.version_list(file(verfile).readline())
elif os.path.isdir(extdir):
# Try to obtain the list of supported versions
# from the extensions in /usr/lib
self.pyversions = isect(py_supported,os.listdir(extdir))
else:
# Otherwise, support all versions
pass
if os.path.isdir(extdir):
# Add the extensions to the file listing
for version in self.pyversions:
self.extend(allfiles(os.path.join(extdir,version)))
# This class emulates the file listing as provided by /usr/share/python-support/$package.private
# with the deprecated layout /usr/share/python-support/$package.dirs
class PrivateDirList(_PrivateList):
def __init__ (self, path):
self.name = path
self.extend(allfiles(path, onlypy=True))
versionfile = os.path.join(path, ".pyversion")
if os.path.isfile(versionfile):
self.pyversion = file(versionfile).readline().strip()
class CachedFileList(dict):
def __getitem__ (self, name):
if name in self and dict.__getitem__(self, name) == None:
if name.startswith("/"):
# The case of old-style private directories
self[name] = PrivateDirList (name)
else:
path = os.path.join (sourcepath, name)
if name.endswith(".public"):
self[name] = SharedFileList (path)
elif name.endswith(".private"):
self[name] = PrivateFileList (path)
elif os.path.isdir(path):
self[name] = SharedDirList (path)
else:
raise Exception("[Internal Error] I don't know what to do with this path: %s"%path)
return dict.__getitem__(self, name)
def bytecompile_all(py,path=None):
if not path:
path=os.path.join(basepath,py)
if not os.path.isdir(path):
return
debug("Byte-compilation of whole %s..."%path)
os.spawnl(os.P_WAIT, '/usr/bin/'+py, py,
os.path.join('/usr/lib/',py,'compileall.py'), '-q', path)
# A function to create the ".path" at the root of the installed directory
# Returns the list of affected directories
def create_dotpath(py):
path=os.path.join(basepath,py)
if not os.path.isdir(path):
return
pathfile=os.path.join(path,".path")
debug("Generation of %s..."%pathfile)
pathlist=[path]
ret=[]
for f in os.listdir(path):
f=os.path.join(path,f)
if f.endswith(".pth") and os.path.isfile(f):
for l in file(f):
l=l.rstrip('\n')
if l.startswith('import'):
# Do not ship lines starting with "import", they are executed! (complete WTF)
continue
pathlist.append(l)
l2=os.path.join(path,l)
pathlist.append(l2)
ret.append(l2)
fd=file(pathfile,"w")
fd.writelines([l+'\n' for l in pathlist])
fd.close()
return ret
def post_change_stuff(py):
# All the changes that need to be done after anything has changed
# in a /usr/lib/pymodules/pythonX.Y directory
# * Cleanup of all dangling symlinks that are left out after a package
# is upgraded/removed.
# * The namespace packages are here because python doesn't consider a
# directory to be able to contain packages if there is no __init__.py
# file (yes, this is completely stupid).
# * The .path file must be created by concatenating all those .pth
# files that extend sys.path (this also badly sucks).
# * Byte-compilation of all .py files that haven't already been
path=os.path.join(basepath,py)
if not os.path.isdir(path):
return
# First, remove any dangling symlinks.
# In the same loop, we find which directories may need a namespace package
dirhash={}
for dir, dirs, files in os.walk(path):
dirhash[dir]=False
files.sort() # We need the .py to appear before the .pyc
for f in files+dirs:
# We also examine dirs as some symlinks are dirs
abspath=os.path.join(dir,f)
islink=os.path.islink(abspath)
if islink:
if not os.path.exists(abspath):
# We refer to a file that was removed
debug("remove "+abspath)
os.remove(abspath)
continue
srcfile = os.readlink (abspath)
# Remove links left here after a change in the supported python versions for a package
removed = False
for package in public_packages:
if srcfile in public_packages[package]:
if py not in public_packages[package].pyversions:
debug("remove "+abspath)
os.remove(abspath)
removed = True
break
else:
# Remove files provided by packages that do not use python-support anymore
debug("remove "+abspath)
os.remove(abspath)
removed = True
if removed:
# Do not go further, the file was removed
continue
if f[-4:] in ['.pyc', '.pyo']:
if not os.path.exists(abspath[:-1]):
debug("remove "+abspath)
os.remove(abspath)
continue
elif f[-3:] in ['.py', '.so']:
if islink or f!='__init__.py':
# List the directory as maybe needing a namespace packages
d=dir
while dirhash.has_key(d) and not dirhash[d]:
dirhash[d]=True
d=os.path.dirname(d)
# Remove the directory if it is empty after our crazy removals
try:
os.removedirs(dir)
except OSError:
pass
dirhash[path]=False
# Then, find which directories belong in a .pth file
# These directories don't need a namespace package, so we
# set them to False in dirhash
for p in create_dotpath (py):
dirhash[p] = False
# Finally, create/remove namespace packages
for dir in dirhash:
initfile=os.path.join(dir,"__init__.py")
noinitfile=os.path.join(dir,".noinit")
if dirhash[dir] and not os.path.exists(noinitfile):
if not os.path.exists(initfile):
debug("create namespace "+initfile)
file(initfile,"w").close()
else:
for e in ['','c','o']:
if os.path.exists(initfile+e):
debug('remove namespace '+initfile+e)
os.remove(initfile+e)
try:
os.removedirs(dir)
except OSError:
pass
bytecompile_all(py)
# A helper function for older $package.dirs files
def dirlist_file(f):
return [ l.rstrip('\n') for l in file(f) if len(l)>1 ]
# End of function definitions - Start of the script itself
# Ensure that the umask is sane
os.umask(022)
# Read all modules listing
public_packages = CachedFileList()
private_packages = CachedFileList()
dirlisting = os.listdir(sourcepath)
for name in dirlisting:
path=os.path.join(sourcepath,name)
if name == "private":
continue
ext = name.split(".")[-1]
if os.path.isdir(path):
if ext in ["public", "private", "dirs"]:
# Presumably a bogus directory, see #528130
warning("%s is a directory"%name)
else:
public_packages[name] = None
continue
if not os.path.isfile(path):
# Ignore whatever is not a file, like dangling symlinks
continue
if ext == "public":
public_packages[name] = None
elif ext == "private":
private_packages[name] = None
elif ext == "dirs":
for dirname in dirlist_file (path):
private_packages[dirname] = None
# Just ignore all other files
# Parse arguments
do_public=[]
do_private=[]
for arg in args:
if arg.startswith(sourcepath):
arg = arg[len(sourcepath):].lstrip("/")
if arg.endswith(".dirs") and arg in dirlisting:
for dirname in dirlist_file(os.path.join(sourcepath, arg)):
do_private.append(private_packages[dirname])
elif arg in public_packages:
do_public.append(public_packages[arg])
elif arg in private_packages:
do_private.append(private_packages[arg])
else:
if options.clean_mode:
warning("%s does not exist.\n Some bytecompiled files may be left behind."%arg)
else:
parser.error("%s is not a recognized python-support module."%arg)
# Check consistency options (although these ones should not exist anymore)
if do_private and options.force_public:
parser.error("Option -i cannot be used with a .private module file.")
if do_public and options.force_private:
parser.error("Option -b cannot be used with a .public module file.")
if options.rebuild_everything:
options.rebuild_all = True
for pyver in py_supported:
dir = os.path.join(basepath,pyver)
if os.path.isdir(dir):
shutil.rmtree(dir)
# Check for changes in installed python versions
need_postinstall = []
for pyver in py_oldversions+py_supported:
dir = os.path.join(basepath,pyver)
# Check for ".path" because sometimes the directory already exists
# while the python version isn't installed, because of some .so's.
if pyver not in py_installed and os.path.isdir(dir):
debug("Removing obsolete directory %s..."%(dir))
shutil.rmtree(dir)
if pyver in py_installed and not os.path.isfile(os.path.join(dir,".path")):
need_postinstall.append(pyver)
if need_postinstall:
debug("Building all modules for %s..."%(" ".join(need_postinstall)))
for package in public_packages:
public_packages[package].install(need_postinstall)
for pyver in need_postinstall:
# Here we need to launch create_dotpath because otherwise we could
# end up without the .path file that is checked 6 lines earlier
create_dotpath(pyver)
if options.rebuild_all:
for package in private_packages:
private_packages[package].bytecompile()
# Now for the processing of what was handed on the command line
for package in do_private:
if not options.clean_mode:
package.bytecompile()
else:
package.clean()
need_dotpath = False
for package in do_public:
need_postinstall = concat (need_postinstall, isect(package.pyversions,py_installed))
if options.clean_mode:
continue
package.install(py_installed)
for f in package:
if f.endswith(".pth"):
need_dotpath = True
# Only do the funny and time-consuming things when the -p option is
# given, e.g when python-support is triggered.
if need_postinstall and 'DPKG_RUNNING_VERSION' in os.environ and not options.post_install:
ret = os.spawnlp(os.P_WAIT, 'dpkg-trigger', 'dpkg-trigger', '--no-await', 'pysupport')
if ret:
sys.stderr.write("ERROR: dpkg-trigger failed\n")
sys.exit(1)
if need_dotpath:
for py in need_postinstall:
create_dotpath (py)
need_postinstall = []
if options.post_install:
# The trigger has been activated; do it for all installed versions
need_postinstall = py_installed
if need_postinstall:
need_dotpath = False
for py in need_postinstall:
post_change_stuff(py)