--- /dev/null
+#!/usr/bin/python
+# Copyright 2009 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+import sys
+# This block ensures that ^C interrupts are handled quietly.
+try:
+ import signal
+
+ def exithandler(signum,frame):
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ sys.exit(1)
+
+ signal.signal(signal.SIGINT, exithandler)
+ signal.signal(signal.SIGTERM, exithandler)
+
+except KeyboardInterrupt:
+ sys.exit(1)
+
+import logging
+import optparse
+import os
+import portage
+import _emerge
+from portage.cache.cache_errors import CacheError, StatCollision
+from portage.util import writemsg_level
+
+def parse_args(args):
+ usage = "egencache [options] --update [atom] ..."
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option("--update",
+ action="store_true",
+ help="update metadata/cache/ (generate as necessary)")
+ parser.add_option("--cache-dir",
+ help="location of the metadata cache",
+ dest="cache_dir")
+ parser.add_option("--config-root",
+ help="location of portage config files",
+ dest="config_root")
+ parser.add_option("--jobs",
+ action="store",
+ help="max ebuild processes to spawn")
+ parser.add_option("--load-average",
+ action="store",
+ help="max load allowed when spawning multiple jobs",
+ dest="load_average")
+ parser.add_option("--rsync",
+ action="store_true",
+ help="enable rsync stat collision workaround " + \
+ "for bug 139134 (use with --update)")
+ options, args = parser.parse_args(args)
+
+ if not options.update:
+ parser.error('No action specified (--update ' + \
+ 'is the only available action)')
+
+ if options.config_root is not None and \
+ not os.path.isdir(options.config_root):
+ parser.error("Not a directory: --config-root='%s'" % \
+ (options.config_root,))
+
+ if options.cache_dir is not None and not os.path.isdir(options.cache_dir):
+ parser.error("Not a directory: --cache-dir='%s'" % \
+ (options.cache_dir,))
+
+ for atom in args:
+ try:
+ atom = portage.dep.Atom(atom)
+ except portage.exception.InvalidAtom:
+ parser.error('Invalid atom: %s' % (atom,))
+
+ if str(atom) != atom.cp:
+ parser.error('Atom is too specific: %s' % (atom,))
+
+ return options, args
+
+class GenCache(object):
+ def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None,
+ rsync=False):
+ self._portdb = portdb
+ # We can globally cleanse stale cache only if we
+ # iterate over every single cp.
+ self._global_cleanse = cp_iter is None
+ if cp_iter is not None:
+ self._cp_set = set(cp_iter)
+ cp_iter = iter(self._cp_set)
+ else:
+ self._cp_set = None
+ self._regen = _emerge.MetadataRegen(portdb, cp_iter=cp_iter,
+ consumer=self._metadata_callback,
+ max_jobs=max_jobs, max_load=max_load)
+ self.returncode = os.EX_OK
+ metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule")
+ self._trg_cache = metadbmodule(portdb.porttree_root,
+ "metadata/cache", portage.auxdbkeys[:])
+ if rsync:
+ self._trg_cache.raise_stat_collision = True
+ self._existing_nodes = set()
+
+ def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata):
+ self._existing_nodes.add(cpv)
+ if metadata is not None:
+ if metadata.get('EAPI') == '0':
+ del metadata['EAPI']
+ try:
+ try:
+ self._trg_cache[cpv] = metadata
+ except StatCollision, sc:
+ # If the content of a cache entry changes and neither the
+ # file mtime nor size changes, it will prevent rsync from
+ # detecting changes. Cache backends may raise this
+ # exception from _setitem() if they detect this type of stat
+ # collision. These exceptions are be handled by bumping the
+ # mtime on the ebuild (and the corresponding cache entry).
+ # See bug #139134.
+ max_mtime = sc.mtime
+ for ec, (loc, ec_mtime) in metadata['_eclasses_'].iteritems():
+ if max_mtime < ec_mtime:
+ max_mtime = ec_mtime
+ if max_mtime == sc.mtime:
+ max_mtime += 1
+ max_mtime = long(max_mtime)
+ try:
+ os.utime(ebuild_path, (max_mtime, max_mtime))
+ except OSError, e:
+ self.returncode |= 1
+ writemsg_level(
+ "%s writing target: %s\n" % (cpv, e),
+ level=logging.ERROR, noiselevel=-1)
+ else:
+ metadata['_mtime_'] = max_mtime
+ self._trg_cache[cpv] = metadata
+ self._portdb.auxdb[repo_path][cpv] = metadata
+
+ except CacheError, ce:
+ self.returncode |= 1
+ writemsg_level(
+ "%s writing target: %s\n" % (cpv, ce),
+ level=logging.ERROR, noiselevel=-1)
+
+ def run(self):
+ self._regen.run()
+ self.returncode |= self._regen.returncode
+
+ trg_cache = self._trg_cache
+ dead_nodes = None
+ if self._global_cleanse:
+ try:
+ dead_nodes = set(trg_cache.iterkeys())
+ except CacheError, ce:
+ self.returncode |= 1
+ writemsg_level(
+ "Error listing cache entries for " + \
+ "'%s/metadata/cache': %s, continuing...\n" % \
+ (self._portdb.porttree_root, ce),
+ level=logging.ERROR, noiselevel=-1)
+
+ else:
+ cp_set = self._cp_set
+ cpv_getkey = portage.cpv_getkey
+ try:
+ dead_nodes = set(cpv for cpv in \
+ trg_cache.iterkeys() \
+ if cpv_getkey(cpv) in cp_set)
+ except CacheError, ce:
+ self.returncode |= 1
+ writemsg_level(
+ "Error listing cache entries for " + \
+ "'%s/metadata/cache': %s, continuing...\n" % \
+ (self._portdb.porttree_root, ce),
+ level=logging.ERROR, noiselevel=-1)
+
+ if dead_nodes:
+ dead_nodes.difference_update(self._existing_nodes)
+ for k in dead_nodes:
+ try:
+ del trg_cache[k]
+ except KeyError:
+ pass
+ except CacheError:
+ self.returncode |= 1
+
+ if not trg_cache.autocommits:
+ try:
+ trg_cache.commit()
+ except CacheError, ce:
+ self.returncode |= 1
+ writemsg_level(
+ "committing target: %s\n" % (ce,),
+ level=logging.ERROR, noiselevel=-1)
+
+def egencache_main(args):
+ options, args = parse_args(args)
+
+ config_root = options.config_root
+ if config_root is None:
+ config_root = '/'
+
+ # The calling environment is ignored, so the program is
+ # completely controlled by commandline arguments.
+ env = {}
+
+ # TODO: Implement --repo for choosing a repo.
+ env['PORTDIR_OVERLAY'] = ''
+
+ if options.cache_dir is not None:
+ env['PORTAGE_DEPCACHEDIR'] = options.cache_dir
+
+ settings = portage.config(config_root=config_root,
+ target_root='/', env=env)
+
+ if 'metadata-transfer' not in settings.features:
+ writemsg_level("ecachegen: error: " + \
+ "FEATURES=metadata-transfer is not enabled\n",
+ level=logging.ERROR, noiselevel=-1)
+ return 1
+
+ portdb = portage.portdbapi(settings["PORTDIR"], mysettings=settings)
+
+ cp_iter = None
+ if args:
+ cp_iter = iter(args)
+
+ gen_cache = GenCache(portdb, cp_iter=cp_iter,
+ max_jobs=options.jobs,
+ max_load=options.load_average,
+ rsync=options.rsync)
+ gen_cache.run()
+ return gen_cache.returncode
+
+if __name__ == "__main__":
+ portage._disable_legacy_globals()
+ portage.util.noiselimit = -1
+ sys.exit(egencache_main(sys.argv[1:]))
# One more pass to help devs track down the source
scanelf -qyRAF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log
vecho -ne '\a\n'
- eqawarn "QA Notice: The following files contain executable stacks"
- eqawarn " Files with executable stacks will not work properly (or at all!)"
- eqawarn " on some architectures/operating systems. A bug should be filed"
- eqawarn " at http://bugs.gentoo.org/ to make sure the file is fixed."
+ eqawarn "QA Notice: The following files contain writable and executable sections"
+ eqawarn " Files with such sections will not work properly (or at all!) on some"
+ eqawarn " architectures/operating systems. A bug should be filed at"
+ eqawarn " http://bugs.gentoo.org/ to make sure the issue is fixed."
eqawarn " For more information, see http://hardened.gentoo.org/gnu-stack.xml"
eqawarn " Please include the following list of files in your report:"
eqawarn " Note: Bugs should be filed for the respective maintainers"
- eqawarn " of the package in question and not hardened@"
+ eqawarn " of the package in question and not hardened@g.o."
eqawarn "${f}"
vecho -ne '\a\n'
die_msg="${die_msg} execstacks"
"""
__slots__ = ("cpv", "ebuild_path", "fd_pipes", "metadata_callback",
- "ebuild_mtime", "portdb", "repo_path", "settings") + \
+ "ebuild_mtime", "metadata", "portdb", "repo_path", "settings") + \
("_raw_metadata",)
_file_names = ("ebuild",)
self.returncode = 1
else:
metadata = izip(portage.auxdbkeys, metadata_lines)
- self.metadata_callback(self.cpv, self.ebuild_path,
- self.repo_path, metadata, self.ebuild_mtime)
+ self.metadata = self.metadata_callback(self.cpv,
+ self.ebuild_path, self.repo_path, metadata,
+ self.ebuild_mtime)
class EbuildProcess(SpawnProcess):
class MetadataRegen(PollScheduler):
- def __init__(self, portdb, max_jobs=None, max_load=None):
+ def __init__(self, portdb, cp_iter=None, consumer=None,
+ max_jobs=None, max_load=None):
PollScheduler.__init__(self)
self._portdb = portdb
+ self._global_cleanse = False
+ if cp_iter is None:
+ cp_iter = self._iter_every_cp()
+ # We can globally cleanse stale cache only if we
+ # iterate over every single cp.
+ self._global_cleanse = True
+ self._cp_iter = cp_iter
+ self._consumer = consumer
if max_jobs is None:
max_jobs = 1
unregister=self._unregister)
self._valid_pkgs = set()
+ self._cp_set = set()
self._process_iter = self._iter_metadata_processes()
self.returncode = os.EX_OK
self._error_count = 0
+ def _iter_every_cp(self):
+ every_cp = self._portdb.cp_all()
+ every_cp.sort(reverse=True)
+ try:
+ while True:
+ yield every_cp.pop()
+ except IndexError:
+ pass
+
def _iter_metadata_processes(self):
portdb = self._portdb
valid_pkgs = self._valid_pkgs
- every_cp = portdb.cp_all()
- every_cp.sort(reverse=True)
+ cp_set = self._cp_set
+ consumer = self._consumer
- while every_cp:
- cp = every_cp.pop()
+ for cp in self._cp_iter:
+ cp_set.add(cp)
portage.writemsg_stdout("Processing %s\n" % cp)
cpv_list = portdb.cp_list(cp)
for cpv in cpv_list:
valid_pkgs.add(cpv)
ebuild_path, repo_path = portdb.findname2(cpv)
- metadata_process = portdb._metadata_process(
+ metadata, st, emtime = portdb._pull_valid_cache(
cpv, ebuild_path, repo_path)
- if metadata_process is None:
+ if metadata is not None:
+ if consumer is not None:
+ consumer(cpv, ebuild_path,
+ repo_path, metadata)
continue
- yield metadata_process
+
+ yield EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path,
+ ebuild_mtime=emtime,
+ metadata_callback=portdb._metadata_callback,
+ portdb=portdb, repo_path=repo_path,
+ settings=portdb.doebuild_settings)
def run(self):
from portage.cache.cache_errors import CacheError
dead_nodes = {}
- for mytree in portdb.porttrees:
- try:
- dead_nodes[mytree] = set(portdb.auxdb[mytree].iterkeys())
- except CacheError, e:
- portage.writemsg("Error listing cache entries for " + \
- "'%s': %s, continuing...\n" % (mytree, e), noiselevel=-1)
- del e
- dead_nodes = None
- break
-
while self._schedule():
self._poll_loop()
while self._jobs:
self._poll_loop()
+ if self._global_cleanse:
+ for mytree in portdb.porttrees:
+ try:
+ dead_nodes[mytree] = set(portdb.auxdb[mytree].iterkeys())
+ except CacheError, e:
+ portage.writemsg("Error listing cache entries for " + \
+ "'%s': %s, continuing...\n" % (mytree, e),
+ noiselevel=-1)
+ del e
+ dead_nodes = None
+ break
+ else:
+ cp_set = self._cp_set
+ cpv_getkey = portage.cpv_getkey
+ for mytree in portdb.porttrees:
+ try:
+ dead_nodes[mytree] = set(cpv for cpv in \
+ portdb.auxdb[mytree].iterkeys() \
+ if cpv_getkey(cpv) in cp_set)
+ except CacheError, e:
+ portage.writemsg("Error listing cache entries for " + \
+ "'%s': %s, continuing...\n" % (mytree, e),
+ noiselevel=-1)
+ del e
+ dead_nodes = None
+ break
+
if dead_nodes:
for y in self._valid_pkgs:
for mytree in portdb.porttrees:
self._error_count += 1
self._valid_pkgs.discard(metadata_process.cpv)
portage.writemsg("Error processing %s, continuing...\n" % \
- (metadata_process.cpv,))
+ (metadata_process.cpv,), noiselevel=-1)
+
+ if self._consumer is not None:
+ # On failure, still notify the consumer (in this case the metadata
+ # argument is None).
+ self._consumer(metadata_process.cpv,
+ metadata_process.ebuild_path,
+ metadata_process.repo_path,
+ metadata_process.metadata)
+
self._schedule()
class UninstallFailure(portage.exception.PortageException):
def cpv_getkey(mycpv):
myslash=mycpv.split("/")
mysplit=pkgsplit(myslash[-1])
+ if mysplit is None:
+ return None
mylen=len(myslash)
if mylen==2:
return myslash[0]+"/"+mysplit[0]
self.info = info
def __str__(self):
return "cache is non-modifiable"+str(self.info)
+
+class StatCollision(CacheError):
+ """
+ If the content of a cache entry changes and neither the file mtime nor
+ size changes, it will prevent rsync from detecting changes. Cache backends
+ may raise this exception from _setitem() if they detect this type of stat
+ collision. See bug #139134.
+ """
+ def __init__(self, key, filename, mtime, size):
+ self.key = key
+ self.filename = filename
+ self.mtime = mtime
+ self.size = size
+
+ def __str__(self):
+ return "%s has stat collision with size %s and mtime %s" % \
+ (self.key, self.size, self.mtime)
+
+ def __repr__(self):
+ return "portage.cache.cache_errors.StatCollision(%s)" % \
+ (', '.join((repr(self.key), repr(self.filename),
+ repr(self.mtime), repr(self.size))),)
# License: GPL2
# $Id$
-import errno, os, re
+import errno, os, re, sys
from portage.cache import cache_errors, flat_hash
import portage.eclass_cache
from portage.cache.template import reconstruct_eclasses
super(database, self).__init__(location, *args, **config)
self.location = os.path.join(loc, "metadata","cache")
self.ec = portage.eclass_cache.cache(loc)
+ self.raise_stat_collision = False
def _parse_data(self, data, cpv):
_hashed_re_match = self._hashed_re.match
values = ProtectedDict(values)
values["INHERITED"] = ' '.join(sorted(values["_eclasses_"]))
+ new_content = []
+ for k in self.auxdbkey_order:
+ new_content.append(unicode(values.get(k, ''), errors='replace'))
+ new_content.append(u'\n')
+ for i in xrange(magic_line_count - len(self.auxdbkey_order)):
+ new_content.append(u'\n')
+ new_content = u''.join(new_content)
+ new_content = new_content.encode(
+ sys.getdefaultencoding(), 'backslashreplace')
+
+ new_fp = os.path.join(self.location, cpv)
+ try:
+ f = open(new_fp, 'rb')
+ except EnvironmentError:
+ pass
+ else:
+ try:
+ try:
+ existing_st = os.fstat(f.fileno())
+ existing_content = f.read()
+ finally:
+ f.close()
+ except EnvironmentError:
+ pass
+ else:
+ existing_mtime = long(existing_st.st_mtime)
+ if values['_mtime_'] == existing_mtime and \
+ existing_content == new_content:
+ return
+
+ if self.raise_stat_collision and \
+ values['_mtime_'] == existing_mtime and \
+ len(new_content) == existing_st.st_size:
+ raise cache_errors.StatCollision(cpv, new_fp,
+ existing_mtime, existing_st.st_size)
+
s = cpv.rfind("/")
fp = os.path.join(self.location,cpv[:s],
".update.%i.%s" % (os.getpid(), cpv[s+1:]))
try:
- myf = open(fp, "w")
+ myf = open(fp, 'wb')
except EnvironmentError, e:
if errno.ENOENT == e.errno:
try:
self._ensure_dirs(cpv)
- myf = open(fp, "w")
+ myf = open(fp, 'wb')
except EnvironmentError, e:
raise cache_errors.CacheCorruption(cpv, e)
else:
raise cache_errors.CacheCorruption(cpv, e)
try:
- for k in self.auxdbkey_order:
- myf.write(values.get(k, "") + "\n")
- for i in xrange(magic_line_count - len(self.auxdbkey_order)):
- myf.write("\n")
+ myf.write(new_content)
finally:
myf.close()
self._ensure_access(fp, mtime=values["_mtime_"])
- new_fp = os.path.join(self.location, cpv)
try:
os.rename(fp, new_fp)
except EnvironmentError, e:
metadata["EAPI"] = "-" + eapi.lstrip("-")
self.auxdb[repo_path][cpv] = metadata
+ return metadata
def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
from email.MIMEText import MIMEText as TextMessage
from email.MIMEMultipart import MIMEMultipart as MultipartMessage
from email.MIMEBase import MIMEBase as BaseMessage
+from email.header import Header
def create_message(sender, recipient, subject, body, attachments=None):
if attachments == None:
mymessage.set_unixfrom(sender)
mymessage["To"] = recipient
mymessage["From"] = sender
- mymessage["Subject"] = subject
+ # Use Header as a workaround so that long subject lines are wrapped
+ # correctly by <=python-2.6 (gentoo bug #263370, python issue #1974).
+ mymessage["Subject"] = Header(subject)
mymessage["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S %z")
return mymessage
projects / portage.git / commitdiff