From 5f4a072a308b91f6399fa9058a325384b72a7f54 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 2 Apr 2009 13:02:59 +0000 Subject: [PATCH] Merged from trunk -r13253:13262 | 13254 | Add a cp_iter keyword argument to the MetadataRegen | | zmedico | constructor, which can be used to do a regen for a subset | | | of packages. TODO: Add support to cleanse cache for the | | | specific cp values that are processed. | | 13255 | Add support in MetadataRegen for cleansing stale cache when | | zmedico | only a subset of packages are processed. | | 13256 | Add a "consumer" keyword parameter to the MetadataRegen | | zmedico | constructor which can be used to pass in a callback that is | | | called for each ebuild that is processed (allowing access | | | to the ebuild metadata). | | 13257 | reword the qa execstack notice since it can be triggered by | | SpankMan | pt_load sections and not stacks | | 13258 | In MetadataRegen._metadata_exit(), notify the consumer of | | zmedico | failures (in this case the metadata argument is None). | | 13259 | When in --quiet mode, still show a message for failures in | | zmedico | MetadataRegen._metadata_exit(). | | 13260 | Bug #261377 - Add a new 'egencache' tool to generate | | zmedico | metadata cache for distribution. It only have the most | | | basic functionality now, and more features will be added | | | later. Usage: egencache [options] --update [atom] ... | | | Options: -h, --help show this help message and exit | | | --update update metadata/cache/ (generate as necessary) | | | --cache-dir=CACHE_DIR location of the metadata cache | | | --config-root=CONFIG_ROOT location of portage config files | | | --jobs=JOBS max ebuild processes to spawn | | | --load-average=LOAD_AVERAGE max load allowed when spawning | | | multiple jobs | | 13261 | Bug #263370 - In create_message(), use email.header.Header | | zmedico | to wrap the subject, as a workaround so that long subject | | | lines are wrapped correctly by <=python-2.6 (gentoo bug | | | #263370, python issue #1974). | | 13262 | Add a new egencache --rsync option which enables a stat | | zmedico | collision workaround for cases in which the content of a | | | cache entry changes and neither the file mtime nor size | | | changes (preventing rsync from detecting changes). See bug | | | #139134. This option should only be needed for distribution | | | via something like rsync, which relies on timestamps and | | | file sizes to detect changes. It's not needed with git | | | since that uses a more thorough mechanism which allows it | | | to detect changed inode numbers (described in racy-git.txt | | | in the git technical docs). | svn path=/main/branches/prefix/; revision=13277 --- bin/egencache | 236 ++++++++++++++++++++++++++++++ bin/misc-functions.sh | 10 +- pym/_emerge/__init__.py | 98 ++++++++++--- pym/portage/__init__.py | 2 + pym/portage/cache/cache_errors.py | 22 +++ pym/portage/cache/metadata.py | 49 ++++++- pym/portage/dbapi/porttree.py | 1 + pym/portage/mail.py | 5 +- 8 files changed, 387 insertions(+), 36 deletions(-) create mode 100755 bin/egencache diff --git a/bin/egencache b/bin/egencache new file mode 100755 index 000000000..e8981deba --- /dev/null +++ b/bin/egencache @@ -0,0 +1,236 @@ +#!/usr/bin/python +# Copyright 2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +import sys +# This block ensures that ^C interrupts are handled quietly. +try: + import signal + + def exithandler(signum,frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + sys.exit(1) + + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + +except KeyboardInterrupt: + sys.exit(1) + +import logging +import optparse +import os +import portage +import _emerge +from portage.cache.cache_errors import CacheError, StatCollision +from portage.util import writemsg_level + +def parse_args(args): + usage = "egencache [options] --update [atom] ..." + parser = optparse.OptionParser(usage=usage) + parser.add_option("--update", + action="store_true", + help="update metadata/cache/ (generate as necessary)") + parser.add_option("--cache-dir", + help="location of the metadata cache", + dest="cache_dir") + parser.add_option("--config-root", + help="location of portage config files", + dest="config_root") + parser.add_option("--jobs", + action="store", + help="max ebuild processes to spawn") + parser.add_option("--load-average", + action="store", + help="max load allowed when spawning multiple jobs", + dest="load_average") + parser.add_option("--rsync", + action="store_true", + help="enable rsync stat collision workaround " + \ + "for bug 139134 (use with --update)") + options, args = parser.parse_args(args) + + if not options.update: + parser.error('No action specified (--update ' + \ + 'is the only available action)') + + if options.config_root is not None and \ + not os.path.isdir(options.config_root): + parser.error("Not a directory: --config-root='%s'" % \ + (options.config_root,)) + + if options.cache_dir is not None and not os.path.isdir(options.cache_dir): + parser.error("Not a directory: --cache-dir='%s'" % \ + (options.cache_dir,)) + + for atom in args: + try: + atom = portage.dep.Atom(atom) + except portage.exception.InvalidAtom: + parser.error('Invalid atom: %s' % (atom,)) + + if str(atom) != atom.cp: + parser.error('Atom is too specific: %s' % (atom,)) + + return options, args + +class GenCache(object): + def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, + rsync=False): + self._portdb = portdb + # We can globally cleanse stale cache only if we + # iterate over every single cp. + self._global_cleanse = cp_iter is None + if cp_iter is not None: + self._cp_set = set(cp_iter) + cp_iter = iter(self._cp_set) + else: + self._cp_set = None + self._regen = _emerge.MetadataRegen(portdb, cp_iter=cp_iter, + consumer=self._metadata_callback, + max_jobs=max_jobs, max_load=max_load) + self.returncode = os.EX_OK + metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule") + self._trg_cache = metadbmodule(portdb.porttree_root, + "metadata/cache", portage.auxdbkeys[:]) + if rsync: + self._trg_cache.raise_stat_collision = True + self._existing_nodes = set() + + def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata): + self._existing_nodes.add(cpv) + if metadata is not None: + if metadata.get('EAPI') == '0': + del metadata['EAPI'] + try: + try: + self._trg_cache[cpv] = metadata + except StatCollision, sc: + # If the content of a cache entry changes and neither the + # file mtime nor size changes, it will prevent rsync from + # detecting changes. Cache backends may raise this + # exception from _setitem() if they detect this type of stat + # collision. These exceptions are be handled by bumping the + # mtime on the ebuild (and the corresponding cache entry). + # See bug #139134. + max_mtime = sc.mtime + for ec, (loc, ec_mtime) in metadata['_eclasses_'].iteritems(): + if max_mtime < ec_mtime: + max_mtime = ec_mtime + if max_mtime == sc.mtime: + max_mtime += 1 + max_mtime = long(max_mtime) + try: + os.utime(ebuild_path, (max_mtime, max_mtime)) + except OSError, e: + self.returncode |= 1 + writemsg_level( + "%s writing target: %s\n" % (cpv, e), + level=logging.ERROR, noiselevel=-1) + else: + metadata['_mtime_'] = max_mtime + self._trg_cache[cpv] = metadata + self._portdb.auxdb[repo_path][cpv] = metadata + + except CacheError, ce: + self.returncode |= 1 + writemsg_level( + "%s writing target: %s\n" % (cpv, ce), + level=logging.ERROR, noiselevel=-1) + + def run(self): + self._regen.run() + self.returncode |= self._regen.returncode + + trg_cache = self._trg_cache + dead_nodes = None + if self._global_cleanse: + try: + dead_nodes = set(trg_cache.iterkeys()) + except CacheError, ce: + self.returncode |= 1 + writemsg_level( + "Error listing cache entries for " + \ + "'%s/metadata/cache': %s, continuing...\n" % \ + (self._portdb.porttree_root, ce), + level=logging.ERROR, noiselevel=-1) + + else: + cp_set = self._cp_set + cpv_getkey = portage.cpv_getkey + try: + dead_nodes = set(cpv for cpv in \ + trg_cache.iterkeys() \ + if cpv_getkey(cpv) in cp_set) + except CacheError, ce: + self.returncode |= 1 + writemsg_level( + "Error listing cache entries for " + \ + "'%s/metadata/cache': %s, continuing...\n" % \ + (self._portdb.porttree_root, ce), + level=logging.ERROR, noiselevel=-1) + + if dead_nodes: + dead_nodes.difference_update(self._existing_nodes) + for k in dead_nodes: + try: + del trg_cache[k] + except KeyError: + pass + except CacheError: + self.returncode |= 1 + + if not trg_cache.autocommits: + try: + trg_cache.commit() + except CacheError, ce: + self.returncode |= 1 + writemsg_level( + "committing target: %s\n" % (ce,), + level=logging.ERROR, noiselevel=-1) + +def egencache_main(args): + options, args = parse_args(args) + + config_root = options.config_root + if config_root is None: + config_root = '/' + + # The calling environment is ignored, so the program is + # completely controlled by commandline arguments. + env = {} + + # TODO: Implement --repo for choosing a repo. + env['PORTDIR_OVERLAY'] = '' + + if options.cache_dir is not None: + env['PORTAGE_DEPCACHEDIR'] = options.cache_dir + + settings = portage.config(config_root=config_root, + target_root='/', env=env) + + if 'metadata-transfer' not in settings.features: + writemsg_level("ecachegen: error: " + \ + "FEATURES=metadata-transfer is not enabled\n", + level=logging.ERROR, noiselevel=-1) + return 1 + + portdb = portage.portdbapi(settings["PORTDIR"], mysettings=settings) + + cp_iter = None + if args: + cp_iter = iter(args) + + gen_cache = GenCache(portdb, cp_iter=cp_iter, + max_jobs=options.jobs, + max_load=options.load_average, + rsync=options.rsync) + gen_cache.run() + return gen_cache.returncode + +if __name__ == "__main__": + portage._disable_legacy_globals() + portage.util.noiselimit = -1 + sys.exit(egencache_main(sys.argv[1:])) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 5ef5f53b4..c7508fbb2 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -151,14 +151,14 @@ install_qa_check() { # One more pass to help devs track down the source scanelf -qyRAF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log vecho -ne '\a\n' - eqawarn "QA Notice: The following files contain executable stacks" - eqawarn " Files with executable stacks will not work properly (or at all!)" - eqawarn " on some architectures/operating systems. A bug should be filed" - eqawarn " at http://bugs.gentoo.org/ to make sure the file is fixed." + eqawarn "QA Notice: The following files contain writable and executable sections" + eqawarn " Files with such sections will not work properly (or at all!) on some" + eqawarn " architectures/operating systems. A bug should be filed at" + eqawarn " http://bugs.gentoo.org/ to make sure the issue is fixed." eqawarn " For more information, see http://hardened.gentoo.org/gnu-stack.xml" eqawarn " Please include the following list of files in your report:" eqawarn " Note: Bugs should be filed for the respective maintainers" - eqawarn " of the package in question and not hardened@" + eqawarn " of the package in question and not hardened@g.o." eqawarn "${f}" vecho -ne '\a\n' die_msg="${die_msg} execstacks" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index f2fb37ead..44ec88b9a 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -3051,7 +3051,7 @@ class EbuildMetadataPhase(SubProcess): """ __slots__ = ("cpv", "ebuild_path", "fd_pipes", "metadata_callback", - "ebuild_mtime", "portdb", "repo_path", "settings") + \ + "ebuild_mtime", "metadata", "portdb", "repo_path", "settings") + \ ("_raw_metadata",) _file_names = ("ebuild",) @@ -3156,8 +3156,9 @@ class EbuildMetadataPhase(SubProcess): self.returncode = 1 else: metadata = izip(portage.auxdbkeys, metadata_lines) - self.metadata_callback(self.cpv, self.ebuild_path, - self.repo_path, metadata, self.ebuild_mtime) + self.metadata = self.metadata_callback(self.cpv, + self.ebuild_path, self.repo_path, metadata, + self.ebuild_mtime) class EbuildProcess(SpawnProcess): @@ -11610,9 +11611,18 @@ class Scheduler(PollScheduler): class MetadataRegen(PollScheduler): - def __init__(self, portdb, max_jobs=None, max_load=None): + def __init__(self, portdb, cp_iter=None, consumer=None, + max_jobs=None, max_load=None): PollScheduler.__init__(self) self._portdb = portdb + self._global_cleanse = False + if cp_iter is None: + cp_iter = self._iter_every_cp() + # We can globally cleanse stale cache only if we + # iterate over every single cp. + self._global_cleanse = True + self._cp_iter = cp_iter + self._consumer = consumer if max_jobs is None: max_jobs = 1 @@ -11625,28 +11635,46 @@ class MetadataRegen(PollScheduler): unregister=self._unregister) self._valid_pkgs = set() + self._cp_set = set() self._process_iter = self._iter_metadata_processes() self.returncode = os.EX_OK self._error_count = 0 + def _iter_every_cp(self): + every_cp = self._portdb.cp_all() + every_cp.sort(reverse=True) + try: + while True: + yield every_cp.pop() + except IndexError: + pass + def _iter_metadata_processes(self): portdb = self._portdb valid_pkgs = self._valid_pkgs - every_cp = portdb.cp_all() - every_cp.sort(reverse=True) + cp_set = self._cp_set + consumer = self._consumer - while every_cp: - cp = every_cp.pop() + for cp in self._cp_iter: + cp_set.add(cp) portage.writemsg_stdout("Processing %s\n" % cp) cpv_list = portdb.cp_list(cp) for cpv in cpv_list: valid_pkgs.add(cpv) ebuild_path, repo_path = portdb.findname2(cpv) - metadata_process = portdb._metadata_process( + metadata, st, emtime = portdb._pull_valid_cache( cpv, ebuild_path, repo_path) - if metadata_process is None: + if metadata is not None: + if consumer is not None: + consumer(cpv, ebuild_path, + repo_path, metadata) continue - yield metadata_process + + yield EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path, + ebuild_mtime=emtime, + metadata_callback=portdb._metadata_callback, + portdb=portdb, repo_path=repo_path, + settings=portdb.doebuild_settings) def run(self): @@ -11654,22 +11682,39 @@ class MetadataRegen(PollScheduler): from portage.cache.cache_errors import CacheError dead_nodes = {} - for mytree in portdb.porttrees: - try: - dead_nodes[mytree] = set(portdb.auxdb[mytree].iterkeys()) - except CacheError, e: - portage.writemsg("Error listing cache entries for " + \ - "'%s': %s, continuing...\n" % (mytree, e), noiselevel=-1) - del e - dead_nodes = None - break - while self._schedule(): self._poll_loop() while self._jobs: self._poll_loop() + if self._global_cleanse: + for mytree in portdb.porttrees: + try: + dead_nodes[mytree] = set(portdb.auxdb[mytree].iterkeys()) + except CacheError, e: + portage.writemsg("Error listing cache entries for " + \ + "'%s': %s, continuing...\n" % (mytree, e), + noiselevel=-1) + del e + dead_nodes = None + break + else: + cp_set = self._cp_set + cpv_getkey = portage.cpv_getkey + for mytree in portdb.porttrees: + try: + dead_nodes[mytree] = set(cpv for cpv in \ + portdb.auxdb[mytree].iterkeys() \ + if cpv_getkey(cpv) in cp_set) + except CacheError, e: + portage.writemsg("Error listing cache entries for " + \ + "'%s': %s, continuing...\n" % (mytree, e), + noiselevel=-1) + del e + dead_nodes = None + break + if dead_nodes: for y in self._valid_pkgs: for mytree in portdb.porttrees: @@ -11709,7 +11754,16 @@ class MetadataRegen(PollScheduler): self._error_count += 1 self._valid_pkgs.discard(metadata_process.cpv) portage.writemsg("Error processing %s, continuing...\n" % \ - (metadata_process.cpv,)) + (metadata_process.cpv,), noiselevel=-1) + + if self._consumer is not None: + # On failure, still notify the consumer (in this case the metadata + # argument is None). + self._consumer(metadata_process.cpv, + metadata_process.ebuild_path, + metadata_process.repo_path, + metadata_process.metadata) + self._schedule() class UninstallFailure(portage.exception.PortageException): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index fee4c04bc..eedb1389d 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -7217,6 +7217,8 @@ def dep_wordreduce(mydeplist,mysettings,mydbapi,mode,use_cache=1): def cpv_getkey(mycpv): myslash=mycpv.split("/") mysplit=pkgsplit(myslash[-1]) + if mysplit is None: + return None mylen=len(myslash) if mylen==2: return myslash[0]+"/"+mysplit[0] diff --git a/pym/portage/cache/cache_errors.py b/pym/portage/cache/cache_errors.py index f63e5994b..e1e8eead0 100644 --- a/pym/portage/cache/cache_errors.py +++ b/pym/portage/cache/cache_errors.py @@ -39,3 +39,25 @@ class ReadOnlyRestriction(CacheError): self.info = info def __str__(self): return "cache is non-modifiable"+str(self.info) + +class StatCollision(CacheError): + """ + If the content of a cache entry changes and neither the file mtime nor + size changes, it will prevent rsync from detecting changes. Cache backends + may raise this exception from _setitem() if they detect this type of stat + collision. See bug #139134. + """ + def __init__(self, key, filename, mtime, size): + self.key = key + self.filename = filename + self.mtime = mtime + self.size = size + + def __str__(self): + return "%s has stat collision with size %s and mtime %s" % \ + (self.key, self.size, self.mtime) + + def __repr__(self): + return "portage.cache.cache_errors.StatCollision(%s)" % \ + (', '.join((repr(self.key), repr(self.filename), + repr(self.mtime), repr(self.size))),) diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index 5222223c6..a8be01095 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -3,7 +3,7 @@ # License: GPL2 # $Id$ -import errno, os, re +import errno, os, re, sys from portage.cache import cache_errors, flat_hash import portage.eclass_cache from portage.cache.template import reconstruct_eclasses @@ -30,6 +30,7 @@ class database(flat_hash.database): super(database, self).__init__(location, *args, **config) self.location = os.path.join(loc, "metadata","cache") self.ec = portage.eclass_cache.cache(loc) + self.raise_stat_collision = False def _parse_data(self, data, cpv): _hashed_re_match = self._hashed_re.match @@ -73,31 +74,63 @@ class database(flat_hash.database): values = ProtectedDict(values) values["INHERITED"] = ' '.join(sorted(values["_eclasses_"])) + new_content = [] + for k in self.auxdbkey_order: + new_content.append(unicode(values.get(k, ''), errors='replace')) + new_content.append(u'\n') + for i in xrange(magic_line_count - len(self.auxdbkey_order)): + new_content.append(u'\n') + new_content = u''.join(new_content) + new_content = new_content.encode( + sys.getdefaultencoding(), 'backslashreplace') + + new_fp = os.path.join(self.location, cpv) + try: + f = open(new_fp, 'rb') + except EnvironmentError: + pass + else: + try: + try: + existing_st = os.fstat(f.fileno()) + existing_content = f.read() + finally: + f.close() + except EnvironmentError: + pass + else: + existing_mtime = long(existing_st.st_mtime) + if values['_mtime_'] == existing_mtime and \ + existing_content == new_content: + return + + if self.raise_stat_collision and \ + values['_mtime_'] == existing_mtime and \ + len(new_content) == existing_st.st_size: + raise cache_errors.StatCollision(cpv, new_fp, + existing_mtime, existing_st.st_size) + s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s], ".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: - myf = open(fp, "w") + myf = open(fp, 'wb') except EnvironmentError, e: if errno.ENOENT == e.errno: try: self._ensure_dirs(cpv) - myf = open(fp, "w") + myf = open(fp, 'wb') except EnvironmentError, e: raise cache_errors.CacheCorruption(cpv, e) else: raise cache_errors.CacheCorruption(cpv, e) try: - for k in self.auxdbkey_order: - myf.write(values.get(k, "") + "\n") - for i in xrange(magic_line_count - len(self.auxdbkey_order)): - myf.write("\n") + myf.write(new_content) finally: myf.close() self._ensure_access(fp, mtime=values["_mtime_"]) - new_fp = os.path.join(self.location, cpv) try: os.rename(fp, new_fp) except EnvironmentError, e: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index c94a2965d..9d4b1523d 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -349,6 +349,7 @@ class portdbapi(dbapi): metadata["EAPI"] = "-" + eapi.lstrip("-") self.auxdb[repo_path][cpv] = metadata + return metadata def _pull_valid_cache(self, cpv, ebuild_path, repo_path): diff --git a/pym/portage/mail.py b/pym/portage/mail.py index 72b411264..5f1cc11a6 100644 --- a/pym/portage/mail.py +++ b/pym/portage/mail.py @@ -7,6 +7,7 @@ import portage.exception, socket, smtplib, os, sys, time from email.MIMEText import MIMEText as TextMessage from email.MIMEMultipart import MIMEMultipart as MultipartMessage from email.MIMEBase import MIMEBase as BaseMessage +from email.header import Header def create_message(sender, recipient, subject, body, attachments=None): if attachments == None: @@ -25,7 +26,9 @@ def create_message(sender, recipient, subject, body, attachments=None): mymessage.set_unixfrom(sender) mymessage["To"] = recipient mymessage["From"] = sender - mymessage["Subject"] = subject + # Use Header as a workaround so that long subject lines are wrapped + # correctly by <=python-2.6 (gentoo bug #263370, python issue #1974). + mymessage["Subject"] = Header(subject) mymessage["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S %z") return mymessage -- 2.26.2