From 23bb604b2bf47816fff1da9588a3d1e1dab34192 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 25 Dec 2008 13:50:16 +0000 Subject: [PATCH] Merged from trunk -r12302:12345 | 12303 | Add support for synchronizing ebuild and eclass timestamps | | zmedico | with the metadata cache timestamps which emerge --sync | | | operates on a git repository. This is necessary since git | | | doesn't preserve timestamps. It is assumed that the | | | timestamps should be synchronized if the relevant ebuilds | | | and eclasses are unmodified relative to the HEAD commit (as | | | reported by git-ls-files). This feature was requested by | | | Daniel Robbins, so that it's possible for the funtoo git | | | repository to distribute pre-generated metadata. | | 12327 | Inside mirror_cache(), handle unsupported EAPI like | | zmedico | portdbapi does. | | 12328 | Don't add any more that one "-" symbol to the front of an | | zmedico | unsupported EAPI. | | 12329 | Handle potential KeyErrors that may be raised from | | zmedico | get_eclass_data(), and remove unnecessary print statements | | | inside get_eclass_data(). | | 12330 | Inside portdbapi._metadata_callback(), preserve _eclasses_ | | zmedico | when the EAPI is unsupported since the eclass timestamps | | | might be needed in order to decide that the cache entry | | | should be regenerated. Also, make sure that all metadata | | | returned from portdbapi.aux_get() is returned as empty | | | strings when the EAPI is unsupported. | | 12331 | Make vardbapi._aux_get() return _mtime_ as a long (never as | | zmedico | a float). | | 12332 | Use long() instead of float() when parsing Package.mtime | | zmedico | from the metadata. | | 12339 | Refactor EbuildMetadataPhase._output_handler() and | | zmedico | _set_returncode(). | | 12341 | Fix typo. | | zmedico | | | 12345 | scan archives of objects for exec stacks | | SpankMan | | svn path=/main/branches/prefix/; revision=12347 --- bin/misc-functions.sh | 6 +- pym/_emerge/__init__.py | 168 ++++++++++++++++++++++++++++++---- pym/portage/cache/metadata.py | 9 +- pym/portage/cache/util.py | 19 +++- pym/portage/dbapi/porttree.py | 35 ++++--- pym/portage/dbapi/vartree.py | 2 +- pym/portage/eclass_cache.py | 7 +- 7 files changed, 199 insertions(+), 47 deletions(-) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index edf64580c..ed077ff6d 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -98,7 +98,7 @@ install_qa_check() { export QA_TEXTRELS="${QA_TEXTRELS} lib*/modules/*.ko" f=$(scanelf -qyRF '%t %p' "${ED}" | grep -v 'usr/lib/debug/') if [[ -n ${f} ]] ; then - scanelf -qyRF '%T %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-textrel.log + scanelf -qyRAF '%T %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-textrel.log vecho -ne '\a\n' eqawarn "QA Notice: The following files contain runtime text relocations" eqawarn " Text relocations force the dynamic linker to perform extra" @@ -136,14 +136,14 @@ install_qa_check() { [[ -n ${QA_STRICT_WX_LOAD} ]] && QA_WX_LOAD="" export QA_EXECSTACK="${QA_EXECSTACK} lib*/modules/*.ko" export QA_WX_LOAD="${QA_WX_LOAD} lib*/modules/*.ko" - f=$(scanelf -qyRF '%e %p' "${ED}" | grep -v 'usr/lib/debug/') + f=$(scanelf -qyRAF '%e %p' "${ED}" | grep -v 'usr/lib/debug/') ;; esac ;; esac if [[ -n ${f} ]] ; then # One more pass to help devs track down the source - scanelf -qyRF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log + scanelf -qyRAF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log vecho -ne '\a\n' eqawarn "QA Notice: The following files contain executable stacks" eqawarn " Files with executable stacks will not work properly (or at all!)" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 2be2b1f8a..466a82fcf 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1593,7 +1593,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): def _set__mtime_(self, k, v): if isinstance(v, basestring): try: - v = float(v.strip()) + v = long(v.strip()) except ValueError: v = 0 self._pkg.mtime = v @@ -2931,28 +2931,25 @@ class EbuildMetadataPhase(SubProcess): if event & PollConstants.POLLIN: self._raw_metadata.append(self._files.ebuild.read()) - - if not self._raw_metadata[-1] or event & PollConstants.POLLHUP: - # Split lines here so they can be counted inside _set_returncode(). - self._raw_metadata = "".join(self._raw_metadata).splitlines() - self._unregister() - self.wait() - - if self.returncode == os.EX_OK: - metadata = izip(portage.auxdbkeys, self._raw_metadata) - self.metadata_callback(self.cpv, self.ebuild_path, - self.repo_path, metadata, self.ebuild_mtime) + if not self._raw_metadata[-1]: + self._unregister() + self.wait() self._unregister_if_appropriate(event) return self._registered def _set_returncode(self, wait_retval): SubProcess._set_returncode(self, wait_retval) - if self.returncode == os.EX_OK and \ - len(portage.auxdbkeys) != len(self._raw_metadata): - # Don't trust bash's returncode if the - # number of lines is incorrect. - self.returncode = 1 + if self.returncode == os.EX_OK: + metadata_lines = "".join(self._raw_metadata).splitlines() + if len(portage.auxdbkeys) != len(metadata_lines): + # Don't trust bash's returncode if the + # number of lines is incorrect. + self.returncode = 1 + else: + metadata = izip(portage.auxdbkeys, metadata_lines) + self.metadata_callback(self.cpv, self.ebuild_path, + self.repo_path, metadata, self.ebuild_mtime) class EbuildProcess(SpawnProcess): @@ -11845,6 +11842,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): vcs_dirs = vcs_dirs.intersection(os.listdir(myportdir)) os.umask(0022) + dosyncuri = syncuri updatecache_flg = False if myaction == "metadata": print "skipping sync" @@ -11867,7 +11865,9 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): msg = ">>> Git pull in %s successful" % myportdir emergelog(xterm_titles, msg) writemsg_level(msg + "\n") - return exitcode + exitcode = git_sync_timestamps(settings, myportdir) + if exitcode == os.EX_OK: + updatecache_flg = True elif syncuri[:8]=="rsync://": for vcs_dir in vcs_dirs: writemsg_level(("!!! %s appears to be under revision " + \ @@ -12337,6 +12337,138 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): display_news_notification(root_config, myopts) return os.EX_OK +def git_sync_timestamps(settings, portdir): + """ + Since git doesn't preserve timestamps, synchronize timestamps between + entries and ebuilds/eclasses. Assume the cache has the correct timestamp + for a given file as long as the file in the working tree is not modified + (relative to HEAD). + """ + cache_dir = os.path.join(portdir, "metadata", "cache") + if not os.path.isdir(cache_dir): + return os.EX_OK + writemsg_level(">>> Synchronizing timestamps...\n") + + from portage.cache.cache_errors import CacheError + try: + cache_db = settings.load_best_module("portdbapi.metadbmodule")( + portdir, "metadata/cache", portage.auxdbkeys[:], readonly=True) + except CacheError, e: + writemsg_level("!!! Unable to instantiate cache: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + return 1 + + ec_dir = os.path.join(portdir, "eclass") + try: + ec_names = set(f[:-7] for f in os.listdir(ec_dir) \ + if f.endswith(".eclass")) + except OSError, e: + writemsg_level("!!! Unable to list eclasses: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + return 1 + + args = [portage.const.BASH_BINARY, "-c", + "cd %s && git ls-files -m --with-tree=HEAD" % \ + portage._shell_quote(portdir)] + import subprocess + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + modified_files = set(l.rstrip("\n") for l in proc.stdout) + rval = proc.wait() + if rval != os.EX_OK: + return rval + + modified_eclasses = set(ec for ec in ec_names \ + if os.path.join("eclass", ec + ".eclass") in modified_files) + + updated_ec_mtimes = {} + + for cpv in cache_db: + cpv_split = portage.catpkgsplit(cpv) + if cpv_split is None: + writemsg_level("!!! Invalid cache entry: %s\n" % (cpv,), + level=logging.ERROR, noiselevel=-1) + continue + + cat, pn, ver, rev = cpv_split + cat, pf = portage.catsplit(cpv) + relative_eb_path = os.path.join(cat, pn, pf + ".ebuild") + if relative_eb_path in modified_files: + continue + + try: + cache_entry = cache_db[cpv] + eb_mtime = cache_entry.get("_mtime_") + ec_mtimes = cache_entry.get("_eclasses_") + except KeyError: + writemsg_level("!!! Missing cache entry: %s\n" % (cpv,), + level=logging.ERROR, noiselevel=-1) + continue + except CacheError, e: + writemsg_level("!!! Unable to access cache entry: %s %s\n" % \ + (cpv, e), level=logging.ERROR, noiselevel=-1) + continue + + if eb_mtime is None: + writemsg_level("!!! Missing ebuild mtime: %s\n" % (cpv,), + level=logging.ERROR, noiselevel=-1) + continue + + try: + eb_mtime = long(eb_mtime) + except ValueError: + writemsg_level("!!! Invalid ebuild mtime: %s %s\n" % \ + (cpv, eb_mtime), level=logging.ERROR, noiselevel=-1) + continue + + if ec_mtimes is None: + writemsg_level("!!! Missing eclass mtimes: %s\n" % (cpv,), + level=logging.ERROR, noiselevel=-1) + continue + + if modified_eclasses.intersection(ec_mtimes): + continue + + missing_eclasses = set(ec_mtimes).difference(ec_names) + if missing_eclasses: + writemsg_level("!!! Non-existent eclass(es): %s %s\n" % \ + (cpv, sorted(missing_eclasses)), level=logging.ERROR, + noiselevel=-1) + continue + + eb_path = os.path.join(portdir, relative_eb_path) + try: + current_eb_mtime = os.stat(eb_path) + except OSError: + writemsg_level("!!! Missing ebuild: %s\n" % \ + (cpv,), level=logging.ERROR, noiselevel=-1) + continue + + inconsistent = False + for ec, (ec_path, ec_mtime) in ec_mtimes.iteritems(): + updated_mtime = updated_ec_mtimes.get(ec) + if updated_mtime is not None and updated_mtime != ec_mtime: + writemsg_level("!!! Inconsistent eclass mtime: %s %s\n" % \ + (cpv, ec), level=logging.ERROR, noiselevel=-1) + inconsistent = True + break + + if inconsistent: + continue + + if current_eb_mtime != eb_mtime: + os.utime(eb_path, (eb_mtime, eb_mtime)) + + for ec, (ec_path, ec_mtime) in ec_mtimes.iteritems(): + if ec in updated_ec_mtimes: + continue + ec_path = os.path.join(ec_dir, ec + ".eclass") + current_mtime = long(os.stat(ec_path).st_mtime) + if current_mtime != ec_mtime: + os.utime(ec_path, (ec_mtime, ec_mtime)) + updated_ec_mtimes[ec] = ec_mtime + + return os.EX_OK + def action_metadata(settings, portdb, myopts): portage.writemsg_stdout("\n>>> Updating Portage cache: ") old_umask = os.umask(0002) diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index 86717e6c5..b602f62bc 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -4,7 +4,7 @@ # $Id$ import os, re, stat, types -from portage.cache import flat_hash +from portage.cache import cache_errors, flat_hash import portage.eclass_cache from portage.cache.template import reconstruct_eclasses from portage.cache.mappings import ProtectedDict @@ -55,7 +55,12 @@ class database(flat_hash.database): if "_eclasses_" not in d: if "INHERITED" in d: - d["_eclasses_"] = self.ec.get_eclass_data(d["INHERITED"].split(), from_master_only=True) + try: + d["_eclasses_"] = self.ec.get_eclass_data( + d["INHERITED"].split(), from_master_only=True) + except KeyError, e: + # INHERITED contains a non-existent eclass. + raise cache_errors.CacheCorruption(cpv, e) del d["INHERITED"] else: d["_eclasses_"] = {} diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py index dc3881b19..7d81f2d05 100644 --- a/pym/portage/cache/util.py +++ b/pym/portage/cache/util.py @@ -10,6 +10,7 @@ from portage.cache import cache_errors def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, verbose_instance=None): + from portage import eapi_is_supported if not src_cache.complete_eclass_entries and not eclass_cache: raise Exception("eclass_cache required for cache's of class %s!" % src_cache.__class__) @@ -89,13 +90,27 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, # Even if _eclasses_ already exists, replace it with data from # eclass_cache, in order to insert local eclass paths. - eclasses = eclass_cache.get_eclass_data(inherited, - from_master_only=True) + try: + eclasses = eclass_cache.get_eclass_data(inherited, + from_master_only=True) + except KeyError: + # INHERITED contains a non-existent eclass. + noise.eclass_stale(x) + continue + if eclasses is None: noise.eclass_stale(x) continue entry["_eclasses_"] = eclasses + eapi = entry.get("EAPI") + if not eapi: + eapi = "0" + if not eapi_is_supported(eapi): + for k in set(entry).difference(("_mtime_", "_eclasses_")): + entry[k] = "" + entry["EAPI"] = "-" + eapi.lstrip("-") + # by this time, if it reaches here, the eclass has been validated, and the entry has # been updated/translated (if needs be, for metadata/cache mainly) try: trg_cache[x] = entry diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 19010bc62..77e6ae494 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -296,17 +296,6 @@ class portdbapi(dbapi): i = metadata.iteritems() metadata = dict(i) - if "EAPI" not in metadata or not metadata["EAPI"].strip(): - metadata["EAPI"] = "0" - - if not eapi_is_supported(metadata["EAPI"]): - # if newer version, wipe everything and negate eapi - eapi = metadata["EAPI"] - metadata = {} - for x in self._known_keys: - metadata.setdefault(x, "") - metadata["EAPI"] = "-" + eapi - if metadata.get("INHERITED", False): metadata["_eclasses_"] = \ self.eclassdb.get_eclass_data(metadata["INHERITED"].split()) @@ -315,6 +304,16 @@ class portdbapi(dbapi): metadata.pop("INHERITED", None) metadata["_mtime_"] = mtime + + eapi = metadata.get("EAPI") + if not eapi or not eapi.strip(): + eapi = "0" + metadata["EAPI"] = eapi + if not eapi_is_supported(eapi): + for k in set(metadata).difference(("_mtime_", "_eclasses_")): + metadata[k] = "" + metadata["EAPI"] = "-" + eapi.lstrip("-") + self.auxdb[repo_path][cpv] = metadata def _pull_valid_cache(self, cpv, ebuild_path, repo_path): @@ -425,15 +424,21 @@ class portdbapi(dbapi): else: mydata["_eclasses_"] = {} - if not mydata.setdefault("EAPI", "0"): - mydata["EAPI"] = "0" - # do we have a origin repository name for the current package mydata["repository"] = self._repository_map.get( os.path.sep.join(myebuild.split(os.path.sep)[:-3]), "") mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", [])) - mydata["_mtime_"] = st.st_mtime + mydata["_mtime_"] = long(st.st_mtime) + + eapi = mydata.get("EAPI") + if not eapi: + eapi = "0" + mydata["EAPI"] = eapi + if not eapi_is_supported(eapi): + for k in set(mydata).difference(("_mtime_", "_eclasses_")): + mydata[k] = "" + mydata["EAPI"] = "-" + eapi.lstrip("-") #finally, we look at our internal cache entry and return the requested data. returnme = [mydata.get(x, "") for x in mylist] diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 5dccef3cf..2a6cd5d46 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1702,7 +1702,7 @@ class vardbapi(dbapi): results = [] for x in wants: if x == "_mtime_": - results.append(st.st_mtime) + results.append(long(st.st_mtime)) continue try: myf = open(os.path.join(mydir, x), "r") diff --git a/pym/portage/eclass_cache.py b/pym/portage/eclass_cache.py index 93c956f8a..670e9fb24 100644 --- a/pym/portage/eclass_cache.py +++ b/pym/portage/eclass_cache.py @@ -99,12 +99,7 @@ class cache(object): def get_eclass_data(self, inherits, from_master_only=False): ec_dict = {} for x in inherits: - try: - ec_dict[x] = self.eclasses[x] - except KeyError: - print "ec=",ec_dict - print "inherits=",inherits - raise + ec_dict[x] = self.eclasses[x] if from_master_only and \ self._eclass_locations[x] != self._master_eclass_root: return None -- 2.26.2