From: Fabian Groffen Date: Tue, 16 Dec 2008 18:25:18 +0000 (-0000) Subject: Merged from trunk -r12182:12200 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=306ff09ccae450e06aac6997b7638a8ffe7da7de;p=portage.git Merged from trunk -r12182:12200 | 12183 | For py3k compat, remove Package.__cmp__() and related cmp() | | zmedico | function call. | | 12184 | For py3k compat, avoid using list.sort(cmp). | | zmedico | | | 12185 | When in --debug mode, make the depgraph call | | zmedico | self.digraph.debug_print() to display the final digraph | | | state. This should help when troubleshooting problems with | | | blockers that don't resolve automatically, by making it | | | easier to see what pulled in unwanted packages. | | 12186 | Bug #250166 - To avoid accidental regeneration of digests | | zmedico | with the incorrect files (such as partially downloaded | | | files), trigger the fetch code if the file exists and it's | | | size doesn't match the current manifest entry. If there | | | really is a legitimate reason for the digest to change, | | | `ebuild --force digest` can be used to avoid triggering this | | | code (or else the old digests can be manually removed from | | | the Manifest). | | 12187 | Simplify the code from bug #250166. | | zmedico | | | 12188 | In digestgen(), don't call fetch for files that already | | zmedico | exist but have invalid zero-size in the manifest. | | 12189 | Bug #250148 - Prevent ebuild.sh subprocess from inheriting | | zmedico | file descriptor 9, since otherwise if a daemon process such | | | as udevd gets spawned then it can inherit the file | | | descriptor and hang emerge. | | 12190 | When attempting to close fd 9 at the end of the depend | | zmedico | phase, use exec so that it really works. | | 12191 | When processing output from the "depend" phase, use the | | zmedico | number of lines as a sanity check so that bash's returncode | | | doesn't have to be trusted. | | 12192 | urlparse.urljoin() only works correctly with recognized | | zmedico | protocols and requires the base url to have a trailing | | | slash, so join PORTAGE_BINHOST manually. Thanks to Lance | | | Albertson for reporting. | | 12193 | When the Binpkg class has fetched a package in --fetchonly | | zmedico | mode, wait until after the package is verified and injected | | | before returning. | | 12194 | Create a BinpkgPrefetcher composite task to handle fetch, | | zmedico | verification, and injection. | | 12195 | Verify binary packages even when not in "strict" mode | | zmedico | (normal distfiles are also verified in this case). | | 12196 | Make the Binpkg class handle fetch logging similar to how | | zmedico | EbuildBuild does, directing output to PORTAGE_LOG_FILE | | | instead of emerge-fetch.log which is only used for | | | prefetchers now. | | 12197 | Make BinpkgVerifier rename files that fail verification, | | zmedico | similar to how distfiles are automatically renamed. | | 12198 | Make Binpkg clean the build dir immediately after locking | | zmedico | it. This ensures that a new PORTAGE_LOG_FILE is created. | | 12199 | For py3k compat, remove Atom.__cmp__() and use str.__lt__() | | zmedico | instead. | | 12200 | Remove ManifestEntry.__cmp__() since it's not needed and | | zmedico | py3k won't use it. | svn path=/main/branches/prefix/; revision=12254 --- diff --git a/bin/dispatch-conf b/bin/dispatch-conf index cf42a1ce5..5d6e67e86 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -329,24 +329,28 @@ class dispatch: We keep ._cfg0002_conf over ._cfg0001_conf and ._cfg0000_conf. """ h = {} - + configs = [] newconfigs.sort () for nconf in newconfigs: nconf = nconf.rstrip () conf = re.sub (r'\._cfg\d+_', '', nconf) - dir = re.match (r'^(.+)/', nconf).group (1) + dirname = os.path.dirname(nconf) + conf_map = { + 'current' : conf, + 'dir' : dirname, + 'new' : nconf, + } if conf in h: mrgconf = re.sub(r'\._cfg', '._mrg', h[conf]['new']) if os.path.exists(mrgconf): os.unlink(mrgconf) os.unlink(h[conf]['new']) - - h [conf] = { 'current' : conf, 'dir' : dir, 'new' : nconf } - - configs = h.values () - configs.sort (lambda a, b: cmp(a ['current'], b ['current'])) + h[conf].update(conf_map) + else: + h[conf] = conf_map + configs.append(conf_map) return configs diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 8971c78d3..e86f2f56a 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -2083,7 +2083,7 @@ ebuild_main() { for f in ${auxdbkeys} ; do echo $(echo ${!f}) 1>&9 || exit $? done - 9>&- + exec 9>&- fi set +f ;; @@ -2101,16 +2101,26 @@ ebuild_main() { fi } -[[ -n $EBUILD_SH_ARGS ]] && ebuild_main +if [[ $EBUILD_PHASE = depend ]] ; then + ebuild_main +elif [[ -n $EBUILD_SH_ARGS ]] ; then + ( + # Don't allow subprocesses to inherit the pipe which + # emerge uses to monitor ebuild.sh. + exec 9>&- -# Save the env only for relevant phases. -if [ -n "${EBUILD_SH_ARGS}" ] && \ - ! hasq ${EBUILD_SH_ARGS} clean depend help info nofetch ; then - # Save current environment and touch a success file. (echo for success) - umask 002 - save_ebuild_env | filter_readonly_variables > "${T}/environment" - chown ${PORTAGE_USER:-portage}:${PORTAGE_GROUP:-portage} "${T}/environment" &>/dev/null - chmod g+w "${T}/environment" &>/dev/null + ebuild_main + + # Save the env only for relevant phases. + if ! hasq "$EBUILD_SH_ARGS" clean help info nofetch ; then + umask 002 + save_ebuild_env | filter_readonly_variables > "$T/environment" + chown ${PORTAGE_USER:-portage}:${PORTAGE_GROUP:-portage} "$T/environment" &>/dev/null + chmod g+w "$T/environment" &>/dev/null + fi + exit 0 + ) + exit $? fi # Do not exit when ebuild.sh is sourced by other scripts. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 1a82da933..ba82bbd9f 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1527,13 +1527,6 @@ class Package(Task): (self.type_name, self.root, self.cpv, self.operation) return self._hash_key - def __cmp__(self, other): - if self > other: - return 1 - elif self < other: - return -1 - return 0 - def __lt__(self, other): if other.cp != self.cp: return False @@ -2901,17 +2894,26 @@ class EbuildMetadataPhase(SubProcess): files = self._files self._raw_metadata.append(files.ebuild.read()) if not self._raw_metadata[-1]: + # Split lines here so they can be counted inside _set_returncode(). + self._raw_metadata = "".join(self._raw_metadata).splitlines() self._unregister() self.wait() if self.returncode == os.EX_OK: - metadata = izip(portage.auxdbkeys, - "".join(self._raw_metadata).splitlines()) + metadata = izip(portage.auxdbkeys, self._raw_metadata) self.metadata_callback(self.cpv, self.ebuild_path, self.repo_path, metadata, self.ebuild_mtime) return self._registered + def _set_returncode(self, wait_retval): + SubProcess._set_returncode(self, wait_retval) + if self.returncode == os.EX_OK and \ + len(portage.auxdbkeys) != len(self._raw_metadata): + # Don't trust bash's returncode if the + # number of lines is incorrect. + self.returncode = 1 + class EbuildProcess(SpawnProcess): __slots__ = ("phase", "pkg", "settings", "tree") @@ -3157,8 +3159,7 @@ class Binpkg(CompositeTask): settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] - self._verify = "strict" in self.settings.features and \ - not self.opts.pretend + self._verify = not self.opts.pretend dir_path = os.path.join(settings["PORTAGE_TMPDIR"], "portage", pkg.category, pkg.pf) @@ -3167,6 +3168,10 @@ class Binpkg(CompositeTask): self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") + settings["EBUILD"] = self._ebuild_path + debug = settings.get("PORTAGE_DEBUG") == "1" + portage.doebuild_environment(self._ebuild_path, "setup", + settings["ROOT"], settings, debug, 1, self._bintree.dbapi) # The prefetcher has already completed or it # could be running now. If it's running now, @@ -3206,8 +3211,17 @@ class Binpkg(CompositeTask): pkg = self.pkg pkg_count = self.pkg_count + if not self.opts.fetchonly: + self._build_dir.lock() + try: + shutil.rmtree(self._build_dir.dir_path) + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise + del e + portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) fetcher = BinpkgFetcher(background=self.background, - logfile=self.scheduler.fetch.log_file, pkg=self.pkg, + logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, scheduler=self.scheduler) pkg_path = fetcher.pkg_path self._pkg_path = pkg_path @@ -3219,13 +3233,7 @@ class Binpkg(CompositeTask): short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) - - if self.background: - fetcher.addExitListener(self._fetcher_exit) - self._current_task = fetcher - self.scheduler.fetch.schedule(fetcher) - else: - self._start_task(fetcher, self._fetcher_exit) + self._start_task(fetcher, self._fetcher_exit) return self._fetcher_exit(fetcher) @@ -3236,25 +3244,19 @@ class Binpkg(CompositeTask): # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = True - if self.opts.fetchonly: - self._final_exit(fetcher) - self.wait() - return - elif self._default_exit(fetcher) != os.EX_OK: + if self._default_exit(fetcher) != os.EX_OK: + self._unlock_builddir() self.wait() return verifier = None if self._verify: - verifier = BinpkgVerifier(background=self.background, - logfile=self.scheduler.fetch.log_file, pkg=self.pkg) - + logfile = None if self.background: - verifier.addExitListener(self._verifier_exit) - self._current_task = verifier - self.scheduler.fetch.schedule(verifier) - else: - self._start_task(verifier, self._verifier_exit) + logfile = self.settings.get("PORTAGE_LOG_FILE") + verifier = BinpkgVerifier(background=self.background, + logfile=logfile, pkg=self.pkg) + self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) @@ -3262,6 +3264,7 @@ class Binpkg(CompositeTask): def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: + self._unlock_builddir() self.wait() return @@ -3273,18 +3276,20 @@ class Binpkg(CompositeTask): if self._fetched_pkg: self._bintree.inject(pkg.cpv, filename=pkg_path) + if self.opts.fetchonly: + self._current_task = None + self.returncode = os.EX_OK + self.wait() + return + msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) - self._build_dir.lock() - phase = "clean" settings = self.settings - settings.setcpv(pkg) - settings["EBUILD"] = self._ebuild_path ebuild_phase = EbuildPhase(background=self.background, pkg=pkg, phase=phase, scheduler=self.scheduler, settings=settings, tree=self._tree) @@ -3385,6 +3390,8 @@ class Binpkg(CompositeTask): self.wait() def _unlock_builddir(self): + if self.opts.fetchonly: + return portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() @@ -3550,6 +3557,12 @@ class BinpkgVerifier(AsynchronousTask): writemsg("!!! Expected: %s\n" % e.value[3], noiselevel=-1) rval = 1 + if rval != os.EX_OK: + pkg_path = bintree.getname(pkg.cpv) + head, tail = os.path.split(pkg_path) + temp_filename = portage._checksum_failure_temp_file(head, tail) + writemsg("File renamed to '%s'\n" % (temp_filename,), + noiselevel=-1) finally: sys.stdout = stdout_orig sys.stderr = stderr_orig @@ -3559,6 +3572,40 @@ class BinpkgVerifier(AsynchronousTask): self.returncode = rval self.wait() +class BinpkgPrefetcher(CompositeTask): + + __slots__ = ("pkg",) + \ + ("pkg_path", "_bintree",) + + def _start(self): + self._bintree = self.pkg.root_config.trees["bintree"] + fetcher = BinpkgFetcher(background=self.background, + logfile=self.scheduler.fetch.log_file, pkg=self.pkg, + scheduler=self.scheduler) + self.pkg_path = fetcher.pkg_path + self._start_task(fetcher, self._fetcher_exit) + + def _fetcher_exit(self, fetcher): + + if self._default_exit(fetcher) != os.EX_OK: + self.wait() + return + + verifier = BinpkgVerifier(background=self.background, + logfile=self.scheduler.fetch.log_file, pkg=self.pkg) + self._start_task(verifier, self._verifier_exit) + + def _verifier_exit(self, verifier): + if self._default_exit(verifier) != os.EX_OK: + self.wait() + return + + self._bintree.inject(self.pkg.cpv, filename=self.pkg_path) + + self._current_task = None + self.returncode = os.EX_OK + self.wait() + class BinpkgExtractorAsync(SpawnProcess): __slots__ = ("image_dir", "pkg", "pkg_path") @@ -6453,6 +6500,12 @@ class depgraph(object): self._process_slot_conflicts() def _serialize_tasks(self): + + if "--debug" in self.myopts: + writemsg("\ndigraph:\n\n", noiselevel=-1) + self.digraph.debug_print() + writemsg("\n", noiselevel=-1) + scheduler_graph = self.digraph.copy() mygraph=self.digraph.copy() # Prune "nomerge" root nodes if nothing depends on them, since @@ -9795,9 +9848,8 @@ class Scheduler(PollScheduler): "--getbinpkg" in self.myopts and \ pkg.root_config.trees["bintree"].isremote(pkg.cpv): - prefetcher = BinpkgFetcher(background=True, - logfile=self._fetch_log, pkg=pkg, - scheduler=self._sched_iface) + prefetcher = BinpkgPrefetcher(background=True, + pkg=pkg, scheduler=self._sched_iface) return prefetcher @@ -9833,7 +9885,7 @@ class Scheduler(PollScheduler): portage.match_from_list( portage.const.PORTAGE_PACKAGE_ATOM, [pkg]): if self._running_portage: - return cmp(pkg, self._running_portage) != 0 + return pkg.cpv != self._running_portage.cpv return True return False diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index c11f672f5..00203cd52 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -537,15 +537,17 @@ class digraph(object): len(self.order) def debug_print(self): + def output(s): + writemsg(s, noiselevel=-1) for node in self.nodes: - print node, + output("%s " % (node,)) if self.nodes[node][0]: - print "depends on" + output("depends on\n") else: - print "(no children)" + output("(no children)\n") for child in self.nodes[node][0]: - print " ",child, - print "(%s)" % self.nodes[node][0][child] + output(" %s (%s)\n" % \ + (child, self.nodes[node][0][child],)) #parse /etc/env.d and generate /etc/profile.env @@ -4262,32 +4264,40 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None required_hash_types.add("size") required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) dist_hashes = mf.fhashdict.get("DIST", {}) - missing_hashes = set() + + # To avoid accidental regeneration of digests with the incorrect + # files (such as partially downloaded files), trigger the fetch + # code if the file exists and it's size doesn't match the current + # manifest entry. If there really is a legitimate reason for the + # digest to change, `ebuild --force digest` can be used to avoid + # triggering this code (or else the old digests can be manually + # removed from the Manifest). + missing_files = [] for myfile in distfiles_map: myhashes = dist_hashes.get(myfile) if not myhashes: - missing_hashes.add(myfile) - continue - if required_hash_types.difference(myhashes): - missing_hashes.add(myfile) + missing_files.append(myfile) continue - if myhashes["size"] == 0: - missing_hashes.add(myfile) - if missing_hashes: - missing_files = [] - for myfile in missing_hashes: - try: - st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e + size = myhashes.get("size") + + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + if size == 0: missing_files.append(myfile) - else: - # If the file is empty then it's obviously invalid. - if st.st_size == 0: - missing_files.append(myfile) - if missing_files: + continue + if required_hash_types.difference(myhashes): + missing_files.append(myfile) + continue + else: + if st.st_size == 0 or size is not None and size != st.st_size: + missing_files.append(myfile) + continue + + if missing_files: mytree = os.path.realpath(os.path.dirname( os.path.dirname(mysettings["O"]))) fetch_settings = config(clone=mysettings) @@ -5542,9 +5552,15 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # shift in order to distinguish it from a return value. (just # like portage.process.spawn() would do). if retval & 0xff: - return (retval & 0xff) << 8 - # Otherwise, return its exit code. - return retval >> 8 + retval = (retval & 0xff) << 8 + else: + # Otherwise, return its exit code. + retval = retval >> 8 + if retval == os.EX_OK and len(dbkey) != len(auxdbkeys): + # Don't trust bash's returncode if the + # number of lines is incorrect. + retval = 1 + return retval elif dbkey: mysettings["dbkey"] = dbkey else: diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index e3822aeff..5a6951619 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -653,7 +653,10 @@ class binarytree(object): import urllib, urlparse rmt_idx = self._new_pkgindex() try: - f = urllib.urlopen(urlparse.urljoin(base_url, "Packages")) + # urlparse.urljoin() only works correctly with recognized + # protocols and requires the base url to have a trailing + # slash, so join manually... + f = urllib.urlopen(base_url.rstrip("/") + "/Packages") try: rmt_idx.readHeader(f) remote_timestamp = rmt_idx.header.get("TIMESTAMP", None) diff --git a/pym/portage/dep.py b/pym/portage/dep.py index c17602117..9c9c14380 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -509,7 +509,8 @@ class Atom(object): _str_methods = ("endswith", "find", "index", "lstrip", "replace", "startswith", "split", "strip", "rindex", "rfind", "rstrip", "__getitem__", - "__eq__", "__hash__", "__len__", "__ne__", "__repr__", "__str__") + "__eq__", "__hash__", "__len__", "__lt__", + "__ne__", "__repr__", "__str__") __slots__ = ("__weakref__", "blocker", "cp", "cpv", "operator", "slot", "use") + _str_methods @@ -556,15 +557,6 @@ class Atom(object): use = None obj_setattr(self, "use", use) - def __cmp__(self, other): - self_str = str(self) - other_str = str(other) - if self_str == other_str: - return 0 - if self_str > other_str: - return 1 - return -1 - def __setattr__(self, name, value): raise AttributeError("Atom instances are immutable", self.__class__, name, value) diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 50ed2f52f..384da250a 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -55,10 +55,6 @@ class ManifestEntry(object): def __init__(self, **kwargs): for k, v in kwargs.iteritems(): setattr(self, k, v) - def __cmp__(self, other): - if str(self) == str(other): - return 0 - return 1 class Manifest2Entry(ManifestEntry): def __str__(self):