From 77ee75ac9a3e07b9843dfbdcfab4a6429f643b44 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 24 Aug 2009 09:40:49 +0000 Subject: [PATCH] Merged from trunk -r14117:14127 | 14118 | Note that package.* and use.* as directories is only | | zmedico | supported since portage-2.1.6.7, and it is not included in | | | PMS at this time. | | 14119 | Avoid potential empty lines in warnings about missing | | arfrever | SONAMEs or NEEDED entries. Found by Zac Medico. | | 14120 | Add support for --selective=n, so it can be used to remove | | zmedico | selective behavior that may have been implied by some other | | | option like --update. | | 14121 | Add support for --keep-going=n so that it can be disabled | | zmedico | on the command line after it's been enabled in | | | EMERGE_DEFAULT_OPTS. | | 14122 | Add some --debug output for backtracking. Also, remove | | zmedico | unused imports. | | 14124 | Bug #282269 - Avoid str() calls with overlay paths, which | | zmedico | can trigger UnicodeEncodeError in python-2.x. | | 14125 | Bug #282306 - Inside dblink._security_check(), fall back to | | zmedico | utf_8 encoding it a path cannot be encoded under the user's | | | chosen encoding. | | 14126 | Bug #282269, comment #2 - Avoid potential | | zmedico | UnicodeEncodeError when writing to sys.stderr inside | | | xtermTitle(). | | 14127 | Support package.provided directory. | | arfrever | | svn path=/main/branches/prefix/; revision=14144 --- bin/misc-functions.sh | 2 ++ man/emerge.1 | 7 +++-- man/portage.5 | 3 ++- pym/_emerge/RepoDisplay.py | 4 +++ pym/_emerge/create_depgraph_params.py | 15 ++++++++++- pym/_emerge/depgraph.py | 38 ++++++++++++++++++++++++--- pym/_emerge/help.py | 9 ++++--- pym/_emerge/main.py | 26 ++++++++++++++++-- pym/portage/__init__.py | 2 +- pym/portage/dbapi/porttree.py | 8 ++---- pym/portage/dbapi/vartree.py | 16 +++++++++++ pym/portage/output.py | 3 +++ 12 files changed, 113 insertions(+), 20 deletions(-) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 4f6478fa6..4990a899d 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -264,6 +264,7 @@ install_qa_check() { set -${shopts} fi fi + sed -e "/^\$/d" -i "${T}"/scanelf-missing-SONAME.log f=$(<"${T}"/scanelf-missing-SONAME.log) if [[ -n ${f} ]] ; then vecho -ne '\a\n' @@ -297,6 +298,7 @@ install_qa_check() { set -${shopts} fi fi + sed -e "/^\$/d" -i "${T}"/scanelf-missing-NEEDED.log f=$(<"${T}"/scanelf-missing-NEEDED.log) if [[ -n ${f} ]] ; then vecho -ne '\a\n' diff --git a/man/emerge.1 b/man/emerge.1 index 3104cc18b..acc2dc8d3 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -343,7 +343,7 @@ Specifies the number of packages to build simultaneously. If this option is given without an argument, emerge will not limit the number of jobs that can run simultaneously. Also see the related \fB\-\-load\-average\fR option. .TP -.BR "\-\-keep\-going" +.BR "\-\-keep\-going[=n]" Continue as much as possible after an error. When an error occurs, dependencies are recalculated for remaining packages and any with unsatisfied dependencies are automatically dropped. Also see @@ -446,9 +446,12 @@ be enabled under normal circumstances. For currently supported \fBDEPEND\fR variable. However, behavior may change for new \fBEAPI\fRs when related extensions are added in the future. .TP -.BR "\-\-selective" +.BR "\-\-selective"[=n] This is similar to the \fB\-\-noreplace\fR option, except that it does not take precedence over options such as \fB\-\-newuse\fR. +Some options, such as \fB\-\-update\fR, imply \fB\-\-selective\fR. +Use \fB\-\-selective=n\fR if you want to forcefully disable +\fB\-\-selective\fR, regardless of options like \fB\-\-update\fR. .TP .BR "\-\-skipfirst" This option is only valid when used with \fB\-\-resume\fR. It removes the diff --git a/man/portage.5 b/man/portage.5 index fe457b79f..ea592a213 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -148,7 +148,8 @@ Any file in this directory, directories of other profiles or top-level "profiles" directory that begins with "package." or "use." can be more than just a flat file. If it is a directory, then all the files in that directory will be sorted in ascending alphabetical order by file name and summed together -as if it were a single file. +as if it were a single file. Note that this behavior is only supported since +portage-2.1.6.7, and it is not included in PMS at this time. .I Example: .nf diff --git a/pym/_emerge/RepoDisplay.py b/pym/_emerge/RepoDisplay.py index a954aebfa..e98be93c4 100644 --- a/pym/_emerge/RepoDisplay.py +++ b/pym/_emerge/RepoDisplay.py @@ -48,6 +48,10 @@ class RepoDisplay(object): return s def __str__(self): + """ + In python-2.x, str() can trigger a UnicodeEncodeError here, + so call __str__() directly. + """ output = [] shown_repos = self._shown_repos unknown_repo = self._unknown_repo diff --git a/pym/_emerge/create_depgraph_params.py b/pym/_emerge/create_depgraph_params.py index 9381fb955..188dc6f96 100644 --- a/pym/_emerge/create_depgraph_params.py +++ b/pym/_emerge/create_depgraph_params.py @@ -2,6 +2,9 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import logging +from portage.util import writemsg_level + def create_depgraph_params(myopts, myaction): #configure emerge engine parameters # @@ -23,7 +26,7 @@ def create_depgraph_params(myopts, myaction): "--newuse" in myopts or \ "--reinstall" in myopts or \ "--noreplace" in myopts or \ - "--selective" in myopts: + myopts.get("--selective", "n") != "n": myparams["selective"] = True if "--emptytree" in myopts: myparams["empty"] = True @@ -34,5 +37,15 @@ def create_depgraph_params(myopts, myaction): myparams["deep"] = myopts["--deep"] if "--complete-graph" in myopts: myparams["complete"] = True + if myopts.get("--selective") == "n": + # --selective=n can be used to remove selective + # behavior that may have been implied by some + # other option like --update. + myparams.pop("selective", None) + + if '--debug' in myopts: + writemsg_level('\n\nmyparams %s\n\n' % myparams, + noiselevel=-1, level=logging.DEBUG) + return myparams diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 2f5774e1c..eab8d5972 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -3,6 +3,7 @@ # $Id$ import gc +import logging import re import sys import textwrap @@ -17,6 +18,7 @@ bad = create_color_func("BAD") from portage.sets import SETPREFIX from portage.sets.base import InternalPackageSet from portage.util import cmp_sort_key, writemsg, writemsg_stdout +from portage.util import writemsg_level from _emerge.AtomArg import AtomArg from _emerge.Blocker import Blocker @@ -24,7 +26,6 @@ from _emerge.BlockerCache import BlockerCache from _emerge.BlockerDepPriority import BlockerDepPriority from _emerge.changelog import calc_changelog from _emerge.countdown import countdown -from _emerge.create_depgraph_params import create_depgraph_params from _emerge.create_world_atom import create_world_atom from _emerge.Dependency import Dependency from _emerge.DependencyArg import DependencyArg @@ -714,6 +715,18 @@ class depgraph(object): dep.parent, {})["missing dependency"] = \ set([(dep.parent, dep.root, dep.atom)]) self._dynamic_config._need_restart = True + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to unsatisfied dep:") + msg.append(" parent: %s" % dep.parent) + msg.append(" priority: %s" % dep.priority) + msg.append(" root: %s" % dep.root) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("".join("%s\n" % l for l in msg), + noiselevel=-1, level=logging.DEBUG) return 0 # In some cases, dep_check will return deps that shouldn't @@ -868,6 +881,19 @@ class depgraph(object): self._dynamic_config._runtime_pkg_mask.setdefault( existing_node, {})["slot conflict"] = parent_atoms self._dynamic_config._need_restart = True + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to slot conflict:") + msg.append(" package: %s" % existing_node) + msg.append(" slot: %s" % existing_node.slot_atom) + msg.append(" parents: %s" % \ + [(str(parent), atom) \ + for parent, atom in parent_atoms]) + msg.append("") + writemsg_level("".join("%s\n" % l for l in msg), + noiselevel=-1, level=logging.DEBUG) return 0 # A slot collision has occurred. Sometimes this coincides @@ -3866,8 +3892,6 @@ class depgraph(object): depth >= mylist[i+1][1]: del mylist[i] - from portage import flatten - from portage.dep import use_reduce, paren_reduce # files to fetch list - avoids counting a same file twice # in size display (verbose mode) myfetchlist=[] @@ -4346,7 +4370,9 @@ class depgraph(object): print print counters if show_repos: - writemsg_stdout(str(repo_display), noiselevel=-1) + # In python-2.x, str() can trigger a UnicodeEncodeError here, + # so call __str__() directly. + writemsg_stdout(repo_display.__str__(), noiselevel=-1) if "--changelog" in self._frozen_config.myopts: writemsg_stdout('\n', noiselevel=-1) @@ -4966,6 +4992,10 @@ def backtrack_depgraph(settings, trees, myopts, myparams, runtime_pkg_mask = mydepgraph.get_runtime_pkg_mask() backtracked += 1 elif backtracked and allow_backtracking: + if "--debug" in myopts: + writemsg_level( + "\n\nbacktracking aborted after %s tries\n\n" % \ + backtracked, noiselevel=-1, level=logging.DEBUG) # Backtracking failed, so disable it and do # a plain dep calculation + error message. allow_backtracking = False diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 448ef1adf..18e4369b6 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -360,7 +360,7 @@ def help(myopts, havecolor=1): for line in wrap(desc, desc_width): print desc_indent + line print - print " "+green("--keep-going") + print " " + green("--keep-going") + "[=%s]" % turquoise("n") desc = "Continue as much as possible after " + \ "an error. When an error occurs, " + \ "dependencies are recalculated for " + \ @@ -467,9 +467,12 @@ def help(myopts, havecolor=1): for line in wrap(desc, desc_width): print desc_indent + line print - print " " + green("--selective") + print " " + green("--selective") + "[=%s]" % turquoise("n") desc = "This is similar to the --noreplace option, except that it " + \ - "does not take precedence over options such as --newuse." + "does not take precedence over options such as --newuse. " + \ + "Some options, such as --update, imply --selective. " + \ + "Use --selective=n if you want to forcefully disable " + \ + "--selective, regardless of options like --update." for line in wrap(desc, desc_width): print desc_indent + line print diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 5757cae87..7eb521056 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -46,14 +46,13 @@ options=[ "--emptytree", "--fetchonly", "--fetch-all-uri", "--ignore-default-opts", -"--keep-going", "--noconfmem", "--newuse", "--nodeps", "--noreplace", "--nospinner", "--oneshot", "--onlydeps", "--pretend", "--quiet", "--resume", -"--searchdesc", "--selective", +"--searchdesc", "--skipfirst", "--tree", "--update", @@ -377,7 +376,9 @@ def insert_optional_args(args): '--getbinpkg' : ('n',), '--getbinpkgonly' : ('n',), '--jobs' : valid_integers, + '--keep-going' : ('n',), '--root-deps' : ('rdeps',), + '--selective' : ('n',), '--usepkg' : ('n',), '--usepkgonly' : ('n',), } @@ -533,6 +534,12 @@ def parse_opts(tmpcmdline, silent=False): "action" : "store" }, + "--keep-going": { + "help" : "continue as much as possible after an error", + "type" : "choice", + "choices" : ("True", "n") + }, + "--load-average": { "help" :"Specifies that no new builds should be started " + \ @@ -585,6 +592,13 @@ def parse_opts(tmpcmdline, silent=False): "choices" :("True", "rdeps") }, + "--selective": { + "help" : "similar to the --noreplace but does not take " + \ + "precedence over options such as --newuse", + "type" : "choice", + "choices" : ("True", "n") + }, + "--usepkg": { "shortopt" : "-k", "help" : "use binary packages", @@ -654,9 +668,17 @@ def parse_opts(tmpcmdline, silent=False): else: myoptions.getbinpkgonly = None + if myoptions.keep_going in ("True",): + myoptions.keep_going = True + else: + myoptions.keep_going = None + if myoptions.root_deps == "True": myoptions.root_deps = True + if myoptions.selective == "True": + myoptions.selective = True + if myoptions.deep is not None: deep = None if myoptions.deep == "True": diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 005da2afc..6955eabf5 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2076,7 +2076,7 @@ class config(object): else: self.punmaskdict[mycatpkg]=[x] - pkgprovidedlines = [grabfile(os.path.join(x, "package.provided")) for x in self.profiles] + pkgprovidedlines = [grabfile(os.path.join(x, "package.provided"), recursive=1) for x in self.profiles] pkgprovidedlines = stack_lists(pkgprovidedlines, incremental=1) has_invalid_data = False for x in range(len(pkgprovidedlines)-1, -1, -1): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 34e9d178e..10baa4b9c 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -604,15 +604,11 @@ class portdbapi(dbapi): mydata, st, emtime = self._pull_valid_cache(mycpv, myebuild, mylocation) doregen = mydata is None - writemsg(_("auxdb is valid: ")+str(not doregen)+" "+str(pkg)+"\n", 2) - if doregen: if myebuild in self._broken_ebuilds: raise KeyError(mycpv) if not self._have_root_eclass_dir: raise KeyError(mycpv) - writemsg("doregen: %s %s\n" % (doregen, mycpv), 2) - writemsg(_("Generating cache entry(0) for: ")+str(myebuild)+"\n", 1) self.doebuild_settings.setcpv(mycpv) mydata = {} @@ -831,7 +827,7 @@ class portdbapi(dbapi): os.path.join(self.mysettings["DISTDIR"], x), mysums[x]) except FileNotFound, e: ok = False - reason = _("File Not Found: '%s'") % str(e) + reason = _("File Not Found: '%s'") % (e,) if not ok: failures[x] = reason if failures: @@ -1105,7 +1101,7 @@ class portdbapi(dbapi): except PortageException, e: writemsg("!!! Error: aux_get('%s', %s)\n" % (mycpv, aux_keys), noiselevel=-1) - writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) del e continue eapi = metadata["EAPI"] diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 192b137d3..31f42b857 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -3849,6 +3849,22 @@ class dblink(object): 0 == i % self._file_merge_yield_interval: scheduler.scheduleYield() + if os is _os_merge: + try: + _unicode_encode(path, + encoding=_encodings['merge'], errors='strict') + except UnicodeEncodeError: + # The package appears to have been merged with a + # different value of sys.getfilesystemencoding(), + # so fall back to utf_8 if appropriate. + try: + _unicode_encode(path, + encoding=_encodings['fs'], errors='strict') + except UnicodeEncodeError: + pass + else: + os = portage.os + try: s = os.lstat(path) except OSError, e: diff --git a/pym/portage/output.py b/pym/portage/output.py index c0839fa53..224e44380 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -252,6 +252,9 @@ def xtermTitle(mystr, raw=False): mystr = mystr[:_max_xtermTitle_len] if not raw: mystr = '\x1b]0;%s\x07' % mystr + if sys.hexversion < 0x3000000 and isinstance(mystr, unicode): + # avoid potential UnicodeEncodeError + mystr = mystr.encode(_encodings['stdio'], 'backslashreplace') sys.stderr.write(mystr) sys.stderr.flush() -- 2.26.2