From: Fabian Groffen Date: Mon, 6 Jul 2009 18:33:42 +0000 (-0000) Subject: Merged from trunk -r13764:13774 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=b5c1b88b3073d53981433fa6fcc58062aebc9b53;p=portage.git Merged from trunk -r13764:13774 | 13765 | Move "import copy" to the top of the file. | | zmedico | | | 13766 | Make --help into an action, and move --help --sync output | | zmedico | to the actions section. Thanks to Marat Radchenko | | | for this patch. | | 13767 | Implement LazyItemsDict.pop() since dict.pop() bypasses our | | zmedico | overridden __getitem__ implementation. This fixes a problem | | | with config.setcpv() failing to set PORTAGE_REPO_NAME when | | | appropriate. Thanks to Arfrever for reporting. | | 13768 | Don't set RDEPEND=${RDEPEND-${DEPEND}} for EAPI=3 and | | arfrever | simplify setting it for lower EAPIs. | | 13769 | Bug #275217 - Part 5 - When a slot conflict occurs, mask | | zmedico | the first package that got pulled in and restart the | | | calculation. Thanks to Sebastian Mingramm (few) | | | for the initial patch which I added | | | some additional features to: * display message about missed | | | updates * cache frozen_config instance, to optimize | | | performance * disable backtracking if it fails, fall back | | | to a normal dep calculation + error message. | | 13770 | Cleanup spinner before calling depgraph.display_problems(). | | zmedico | | | 13771 | Disable depgraph._show_missed_update() when --quiet is | | zmedico | enabled and --debug is not enabled. | | 13772 | Remove the --avoid-update option and make it the default | | zmedico | behavior when --update is not specified, as suggested by | | | Allen Brooker (AllenJB) in bug | | | #275945, comment #2. | | 13773 | Make LazyItemsDict derive from UserDict instead of dict so | | zmedico | that we don't have to implement the pop method (and | | | possibly others such as setdefault). | | 13774 | Delete runtime_pkg_mask to avoid memory leak. | | zmedico | | Merged from trunk -r13774:13784 | 13775 | Don't pass myopts into profile_check() since it's no longer | | zmedico | needed. Thanks to Radchenko | | | for this patch. | | 13776 | Update help() usage since the myopts parameter is gone. | | zmedico | Thanks to Marat Radchenko for | | | this patch. | | 13777 | Correct --help usage since it no longer takes system, world, | | zmedico | or --sync as input. Thanks to Marat Radchenko | | | for this patch. | | 13778 | Correct --help usage since it no longer takes system, world, | | zmedico | or --sync as input. Thanks to Marat Radchenko | | | for this patch. | | 13779 | As a performance optimization, use StringIO instead of | | zmedico | _insert_newline_eof to solve bug #228117. Thanks to Marat | | | Radchenko for this patch. | | 13780 | Make getconfig() use codecs.option() for py3k compatible | | zmedico | unicode handling. | | 13781 | Make code from previous commit safe for empty files. | | zmedico | | | 13782 | Make grablines() use codecs.open() for py3k compatible | | zmedico | unicode handling. | | 13783 | Make atomic_ofstream() use codecs.open() for py3k compatible | | zmedico | unicode handling when mode does not contain 'b'. | | 13784 | Remove the workaround from bug #263081 since most strings | | zmedico | should be unicode now. | svn path=/main/branches/prefix/; revision=13796 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index d81b6efd6..ff47591e5 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1770,13 +1770,10 @@ _source_ebuild() { rm "$PORTAGE_BUILDDIR/.ebuild_changed" fi - # This next line is not the same as export RDEPEND=${RDEPEND:-${DEPEND}} - # That will test for unset *or* NULL (""). We want just to set for unset... - # turn off glob expansion from here on in to prevent *'s and ? in the - # DEPEND syntax from getting expanded :) - set -f - if [ "${RDEPEND-unset}" == "unset" ] ; then - export RDEPEND=${DEPEND} + [[ -n $EAPI ]] || EAPI=0 + + if has "$EAPI" 0 1 2 ; then + export RDEPEND=${RDEPEND-${DEPEND}} debug-print "RDEPEND: not set... Setting to: ${DEPEND}" fi @@ -1787,9 +1784,6 @@ _source_ebuild() { PDEPEND="${PDEPEND} ${E_PDEPEND}" unset ECLASS E_IUSE E_DEPEND E_RDEPEND E_PDEPEND - set +f - - [[ -n $EAPI ]] || EAPI=0 # alphabetically ordered by $EBUILD_PHASE value local f valid_phases diff --git a/man/emerge.1 b/man/emerge.1 index a36302650..787e2e37f 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -16,7 +16,7 @@ emerge \- Command\-line interface to the Portage system \fB\-\-search\fR \fIsomestring\fR .TP .BR emerge -\fB\-\-help\fR [\fB\-\-sync\fR] +\fB\-\-help\fR [\fB\-\-verbose\fR] .SH "DESCRIPTION" \fBemerge\fR is the definitive command\-line interface to the Portage system. It is primarily used for installing packages, and \fBemerge\fR @@ -90,15 +90,6 @@ option if you want to install a tbz2\fR. The packages are added to the \fBworld\fR file at the end, so that they are considered for later updating. .TP -.BR "\-\-avoid\-update " -Tries to prevent package updates. This may not always be -possible since new packages or new dependencies due to USE -flag changes may require a newer version of an installed -package. Furthermore, this option may lead to slot conflicts -(Multiple package instances within a single package slot -have been pulled into the dependency graph). It is not -possible to use this option in such a case. -.TP .BR "\-\-clean " (\fB\-c\fR) Cleans up the system by examining the installed packages and removing older packages. This is accomplished by looking at each installed package and separating @@ -373,9 +364,6 @@ A USE flag was turned off for a package. USE flags may be toggled by your profile as well as your USE and package.use settings. - -\fB\-\-avoid\-update\fR may prevent updates of installed packages without -use flag changes. Note that \fB\-\-avoid-\update\fR may lead to slot conflicts. .TP .BR "\-\-noconfmem" Causes portage to disregard merge records indicating that a config file diff --git a/pym/_emerge/AbstractDepPriority.py b/pym/_emerge/AbstractDepPriority.py index 53d6a8a09..e51999aa0 100644 --- a/pym/_emerge/AbstractDepPriority.py +++ b/pym/_emerge/AbstractDepPriority.py @@ -2,7 +2,9 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import copy from _emerge.SlotObject import SlotObject + class AbstractDepPriority(SlotObject): __slots__ = ("buildtime", "runtime", "runtime_post") @@ -25,6 +27,4 @@ class AbstractDepPriority(SlotObject): return self.__int__() >= other def copy(self): - import copy return copy.copy(self) - diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index c70183ba5..0ad504ae1 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -42,7 +42,7 @@ from _emerge.clear_caches import clear_caches from _emerge.countdown import countdown from _emerge.create_depgraph_params import create_depgraph_params from _emerge.Dependency import Dependency -from _emerge.depgraph import depgraph, resume_depgraph +from _emerge.depgraph import depgraph, resume_depgraph, _frozen_depgraph_config from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange from _emerge.emergelog import emergelog from _emerge.is_valid_package_atom import is_valid_package_atom @@ -299,25 +299,51 @@ def action_build(settings, trees, mtimedb, print darkgreen("emerge: It seems we have nothing to resume...") return os.EX_OK - myparams = create_depgraph_params(myopts, myaction) if "--quiet" not in myopts and "--nodeps" not in myopts: print "Calculating dependencies ", sys.stdout.flush() - mydepgraph = depgraph(settings, trees, myopts, myparams, spinner) - try: - retval, favorites = mydepgraph.select_files(myfiles) - except portage.exception.PackageNotFound, e: - portage.writemsg("\n!!! %s\n" % str(e), noiselevel=-1) - return 1 - except portage.exception.PackageSetNotFound, e: - root_config = trees[settings["ROOT"]]["root_config"] - display_missing_pkg_set(root_config, e.value) - return 1 + + runtime_pkg_mask = None + allow_backtracking = True + backtracked = False + frozen_config = _frozen_depgraph_config(settings, trees, + myopts, spinner) + myparams = create_depgraph_params(myopts, myaction) + while True: + mydepgraph = depgraph(settings, trees, myopts, myparams, spinner, + frozen_config=frozen_config, + allow_backtracking=allow_backtracking, + runtime_pkg_mask=runtime_pkg_mask) + try: + retval, favorites = mydepgraph.select_files(myfiles) + except portage.exception.PackageNotFound, e: + portage.writemsg("\n!!! %s\n" % str(e), noiselevel=-1) + return 1 + except portage.exception.PackageSetNotFound, e: + root_config = trees[settings["ROOT"]]["root_config"] + display_missing_pkg_set(root_config, e.value) + return 1 + if not retval: + if mydepgraph.need_restart(): + runtime_pkg_mask = mydepgraph.get_runtime_pkg_mask() + backtracked = True + elif backtracked and allow_backtracking: + # Backtracking failed, so disable it and do + # a plain dep calculation + error message. + allow_backtracking = False + runtime_pkg_mask = None + else: + if show_spinner: + print "\b\b... done!" + mydepgraph.display_problems() + return 1 + else: + break + + del frozen_config, runtime_pkg_mask + if show_spinner: print "\b\b... done!" - if not retval: - mydepgraph.display_problems() - return 1 if "--pretend" not in myopts and \ ("--ask" in myopts or "--tree" in myopts or \ diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 0e4f8510b..96a0967c5 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -93,8 +93,10 @@ class _frozen_depgraph_config(object): class _dynamic_depgraph_config(object): - def __init__(self, depgraph, myparams): - self.myparams = myparams + def __init__(self, depgraph, myparams, allow_backtracking, + runtime_pkg_mask): + self.myparams = myparams.copy() + self._allow_backtracking = allow_backtracking # Maps slot atom to package for each Package added to the graph. self._slot_pkg_map = {} # Maps nodes to the reasons they were selected for reinstallation. @@ -157,6 +159,13 @@ class _dynamic_depgraph_config(object): self._initially_unsatisfied_deps = [] self._ignored_deps = [] self._highest_pkg_cache = {} + if runtime_pkg_mask is None: + runtime_pkg_mask = {} + else: + runtime_pkg_mask = dict((k, v.copy()) for (k, v) in \ + runtime_pkg_mask.iteritems()) + self._runtime_pkg_mask = runtime_pkg_mask + self._need_restart = False for myroot in depgraph._frozen_config.trees: self._slot_pkg_map[myroot] = {} @@ -247,16 +256,67 @@ class depgraph(object): _dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"] def __init__(self, settings, trees, myopts, myparams, spinner, - frozen_config=None): + frozen_config=None, runtime_pkg_mask=None, allow_backtracking=False): if frozen_config is None: frozen_config = _frozen_depgraph_config(settings, trees, myopts, spinner) self._frozen_config = frozen_config - self._dynamic_config = _dynamic_depgraph_config(self, myparams) + self._dynamic_config = _dynamic_depgraph_config(self, myparams, + allow_backtracking, runtime_pkg_mask) self._select_atoms = self._select_atoms_highest_available self._select_package = self._select_pkg_highest_available + def _show_missed_update(self): + + if '--quiet' in self._frozen_config.myopts and \ + '--debug' not in self._frozen_config.myopts: + return + + missed_updates = {} + for pkg, mask_reasons in \ + self._dynamic_config._runtime_pkg_mask.iteritems(): + if mask_reasons.get("slot conflict"): + if pkg.slot_atom in missed_updates: + other_pkg, parent_atoms = missed_updates[pkg.slot_atom] + if other_pkg > pkg: + continue + missed_updates[pkg.slot_atom] = \ + (pkg, mask_reasons["slot conflict"]) + + if not missed_updates: + return + + msg = [] + msg.append("\n!!! One or more updates have been skipped due to " + \ + "a dependency conflict:\n\n") + + indent = " " + for pkg, parent_atoms in missed_updates.itervalues(): + msg.append(str(pkg.slot_atom)) + msg.append("\n\n") + + for parent, atom in parent_atoms: + msg.append(indent) + msg.append(str(pkg)) + + msg.append(" conflicts with\n") + for parent, atom in parent_atoms: + msg.append(2*indent) + if isinstance(parent, + (PackageArg, AtomArg)): + # For PackageArg and AtomArg types, it's + # redundant to display the atom attribute. + msg.append(str(parent)) + else: + # Display the specific atom from SetArg or + # Package types. + msg.append("%s required by %s" % (atom, parent)) + msg.append("\n") + msg.append("\n") + sys.stderr.write("".join(msg)) + sys.stderr.flush() + def _show_slot_collision_notice(self): """Show an informational message advising the user to mask one of the the packages. In some cases it may be possible to resolve this @@ -698,6 +758,31 @@ class depgraph(object): (dep.parent, dep.atom)) return 1 else: + # A slot conflict has occurred. + if self._dynamic_config._allow_backtracking and \ + not self._accept_blocker_conflicts(): + self._add_slot_conflict(pkg) + if dep.atom is not None and dep.parent is not None: + self._add_parent_atom(pkg, (dep.parent, dep.atom)) + if arg_atoms: + for parent_atom in arg_atoms: + parent, atom = parent_atom + self._add_parent_atom(pkg, parent_atom) + self._process_slot_conflicts() + + parent_atoms = \ + self._dynamic_config._parent_atoms.get(pkg, set()) + if parent_atoms: + parent_atoms = self._dynamic_config._slot_conflict_parent_atoms.intersection(parent_atoms) + if pkg >= existing_node: + # We only care about the parent atoms + # when they trigger a downgrade. + parent_atoms = set() + + self._dynamic_config._runtime_pkg_mask.setdefault( + existing_node, {})["slot conflict"] = parent_atoms + self._dynamic_config._need_restart = True + return 0 # A slot collision has occurred. Sometimes this coincides # with unresolvable blockers, so the slot collision will be @@ -1382,8 +1467,10 @@ class depgraph(object): if isinstance(arg, PackageArg): if not self._add_pkg(arg.package, dep) or \ not self._create_graph(): - sys.stderr.write(("\n\n!!! Problem resolving " + \ - "dependencies for %s\n") % arg.arg) + if not self._dynamic_config._need_restart: + sys.stderr.write(("\n\n!!! Problem " + \ + "resolving dependencies for %s\n") % \ + arg.arg) return 0, myfavorites continue if debug: @@ -1443,7 +1530,9 @@ class depgraph(object): # so that later dep_check() calls can use it as feedback # for making more consistent atom selections. if not self._add_pkg(pkg, dep): - if isinstance(arg, SetArg): + if self._dynamic_config._need_restart: + pass + elif isinstance(arg, SetArg): sys.stderr.write(("\n\n!!! Problem resolving " + \ "dependencies for %s from %s\n") % \ (atom, arg.arg)) @@ -1920,7 +2009,7 @@ class depgraph(object): selective = "selective" in self._dynamic_config.myparams reinstall = False noreplace = "--noreplace" in self._frozen_config.myopts - avoid_update = "--avoid-update" in self._frozen_config.myopts + avoid_update = "--update" not in self._frozen_config.myopts # Behavior of the "selective" parameter depends on # whether or not a package matches an argument atom. # If an installed package provides an old-style @@ -1947,6 +2036,9 @@ class depgraph(object): for pkg in self._iter_match_pkgs(root_config, pkg_type, atom, onlydeps=onlydeps): + if pkg in self._dynamic_config._runtime_pkg_mask: + # The package has been masked by the backtracking logic + continue cpv = pkg.cpv # Make --noreplace take precedence over --newuse. if not pkg.installed and noreplace and \ @@ -4214,8 +4306,10 @@ class depgraph(object): if self._dynamic_config._unsatisfied_blockers_for_display is not None: self._show_unsatisfied_blockers( self._dynamic_config._unsatisfied_blockers_for_display) - else: + elif self._dynamic_config._slot_collision_info: self._show_slot_collision_notice() + else: + self._show_missed_update() # TODO: Add generic support for "set problem" handlers so that # the below warnings aren't special cases for world only. @@ -4574,6 +4668,12 @@ class depgraph(object): graph in order to avoid making a potentially unsafe decision. """ + def need_restart(self): + return self._dynamic_config._need_restart + + def get_runtime_pkg_mask(self): + return self._dynamic_config._runtime_pkg_mask.copy() + class _dep_check_composite_db(portage.dbapi): """ A dbapi-like interface that is optimized for use in dep_check() calls. diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 1f4b5201e..34a4bda29 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -11,7 +11,7 @@ def shorthelp(): print " "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] < "+turquoise("system")+" | "+turquoise("world")+" >" print " "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >" print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]" - print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] " + print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("--verbose")+" ] " print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhjkKlnNoOpqPsStuvV")+"]" print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]" print " [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ]" @@ -21,17 +21,17 @@ def shorthelp(): print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]" print bold("Actions:")+" [ "+green("--depclean")+" | "+green("--list-sets")+" | "+green("--search")+" | "+green("--sync")+" | "+green("--version")+" ]" -def help(myaction,myopts,havecolor=1): +def help(myopts, havecolor=1): # TODO: Implement a wrap() that accounts for console color escape codes. from textwrap import wrap desc_left_margin = 14 desc_indent = desc_left_margin * " " desc_width = 80 - desc_left_margin - 5 - if not myaction and ("--verbose" not in myopts): + if "--verbose" not in myopts: shorthelp() print print " For more help try 'emerge --help --verbose' or consult the man page." - elif not myaction: + else: shorthelp() print print turquoise("Help (this screen):") @@ -195,6 +195,20 @@ def help(myaction,myopts,havecolor=1): print " emerge -S applet" print " emerge -S 'perl.*module'" print + print " "+green("--sync") + print " Tells emerge to update the Portage tree as specified in" + print " The SYNC variable found in /etc/make.conf. By default, SYNC instructs" + print " emerge to perform an rsync-style update with rsync.gentoo.org." + print + print " 'emerge-webrsync' exists as a helper app to emerge --sync, providing a" + print " method to receive the entire portage tree as a tarball that can be" + print " extracted and used. First time syncs would benefit greatly from this." + print + print " "+turquoise("WARNING:") + print " If using our rsync server, emerge will clean out all files that do not" + print " exist on it, including ones that you may have created. The exceptions" + print " to this are the distfiles, local and packages directories." + print print " "+green("--unmerge")+" ("+green("-C")+" short option)" print " "+turquoise("WARNING: This action can remove important packages!") print " Removes all matching packages. This does no checking of" @@ -235,17 +249,6 @@ def help(myaction,myopts,havecolor=1): print " to the prompt, so an accidental press of the \"Enter\" key at any" print " time prior to the prompt will be interpreted as a choice!" print - print " "+green("--avoid-update") - desc = "Tries to prevent package updates. This may not always be" + \ - " possible since new packages or new dependencies due to USE" + \ - " flag changes may require a newer version of an installed" + \ - " package. Furthermore, this option may lead to slot conflicts" + \ - " (Multiple package instances within a single package slot" + \ - " have been pulled into the dependency graph). It is not" + \ - " possible to use this option in such a case." - for line in wrap(desc, desc_width): - print desc_indent + line - print print " "+green("--buildpkg")+" ("+green("-b")+" short option)" desc = "Tells emerge to build binary packages for all ebuilds processed in" + \ " addition to actually merging the packages. Useful for maintainers" + \ @@ -500,20 +503,3 @@ def help(myaction,myopts,havecolor=1): print " added to EMERGE_DEFAULT_OPTS (see make.conf(5)) and later" print " overridden via the command line." print - elif myaction == "sync": - print - print bold("Usage: ")+turquoise("emerge")+" "+turquoise("--sync") - print - print " 'emerge --sync' tells emerge to update the Portage tree as specified in" - print " The SYNC variable found in /etc/make.conf. By default, SYNC instructs" - print " emerge to perform an rsync-style update with rsync.gentoo.org." - print - print " 'emerge-webrsync' exists as a helper app to emerge --sync, providing a" - print " method to receive the entire portage tree as a tarball that can be" - print " extracted and used. First time syncs would benefit greatly from this." - print - print " "+turquoise("WARNING:") - print " If using our rsync server, emerge will clean out all files that do not" - print " exist on it, including ones that you may have created. The exceptions" - print " to this are the distfiles, local and packages directories." - print diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index a5e57adb9..c71f67b74 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -67,7 +67,7 @@ options=[ "--emptytree", "--fetchonly", "--fetch-all-uri", "--getbinpkg", "--getbinpkgonly", -"--help", "--ignore-default-opts", +"--ignore-default-opts", "--keep-going", "--noconfmem", "--newuse", @@ -78,7 +78,6 @@ options=[ "--searchdesc", "--selective", "--skipfirst", "--tree", -"--avoid-update", "--update", "--usepkg", "--usepkgonly", "--verbose", @@ -216,7 +215,6 @@ def chk_updated_info_files(root, infodirs, prev_mtimes, retval): if icount > 0: out.einfo("Processed %d info files." % (icount,)) - def display_preserved_libs(vardbapi): MAX_DISPLAY = 3 @@ -287,11 +285,10 @@ def display_preserved_libs(vardbapi): print colorize("WARN", " * ") + " used by %d other files" % (len(consumers) - MAX_DISPLAY) print "Use " + colorize("GOOD", "emerge @preserved-rebuild") + " to rebuild packages using these libraries" - def post_emerge(root_config, myopts, mtimedb, retval): """ Misc. things to run at the end of a merge session. - + Update Info Files Update Config Files Update News Items @@ -360,14 +357,13 @@ def post_emerge(root_config, myopts, mtimedb, retval): portage.locks.unlockdir(vdb_lock) chk_updated_cfg_files(target_root + EPREFIX, config_protect) - + display_news_notification(root_config, myopts) if retval in (None, os.EX_OK) or (not "--pretend" in myopts): display_preserved_libs(vardbapi) sys.exit(retval) - def multiple_actions(action1, action2): sys.stderr.write("\n!!! Multiple actions requested... Please choose one only.\n") sys.stderr.write("!!! '%s' or '%s'\n\n" % (action1, action2)) @@ -452,7 +448,7 @@ def parse_opts(tmpcmdline, silent=False): global options, shortmapping actions = frozenset([ - "clean", "config", "depclean", + "clean", "config", "depclean", "help", "info", "list-sets", "metadata", "prune", "regen", "search", "sync", "unmerge", "version", @@ -679,7 +675,7 @@ def expand_set_arguments(myfiles, myaction, root_config): IS_OPERATOR = "/@" DIFF_OPERATOR = "-@" UNION_OPERATOR = "+@" - + for i in range(0, len(myfiles)): if myfiles[i].startswith(SETPREFIX): start = 0 @@ -692,7 +688,7 @@ def expand_set_arguments(myfiles, myaction, root_config): if start > 0 and start < end: namepart = x[:start] argpart = x[start+1:end] - + # TODO: implement proper quoting args = argpart.split(",") options = {} @@ -709,13 +705,13 @@ def expand_set_arguments(myfiles, myaction, root_config): newset += x x = "" myfiles[i] = SETPREFIX+newset - + sets = setconfig.getSets() # display errors that occured while loading the SetConfig instance for e in setconfig.errors: print colorize("BAD", "Error during set creation: %s" % e) - + # emerge relies on the existance of sets with names "world" and "system" required_sets = ("world", "system") missing_sets = [] @@ -882,10 +878,8 @@ def config_protect_check(trees): msg += " for '%s'" % root writemsg_level(msg, level=logging.WARN, noiselevel=-1) -def profile_check(trees, myaction, myopts): - if myaction in ("info", "sync"): - return os.EX_OK - elif "--version" in myopts or "--help" in myopts: +def profile_check(trees, myaction): + if myaction in ("help", "info", "sync", "version"): return os.EX_OK for root, root_trees in trees.iteritems(): if root_trees["root_config"].settings.profiles: @@ -923,7 +917,7 @@ def emerge_main(): os.umask(022) settings, trees, mtimedb = load_emerge_config() portdb = trees[settings["ROOT"]]["porttree"].dbapi - rval = profile_check(trees, myaction, myopts) + rval = profile_check(trees, myaction) if rval != os.EX_OK: return rval @@ -1074,11 +1068,6 @@ def emerge_main(): noiselevel=-1) return 1 - if "--avoid-update" in myopts and "--update" in myopts: - writemsg("!!! conflicting options given: " + \ - "--update and --avoid-update\n", noiselevel=-1) - return 1 - if settings.get("PORTAGE_DEBUG", "") == "1": spinner.update = spinner.update_quiet portage.debug=1 @@ -1095,8 +1084,8 @@ def emerge_main(): settings.profile_path, settings["CHOST"], trees[settings["ROOT"]]["vartree"].dbapi) return 0 - elif "--help" in myopts: - _emerge.help.help(myaction, myopts, portage.output.havecolor) + elif myaction == "help": + _emerge.help.help(myopts, portage.output.havecolor) return 0 if "--debug" in myopts: @@ -1104,7 +1093,7 @@ def emerge_main(): print "myopts", myopts if not myaction and not myfiles and "--resume" not in myopts: - _emerge.help.help(myaction, myopts, portage.output.havecolor) + _emerge.help.help(myopts, portage.output.havecolor) return 1 pretend = "--pretend" in myopts diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index 5657617b3..a0b11165f 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -165,18 +165,6 @@ class database(fs_template.FsBased): if k not in internal_columns: d[k] = result[0][column_index] - # As a temporary workaround for bug #263081, convert unicode strings - # to raw byte strings. This avoids potential unicode errors that can - # be triggered elsewhere when attempting to join unicode strings with - # raw byte strings. TODO: Convert all string handling code to use - # unicode strings instead of raw byte strings (required for py3k - # compatibility). - for k, v in d.iteritems(): - try: - d[k] = str(v) - except UnicodeEncodeError: - pass - return d def _setitem(self, cpv, values): diff --git a/pym/portage/util.py b/pym/portage/util.py index a1413c64d..c7c2387d4 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -12,6 +12,7 @@ __all__ = ['apply_permissions', 'apply_recursive_permissions', 'unique_array', 'varexpand', 'write_atomic', 'writedict', 'writemsg', 'writemsg_level', 'writemsg_stdout'] +import codecs import os import errno import logging @@ -25,6 +26,7 @@ from portage.exception import PortageException, FileNotFound, \ import portage.exception from portage.dep import isvalidatom from portage.proxy.objectproxy import ObjectProxy +from portage.cache.mappings import UserDict from portage.const import EPREFIX, EPREFIX_LSTRIP try: @@ -316,7 +318,7 @@ def grablines(myfilename,recursive=0): os.path.join(myfilename, f), recursive)) else: try: - myfile = open(myfilename, "r") + myfile = codecs.open(myfilename, mode='r', errors='replace') mylines = myfile.readlines() myfile.close() except IOError, e: @@ -353,65 +355,6 @@ class _tolerant_shlex(shlex.shlex): (self.infile, str(e)), noiselevel=-1) return (newfile, StringIO()) -class _insert_newline_eof(ObjectProxy): - """ - Read functions insert anywhere from 0 and 2 newlines just before eof. - This is useful as a workaround for avoiding a silent error in shlex that - is triggered by a source statement at the end of the file without a - trailing newline after the source statement. - """ - - def __init__(self, *pargs, **kargs): - ObjectProxy.__init__(self) - object.__setattr__(self, '_file', open(*pargs, **kargs)) - - def _get_target(self): - return object.__getattribute__(self, '_file') - - def __getattribute__(self, attr): - if attr in ('read', 'readline', 'readlines'): - return object.__getattribute__(self, attr) - return getattr(object.__getattribute__(self, '_file'), attr) - - def read(self, *args): - try: - object.__getattribute__(self, '_got_eof') - return "" - except AttributeError: - pass - rval = object.__getattribute__(self, '_file').read(*args) - if rval and not args and rval[-1:] != "\n": - rval += "\n" - if not rval: - object.__setattr__(self, '_got_eof', True) - return "\n" - return rval - - def readline(self, *args): - try: - object.__getattribute__(self, '_got_eof') - return "" - except AttributeError: - pass - rval = object.__getattribute__(self, '_file').readline(*args) - if rval and rval[-1:] != "\n": - rval += "\n" - if not rval: - object.__setattr__(self, '_got_eof', True) - rval = "\n" - return rval - - def readlines(self, *args): - try: - object.__getattribute__(self, '_got_eof') - return [] - except AttributeError: - pass - lines = object.__getattribute__(self, '_file').readlines(*args) - if lines and lines[-1][-1:] != "\n": - lines[-1] += "\n" - return lines - def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if isinstance(expand, dict): # Some existing variable definitions have been @@ -422,7 +365,13 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): expand_map = {} mykeys = {} try: - f = _insert_newline_eof(mycfg) + # Workaround for avoiding a silent error in shlex that + # is triggered by a source statement at the end of the file without a + # trailing newline after the source statement + content = codecs.open(mycfg, mode='r', errors='replace').read() + if content and content[-1] != u'\n': + content += u'\n' + f = StringIO(content) except IOError, e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mycfg) @@ -921,6 +870,11 @@ class atomic_ofstream(ObjectProxy): """Opens a temporary filename.pid in the same directory as filename.""" ObjectProxy.__init__(self) object.__setattr__(self, '_aborted', False) + if 'b' in mode: + open_func = open + else: + open_func = codecs.open + kargs.setdefault('errors', 'replace') if follow_links: canonical_path = os.path.realpath(filename) @@ -928,7 +882,7 @@ class atomic_ofstream(ObjectProxy): tmp_name = "%s.%i" % (canonical_path, os.getpid()) try: object.__setattr__(self, '_file', - open(tmp_name, mode=mode, **kargs)) + open_func(tmp_name, mode=mode, **kargs)) return except IOError, e: if canonical_path == filename: @@ -939,7 +893,8 @@ class atomic_ofstream(ObjectProxy): object.__setattr__(self, '_real_name', filename) tmp_name = "%s.%i" % (filename, os.getpid()) - object.__setattr__(self, '_file', open(tmp_name, mode=mode, **kargs)) + object.__setattr__(self, '_file', + open_func(tmp_name, mode=mode, **kargs)) def _get_target(self): return object.__getattribute__(self, '_file') @@ -998,10 +953,10 @@ class atomic_ofstream(ObjectProxy): if base_destructor is not None: base_destructor(self) -def write_atomic(file_path, content): +def write_atomic(file_path, content, **kwargs): f = None try: - f = atomic_ofstream(file_path) + f = atomic_ofstream(file_path, **kwargs) f.write(content) f.close() except (IOError, OSError), e: @@ -1044,7 +999,7 @@ def ensure_dirs(dir_path, *args, **kwargs): perms_modified = apply_permissions(dir_path, *args, **kwargs) return created_dir or perms_modified -class LazyItemsDict(dict): +class LazyItemsDict(UserDict): """A mapping object that behaves like a standard dict except that it allows for lazy initialization of values via callable objects. Lazy items can be overwritten and deleted just as normal items.""" @@ -1053,19 +1008,8 @@ class LazyItemsDict(dict): def __init__(self, *args, **kwargs): - if len(args) > 1: - raise TypeError( - "expected at most 1 positional argument, got " + \ - repr(len(args))) - - dict.__init__(self) self.lazy_items = {} - - if args: - self.update(args[0]) - - if kwargs: - self.update(kwargs) + UserDict.__init__(self, *args, **kwargs) def addLazyItem(self, item_key, value_callable, *pargs, **kwargs): """Add a lazy item for the given key. When the item is requested, @@ -1073,7 +1017,7 @@ class LazyItemsDict(dict): self.lazy_items[item_key] = \ self._LazyItem(value_callable, pargs, kwargs, False) # make it show up in self.keys(), etc... - dict.__setitem__(self, item_key, None) + UserDict.__setitem__(self, item_key, None) def addLazySingleton(self, item_key, value_callable, *pargs, **kwargs): """This is like addLazyItem except value_callable will only be called @@ -1081,7 +1025,7 @@ class LazyItemsDict(dict): self.lazy_items[item_key] = \ self._LazyItem(value_callable, pargs, kwargs, True) # make it show up in self.keys(), etc... - dict.__setitem__(self, item_key, None) + UserDict.__setitem__(self, item_key, None) def update(self, *args, **kwargs): if len(args) > 1: @@ -1097,14 +1041,14 @@ class LazyItemsDict(dict): elif isinstance(map_obj, LazyItemsDict): for k in map_obj: if k in map_obj.lazy_items: - dict.__setitem__(self, k, None) + UserDict.__setitem__(self, k, None) else: - dict.__setitem__(self, k, map_obj[k]) + UserDict.__setitem__(self, k, map_obj[k]) self.lazy_items.update(map_obj.lazy_items) else: - dict.update(self, map_obj) + UserDict.update(self, map_obj) if kwargs: - dict.update(self, kwargs) + UserDict.update(self, kwargs) def __getitem__(self, item_key): if item_key in self.lazy_items: @@ -1121,20 +1065,21 @@ class LazyItemsDict(dict): return result else: - return dict.__getitem__(self, item_key) + return UserDict.__getitem__(self, item_key) def __setitem__(self, item_key, value): if item_key in self.lazy_items: del self.lazy_items[item_key] - dict.__setitem__(self, item_key, value) + UserDict.__setitem__(self, item_key, value) + def __delitem__(self, item_key): if item_key in self.lazy_items: del self.lazy_items[item_key] - dict.__delitem__(self, item_key) + UserDict.__delitem__(self, item_key) def clear(self): self.lazy_items.clear() - dict.clear(self) + UserDict.clear(self) def copy(self): return self.__copy__() @@ -1176,11 +1121,12 @@ class LazyItemsDict(dict): except TypeError: if not lazy_item.singleton: raise - dict.__setitem__(result, k_copy, deepcopy(self[k], memo)) + UserDict.__setitem__(result, + k_copy, deepcopy(self[k], memo)) else: - dict.__setitem__(result, k_copy, None) + UserDict.__setitem__(result, k_copy, None) else: - dict.__setitem__(result, k_copy, deepcopy(self[k], memo)) + UserDict.__setitem__(result, k_copy, deepcopy(self[k], memo)) return result class _LazyItem(object):