From b2a967244a2593494406744ebc30aacce5f384bc Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 1 Sep 2009 17:20:21 +0000 Subject: [PATCH] Merged from trunk -r14160:14170 | 14161 | Fix mkdir() to call the local setfscreate() with strict | | zmedico | returncode checking. Thanks to Chris PeBenito | | | for this patch. | | 14162 | Use plain ascii encoding for this file, in order to avoid | | zmedico | error messages like this which building stages (happens | | | when python is built with USE=build): * Byte compiling | | | python modules for python-2.6 .. ... Compiling | | | //usr/lib64/portage/pym/portage/cache/ebuild_xattr.py ... | | | SyntaxError: ('unknown encoding: UTF8', | | | ('//usr/lib64/portage/pym/portage/cache/ ebuild_xattr.py', | | | 0, 0, None)) | | 14163 | Fix tar_contents() to handle UnicodeEncodeError by falling | | zmedico | back to utf_8 if appropriate. | | 14164 | Fix the code from bug #275796 to ensure that choices always | | zmedico | go into the preferred_in_graph slot when appropriate. | | 14165 | Bug #278729 - Inside dep_zapdeps(), account for USE | | zmedico | dependencies in some cases where USE settings can adversely | | | affect || preference evaluation. This requires invalid | | | atoms to be dropped inside _expand_new_virtuals() since we | | | only want real Atom instances inside dep_zapdeps(). Unlike | | | previous attempts to solve this bug, cases such as || ( | | | foo[a] foo[b] ) should now be correctly handled. | | 14166 | Improve logic for bug #278729. | | zmedico | | | 14167 | Bug #281834 - In getconfig(), do not allow definition of | | zmedico | variables that have invalid names according to shell | | | standards (such as names containing hyphens). | | 14168 | Use KeyValuePairFileLoader instead of getconfig() for | | zmedico | /etc/portage/modules, since getconfig() is too strict about | | | variable names now. Thanks to Arfrever for reporting. | | 14169 | Use KeyValuePairFileLoader intead on getconfig(), since | | zmedico | getconfig() is too strict about variable names now. | | 14170 | Many packages use setuptools at run-time, so remove it from | | arfrever | list of suspected run-time dependencies. | svn path=/main/branches/prefix/; revision=14177 --- bin/repoman | 1 - pym/portage/__init__.py | 82 ++++++++++++++++++++++++------- pym/portage/_selinux.py | 4 +- pym/portage/cache/ebuild_xattr.py | 3 +- pym/portage/dbapi/vartree.py | 23 ++++++++- pym/portage/dispatch_conf.py | 14 ++++-- pym/portage/util.py | 13 +++++ 7 files changed, 112 insertions(+), 28 deletions(-) diff --git a/bin/repoman b/bin/repoman index 410dcf8a1..91fb16d18 100755 --- a/bin/repoman +++ b/bin/repoman @@ -403,7 +403,6 @@ suspect_rdepend = frozenset([ "dev-lang/swig", "dev-lang/yasm", "dev-perl/extutils-pkgconfig", - "dev-python/setuptools", "dev-util/byacc", "dev-util/cmake", "dev-util/ftjam", diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 6955eabf5..5e00b13ba 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -69,6 +69,7 @@ try: 'get_operator,isjustname,isspecific,isvalidatom,' + \ 'match_from_list,match_to_list', 'portage.eclass_cache', + 'portage.env.loaders', 'portage.exception', 'portage.getbinpkg', 'portage.locks', @@ -1632,8 +1633,10 @@ class config(object): self.module_priority = ["user","default"] self.modules = {} - self.modules["user"] = getconfig( - os.path.join(config_root, MODULES_FILE_PATH)) + modules_loader = portage.env.loaders.KeyValuePairFileLoader( + os.path.join(config_root, MODULES_FILE_PATH), None, None) + modules_dict, modules_errors = modules_loader.load() + self.modules["user"] = modules_dict if self.modules["user"] is None: self.modules["user"] = {} self.modules["default"] = { @@ -7477,6 +7480,9 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", if portage.dep._dep_check_strict: raise portage.exception.ParseError( _("invalid atom: '%s'") % x) + else: + # Only real Atom instances are allowed past this point. + continue else: if x.blocker and x.blocker.overlap.forbid and \ eapi in ("0", "1") and portage.dep._dep_check_strict: @@ -7665,6 +7671,9 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): preferred_in_graph = [] preferred_any_slot = [] preferred_non_installed = [] + unsat_use_in_graph = [] + unsat_use_installed = [] + unsat_use_non_installed = [] other = [] # Alias the trees we'll be checking availability against @@ -7694,19 +7703,35 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): continue all_available = True + all_use_satisfied = True versions = {} for atom in atoms: if atom[:1] == "!": continue - avail_pkg = mydbapi.match(atom) + # Ignore USE dependencies here since we don't want USE + # settings to adversely affect || preference evaluation. + avail_pkg = mydbapi.match(atom.without_use) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) avail_slot = "%s:%s" % (dep_getkey(atom), mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) if not avail_pkg: all_available = False + all_use_satisfied = False break + if atom.use: + avail_pkg_use = mydbapi.match(atom) + if not avail_pkg_use: + all_use_satisfied = False + else: + # highest (ascending order) + avail_pkg_use = avail_pkg_use[-1] + if avail_pkg_use != avail_pkg: + avail_pkg = avail_pkg_use + avail_slot = "%s:%s" % (dep_getkey(atom), + mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) + versions[avail_slot] = avail_pkg this_choice = (atoms, versions, all_available) @@ -7730,13 +7755,20 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): not slot_atom.startswith("virtual/"): all_installed_slots = False break - if all_installed: - if all_installed_slots: - preferred_installed.append(this_choice) + if graph_db is None: + if all_use_satisfied: + if all_installed: + if all_installed_slots: + preferred_installed.append(this_choice) + else: + preferred_any_slot.append(this_choice) + else: + preferred_non_installed.append(this_choice) else: - preferred_any_slot.append(this_choice) - elif graph_db is None: - preferred_non_installed.append(this_choice) + if all_installed_slots: + unsat_use_installed.append(this_choice) + else: + unsat_use_non_installed.append(this_choice) else: all_in_graph = True for slot_atom in versions: @@ -7745,9 +7777,10 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): not slot_atom.startswith("virtual/"): all_in_graph = False break + circular_atom = None if all_in_graph: if parent is None or priority is None: - preferred_in_graph.append(this_choice) + pass elif priority.buildtime: # Check if the atom would result in a direct circular # dependency and try to avoid that if it seems likely @@ -7755,7 +7788,6 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): # buildtime deps that aren't already satisfied by an # installed package. cpv_slot_list = [parent] - circular_atom = None for atom in atoms: if "!" == atom[:1]: continue @@ -7768,19 +7800,35 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): if match_from_list(atom, cpv_slot_list): circular_atom = atom break - if circular_atom is None: + if circular_atom is not None: + other.append(this_choice) + else: + if all_use_satisfied: + if all_in_graph: preferred_in_graph.append(this_choice) + elif all_installed: + if all_installed_slots: + preferred_installed.append(this_choice) + else: + preferred_any_slot.append(this_choice) else: - other.append(this_choice) + preferred_non_installed.append(this_choice) else: - preferred_in_graph.append(this_choice) - else: - preferred_non_installed.append(this_choice) + if all_in_graph: + unsat_use_in_graph.append(this_choice) + elif all_installed_slots: + unsat_use_installed.append(this_choice) + else: + unsat_use_non_installed.append(this_choice) else: other.append(this_choice) + # unsat_use_* must come after preferred_non_installed + # for correct ordering in cases like || ( foo[a] foo[b] ). preferred = preferred_in_graph + preferred_installed + \ - preferred_any_slot + preferred_non_installed + other + preferred_any_slot + preferred_non_installed + \ + unsat_use_in_graph + unsat_use_installed + unsat_use_non_installed + \ + other for allow_masked in (False, True): for atoms, versions, all_available in preferred: diff --git a/pym/portage/_selinux.py b/pym/portage/_selinux.py index ca6ec4dec..d013fa969 100644 --- a/pym/portage/_selinux.py +++ b/pym/portage/_selinux.py @@ -43,11 +43,11 @@ def mkdir(target, refdir): _("mkdir: Failed getting context of reference directory \"%s\".") \ % refdir) - selinux.setfscreatecon(ctx) + setfscreate(ctx) try: os.mkdir(target) finally: - selinux.setfscreatecon() + setfscreate() def rename(src, dest): src = _unicode_encode(src, encoding=_encodings['fs'], errors='strict') diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py index 4406b4e77..fa3937e2f 100644 --- a/pym/portage/cache/ebuild_xattr.py +++ b/pym/portage/cache/ebuild_xattr.py @@ -1,6 +1,5 @@ -# -*- coding: UTF8 -*- # Copyright: 2009 Gentoo Foundation -# Author(s): Petteri Räty (betelgeuse@gentoo.org) +# Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 # $Id$ diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 04fccd128..bb3795193 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -5131,6 +5131,26 @@ def write_contents(contents, root, f): def tar_contents(contents, root, tar, protect=None, onProgress=None): os = _os_merge + + try: + for x in contents: + _unicode_encode(x, + encoding=_encodings['merge'], + errors='strict') + except UnicodeEncodeError: + # The package appears to have been merged with a + # different value of sys.getfilesystemencoding(), + # so fall back to utf_8 if appropriate. + try: + for x in contents: + _unicode_encode(x, + encoding=_encodings['fs'], + errors='strict') + except UnicodeEncodeError: + pass + else: + os = portage.os + from portage.util import normalize_path import tarfile root = normalize_path(root).rstrip(os.path.sep) + os.path.sep @@ -5178,7 +5198,8 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): tar.addfile(tarinfo) else: f = open(_unicode_encode(path, - encoding=_encodings['merge'], errors='strict'), 'rb') + encoding=object.__getattribute__(os, '_encoding'), + errors='strict'), 'rb') try: tar.addfile(tarinfo, f) finally: diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py index d61266b31..98e2a57dd 100644 --- a/pym/portage/dispatch_conf.py +++ b/pym/portage/dispatch_conf.py @@ -21,15 +21,19 @@ RCS_MERGE = "rcsmerge -p -r" + RCS_BRANCH + " '%s' > '%s'" DIFF3_MERGE = "diff3 -mE '%s' '%s' '%s' > '%s'" def read_config(mandatory_opts): - try: - opts = portage.getconfig(portage.const.EPREFIX+'/etc/dispatch-conf.conf') - except: - opts = None - + loader = portage.env.loaders.KeyValuePairFileLoader( + portage.const.EPREFIX+'/etc/dispatch-conf.conf', None) + opts, errors = loader.load() if not opts: print >> sys.stderr, _('dispatch-conf: Error reading %s/etc/dispatch-conf.conf; fatal') % (portage.const.EPREFIX,) sys.exit(1) + # Handle quote removal here, since KeyValuePairFileLoader doesn't do that. + quotes = "\"'" + for k, v in opts.iteritems(): + if v[:1] in quotes and v[:1] == v[-1:]: + opts[k] = v[1:-1] + for key in mandatory_opts: if key not in opts: if key == "merge": diff --git a/pym/portage/util.py b/pym/portage/util.py index 685843cac..a56c8711e 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -16,6 +16,7 @@ import commands import codecs import errno import logging +import re import shlex import stat import string @@ -380,6 +381,8 @@ class _tolerant_shlex(shlex.shlex): (self.infile, str(e)), noiselevel=-1) return (newfile, StringIO()) +_invalid_var_name_re = re.compile(r'^\d|\W') + def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if isinstance(expand, dict): # Some existing variable definitions have been @@ -463,6 +466,16 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): return mykeys key = _unicode_decode(key) val = _unicode_decode(val) + + if _invalid_var_name_re.search(key) is not None: + if not tolerant: + raise Exception(_( + "ParseError: Invalid variable name '%s': line %s") % \ + (key, lex.lineno - 1)) + writemsg(_("!!! Invalid variable name '%s': line %s in %s\n") \ + % (key, lex.lineno - 1, mycfg), noiselevel=-1) + continue + if expand: mykeys[key] = varexpand(val, expand_map) expand_map[key] = mykeys[key] -- 2.26.2