From d8bbebf01041a25dba856d7cf676348e9f337fd6 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 27 May 2008 15:30:44 +0000 Subject: [PATCH] Merged from trunk 10418:10438 | 10419 | * Add support for Package instances and USE deps in | | zmedico | match_from_list(). * Add USE dep matching support to | | | depgraph._iter_atoms_for_pkg(). | | 10420 | Remove the Package.cpv_slot attribute. | | zmedico | | | 10421 | Use the metadata wrapper to initialize Package slot and use | | zmedico | attributes. | | 10422 | Remove unused code. | | zmedico | | | 10423 | Fix breakage in Package constructor attribute | | zmedico | initialization. | | 10425 | Fix a broken reference to the Package.cpv_slot attribute. | | zmedico | | | 10427 | * Enable config.setcpv() to use a Package instance in place | | zmedico | of a cpv. * Make depgraph._select_package() pass Package | | | instances into setcpv() calls. * Enable the Package | | | constructor to use an iterable for metadata items. | | 10429 | Skip source_all_bashrcs() when $EBUILD_PHASE is not set. | | zmedico | | | 10431 | Avoid triggering "cPickle.UnpicklingError: Global and | | zmedico | instance pickles are not supported." errors that are | | | triggered when the pickle contains custom classes. | | 10433 | Don't silently swallow instances of UnpicklingError since | | zmedico | they are easily triggered by storing instances of custom | | | classes in a pickle. | | 10434 | Don't silently swallow instances of UnpicklingError since | | zmedico | they are easily triggered by storing instances of custom | | | classes in a pickle. | | 10436 | Add some debug output for arguments and their associated | | zmedico | atoms, to help in debugging problems similar to bug #223735. | | 10438 | Bug #223685 - Use the finally clause to collect elog | | zmedico | messages just before releasing the build dir lock, so they | | | never get missed. | svn path=/main/branches/prefix/; revision=10457 --- bin/ebuild.sh | 1 + pym/_emerge/__init__.py | 150 +++++++++++++++++++++++------------ pym/portage/__init__.py | 20 ++++- pym/portage/dbapi/vartree.py | 7 +- pym/portage/dep.py | 51 ++++++++++-- pym/portage/sets/base.py | 4 +- 6 files changed, 168 insertions(+), 65 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 064f5313d..eb7bc58fb 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1356,6 +1356,7 @@ remove_path_entry() { } source_all_bashrcs() { + [ -n "$EBUILD_PHASE" ] || return local OCC="${CC}" OCXX="${CXX}" # source the existing profile.bashrc's. save_IFS diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 17482b6a3..50897042b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1295,7 +1295,8 @@ class Package(Task): __slots__ = ("built", "cpv", "depth", "installed", "metadata", "onlydeps", "operation", "root", "type_name", - "category", "cp", "cpv_slot", "pf", "pv_split", "slot_atom") + "category", "cp", "cpv_split", + "pf", "pv_split", "slot", "slot_atom", "use") metadata_keys = [ "CHOST", "COUNTER", "DEPEND", "EAPI", "IUSE", "KEYWORDS", @@ -1304,11 +1305,38 @@ class Package(Task): def __init__(self, **kwargs): Task.__init__(self, **kwargs) + self.metadata = self._metadata_wrapper(self, self.metadata) self.cp = portage.cpv_getkey(self.cpv) - self.slot_atom = "%s:%s" % (self.cp, self.metadata["SLOT"]) - self.cpv_slot = "%s:%s" % (self.cpv, self.metadata["SLOT"]) + self.slot_atom = portage.dep.Atom("%s:%s" % (self.cp, self.slot)) self.category, self.pf = portage.catsplit(self.cpv) - self.pv_split = portage.catpkgsplit(self.cpv)[1:] + self.cpv_split = portage.catpkgsplit(self.cpv) + self.pv_split = self.cpv_split[1:] + + class _use(object): + def __init__(self, use): + self.enabled = frozenset(use) + + class _metadata_wrapper(dict): + """ + Detect metadata updates and synchronize Package attributes. + """ + def __init__(self, pkg, metadata): + dict.__init__(self) + self._pkg = pkg + i = getattr(metadata, "iteritems", None) + if i is None: + i = metadata + else: + i = i() + for k, v in i: + self[k] = v + + def __setitem__(self, k, v): + dict.__setitem__(self, k, v) + if k == "USE": + self._pkg.use = self._pkg._use(v.split()) + elif k == "SLOT": + self._pkg.slot = v def _get_hash_key(self): hash_key = getattr(self, "_hash_key", None) @@ -1422,8 +1450,12 @@ class BlockerCache(DictMixin): self._cache_data = mypickle.load() f.close() del f - except (IOError, OSError, EOFError, cPickle.UnpicklingError): - pass + except (IOError, OSError, EOFError, cPickle.UnpicklingError), e: + if isinstance(e, cPickle.UnpicklingError): + writemsg("!!! Error loading '%s': %s\n" % \ + (self._cache_filename, str(e)), noiselevel=-1) + del e + cache_valid = self._cache_data and \ isinstance(self._cache_data, dict) and \ self._cache_data.get("version") == self._cache_version and \ @@ -1451,7 +1483,7 @@ class BlockerCache(DictMixin): if not isinstance(counter, (int, long)): invalid_items.add(k) continue - if not isinstance(atoms, list): + if not isinstance(atoms, (list, tuple)): invalid_items.add(k) continue invalid_atom = False @@ -1518,7 +1550,7 @@ class BlockerCache(DictMixin): @type blocker_data: BlockerData """ self._cache_data["blockers"][cpv] = \ - (blocker_data.counter, blocker_data.atoms) + (blocker_data.counter, tuple(str(x) for x in blocker_data.atoms)) self._modified = True def __iter__(self): @@ -2466,6 +2498,7 @@ class depgraph(object): def select_files(self, myfiles): """Given a list of .tbz2s, .ebuilds sets, and deps, create the appropriate depgraph and return a favorite list.""" + debug = "--debug" in self.myopts root_config = self.roots[self.target_root] sets = root_config.sets getSetAtoms = root_config.setconfig.getSetAtoms @@ -2692,6 +2725,8 @@ class depgraph(object): if arg not in refs: refs.append(arg) pprovideddict = pkgsettings.pprovideddict + if debug: + portage.writemsg("\n", noiselevel=-1) # Order needs to be preserved since a feature of --nodeps # is to allow the user to force a specific merge order. args.reverse() @@ -2713,6 +2748,9 @@ class depgraph(object): "dependencies for %s\n") % arg.arg) return 0, myfavorites continue + if debug: + portage.writemsg(" Arg: %s\n Atom: %s\n" % \ + (arg, atom), noiselevel=-1) pkg, existing_node = self._select_package( myroot, atom, onlydeps=onlydeps) if not pkg: @@ -3050,34 +3088,24 @@ class depgraph(object): if pkg is None: calculated_use = False try: - metadata = dict(izip(self._mydbapi_keys, - db.aux_get(cpv, self._mydbapi_keys))) + metadata = zip(self._mydbapi_keys, + db.aux_get(cpv, self._mydbapi_keys)) except KeyError: continue + pkg = Package(built=built, cpv=cpv, + installed=installed, metadata=metadata, + onlydeps=onlydeps, root=root, type_name=pkg_type) + metadata = pkg.metadata if not built and ("?" in metadata["LICENSE"] or \ "?" in metadata["PROVIDE"]): # This is avoided whenever possible because # it's expensive. It only needs to be done here # if it has an effect on visibility. - pkgsettings.setcpv(cpv, mydb=metadata) + pkgsettings.setcpv(pkg) metadata["USE"] = pkgsettings["PORTAGE_USE"] calculated_use = True - pkg = Package(built=built, cpv=cpv, - installed=installed, metadata=metadata, - onlydeps=onlydeps, root=root, type_name=pkg_type) self._pkg_cache[pkg] = pkg - myarg = None - if root == self.target_root: - try: - myarg = self._iter_atoms_for_pkg(pkg).next() - except StopIteration: - pass - except portage.exception.InvalidDependString: - if not installed: - # masked by corruption - continue - if not installed and myarg: - found_available_arg = True + if not installed or (installed and matched_packages): # Only enforce visibility on installed packages # if there is at least one other visible package @@ -3114,8 +3142,24 @@ class depgraph(object): if not pkg.built and not calculated_use: # This is avoided whenever possible because # it's expensive. - pkgsettings.setcpv(cpv, mydb=pkg.metadata) + pkgsettings.setcpv(pkg) pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"] + + myarg = None + if root == self.target_root: + try: + # Ebuild USE must have been calculated prior + # to this point, in case atoms have USE deps. + myarg = self._iter_atoms_for_pkg(pkg).next() + except StopIteration: + pass + except portage.exception.InvalidDependString: + if not installed: + # masked by corruption + continue + if not installed and myarg: + found_available_arg = True + if atom.use and not pkg.built: use = pkg.metadata["USE"].split() if atom.use.enabled.difference(use): @@ -3136,9 +3180,7 @@ class depgraph(object): e_pkg = self._slot_pkg_map[root].get(pkg.slot_atom) if not e_pkg: break - cpv_slot = "%s:%s" % \ - (e_pkg.cpv, e_pkg.metadata["SLOT"]) - if portage.dep.match_from_list(atom, [cpv_slot]): + if portage.dep.match_from_list(atom, [e_pkg]): if highest_version and \ e_pkg.cp == atom_cp and \ e_pkg < highest_version and \ @@ -3158,14 +3200,11 @@ class depgraph(object): "--reinstall" in self.myopts): iuses = set(filter_iuse_defaults( pkg.metadata["IUSE"].split())) - old_use = pkg.metadata["USE"].split() - mydb = pkg.metadata - if myeb and not usepkgonly: - mydb = portdb + old_use = pkg.use.enabled if myeb: - pkgsettings.setcpv(myeb, mydb=mydb) + pkgsettings.setcpv(myeb) else: - pkgsettings.setcpv(cpv, mydb=mydb) + pkgsettings.setcpv(pkg) now_use = pkgsettings["PORTAGE_USE"].split() forced_flags = set() forced_flags.update(pkgsettings.useforce) @@ -3173,8 +3212,7 @@ class depgraph(object): cur_iuse = iuses if myeb and not usepkgonly: cur_iuse = set(filter_iuse_defaults( - portdb.aux_get(myeb, - ["IUSE"])[0].split())) + myeb.metadata["IUSE"].split())) if self._reinstall_for_flags(forced_flags, old_use, iuses, now_use, cur_iuse): @@ -3185,7 +3223,7 @@ class depgraph(object): ("--newuse" in self.myopts or \ "--reinstall" in self.myopts) and \ cpv in vardb.match(atom): - pkgsettings.setcpv(cpv, mydb=pkg.metadata) + pkgsettings.setcpv(pkg) forced_flags = set() forced_flags.update(pkgsettings.useforce) forced_flags.update(pkgsettings.usemask) @@ -3202,7 +3240,7 @@ class depgraph(object): if reinstall_for_flags: reinstall = True if not built: - myeb = cpv + myeb = pkg matched_packages.append(pkg) if reinstall_for_flags: self._reinstall_nodes[pkg] = \ @@ -3214,7 +3252,8 @@ class depgraph(object): if "--debug" in self.myopts: for pkg in matched_packages: - print (pkg.type_name + ":").rjust(10), pkg.cpv + portage.writemsg("%s %s\n" % \ + ((pkg.type_name + ":").rjust(10), pkg.cpv), noiselevel=-1) # Filter out any old-style virtual matches if they are # mixed with new-style virtual matches. @@ -5195,23 +5234,23 @@ class depgraph(object): continue mydb = trees[myroot][self.pkg_tree_map[pkg_type]].dbapi try: - metadata = dict(izip(self._mydbapi_keys, - mydb.aux_get(pkg_key, self._mydbapi_keys))) + metadata = zip(self._mydbapi_keys, + mydb.aux_get(pkg_key, self._mydbapi_keys)) except KeyError: # It does no exist or it is corrupt. if action == "uninstall": continue raise portage.exception.PackageNotFound(pkg_key) - if pkg_type == "ebuild": - pkgsettings = self.pkgsettings[myroot] - pkgsettings.setcpv(pkg_key, mydb=metadata) - metadata["USE"] = pkgsettings["PORTAGE_USE"] installed = action == "uninstall" built = pkg_type != "ebuild" pkg = Package(built=built, cpv=pkg_key, installed=installed, metadata=metadata, operation=action, root=myroot, type_name=pkg_type) + if pkg_type == "ebuild": + pkgsettings = self.pkgsettings[myroot] + pkgsettings.setcpv(pkg) + pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"] self._pkg_cache[pkg] = pkg root_config = self.roots[pkg.root] @@ -5957,9 +5996,6 @@ class MergeTask(object): pkgsettings, self.edebug, mydbapi=portdb, tree="porttree") del pkgsettings["PORTAGE_BINPKG_TMPFILE"] - if retval != os.EX_OK or \ - "--buildpkgonly" in self.myopts: - elog_process(pkg_key, pkgsettings, phasefilter=filter_mergephases) if retval != os.EX_OK: return retval bintree = self.trees[myroot]["bintree"] @@ -6011,6 +6047,8 @@ class MergeTask(object): return retval finally: if builddir_lock: + elog_process(pkg.cpv, pkgsettings, + phasefilter=filter_mergephases) portage.locks.unlockdir(builddir_lock) try: if not catdir_lock: @@ -8534,7 +8572,17 @@ def action_build(settings, trees, mtimedb, # XXX: Stored as a list for backward compatibility. mtimedb["resume"]["myopts"] = \ [k for k in myopts if myopts[k] is True] - mtimedb["resume"]["favorites"]=favorites + + # Convert Atom instances to plain str since the mtimedb loader + # sets unpickler.find_global = None which causes unpickler.load() + # to raise the following exception: + # + # cPickle.UnpicklingError: Global and instance pickles are not supported. + # + # TODO: Maybe stop setting find_global = None, or find some other + # way to avoid accidental triggering of the above UnpicklingError. + mtimedb["resume"]["favorites"] = [str(x) for x in favorites] + if ("--digest" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts): for pkgline in mydepgraph.altlist(): if pkgline[0]=="ebuild" and pkgline[3]=="merge": diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e75c84274..5ec42868e 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1924,6 +1924,13 @@ class config(object): """ self.modifying() + + pkg = None + if not isinstance(mycpv, basestring): + pkg = mycpv + mycpv = pkg.cpv + mydb = pkg.metadata + if self.mycpv == mycpv: return ebuild_phase = self.get("EBUILD_PHASE") @@ -1939,7 +1946,10 @@ class config(object): iuse = mydb["IUSE"] else: slot, iuse = mydb.aux_get(self.mycpv, ["SLOT", "IUSE"]) - cpv_slot = "%s:%s" % (self.mycpv, slot) + if pkg is None: + cpv_slot = "%s:%s" % (self.mycpv, slot) + else: + cpv_slot = pkg pkginternaluse = [] for x in iuse.split(): if x.startswith("+"): @@ -5696,7 +5706,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): # Check if the atom would result in a direct circular # dependency and try to avoid that if it seems likely # to be unresolvable. - cpv_slot_list = [parent.cpv_slot] + cpv_slot_list = [parent] circular_atom = None for atom in atoms: if "!" == atom[:1]: @@ -6625,7 +6635,11 @@ class MtimeDB(dict): d = mypickle.load() f.close() del f - except (IOError, OSError, EOFError, cPickle.UnpicklingError): + except (IOError, OSError, EOFError, cPickle.UnpicklingError), e: + if isinstance(e, cPickle.UnpicklingError): + writemsg("!!! Error loading '%s': %s\n" % \ + (filename, str(e)), noiselevel=-1) + del e d = {} if "old" in d: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 94b80591b..9539e46b4 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -705,8 +705,11 @@ class vardbapi(dbapi): self._aux_cache = mypickle.load() f.close() del f - except (IOError, OSError, EOFError, cPickle.UnpicklingError): - pass + except (IOError, OSError, EOFError, cPickle.UnpicklingError), e: + if isinstance(e, cPickle.UnpicklingError): + writemsg("!!! Error loading '%s': %s\n" % \ + (self._aux_cache_filename, str(e)), noiselevel=-1) + del e if not self._aux_cache or \ not isinstance(self._aux_cache, dict) or \ self._aux_cache.get("version") != self._aux_cache_version or \ diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 75d4f4c47..ae6363d07 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -759,6 +759,9 @@ def match_from_list(mydep, candidate_list): @return: A list of package atoms that match the given package atom """ + if not candidate_list: + return [] + from portage.util import writemsg if "!" == mydep[:1]: mydep = mydep[1:] @@ -791,13 +794,21 @@ def match_from_list(mydep, candidate_list): if operator is None: for x in candidate_list: - if dep_getkey(x) != mycpv: + cp = getattr(x, "cp", None) + if cp is None: + cp = dep_getkey(x) + if cp != mycpv: continue mylist.append(x) elif operator == "=": # Exact match - mylist = [cpv for cpv in candidate_list if \ - cpvequal(remove_slot(cpv), mycpv)] + for x in candidate_list: + xcpv = getattr(x, "cpv", None) + if xcpv is None: + xcpv = dep_getcpv(x) + if not cpvequal(xcpv, mycpv): + continue + mylist.append(x) elif operator == "=*": # glob match # XXX: Nasty special casing for leading zeros @@ -809,7 +820,9 @@ def match_from_list(mydep, candidate_list): myver = "0"+myver mycpv = mysplit[0]+"/"+mysplit[1]+"-"+myver for x in candidate_list: - xs = catpkgsplit(remove_slot(x)) + xs = getattr(x, "cpv_split", None) + if xs is None: + xs = catpkgsplit(remove_slot(x)) myver = xs[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver @@ -819,7 +832,9 @@ def match_from_list(mydep, candidate_list): elif operator == "~": # version, any revision, match for x in candidate_list: - xs = catpkgsplit(remove_slot(x)) + xs = getattr(x, "cpv_split", None) + if xs is None: + xs = catpkgsplit(remove_slot(x)) if xs is None: raise InvalidData(x) if not cpvequal(xs[0]+"/"+xs[1]+"-"+xs[2], mycpv_cps[0]+"/"+mycpv_cps[1]+"-"+mycpv_cps[2]): @@ -831,8 +846,13 @@ def match_from_list(mydep, candidate_list): elif operator in [">", ">=", "<", "<="]: mysplit = ["%s/%s" % (cat, pkg), ver, rev] for x in candidate_list: + xs = getattr(x, "cpv_split", None) + if xs is None: + xs = catpkgsplit(remove_slot(x)) + xcat, xpkg, xver, xrev = xs + xs = ["%s/%s" % (xcat, xpkg), xver, xrev] try: - result = pkgcmp(pkgsplit(remove_slot(x)), mysplit) + result = pkgcmp(xs, mysplit) except ValueError: # pkgcmp may return ValueError during int() conversion writemsg("\nInvalid package name: %s\n" % x, noiselevel=-1) raise @@ -859,9 +879,26 @@ def match_from_list(mydep, candidate_list): candidate_list = mylist mylist = [] for x in candidate_list: - xslot = dep_getslot(x) + xslot = getattr(x, "slot", None) + if xslot is None and isinstance(x, basestring): + xslot = dep_getslot(x) if xslot is not None and xslot != slot: continue mylist.append(x) + if mydep.use: + candidate_list = mylist + mylist = [] + for x in candidate_list: + # Note: IUSE intersection is neglected here since there + # is currently no way to access implicit IUSE. However, IUSE + # filtering can be added elsewhere in the chain. + use = getattr(x, "use", None) + if use is not None: + if mydep.use.enabled.difference(use.enabled): + continue + if mydep.use.disabled.intersection(use.enabled): + continue + mylist.append(x) + return mylist diff --git a/pym/portage/sets/base.py b/pym/portage/sets/base.py index ac77e95c3..3f85965bc 100644 --- a/pym/portage/sets/base.py +++ b/pym/portage/sets/base.py @@ -108,7 +108,7 @@ class PackageSet(object): atoms = list(self.iterAtomsForPackage(pkg)) if not atoms: return None - return best_match_to_list(pkg.cpv_slot, atoms) + return best_match_to_list(pkg, atoms) def iterAtomsForPackage(self, pkg): """ @@ -116,7 +116,7 @@ class PackageSet(object): arguments against the PROVIDE metadata. This will raise an InvalidDependString exception if PROVIDE is invalid. """ - cpv_slot_list = ["%s:%s" % (pkg.cpv, pkg.metadata["SLOT"])] + cpv_slot_list = [pkg] cp = cpv_getkey(pkg.cpv) self._load() # make sure the atoms are loaded atoms = self._atommap.get(cp) -- 2.26.2