1 # Copyright 2010-2013 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
4 from __future__ import unicode_literals
6 __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps']
11 from portage.dep import Atom, match_from_list, use_reduce
12 from portage.exception import InvalidDependString, ParseError
13 from portage.localization import _
14 from portage.util import writemsg, writemsg_level
15 from portage.versions import vercmp, _pkg_str
17 def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/",
18 trees=None, use_mask=None, use_force=None, **kwargs):
20 In order to solve bug #141118, recursively expand new-style virtuals so
21 as to collapse one or more levels of indirection, generating an expanded
22 search space. In dep_zapdeps, new-style virtuals will be assigned
23 zero cost regardless of whether or not they are currently installed. Virtual
24 blockers are supported but only when the virtual expands to a single
25 atom because it wouldn't necessarily make sense to block all the components
26 of a compound virtual. When more than one new-style virtual is matched,
27 the matches are sorted from highest to lowest versions and the atom is
28 expanded to || ( highest match ... lowest match )."""
30 mytrees = trees[myroot]
31 portdb = mytrees["porttree"].dbapi
32 pkg_use_enabled = mytrees.get("pkg_use_enabled")
33 # Atoms are stored in the graph as (atom, id(atom)) tuples
34 # since each atom is considered to be a unique entity. For
35 # example, atoms that appear identical may behave differently
36 # in USE matching, depending on their unevaluated form. Also,
37 # specially generated virtual atoms may appear identical while
38 # having different _orig_atom attributes.
39 atom_graph = mytrees.get("atom_graph")
40 parent = mytrees.get("parent")
41 virt_parent = mytrees.get("virt_parent")
43 if parent is not None:
44 if virt_parent is not None:
45 graph_parent = virt_parent
49 repoman = not mysettings.local_config
50 if kwargs["use_binaries"]:
51 portdb = trees[myroot]["bintree"].dbapi
52 pprovideddict = mysettings.pprovideddict
53 myuse = kwargs["myuse"]
58 elif isinstance(x, list):
59 newsplit.append(_expand_new_virtuals(x, edebug, mydbapi,
60 mysettings, myroot=myroot, trees=trees, use_mask=use_mask,
61 use_force=use_force, **kwargs))
64 if not isinstance(x, Atom):
66 _("invalid token: '%s'") % x)
69 x = x._eval_qa_conditionals(use_mask, use_force)
72 if not mykey.startswith("virtual/"):
74 if atom_graph is not None:
75 atom_graph.add((x, id(x)), graph_parent)
79 # Virtual blockers are no longer expanded here since
80 # the un-expanded virtual atom is more useful for
81 # maintaining a cache of blocker atoms.
83 if atom_graph is not None:
84 atom_graph.add((x, id(x)), graph_parent)
87 if repoman or not hasattr(portdb, 'match_pkgs') or \
88 pkg_use_enabled is None:
89 if portdb.cp_list(x.cp):
92 # TODO: Add PROVIDE check for repoman.
94 myvartree = mytrees.get("vartree")
95 if myvartree is not None:
96 mysettings._populate_treeVirtuals_if_needed(myvartree)
97 mychoices = mysettings.getvirtuals().get(mykey, [])
99 a.append(Atom(x.replace(x.cp, y.cp, 1)))
103 newsplit.append(a[0])
105 newsplit.append(['||'] + a)
109 # Ignore USE deps here, since otherwise we might not
110 # get any matches. Choices with correct USE settings
111 # will be preferred in dep_zapdeps().
112 matches = portdb.match_pkgs(x.without_use)
113 # Use descending order to prefer higher versions.
116 # only use new-style matches
117 if pkg.cp.startswith("virtual/"):
121 if not pkgs and not portdb.cp_list(x.cp):
122 myvartree = mytrees.get("vartree")
123 if myvartree is not None:
124 mysettings._populate_treeVirtuals_if_needed(myvartree)
125 mychoices = mysettings.getvirtuals().get(mykey, [])
127 if not (pkgs or mychoices):
128 # This one couldn't be expanded as a new-style virtual. Old-style
129 # virtuals have already been expanded by dep_virtual, so this one
130 # is unavailable and dep_zapdeps will identify it as such. The
131 # atom is not eliminated here since it may still represent a
132 # dependency that needs to be satisfied.
134 if atom_graph is not None:
135 atom_graph.add((x, id(x)), graph_parent)
140 virt_atom = '=' + pkg.cpv
141 if x.unevaluated_atom.use:
142 virt_atom += str(x.unevaluated_atom.use)
143 virt_atom = Atom(virt_atom)
146 virt_atom = virt_atom.evaluate_conditionals(
147 mysettings.get("PORTAGE_USE", "").split())
149 virt_atom = virt_atom.evaluate_conditionals(myuse)
151 virt_atom = virt_atom.evaluate_conditionals(
152 pkg_use_enabled(parent))
154 virt_atom = Atom(virt_atom)
156 # Allow the depgraph to map this atom back to the
157 # original, in order to avoid distortion in places
158 # like display or conflict resolution code.
159 virt_atom.__dict__['_orig_atom'] = x
161 # According to GLEP 37, RDEPEND is the only dependency
162 # type that is valid for new-style virtuals. Repoman
163 # should enforce this.
164 depstring = pkg._metadata['RDEPEND']
165 pkg_kwargs = kwargs.copy()
166 pkg_kwargs["myuse"] = pkg_use_enabled(pkg)
168 writemsg_level(_("Virtual Parent: %s\n") \
169 % (pkg,), noiselevel=-1, level=logging.DEBUG)
170 writemsg_level(_("Virtual Depstring: %s\n") \
171 % (depstring,), noiselevel=-1, level=logging.DEBUG)
173 # Set EAPI used for validation in dep_check() recursion.
174 mytrees["virt_parent"] = pkg
177 mycheck = dep_check(depstring, mydbapi, mysettings,
178 myroot=myroot, trees=trees, **pkg_kwargs)
180 # Restore previous EAPI after recursion.
181 if virt_parent is not None:
182 mytrees["virt_parent"] = virt_parent
184 del mytrees["virt_parent"]
187 raise ParseError("%s: %s '%s'" % \
188 (pkg, mycheck[1], depstring))
190 # pull in the new-style virtual
191 mycheck[1].append(virt_atom)
193 if atom_graph is not None:
194 virt_atom_node = (virt_atom, id(virt_atom))
195 atom_graph.add(virt_atom_node, graph_parent)
196 atom_graph.add(pkg, virt_atom_node)
197 # Plain old-style virtuals. New-style virtuals are preferred.
200 new_atom = Atom(x.replace(x.cp, y.cp, 1))
201 matches = portdb.match(new_atom)
202 # portdb is an instance of depgraph._dep_check_composite_db, so
203 # USE conditionals are already evaluated.
204 if matches and mykey in \
205 portdb.aux_get(matches[-1], ['PROVIDE'])[0].split():
207 if atom_graph is not None:
208 atom_graph.add((new_atom, id(new_atom)),
211 if not a and mychoices:
212 # Check for a virtual package.provided match.
214 new_atom = Atom(x.replace(x.cp, y.cp, 1))
215 if match_from_list(new_atom,
216 pprovideddict.get(new_atom.cp, [])):
218 if atom_graph is not None:
219 atom_graph.add((new_atom, id(new_atom)), graph_parent)
223 if atom_graph is not None:
224 atom_graph.add((x, id(x)), graph_parent)
226 newsplit.append(a[0])
228 newsplit.append(['||'] + a)
232 def dep_eval(deplist):
236 #or list; we just need one "1"
237 for x in deplist[1:]:
238 if isinstance(x, list):
243 #XXX: unless there's no available atoms in the list
244 #in which case we need to assume that everything is
245 #okay as some ebuilds are relying on an old bug.
246 if len(deplist) == 1:
251 if isinstance(x, list):
258 def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None):
260 Takes an unreduced and reduced deplist and removes satisfied dependencies.
261 Returned deplist contains steps that must be taken to satisfy dependencies.
265 writemsg("ZapDeps -- %s\n" % (use_binaries), 2)
266 if not reduced or unreduced == ["||"] or dep_eval(reduced):
269 if unreduced[0] != "||":
271 for x, satisfied in zip(unreduced, reduced):
272 if isinstance(x, list):
273 unresolved += dep_zapdeps(x, satisfied, myroot,
274 use_binaries=use_binaries, trees=trees)
279 # We're at a ( || atom ... ) type level and need to make a choice
281 satisfieds = reduced[1:]
283 # Our preference order is for an the first item that:
284 # a) contains all unmasked packages with the same key as installed packages
285 # b) contains all unmasked packages
286 # c) contains masked installed packages
287 # d) is the first item
289 preferred_installed = []
290 preferred_in_graph = []
291 preferred_any_slot = []
292 preferred_non_installed = []
293 unsat_use_in_graph = []
294 unsat_use_installed = []
295 unsat_use_non_installed = []
297 other_installed_some = []
300 # unsat_use_* must come after preferred_non_installed
301 # for correct ordering in cases like || ( foo[a] foo[b] ).
306 preferred_non_installed,
309 unsat_use_non_installed,
311 other_installed_some,
315 # Alias the trees we'll be checking availability against
316 parent = trees[myroot].get("parent")
317 priority = trees[myroot].get("priority")
318 graph_db = trees[myroot].get("graph_db")
319 graph = trees[myroot].get("graph")
320 want_update_pkg = trees[myroot].get("want_update_pkg")
322 if "vartree" in trees[myroot]:
323 vardb = trees[myroot]["vartree"].dbapi
325 mydbapi = trees[myroot]["bintree"].dbapi
327 mydbapi = trees[myroot]["porttree"].dbapi
330 mydbapi_match_pkgs = mydbapi.match_pkgs
331 except AttributeError:
332 def mydbapi_match_pkgs(atom):
333 return [mydbapi._pkg_str(cpv, atom.repo)
334 for cpv in mydbapi.match(atom)]
336 # Sort the deps into installed, not installed but already
337 # in the graph and other, not installed and not in the graph
338 # and other, with values of [[required_atom], availablility]
339 for x, satisfied in zip(deps, satisfieds):
340 if isinstance(x, list):
341 atoms = dep_zapdeps(x, satisfied, myroot,
342 use_binaries=use_binaries, trees=trees)
346 # When called by repoman, we can simply return the first choice
347 # because dep_eval() handles preference selection.
351 all_use_satisfied = True
357 # Ignore USE dependencies here since we don't want USE
358 # settings to adversely affect || preference evaluation.
359 avail_pkg = mydbapi_match_pkgs(atom.without_use)
361 avail_pkg = avail_pkg[-1] # highest (ascending order)
362 avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot))
364 all_available = False
365 all_use_satisfied = False
369 avail_pkg_use = mydbapi_match_pkgs(atom)
370 if not avail_pkg_use:
371 all_use_satisfied = False
373 # highest (ascending order)
374 avail_pkg_use = avail_pkg_use[-1]
375 if avail_pkg_use != avail_pkg:
376 avail_pkg = avail_pkg_use
377 avail_slot = Atom("%s:%s" % (atom.cp, avail_pkg.slot))
379 slot_map[avail_slot] = avail_pkg
380 highest_cpv = cp_map.get(avail_pkg.cp)
381 if highest_cpv is None or \
382 vercmp(avail_pkg.version, highest_cpv.version) > 0:
383 cp_map[avail_pkg.cp] = avail_pkg
385 this_choice = (atoms, slot_map, cp_map, all_available)
387 # The "all installed" criterion is not version or slot specific.
388 # If any version of a package is already in the graph then we
389 # assume that it is preferred over other possible packages choices.
391 for atom in set(Atom(atom.cp) for atom in atoms \
392 if not atom.blocker):
393 # New-style virtuals have zero cost to install.
394 if not vardb.match(atom) and not atom.startswith("virtual/"):
395 all_installed = False
397 all_installed_slots = False
399 all_installed_slots = True
400 for slot_atom in slot_map:
401 # New-style virtuals have zero cost to install.
402 if not vardb.match(slot_atom) and \
403 not slot_atom.startswith("virtual/"):
404 all_installed_slots = False
407 if all_use_satisfied:
409 if all_installed_slots:
410 preferred_installed.append(this_choice)
412 preferred_any_slot.append(this_choice)
414 preferred_non_installed.append(this_choice)
416 if all_installed_slots:
417 unsat_use_installed.append(this_choice)
419 unsat_use_non_installed.append(this_choice)
422 for slot_atom in slot_map:
423 # New-style virtuals have zero cost to install.
424 if slot_atom.startswith("virtual/"):
426 # We check if the matched package has actually been
427 # added to the digraph, in order to distinguish between
428 # those packages and installed packages that may need
429 # to be uninstalled in order to resolve blockers.
430 graph_matches = graph_db.match_pkgs(slot_atom)
431 if not graph_matches or graph_matches[-1] not in graph:
436 if parent is None or priority is None:
438 elif priority.buildtime and \
439 not (priority.satisfied or priority.optional):
440 # Check if the atom would result in a direct circular
441 # dependency and try to avoid that if it seems likely
442 # to be unresolvable. This is only relevant for
443 # buildtime deps that aren't already satisfied by an
445 cpv_slot_list = [parent]
449 if vardb.match(atom):
450 # If the atom is satisfied by an installed
451 # version then it's not a circular dep.
453 if atom.cp != parent.cp:
455 if match_from_list(atom, cpv_slot_list):
458 if circular_atom is not None:
459 other.append(this_choice)
461 if all_use_satisfied:
463 preferred_in_graph.append(this_choice)
465 if all_installed_slots:
466 preferred_installed.append(this_choice)
467 elif parent is None or want_update_pkg is None:
468 preferred_any_slot.append(this_choice)
470 # When appropriate, prefer a slot that is not
471 # installed yet for bug #478188.
473 for slot_atom, avail_pkg in slot_map.items():
474 if avail_pkg in graph:
476 # New-style virtuals have zero cost to install.
477 if slot_atom.startswith("virtual/") or \
478 vardb.match(slot_atom):
480 if not want_update_pkg(parent, avail_pkg):
485 preferred_installed.append(this_choice)
487 preferred_any_slot.append(this_choice)
489 preferred_non_installed.append(this_choice)
492 unsat_use_in_graph.append(this_choice)
493 elif all_installed_slots:
494 unsat_use_installed.append(this_choice)
496 unsat_use_non_installed.append(this_choice)
499 some_installed = False
502 if vardb.match(atom):
503 some_installed = True
505 all_installed = False
508 other_installed.append(this_choice)
510 other_installed_some.append(this_choice)
512 other.append(this_choice)
514 # Prefer choices which contain upgrades to higher slots. This helps
515 # for deps such as || ( foo:1 foo:2 ), where we want to prefer the
516 # atom which matches the higher version rather than the atom furthest
517 # to the left. Sorting is done separately for each of choice_bins, so
518 # as not to interfere with the ordering of the bins. Because of the
519 # bin separation, the main function of this code is to allow
520 # --depclean to remove old slots (rather than to pull in new slots).
521 for choices in choice_bins:
524 for choice_1 in choices[1:]:
525 atoms_1, slot_map_1, cp_map_1, all_available_1 = choice_1
527 for choice_2 in choices:
528 if choice_1 is choice_2:
529 # choice_1 will not be promoted, so move on
531 atoms_2, slot_map_2, cp_map_2, all_available_2 = choice_2
532 intersecting_cps = cps.intersection(cp_map_2)
533 if not intersecting_cps:
536 has_downgrade = False
537 for cp in intersecting_cps:
538 version_1 = cp_map_1[cp]
539 version_2 = cp_map_2[cp]
540 difference = vercmp(version_1.version, version_2.version)
547 if has_upgrade and not has_downgrade:
548 # promote choice_1 in front of choice_2
549 choices.remove(choice_1)
550 index_2 = choices.index(choice_2)
551 choices.insert(index_2, choice_1)
554 for allow_masked in (False, True):
555 for choices in choice_bins:
556 for atoms, slot_map, cp_map, all_available in choices:
557 if all_available or allow_masked:
560 assert(False) # This point should not be reachable
562 def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None,
563 use_cache=1, use_binaries=0, myroot=None, trees=None):
565 Takes a depend string, parses it, and selects atoms.
566 The myroot parameter is unused (use mysettings['EROOT'] instead).
568 myroot = mysettings['EROOT']
569 edebug = mysettings.get("PORTAGE_DEBUG", None) == "1"
570 #check_config_instance(mysettings)
572 trees = globals()["db"]
576 myusesplit = mysettings["PORTAGE_USE"].split()
579 # We've been given useflags to use.
580 #print "USE FLAGS PASSED IN."
582 #if "bindist" in myusesplit:
583 # print "BINDIST is set!"
585 # print "BINDIST NOT set."
587 #we are being run by autouse(), don't consult USE vars yet.
588 # WE ALSO CANNOT USE SETTINGS
594 # This is only for repoman, in order to constrain the use_reduce
595 # matchall behavior to account for profile use.mask/force. The
596 # ARCH/archlist code here may be redundant, since the profile
597 # really should be handling ARCH masking/forcing itself.
598 mymasks.update(mysettings.usemask)
599 mymasks.update(mysettings.archlist())
600 mymasks.discard(mysettings["ARCH"])
601 useforce.add(mysettings["ARCH"])
602 useforce.update(mysettings.useforce)
603 useforce.difference_update(mymasks)
605 # eapi code borrowed from _expand_new_virtuals()
606 mytrees = trees[myroot]
607 parent = mytrees.get("parent")
608 virt_parent = mytrees.get("virt_parent")
609 current_parent = None
611 if parent is not None:
612 if virt_parent is not None:
613 current_parent = virt_parent
615 current_parent = parent
617 if current_parent is not None:
618 # Don't pass the eapi argument to use_reduce() for installed packages
619 # since previous validation will have already marked them as invalid
620 # when necessary and now we're more interested in evaluating
621 # dependencies so that things like --depclean work as well as possible
622 # in spite of partial invalidity.
623 if not current_parent.installed:
624 eapi = current_parent.eapi
626 if isinstance(depstring, list):
630 mysplit = use_reduce(depstring, uselist=myusesplit,
631 masklist=mymasks, matchall=(use=="all"), excludeall=useforce,
632 opconvert=True, token_class=Atom, eapi=eapi)
633 except InvalidDependString as e:
634 return [0, "%s" % (e,)]
637 #dependencies were reduced to nothing
640 # Recursively expand new-style virtuals so as to
641 # collapse one or more levels of indirection.
643 mysplit = _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings,
644 use=use, mode=mode, myuse=myuse,
645 use_force=useforce, use_mask=mymasks, use_cache=use_cache,
646 use_binaries=use_binaries, myroot=myroot, trees=trees)
647 except ParseError as e:
648 return [0, "%s" % (e,)]
650 mysplit2 = dep_wordreduce(mysplit,
651 mysettings, mydbapi, mode, use_cache=use_cache)
653 return [0, _("Invalid token")]
655 writemsg("\n\n\n", 1)
656 writemsg("mysplit: %s\n" % (mysplit), 1)
657 writemsg("mysplit2: %s\n" % (mysplit2), 1)
659 selected_atoms = dep_zapdeps(mysplit, mysplit2, myroot,
660 use_binaries=use_binaries, trees=trees)
662 return [1, selected_atoms]
664 def dep_wordreduce(mydeplist,mysettings,mydbapi,mode,use_cache=1):
665 "Reduces the deplist to ones and zeros"
667 for mypos, token in enumerate(deplist):
668 if isinstance(deplist[mypos], list):
670 deplist[mypos]=dep_wordreduce(deplist[mypos],mysettings,mydbapi,mode,use_cache=use_cache)
671 elif deplist[mypos]=="||":
673 elif token[:1] == "!":
674 deplist[mypos] = False
676 mykey = deplist[mypos].cp
677 if mysettings and mykey in mysettings.pprovideddict and \
678 match_from_list(deplist[mypos], mysettings.pprovideddict[mykey]):
680 elif mydbapi is None:
681 # Assume nothing is satisfied. This forces dep_zapdeps to
682 # return all of deps the deps that have been selected
683 # (excluding those satisfied by package.provided).
684 deplist[mypos] = False
687 x = mydbapi.xmatch(mode, deplist[mypos])
688 if mode.startswith("minimum-"):
695 mydep=mydbapi.match(deplist[mypos],use_cache=use_cache)
698 if deplist[mypos][0]=="!":
702 #encountered invalid string