From be78021ab54aefa2f875629291fc5473f68f7c02 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 19 Feb 2009 17:43:48 +0000 Subject: [PATCH] Merged from trunk -r12608:12619 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 12609 | Automatically export QA_PRESTRIPPED if it's set. Thanks to | | zmedico | Raúl Porcel for reporting. | | 12610 | For the --buildpkgonly digraph.hasallzeros() check, remove | | zmedico | "nomerge" nodes from the graph. This makes it unnecessary to | | | tweak the dependency priority for onlydeps packages inside | | | depgraph._add_pkg_dep(). | | 12611 | In depgraph._serialize_tasks(), when appropriate, execute | | zmedico | uninstall tasks sooner. This solves some cases of bug | | | #256870 since there is a smaller window of time for some | | | other failure to cause the uninstall to get discarded. | | 12612 | Bug #250020 - When calculating merge order, try to ensure | | zmedico | that packages listed in DEPEND are updated before whenever | | | possible (even though the DEPEND may already be satisfied by | | | an installed instance). The changes to the merge order | | | algorithm should also account for many common cases of bug | | | #199856, but does not necessarily solve all cases. Whenever | | | possible, the new algorithm avoids dropping dependencies | | | that are satisfied by installed packages. Such dependencies | | | are only dropped in a couple of cases: * when solving | | | circular dependencies * when promoting packages to in the | | | merge list (either due an unsatisfied PDEPEND or a portage | | | upgrade) | | 12613 | Fix merge order calculation so that the circular runtime | | zmedico | path isn't entered when it's possible to select a node that | | | only has unsatisfied PDEPEND. This reverts an unintended | | | change from the patch for bug #250020, and solves a problem | | | with libperl being merged after perl when building stage 1. | | 12614 | Fix isvalidatom to properly identify an invalid atom such as | | zmedico | 'dev-java/nanoxml*'. Thanks to Vlastimil Babka | | | for reporting. | | 12615 | Fix isvalidatom() logic from previous commit to avoid false | | zmedico | positives. | | 12616 | Fix more false positives in isvalidatom() from previous | | zmedico | commits. | | 12617 | Add a test case for the new case which is detected by | | zmedico | isvalidatom(). | | 12618 | In dyn_package(), show a more informative die message if an | | zmedico | error occurs when creating the tarball. | | 12619 | Fix broken references to DepPriority attributes. | | zmedico | | svn path=/main/branches/prefix/; revision=12650 --- bin/ebuild.sh | 3 + bin/misc-functions.sh | 2 +- pym/_emerge/__init__.py | 335 +++++++++++++++------- pym/portage/dep.py | 5 +- pym/portage/tests/dep/test_isvalidatom.py | 1 + 5 files changed, 238 insertions(+), 108 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 1add4fb1f..4b0845fda 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1907,6 +1907,9 @@ if ! hasq "$EBUILD_PHASE" clean cleanrm && \ unset ECLASS E_IUSE E_DEPEND E_RDEPEND E_PDEPEND set +f + + # This needs to be exported since prepstrip is a separate shell script. + [[ -n $QA_PRESTRIPPED ]] && export QA_PRESTRIPPED fi # Set default EAPI if necessary, so that most diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 52594be6b..0ad072116 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -762,7 +762,7 @@ dyn_package() { mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed" tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \ bzip2 -f > "$PORTAGE_BINPKG_TMPFILE" - assert + assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'" PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ python -c "from portage import xpak; t=xpak.tbz2('${PORTAGE_BINPKG_TMPFILE}'); t.recompose('${PORTAGE_BUILDDIR}/build-info')" if [ $? -ne 0 ]; then diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 84ab989f2..e210fedb5 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -943,72 +943,21 @@ class AbstractDepPriority(SlotObject): return copy.copy(self) class DepPriority(AbstractDepPriority): - """ - This class generates an integer priority level based of various - attributes of the dependency relationship. Attributes can be assigned - at any time and the new integer value will be generated on calls to the - __int__() method. Rich comparison operators are supported. - - The boolean attributes that affect the integer value are "satisfied", - "buildtime", "runtime", and "system". Various combinations of - attributes lead to the following priority levels: - - Combination of properties Priority Category - - not satisfied and buildtime 0 HARD - not satisfied and runtime -1 MEDIUM - not satisfied and runtime_post -2 MEDIUM_SOFT - satisfied and buildtime and rebuild -3 SOFT - satisfied and buildtime -4 SOFT - satisfied and runtime -5 SOFT - satisfied and runtime_post -6 SOFT - optional -7 SOFT - (none of the above) -7 SOFT - - Several integer constants are defined for categorization of priority - levels: - - MEDIUM The upper boundary for medium dependencies. - MEDIUM_SOFT The upper boundary for medium-soft dependencies. - SOFT The upper boundary for soft dependencies. - MIN The lower boundary for soft dependencies. - """ + __slots__ = ("satisfied", "optional", "rebuild") - MEDIUM = -1 - MEDIUM_SOFT = -2 - SOFT = -3 - MIN = -7 def __int__(self): - if self.optional: - return -7 - if not self.satisfied: - if self.buildtime: - return 0 - if self.runtime: - return -1 - if self.runtime_post: - return -2 - if self.buildtime: - if self.rebuild: - return -3 - return -4 - if self.runtime: - return -5 - if self.runtime_post: - return -6 - return -7 + return 0 def __str__(self): if self.optional: return "optional" - myvalue = self.__int__() - if myvalue > self.MEDIUM: - return "hard" - if myvalue > self.MEDIUM_SOFT: - return "medium" - if myvalue > self.SOFT: - return "medium-soft" + if self.buildtime: + return "buildtime" + if self.runtime: + return "runtime" + if runtime_post: + return "runtime_post" return "soft" class BlockerDepPriority(DepPriority): @@ -1048,6 +997,145 @@ class UnmergeDepPriority(AbstractDepPriority): return "hard" return "soft" +class DepPriorityNormalRange(object): + """ + DepPriority properties Index Category + + buildtime HARD + runtime 3 MEDIUM + runtime_post 2 MEDIUM_SOFT + optional 1 SOFT + (none of the above) 0 NONE + """ + MEDIUM = 3 + MEDIUM_SOFT = 2 + SOFT = 1 + NONE = 0 + + @classmethod + def _ignore_optional(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.optional) + + @classmethod + def _ignore_runtime_post(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.optional or priority.runtime_post) + + @classmethod + def _ignore_runtime(cls, priority): + if priority.__class__ is not DepPriority: + return False + return not priority.buildtime + + ignore_medium = _ignore_runtime + ignore_medium_soft = _ignore_runtime_post + ignore_soft = _ignore_optional + +DepPriorityNormalRange.ignore_priority = ( + None, + DepPriorityNormalRange._ignore_optional, + DepPriorityNormalRange._ignore_runtime_post, + DepPriorityNormalRange._ignore_runtime +) + +class DepPrioritySatisfiedRange(object): + """ + DepPriority Index Category + + not satisfied and buildtime HARD + not satisfied and runtime 7 MEDIUM + not satisfied and runtime_post 6 MEDIUM_SOFT + satisfied and buildtime and rebuild 5 SOFT + satisfied and buildtime 4 SOFT + satisfied and runtime 3 SOFT + satisfied and runtime_post 2 SOFT + optional 1 SOFT + (none of the above) 0 NONE + """ + MEDIUM = 7 + MEDIUM_SOFT = 6 + SOFT = 5 + NONE = 0 + + @classmethod + def _ignore_optional(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.optional) + + @classmethod + def _ignore_satisfied_runtime_post(cls, priority): + if priority.__class__ is not DepPriority: + return False + if priority.optional: + return True + if not priority.satisfied: + return False + return bool(priority.runtime_post) + + @classmethod + def _ignore_satisfied_runtime(cls, priority): + if priority.__class__ is not DepPriority: + return False + if priority.optional: + return True + if not priority.satisfied: + return False + return not priority.buildtime + + @classmethod + def _ignore_satisfied_buildtime(cls, priority): + if priority.__class__ is not DepPriority: + return False + if priority.optional: + return True + if not priority.satisfied: + return False + if priority.buildtime: + return not priority.rebuild + return True + + @classmethod + def _ignore_satisfied_buildtime_rebuild(cls, priority): + if priority.__class__ is not DepPriority: + return False + if priority.optional: + return True + return bool(priority.satisfied) + + @classmethod + def _ignore_runtime_post(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.optional or \ + priority.satisfied or \ + priority.runtime_post) + + @classmethod + def _ignore_runtime(cls, priority): + if priority.__class__ is not DepPriority: + return False + return bool(priority.satisfied or \ + not priority.buildtime) + + ignore_medium = _ignore_runtime + ignore_medium_soft = _ignore_runtime_post + ignore_soft = _ignore_satisfied_buildtime_rebuild + +DepPrioritySatisfiedRange.ignore_priority = ( + None, + DepPrioritySatisfiedRange._ignore_optional, + DepPrioritySatisfiedRange._ignore_satisfied_runtime_post, + DepPrioritySatisfiedRange._ignore_satisfied_runtime, + DepPrioritySatisfiedRange._ignore_satisfied_buildtime, + DepPrioritySatisfiedRange._ignore_satisfied_buildtime_rebuild, + DepPrioritySatisfiedRange._ignore_runtime_post, + DepPrioritySatisfiedRange._ignore_runtime +) + class FakeVartree(portage.vartree): """This is implements an in-memory copy of a vartree instance that provides all the interfaces required for use by the depgraph. The vardb is locked @@ -5131,10 +5219,6 @@ class depgraph(object): strict = mytype != "installed" try: for dep_root, dep_string, dep_priority in deps: - if pkg.onlydeps: - # Decrease priority so that --buildpkgonly - # hasallzeros() works correctly. - dep_priority = DepPriority() if not dep_string: continue if debug: @@ -6819,9 +6903,9 @@ class depgraph(object): dependency relationship. """ n1_n2_medium = n2 in mygraph.child_nodes(n1, - ignore_priority=DepPriority.MEDIUM_SOFT) + ignore_priority=priority_range.ignore_medium_soft) n2_n1_medium = n1 in mygraph.child_nodes(n2, - ignore_priority=DepPriority.MEDIUM_SOFT) + ignore_priority=priority_range.ignore_medium_soft) if n1_n2_medium == n2_n1_medium: return 0 elif n1_n2_medium: @@ -6906,7 +6990,7 @@ class depgraph(object): return False if node == replacement_portage and \ mygraph.child_nodes(node, - ignore_priority=DepPriority.MEDIUM_SOFT): + ignore_priority=priority_range.ignore_medium_soft): # Make sure that portage always has all of it's # RDEPENDs installed first. return False @@ -6921,16 +7005,13 @@ class depgraph(object): def ignore_uninst_or_med(priority): if priority is BlockerDepPriority.instance: return True - return priority <= DepPriority.MEDIUM + return priority_range.ignore_medium(priority) def ignore_uninst_or_med_soft(priority): if priority is BlockerDepPriority.instance: return True - return priority <= DepPriority.MEDIUM_SOFT + return priority_range.ignore_medium_soft(priority) - ignore_priority_soft_range = [None] - ignore_priority_soft_range.extend( - xrange(DepPriority.MIN, DepPriority.MEDIUM_SOFT + 1)) tree_mode = "--tree" in self.myopts # Tracks whether or not the current iteration should prefer asap_nodes # if available. This is set to False when the previous iteration @@ -6938,12 +7019,20 @@ class depgraph(object): # successfully selected. prefer_asap = True + # Controls whether or not the current iteration should drop edges that + # are "satisfied" by installed packages, in order to solve circular + # dependencies. The deep runtime dependencies of installed packages are + # not checked in this case (bug #199856), so it must be avoided + # whenever possible. + drop_satisfied = False + # State of variables for successive iterations that loosen the # criteria for node selection. # - # iteration prefer_asap - # 1 True - # 2 False + # iteration prefer_asap drop_satisfied + # 1 True False + # 2 False False + # 3 False True # # If no nodes are selected on the last iteration, it is due to # unresolved blockers or circular dependencies. @@ -6952,6 +7041,10 @@ class depgraph(object): self.spinner.update() selected_nodes = None ignore_priority = None + if drop_satisfied or (prefer_asap and asap_nodes): + priority_range = DepPrioritySatisfiedRange + else: + priority_range = DepPriorityNormalRange if prefer_asap and asap_nodes: # ASAP nodes are merged before their soft deps. Go ahead and # select root nodes here if necessary, since it's typical for @@ -6960,13 +7053,15 @@ class depgraph(object): if mygraph.contains(node)] for node in asap_nodes: if not mygraph.child_nodes(node, - ignore_priority=DepPriority.SOFT): + ignore_priority=priority_range.ignore_soft): selected_nodes = [node] asap_nodes.remove(node) break if not selected_nodes and \ not (prefer_asap and asap_nodes): - for ignore_priority in ignore_priority_soft_range: + for i in xrange(priority_range.NONE, + priority_range.MEDIUM_SOFT + 1): + ignore_priority = priority_range.ignore_priority[i] nodes = get_nodes(ignore_priority=ignore_priority) if nodes: # If there is a mix of uninstall nodes with other @@ -6976,10 +7071,21 @@ class depgraph(object): # and we want to avoid executing a separate uninstall # task in that case. if len(nodes) > 1: - non_uninstalls = [node for node in nodes \ - if node.operation != "uninstall"] - if non_uninstalls: - nodes = non_uninstalls + good_uninstalls = [] + with_some_uninstalls_excluded = [] + for node in nodes: + if node.operation == "uninstall": + slot_node = self.mydbapi[node.root + ].match_pkgs(node.slot_atom) + if slot_node and \ + slot_node[0].operation == "merge": + continue + good_uninstalls.append(node) + with_some_uninstalls_excluded.append(node) + if good_uninstalls: + nodes = good_uninstalls + elif with_some_uninstalls_excluded: + nodes = with_some_uninstalls_excluded else: nodes = nodes @@ -6994,9 +7100,11 @@ class depgraph(object): # * Only pop one node. # * Removing a root node (node without a parent) # will not produce a leaf node, so avoid it. + # * It's normal for a selected uninstall to be a + # root node, so don't check them for parents. for node in nodes: - if mygraph.parent_nodes(node): - # found a non-root node + if node.operation == "uninstall" or \ + mygraph.parent_nodes(node): selected_nodes = [node] break @@ -7004,13 +7112,14 @@ class depgraph(object): break if not selected_nodes: - nodes = get_nodes(ignore_priority=DepPriority.MEDIUM) + nodes = get_nodes(ignore_priority=priority_range.ignore_medium) if nodes: mergeable_nodes = set(nodes) if prefer_asap and asap_nodes: nodes = asap_nodes - for ignore_priority in xrange(DepPriority.SOFT, - DepPriority.MEDIUM_SOFT + 1): + for i in xrange(priority_range.SOFT, + priority_range.MEDIUM_SOFT + 1): + ignore_priority = priority_range.ignore_priority[i] for node in nodes: if not mygraph.parent_nodes(node): continue @@ -7023,30 +7132,24 @@ class depgraph(object): if selected_nodes: break - # If any nodes have been selected here, it's always - # possible that anything up to a MEDIUM_SOFT priority - # relationship has been ignored. This state is recorded - # in ignore_priority so that relevant nodes will be - # added to asap_nodes when appropriate. - if selected_nodes: - ignore_priority = DepPriority.MEDIUM_SOFT - if prefer_asap and asap_nodes and not selected_nodes: # We failed to find any asap nodes to merge, so ignore # them for the next iteration. prefer_asap = False continue - if selected_nodes and ignore_priority > DepPriority.SOFT: - # Try to merge ignored medium deps as soon as possible. + if selected_nodes and ignore_priority is not None: + # Try to merge ignored medium_soft deps as soon as possible + # if they're not satisfied by installed packages. for node in selected_nodes: children = set(mygraph.child_nodes(node)) soft = children.difference( mygraph.child_nodes(node, - ignore_priority=DepPriority.SOFT)) + ignore_priority=DepPrioritySatisfiedRange.ignore_soft)) medium_soft = children.difference( mygraph.child_nodes(node, - ignore_priority=DepPriority.MEDIUM_SOFT)) + ignore_priority = \ + DepPrioritySatisfiedRange.ignore_medium_soft)) medium_soft.difference_update(soft) for child in medium_soft: if child in selected_nodes: @@ -7064,6 +7167,11 @@ class depgraph(object): # An Uninstall task needs to be executed in order to # avoid conflict if possible. + if drop_satisfied: + priority_range = DepPrioritySatisfiedRange + else: + priority_range = DepPriorityNormalRange + mergeable_nodes = get_nodes( ignore_priority=ignore_uninst_or_med) @@ -7196,7 +7304,7 @@ class depgraph(object): parent_deps = set() for parent in mygraph.parent_nodes(task): parent_deps.update(mygraph.child_nodes(parent, - ignore_priority=DepPriority.MEDIUM_SOFT)) + ignore_priority=priority_range.ignore_medium_soft)) if parent in mergeable_nodes and \ gather_deps(ignore_uninst_or_med_soft, mergeable_nodes, set(), parent): @@ -7233,6 +7341,7 @@ class depgraph(object): # Reset the state variables for leaf node selection and # continue trying to select leaf nodes. prefer_asap = True + drop_satisfied = False continue if not selected_nodes: @@ -7242,6 +7351,10 @@ class depgraph(object): # the nodes must be isolated, ignore_priority is not needed. selected_nodes = get_nodes() + if not selected_nodes and not drop_satisfied: + drop_satisfied = True + continue + if not selected_nodes and not myblocker_uninstalls.is_empty(): # If possible, drop an uninstall task here in order to avoid # the circular deps code path. The corresponding blocker will @@ -7261,6 +7374,7 @@ class depgraph(object): # Reset the state variables for leaf node selection and # continue trying to select leaf nodes. prefer_asap = True + drop_satisfied = False continue if not selected_nodes: @@ -7270,6 +7384,7 @@ class depgraph(object): # At this point, we've succeeded in selecting one or more nodes, so # reset state variables for leaf node selection. prefer_asap = True + drop_satisfied = False mygraph.difference_update(selected_nodes) @@ -7366,7 +7481,7 @@ class depgraph(object): # have no parents and thus can not be part of a cycle. while True: root_nodes = mygraph.root_nodes( - ignore_priority=DepPriority.MEDIUM_SOFT) + ignore_priority=DepPrioritySatisfiedRange.ignore_medium_soft) if not root_nodes: break mygraph.difference_update(root_nodes) @@ -13965,7 +14080,7 @@ def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): continue unsatisfied = \ graph.child_nodes(parent_node, - ignore_priority=DepPriority.SOFT) + ignore_priority=DepPrioritySatisfiedRange.ignore_soft) if pkg in unsatisfied: unsatisfied_parents[parent_node] = parent_node unsatisfied_stack.append(parent_node) @@ -14335,20 +14450,28 @@ def action_build(settings, trees, mtimedb, return retval if "--buildpkgonly" in myopts: graph_copy = mydepgraph.digraph.clone() + removed_nodes = set() for node in list(graph_copy.order): - if not isinstance(node, Package): - graph_copy.remove(node) - if not graph_copy.hasallzeros(ignore_priority=DepPriority.MEDIUM): + if not isinstance(node, Package) or \ + node.operation == "nomerge": + removed_nodes.add(node) + graph_copy.difference_update(removed_nodes) + if not graph_copy.hasallzeros(ignore_priority = \ + DepPrioritySatisfiedRange.ignore_medium): print "\n!!! --buildpkgonly requires all dependencies to be merged." print "!!! You have to merge the dependencies before you can build this package.\n" return 1 else: if "--buildpkgonly" in myopts: graph_copy = mydepgraph.digraph.clone() + removed_nodes = set() for node in list(graph_copy.order): - if not isinstance(node, Package): - graph_copy.remove(node) - if not graph_copy.hasallzeros(ignore_priority=DepPriority.MEDIUM): + if not isinstance(node, Package) or \ + node.operation == "nomerge": + removed_nodes.add(node) + graph_copy.difference_update(removed_nodes) + if not graph_copy.hasallzeros(ignore_priority = \ + DepPrioritySatisfiedRange.ignore_medium): print "\n!!! --buildpkgonly requires all dependencies to be merged." print "!!! Cannot merge requested packages. Merge deps and try again.\n" return 1 diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 9c9c14380..831a8a324 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -782,6 +782,7 @@ def isvalidatom(atom, allow_blockers=False): cpv = dep_getcpv(atom) cpv_catsplit = catsplit(cpv) + without_slot = remove_slot(atom) mycpv_cps = None if cpv: if len(cpv_catsplit) == 2: @@ -795,10 +796,12 @@ def isvalidatom(atom, allow_blockers=False): mycpv_cps[0] = "null" if not mycpv_cps: mycpv_cps = catpkgsplit(cpv) + if mycpv_cps is None and cpv != without_slot: + return 0 operator = get_operator(atom) if operator: - if operator[0] in "<>" and remove_slot(atom).endswith("*"): + if operator[0] in "<>" and without_slot[-1:] == "*": return 0 if mycpv_cps: if len(cpv_catsplit) == 2: diff --git a/pym/portage/tests/dep/test_isvalidatom.py b/pym/portage/tests/dep/test_isvalidatom.py index cd501e90d..1364a92a2 100644 --- a/pym/portage/tests/dep/test_isvalidatom.py +++ b/pym/portage/tests/dep/test_isvalidatom.py @@ -57,6 +57,7 @@ class IsValidAtom(TestCase): ( "null/portage", True ), ( "=null/portage", False ), ( "=null/portage*", False ), + ( "null/portage*:0", False ), ( ">=null/portage-2.1", True ), ( "~null/portage-2.1", True ), ( "=null/portage-2.1*", True ),] -- 2.26.2