From 59f98756b954ee97ab1fd6519bbc86cb4aafbd37 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 5 Mar 2009 18:44:49 +0000 Subject: [PATCH] Merged from trunk -r12704:12715 | 12705 | Bug #259882 - Disable the ignore-previously-merged option | | zmedico | by default since emerge's confmem already provides similar | | | functionality which is a little safer. | | 12706 | In mirror_cache(), normalize EAPI by mapping EAPI=0 to | | zmedico | empty. | | 12707 | Implement lazy evaluation of USE_EXPAND variables in | | zmedico | config.setcpv(), since those variables are only needed when | | | an ebuild shell is spawned. Thanks to Marat Radchenko | | | for testing this patch. He | | | reports a 14% reduction in the time consumed for `emerge | | | -vpe world` on an ARM device (from 3m12.008s to 2m45.055s). | | 12708 | Fix typo in DepPriority.__str__(). | | zmedico | | | 12709 | Implement BlockerDepPriority.__str__(). | | zmedico | | | 12710 | Inside depgraph._merge_order_bias(), promote deep system | | zmedico | runtime deps toward the front of the merge list. This | | | should help optimize merge order to account for implicit | | | system dependencies. | | 12711 | Remove stray parenthesis. | | zmedico | | | 12712 | reject all set*id binaries that have DT_RPATH:$ORIGIN | | SpankMan | #260331 | | 12713 | Bug #259954 - Do not spawn parallel build when the system | | zmedico | is in a fragile state due to a system package having an | | | unsatisfied runtime dependency (such as sys-libs/pam having | | | an unsatisfied PDEPEND on sys-auth/pambase). | | 12714 | In traversing deps to add to | | zmedico | Scheduler._unsatisfied_system_deps, only traverse completed | | | 'merge' nodes since those are the only ones that need to be | | | checked for unsatisfied runtime deps, and it's normal for | | | nodes that aren't yet complete to have unsatisfied runtime | | | deps. | | 12715 | Remove recursion code from | | zmedico | Scheduler._system_merge_started() since indirect deps are | | | checked when the corresponding parent is merged. | svn path=/main/branches/prefix/; revision=12762 --- bin/misc-functions.sh | 11 ++- cnf/dispatch-conf.conf | 4 +- pym/_emerge/__init__.py | 145 ++++++++++++++++++++++++-------- pym/portage/__init__.py | 169 +++++++++++++++++++++++++------------- pym/portage/cache/util.py | 7 +- 5 files changed, 237 insertions(+), 99 deletions(-) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index eedb21364..59217fdce 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -73,14 +73,19 @@ install_qa_check() { # (older, broken libtools would do this). Also check for null paths # because the loader will search $PWD when it finds null paths. f=$(scanelf -qyRF '%r %p' "${ED}" | grep -E "(${PORTAGE_BUILDDIR}|: |::|^:|^ )") - if [[ -n ${f} ]] ; then + # Reject set*id binaries with $ORIGIN in RPATH #260331 + x=$( + find "${D}" -type f -perm /6000 -print0 | xargs -0 \ + scanelf -qyRF '%r %p' | grep '$ORIGIN' + ) + if [[ -n ${f}${x} ]] ; then vecho -ne '\a\n' eqawarn "QA Notice: The following files contain insecure RUNPATH's" eqawarn " Please file a bug about this at http://bugs.gentoo.org/" eqawarn " with the maintaining herd of the package." - eqawarn "${f}" + eqawarn "${f}${f:+${x:+\n}}${x}" vecho -ne '\a\n' - if has stricter ${FEATURES} ; then + if [[ -n ${x} ]] || has stricter ${FEATURES} ; then insecure_rpath=1 else vecho "Auto fixing rpaths for ${f}" diff --git a/cnf/dispatch-conf.conf b/cnf/dispatch-conf.conf index 6a94a52ec..b1a535dc9 100644 --- a/cnf/dispatch-conf.conf +++ b/cnf/dispatch-conf.conf @@ -34,8 +34,10 @@ replace-unmodified=no # Ignore a version that is identical to the previously merged version, # even though it is different from the current user modified version +# Note that emerge already has a similar feature enabled by default, +# which can be disabled by the emerge --noconfmem option. # (yes or no) -ignore-previously-merged=yes +ignore-previously-merged=no # Per-session log file of changes made to configuration files #log-file=/var/log/dispatch-conf.log diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index beb4c710d..196e9109b 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -955,7 +955,7 @@ class DepPriority(AbstractDepPriority): return "buildtime" if self.runtime: return "runtime" - if runtime_post: + if self.runtime_post: return "runtime_post" return "soft" @@ -964,6 +964,9 @@ class BlockerDepPriority(DepPriority): def __int__(self): return 0 + def __str__(self): + return 'blocker' + BlockerDepPriority.instance = BlockerDepPriority() class UnmergeDepPriority(AbstractDepPriority): @@ -1135,6 +1138,38 @@ DepPrioritySatisfiedRange.ignore_priority = ( DepPrioritySatisfiedRange._ignore_runtime ) +def _find_deep_system_runtime_deps(graph): + deep_system_deps = set() + node_stack = [] + for node in graph: + if not isinstance(node, Package) or \ + node.operation == 'uninstall': + continue + if node.root_config.sets['system'].findAtomForPackage(node): + node_stack.append(node) + + def ignore_priority(priority): + """ + Ignore non-runtime priorities. + """ + if isinstance(priority, DepPriority) and \ + (priority.runtime or priority.runtime_post): + return False + return True + + while node_stack: + node = node_stack.pop() + if node in deep_system_deps: + continue + deep_system_deps.add(node) + for child in graph.child_nodes(node, ignore_priority=ignore_priority): + if not isinstance(child, Package) or \ + child.operation == 'uninstall': + continue + node_stack.append(child) + + return deep_system_deps + class FakeVartree(portage.vartree): """This is implements an in-memory copy of a vartree instance that provides all the interfaces required for use by the depgraph. The vardb is locked @@ -1811,8 +1846,8 @@ class AsynchronousTask(SlotObject): """ Start an asynchronous task and then return as soon as possible. """ - self._start() self._start_hook() + self._start() def _start(self): raise NotImplementedError(self) @@ -6810,13 +6845,37 @@ class depgraph(object): return acceptable def _merge_order_bias(self, mygraph): - """Order nodes from highest to lowest overall reference count for - optimal leaf node selection.""" + """ + For optimal leaf node selection, promote deep system runtime deps and + order nodes from highest to lowest overall reference count. + """ + node_info = {} for node in mygraph.order: node_info[node] = len(mygraph.parent_nodes(node)) + deep_system_deps = _find_deep_system_runtime_deps(mygraph) + def cmp_merge_preference(node1, node2): + + if node1.operation == 'uninstall': + if node2.operation == 'uninstall': + return 0 + return 1 + + if node2.operation == 'uninstall': + if node1.operation == 'uninstall': + return 0 + return -1 + + node1_sys = node1 in deep_system_deps + node2_sys = node2 in deep_system_deps + if node1_sys != node2_sys: + if node1_sys: + return -1 + return 1 + return node_info[node2] - node_info[node1] + mygraph.order.sort(key=cmp_sort_key(cmp_merge_preference)) def altlist(self, reversed=False): @@ -10002,6 +10061,13 @@ class Scheduler(PollScheduler): # when no other packages are building. self._deep_system_deps = set() + # Holds packages to merge which will satisfy currently unsatisfied + # deep runtime dependencies of system packages. If this is not empty + # then no parallel builds will be spawned until it is empty. This + # minimizes the possibility that a build will fail due to the system + # being in a fragile state. For example, see bug #259954. + self._unsatisfied_system_deps = set() + self._status_display = JobStatusDisplay() self._max_load = myopts.get("--load-average") max_jobs = myopts.get("--jobs") @@ -10186,38 +10252,10 @@ class Scheduler(PollScheduler): merged, these packages go to merge_wait_queue, to be merged when no other packages are building. """ - graph = self._digraph deep_system_deps = self._deep_system_deps deep_system_deps.clear() - node_stack = [] - for node in graph.order: - if not isinstance(node, Package) or \ - node.operation == "uninstall": - continue - system_set = node.root_config.sets["system"] - if system_set.findAtomForPackage(node): - node_stack.append(node) - - def ignore_priority(priority): - """ - Ignore non-runtime priorities. - """ - if isinstance(priority, DepPriority) and \ - (priority.runtime or priority.runtime_post): - return False - return True - - while node_stack: - node = node_stack.pop() - if node in deep_system_deps: - continue - deep_system_deps.add(node) - for child in graph.child_nodes(node, ignore_priority=ignore_priority): - if not isinstance(child, Package) or \ - child.operation == "uninstall": - continue - node_stack.append(child) - + deep_system_deps.update( + _find_deep_system_runtime_deps(self._digraph)) deep_system_deps.difference_update([pkg for pkg in \ deep_system_deps if pkg.operation != "merge"]) @@ -10779,6 +10817,41 @@ class Scheduler(PollScheduler): elif isinstance(pkg, Blocker): pass + def _system_merge_started(self, merge): + """ + Add any unsatisfied runtime deps to self._unsatisfied_system_deps. + """ + graph = self._digraph + if graph is None: + return + pkg = merge.merge.pkg + completed_tasks = self._completed_tasks + unsatisfied = self._unsatisfied_system_deps + + def ignore_non_runtime_or_satisfied(priority): + """ + Ignore non-runtime and satisfied runtime priorities. + """ + if isinstance(priority, DepPriority) and \ + not priority.satisfied and \ + (priority.runtime or priority.runtime_post): + return False + return True + + # When checking for unsatisfied runtime deps, only check + # direct deps since indirect deps are checked when the + # corresponding parent is merged. + for child in graph.child_nodes(pkg, + ignore_priority=ignore_non_runtime_or_satisfied): + if not isinstance(child, Package) or \ + child.operation == 'uninstall': + continue + if child is pkg: + continue + if child.operation == 'merge' and \ + child not in completed_tasks: + unsatisfied.add(child) + def _merge_wait_exit_handler(self, task): self._merge_wait_scheduled.remove(task) self._merge_exit(task) @@ -10840,6 +10913,7 @@ class Scheduler(PollScheduler): # Since dependencies on system packages are frequently # unspecified, merge them only when no builds are executing. self._merge_wait_queue.append(merge) + merge.addStartListener(self._system_merge_started) else: merge.addExitListener(self._merge_exit) self._task_queues.merge.add(merge) @@ -10866,6 +10940,7 @@ class Scheduler(PollScheduler): def _task_complete(self, pkg): self._completed_tasks.add(pkg) + self._unsatisfied_system_deps.discard(pkg) self._choose_pkg_return_early = False def _merge(self): @@ -10893,6 +10968,7 @@ class Scheduler(PollScheduler): del self._pkg_queue[:] self._completed_tasks.clear() self._deep_system_deps.clear() + self._unsatisfied_system_deps.clear() self._choose_pkg_return_early = False self._status_display.reset() self._digraph = None @@ -11087,6 +11163,7 @@ class Scheduler(PollScheduler): if self._choose_pkg_return_early or \ self._merge_wait_scheduled or \ + (self._jobs and self._unsatisfied_system_deps) or \ not self._can_add_job() or \ self._job_delay(): return bool(state_change) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 3a039647a..e75e875f7 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1212,6 +1212,7 @@ class config(object): "auto": self.configlist[6], "backupenv": self.configlist[7], "env": self.configlist[8] } + self._use_expand_dict = copy.deepcopy(clone._use_expand_dict) self.profiles = copy.deepcopy(clone.profiles) self.backupenv = self.configdict["backupenv"] self.pusedict = copy.deepcopy(clone.pusedict) @@ -1277,6 +1278,7 @@ class config(object): # back up our incremental variables: self.configdict={} + self._use_expand_dict = {} # configlist will contain: [ env.d, globals, defaults, conf, pkg, auto, backupenv, env ] self.configlist.append({}) self.configdict["env.d"] = self.configlist[-1] @@ -1462,7 +1464,7 @@ class config(object): pass del k, v - self.configdict["env"] = self.backupenv.copy() + self.configdict["env"] = util.LazyItemsDict(self.backupenv) # make.globals should not be relative to config_root # because it only contains constants. @@ -1980,6 +1982,86 @@ class config(object): DeprecationWarning) return 1 + class _lazy_use_expand(object): + """ + Lazily evaluate USE_EXPAND variables since they are only needed when + an ebuild shell is spawned. Variables values are made consistent with + the previously calculated USE settings. + """ + + def __init__(self, use, usemask, iuse_implicit, + use_expand_split, use_expand_dict): + self._use = use + self._usemask = usemask + self._iuse_implicit = iuse_implicit + self._use_expand_split = use_expand_split + self._use_expand_dict = use_expand_dict + + def __getitem__(self, key): + prefix = key.lower() + '_' + prefix_len = len(prefix) + expand_flags = set( x[prefix_len:] for x in self._use \ + if x[:prefix_len] == prefix ) + var_split = self._use_expand_dict.get(key, '').split() + # Preserve the order of var_split because it can matter for things + # like LINGUAS. + var_split = [ x for x in var_split if x in expand_flags ] + var_split.extend(expand_flags.difference(var_split)) + has_wildcard = '*' in expand_flags + if has_wildcard: + var_split = [ x for x in var_split if x != "*" ] + has_iuse = set() + for x in self._iuse_implicit: + if x[:prefix_len] == prefix: + has_iuse.add(x[prefix_len:]) + if has_wildcard: + # * means to enable everything in IUSE that's not masked + if has_iuse: + usemask = self._usemask + for suffix in has_iuse: + x = prefix + suffix + if x not in usemask: + if suffix not in expand_flags: + var_split.append(suffix) + else: + # If there is a wildcard and no matching flags in IUSE then + # LINGUAS should be unset so that all .mo files are + # installed. + var_split = [] + # Make the flags unique and filter them according to IUSE. + # Also, continue to preserve order for things like LINGUAS + # and filter any duplicates that variable may contain. + filtered_var_split = [] + remaining = has_iuse.intersection(var_split) + for x in var_split: + if x in remaining: + remaining.remove(x) + filtered_var_split.append(x) + var_split = filtered_var_split + + if var_split: + value = ' '.join(var_split) + else: + # Don't export empty USE_EXPAND vars unless the user config + # exports them as empty. This is required for vars such as + # LINGUAS, where unset and empty have different meanings. + if has_wildcard: + # ebuild.sh will see this and unset the variable so + # that things like LINGUAS work properly + value = '*' + else: + if has_iuse: + value = '' + else: + # It's not in IUSE, so just allow the variable content + # to pass through if it is defined somewhere. This + # allows packages that support LINGUAS but don't + # declare it in IUSE to use the variable outside of the + # USE_EXPAND context. + value = None + + return value + def setcpv(self, mycpv, use_cache=1, mydb=None): """ Load a particular CPV into the config, this lets us see the @@ -2141,69 +2223,34 @@ class config(object): # Use the calculated USE flags to regenerate the USE_EXPAND flags so # that they are consistent. For optimal performance, use slice # comparison instead of startswith(). - use_expand = self.get("USE_EXPAND", "").split() - for var in use_expand: - prefix = var.lower() + "_" + use_expand_split = self.get("USE_EXPAND", "").split() + lazy_use_expand = self._lazy_use_expand(use, self.usemask, + iuse_implicit, use_expand_split, self._use_expand_dict) + use_expand_iuse = set() + for key in use_expand_split: + prefix = key.lower() + '_' prefix_len = len(prefix) expand_flags = set( x[prefix_len:] for x in use \ if x[:prefix_len] == prefix ) - var_split = self.get(var, "").split() - # Preserve the order of var_split because it can matter for things - # like LINGUAS. - var_split = [ x for x in var_split if x in expand_flags ] - var_split.extend(expand_flags.difference(var_split)) - has_wildcard = "*" in var_split - if has_wildcard: - var_split = [ x for x in var_split if x != "*" ] - has_iuse = set() + use_expand_iuse.clear() for x in iuse_implicit: if x[:prefix_len] == prefix: - has_iuse.add(x[prefix_len:]) - if has_wildcard: - # * means to enable everything in IUSE that's not masked - if has_iuse: - for suffix in has_iuse: - x = prefix + suffix - if x not in usemask: - if suffix not in expand_flags: - var_split.append(suffix) - use.add(x) - else: - # If there is a wildcard and no matching flags in IUSE then - # LINGUAS should be unset so that all .mo files are - # installed. - var_split = [] - # Make the flags unique and filter them according to IUSE. - # Also, continue to preserve order for things like LINGUAS - # and filter any duplicates that variable may contain. - filtered_var_split = [] - remaining = has_iuse.intersection(var_split) - for x in var_split: - if x in remaining: - remaining.remove(x) - filtered_var_split.append(x) - var_split = filtered_var_split - - if var_split: - self[var] = " ".join(var_split) + use_expand_iuse.add(x) + # * means to enable everything in IUSE that's not masked + if use_expand_iuse and '*' in expand_flags: + for x in use_expand_iuse: + if x not in usemask: + use.add(x) + if use_expand_iuse: + self.configdict['env'].addLazySingleton( + key, lazy_use_expand.__getitem__, key) else: - # Don't export empty USE_EXPAND vars unless the user config - # exports them as empty. This is required for vars such as - # LINGUAS, where unset and empty have different meanings. - if has_wildcard: - # ebuild.sh will see this and unset the variable so - # that things like LINGUAS work properly - self[var] = "*" - else: - if has_iuse: - self[var] = "" - else: - # It's not in IUSE, so just allow the variable content - # to pass through if it is defined somewhere. This - # allows packages that support LINGUAS but don't - # declare it in IUSE to use the variable outside of the - # USE_EXPAND context. - pass + # It's not in IUSE, so just allow the variable content + # to pass through if it is defined somewhere. This + # allows packages that support LINGUAS but don't + # declare it in IUSE to use the variable outside of the + # USE_EXPAND context. + pass # Filtered for the ebuild environment. Store this in a separate # attribute since we still want to be able to see global USE @@ -2689,6 +2736,12 @@ class config(object): self.configdict["auto"]["USE"] = "" use_expand = self.get("USE_EXPAND", "").split() + use_expand_dict = self._use_expand_dict + use_expand_dict.clear() + for k in use_expand: + v = self.get(k) + if v is not None: + use_expand_dict[k] = v if not self.uvlist: for x in self["USE_ORDER"].split(":"): diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py index 7d81f2d05..86f93440e 100644 --- a/pym/portage/cache/util.py +++ b/pym/portage/cache/util.py @@ -53,13 +53,14 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, except cache_errors.CacheError: pass + for d in (entry, trg): + if d is not None and d.get('EAPI') in ('', '0'): + del d['EAPI'] + if trg and not write_it: """ We don't want to skip the write unless we're really sure that the existing cache is identical, so don't trust _mtime_ and _eclasses_ alone.""" - for d in (entry, trg): - if "EAPI" in d and d["EAPI"] in ("", "0"): - del d["EAPI"] for k in set(chain(entry, trg)).difference( ("_mtime_", "_eclasses_")): if trg.get(k, "") != entry.get(k, ""): -- 2.26.2