-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2011 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-# $Id$
from __future__ import print_function
-import codecs
+import gc
+import gzip
import logging
+import shutil
+import signal
import sys
+import tempfile
import textwrap
import time
+import warnings
import weakref
+import zlib
import portage
from portage import StringIO
from portage import os
from portage import _encodings
-from portage import _unicode_decode
-from portage import _unicode_encode
+from portage import _unicode_decode, _unicode_encode
from portage.cache.mappings import slot_dict_class
+from portage.const import LIBC_PACKAGE_ATOM
from portage.elog.messages import eerror
-from portage.output import colorize, create_color_func, darkgreen, red
+from portage.localization import _
+from portage.output import colorize, create_color_func, red
bad = create_color_func("BAD")
-from portage.sets import SETPREFIX
-from portage.sets.base import InternalPackageSet
+from portage._sets import SETPREFIX
+from portage._sets.base import InternalPackageSet
from portage.util import writemsg, writemsg_level
from portage.package.ebuild.digestcheck import digestcheck
from portage.package.ebuild.digestgen import digestgen
+from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs
+from _emerge.BinpkgFetcher import BinpkgFetcher
from _emerge.BinpkgPrefetcher import BinpkgPrefetcher
+from _emerge.BinpkgVerifier import BinpkgVerifier
from _emerge.Blocker import Blocker
from _emerge.BlockerDB import BlockerDB
from _emerge.clear_caches import clear_caches
from _emerge.EbuildFetcher import EbuildFetcher
from _emerge.EbuildPhase import EbuildPhase
from _emerge.emergelog import emergelog, _emerge_log_dir
+from _emerge.FakeVartree import FakeVartree
from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_deps
from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo
from _emerge.JobStatusDisplay import JobStatusDisplay
class Scheduler(PollScheduler):
+ # max time between display status updates (milliseconds)
+ _max_display_latency = 3000
+
_opts_ignore_blockers = \
frozenset(["--buildpkgonly",
"--fetchonly", "--fetch-all-uri",
class _iface_class(SlotObject):
__slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge",
- "dblinkElog", "dblinkEmergeLog", "fetch", "register", "schedule",
+ "dblinkElog", "dblinkEmergeLog", "fetch",
+ "output", "register", "schedule",
"scheduleSetup", "scheduleUnpack", "scheduleYield",
"unregister")
portage.exception.PortageException.__init__(self, value)
def __init__(self, settings, trees, mtimedb, myopts,
- spinner, mergelist, favorites, digraph):
+ spinner, mergelist=None, favorites=None, graph_config=None):
PollScheduler.__init__(self)
+
+ if mergelist is not None:
+ warnings.warn("The mergelist parameter of the " + \
+ "_emerge.Scheduler constructor is now unused. Use " + \
+ "the graph_config parameter instead.",
+ DeprecationWarning, stacklevel=2)
+
self.settings = settings
self.target_root = settings["ROOT"]
self.trees = trees
self.myopts = myopts
self._spinner = spinner
self._mtimedb = mtimedb
- self._mergelist = mergelist
self._favorites = favorites
- self._args_set = InternalPackageSet(favorites)
+ self._args_set = InternalPackageSet(favorites, allow_repo=True)
self._build_opts = self._build_opts_class()
for k in self._build_opts.__slots__:
setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts)
self.edebug = 1
self.pkgsettings = {}
self._config_pool = {}
- self._blocker_db = {}
- for root in trees:
+ for root in self.trees:
self._config_pool[root] = []
- self._blocker_db[root] = BlockerDB(trees[root]["root_config"])
fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
schedule=self._schedule_fetch)
dblinkDisplayMerge=self._dblink_display_merge,
dblinkElog=self._dblink_elog,
dblinkEmergeLog=self._dblink_emerge_log,
- fetch=fetch_iface, register=self._register,
+ fetch=fetch_iface, output=self._task_output,
+ register=self._register,
schedule=self._schedule_wait,
scheduleSetup=self._schedule_setup,
scheduleUnpack=self._schedule_unpack,
self._prefetchers = weakref.WeakValueDictionary()
self._pkg_queue = []
+ self._running_tasks = set()
self._completed_tasks = set()
self._failed_pkgs = []
self._failed_pkgs_die_msgs = []
self._post_mod_echo_msgs = []
self._parallel_fetch = False
- merge_count = len([x for x in mergelist \
+ self._init_graph(graph_config)
+ merge_count = len([x for x in self._mergelist \
if isinstance(x, Package) and x.operation == "merge"])
self._pkg_count = self._pkg_count_class(
curval=0, maxval=merge_count)
self._job_delay_exp = 1.5
self._previous_job_start_time = None
- self._set_digraph(digraph)
-
# This is used to memoize the _choose_pkg() result when
# no packages can be chosen until one of the existing
# jobs completes.
"thus parallel-fetching is being disabled"+"\n",
noiselevel=-1)
portage.writemsg(red("!!!")+"\n", noiselevel=-1)
- elif len(mergelist) > 1:
+ elif merge_count > 1:
self._parallel_fetch = True
if self._parallel_fetch:
self._running_portage = self._pkg(cpv, "installed",
self._running_root, installed=True)
+ def _terminate_tasks(self):
+ self._status_display.quiet = True
+ # Remove running_tasks that have been added to queues but
+ # haven't been started yet, since we're going to discard
+ # them and their start/exit handlers won't be called.
+ for build in self._task_queues.jobs._task_queue:
+ self._running_tasks.remove(build.pkg)
+ if self._merge_wait_queue:
+ for merge in self._merge_wait_queue:
+ self._running_tasks.remove(merge.merge.pkg)
+ del self._merge_wait_queue[:]
+ for merge in self._task_queues.merge._task_queue:
+ # Setup phases may be scheduled in this queue, but
+ # we're only interested in the PackageMerge instances.
+ if isinstance(merge, PackageMerge):
+ self._running_tasks.remove(merge.merge.pkg)
+ for q in self._task_queues.values():
+ q.clear()
+
+ def _init_graph(self, graph_config):
+ """
+ Initialization structures used for dependency calculations
+ involving currently installed packages.
+ """
+ # TODO: Replace the BlockerDB with a depgraph of installed packages
+ # that's updated incrementally with each upgrade/uninstall operation
+ # This will be useful for making quick and safe decisions with respect
+ # to aggressive parallelization discussed in bug #279623.
+ self._set_graph_config(graph_config)
+ self._blocker_db = {}
+ for root in self.trees:
+ if graph_config is None:
+ fake_vartree = FakeVartree(self.trees[root]["root_config"],
+ pkg_cache=self._pkg_cache)
+ else:
+ fake_vartree = graph_config.trees[root]['vartree']
+ self._blocker_db[root] = BlockerDB(fake_vartree)
+
+ def _destroy_graph(self):
+ """
+ Use this to free memory at the beginning of _calc_resume_list().
+ After _calc_resume_list(), the _init_graph() method
+ must to be called in order to re-generate the structures that
+ this method destroys.
+ """
+ self._blocker_db = None
+ self._set_graph_config(None)
+ gc.collect()
+
def _poll(self, timeout=None):
+
self._schedule()
- PollScheduler._poll(self, timeout=timeout)
+
+ if timeout is None:
+ while True:
+ if not self._poll_event_handlers:
+ self._schedule()
+ if not self._poll_event_handlers:
+ raise StopIteration(
+ "timeout is None and there are no poll() event handlers")
+ previous_count = len(self._poll_event_queue)
+ PollScheduler._poll(self, timeout=self._max_display_latency)
+ self._status_display.display()
+ if previous_count != len(self._poll_event_queue):
+ break
+
+ elif timeout <= self._max_display_latency:
+ PollScheduler._poll(self, timeout=timeout)
+ if timeout == 0:
+ # The display is updated by _schedule() above, so it would be
+ # redundant to update it here when timeout is 0.
+ pass
+ else:
+ self._status_display.display()
+
+ else:
+ remaining_timeout = timeout
+ start_time = time.time()
+ while True:
+ previous_count = len(self._poll_event_queue)
+ PollScheduler._poll(self,
+ timeout=min(self._max_display_latency, remaining_timeout))
+ self._status_display.display()
+ if previous_count != len(self._poll_event_queue):
+ break
+ elapsed_time = time.time() - start_time
+ if elapsed_time < 0:
+ # The system clock has changed such that start_time
+ # is now in the future, so just assume that the
+ # timeout has already elapsed.
+ break
+ remaining_timeout = timeout - 1000 * elapsed_time
+ if remaining_timeout <= 0:
+ break
def _set_max_jobs(self, max_jobs):
self._max_jobs = max_jobs
interactive_tasks.append(task)
return interactive_tasks
- def _set_digraph(self, digraph):
+ def _set_graph_config(self, graph_config):
+
+ if graph_config is None:
+ self._graph_config = None
+ self._pkg_cache = {}
+ self._digraph = None
+ self._mergelist = []
+ self._deep_system_deps.clear()
+ return
+
+ self._graph_config = graph_config
+ self._pkg_cache = graph_config.pkg_cache
+ self._digraph = graph_config.graph
+ self._mergelist = graph_config.mergelist
+
if "--nodeps" in self.myopts or \
(self._max_jobs is not True and self._max_jobs < 2):
# save some memory
self._digraph = None
+ graph_config.graph = None
+ graph_config.pkg_cache.clear()
+ self._deep_system_deps.clear()
+ for pkg in self._mergelist:
+ self._pkg_cache[pkg] = pkg
return
- self._digraph = digraph
self._find_system_deps()
self._prune_digraph()
self._prevent_builddir_collisions()
+ self._implicit_libc_deps()
+ if '--debug' in self.myopts:
+ writemsg("\nscheduler digraph:\n\n", noiselevel=-1)
+ self._digraph.debug_print()
+ writemsg("\n", noiselevel=-1)
def _find_system_deps(self):
"""
priority=DepPriority(buildtime=True))
cpv_map[pkg.cpv].append(pkg)
+ def _implicit_libc_deps(self):
+ """
+ Create implicit dependencies on libc, in order to ensure that libc
+ is installed as early as possible (see bug #303567). If the merge
+ list contains both a new-style virtual and an old-style PROVIDE
+ virtual, the new-style virtual is used.
+ """
+ implicit_libc_roots = set([self._running_root.root])
+ libc_set = InternalPackageSet([LIBC_PACKAGE_ATOM])
+ norm_libc_pkgs = {}
+ virt_libc_pkgs = {}
+ for pkg in self._mergelist:
+ if not isinstance(pkg, Package):
+ # a satisfied blocker
+ continue
+ if pkg.installed:
+ continue
+ if pkg.root in implicit_libc_roots and \
+ pkg.operation == 'merge':
+ if libc_set.findAtomForPackage(pkg):
+ if pkg.category == 'virtual':
+ d = virt_libc_pkgs
+ else:
+ d = norm_libc_pkgs
+ if pkg.root in d:
+ raise AssertionError(
+ "found 2 libc matches: %s and %s" % \
+ (d[pkg.root], pkg))
+ d[pkg.root] = pkg
+
+ # Prefer new-style virtuals over old-style PROVIDE virtuals.
+ libc_pkg_map = norm_libc_pkgs.copy()
+ libc_pkg_map.update(virt_libc_pkgs)
+
+ # Only add a dep when the version changes.
+ for libc_pkg in list(libc_pkg_map.values()):
+ if libc_pkg.root_config.trees['vartree'].dbapi.cpv_exists(
+ libc_pkg.cpv):
+ del libc_pkg_map[pkg.root]
+
+ if not libc_pkg_map:
+ return
+
+ libc_pkgs = set(libc_pkg_map.values())
+ earlier_libc_pkgs = set()
+
+ for pkg in self._mergelist:
+ if not isinstance(pkg, Package):
+ # a satisfied blocker
+ continue
+ if pkg.installed:
+ continue
+ if pkg.root in implicit_libc_roots and \
+ pkg.operation == 'merge':
+ if pkg in libc_pkgs:
+ earlier_libc_pkgs.add(pkg)
+ else:
+ my_libc = libc_pkg_map.get(pkg.root)
+ if my_libc is not None and \
+ my_libc in earlier_libc_pkgs:
+ self._digraph.add(my_libc, pkg,
+ priority=DepPriority(buildtime=True))
+
class _pkg_failure(portage.exception.PortageException):
"""
An instance of this class is raised by unmerge() when
# Call gc.collect() here to avoid heap overflow that
# triggers 'Cannot allocate memory' errors (reported
# with python-2.5).
- import gc
gc.collect()
blocker_db = self._blocker_db[new_pkg.root]
type_name = RootConfig.tree_pkg_map[pkg_dblink.treetype]
root_config = self.trees[pkg_dblink.myroot]["root_config"]
installed = type_name == "installed"
- return self._pkg(cpv, type_name, root_config, installed=installed)
-
- def _append_to_log_path(self, log_path, msg):
-
- f = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'],
- errors='backslashreplace')
- try:
- f.write(_unicode_decode(msg,
- encoding=_encodings['content'], errors='replace'))
- finally:
- f.close()
+ repo = pkg_dblink.settings.get("PORTAGE_REPO_NAME")
+ return self._pkg(cpv, type_name, root_config,
+ installed=installed, myrepo=repo)
def _dblink_elog(self, pkg_dblink, phase, func, msgs):
log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE")
- background = self._background
out = StringIO()
for msg in msgs:
out_str = out.getvalue()
- if not background:
- portage.util.writemsg_stdout(out_str, noiselevel=-1)
-
- if log_path is not None:
- self._append_to_log_path(log_path, out_str)
+ self._task_output(out_str, log_path=log_path)
def _dblink_emerge_log(self, msg):
self._logger.log(msg)
portage.util.writemsg_level(msg,
level=level, noiselevel=noiselevel)
else:
- if not background:
- portage.util.writemsg_level(msg,
- level=level, noiselevel=noiselevel)
- self._append_to_log_path(log_path, msg)
+ self._task_output(msg, log_path=log_path)
def _dblink_ebuild_phase(self,
pkg_dblink, pkg_dbapi, ebuild_path, phase):
scheduler = self._sched_iface
settings = pkg_dblink.settings
- pkg = self._dblink_pkg(pkg_dblink)
background = self._background
log_path = settings.get("PORTAGE_LOG_FILE")
if phase in ('die_hooks', 'success_hooks'):
ebuild_phase = MiscFunctionsProcess(background=background,
- commands=[phase], phase=phase, pkg=pkg,
+ commands=[phase], phase=phase,
scheduler=scheduler, settings=settings)
else:
ebuild_phase = EbuildPhase(background=background,
- pkg=pkg, phase=phase, scheduler=scheduler,
- settings=settings, tree=pkg_dblink.treetype)
+ phase=phase, scheduler=scheduler,
+ settings=settings)
ebuild_phase.start()
ebuild_phase.wait()
digest = '--digest' in self.myopts
if not digest:
for pkgsettings in self.pkgsettings.values():
+ if pkgsettings.mycpv is not None:
+ # ensure that we are using global features
+ # settings rather than those from package.env
+ pkgsettings.reset()
if 'digest' in pkgsettings.features:
digest = True
break
x.operation != 'merge':
continue
pkgsettings = self.pkgsettings[x.root]
+ if pkgsettings.mycpv is not None:
+ # ensure that we are using global features
+ # settings rather than those from package.env
+ pkgsettings.reset()
if '--digest' not in self.myopts and \
'digest' not in pkgsettings.features:
continue
portdb = x.root_config.trees['porttree'].dbapi
- ebuild_path = portdb.findname(x.cpv)
+ ebuild_path = portdb.findname(x.cpv, myrepo=x.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % x.cpv)
pkgsettings['O'] = os.path.dirname(ebuild_path)
return os.EX_OK
+ def _env_sanity_check(self):
+ """
+ Verify a sane environment before trying to build anything from source.
+ """
+ have_src_pkg = False
+ for x in self._mergelist:
+ if isinstance(x, Package) and not x.built:
+ have_src_pkg = True
+ break
+
+ if not have_src_pkg:
+ return os.EX_OK
+
+ for settings in self.pkgsettings.values():
+ for var in ("ARCH", ):
+ value = settings.get(var)
+ if value and value.strip():
+ continue
+ msg = _("%(var)s is not set... "
+ "Are you missing the '%(configroot)setc/make.profile' symlink? "
+ "Is the symlink correct? "
+ "Is your portage tree complete?") % \
+ {"var": var, "configroot": settings["PORTAGE_CONFIGROOT"]}
+
+ out = portage.output.EOutput()
+ for line in textwrap.wrap(msg, 70):
+ out.eerror(line)
+ return 1
+
+ return os.EX_OK
+
def _check_manifests(self):
# Verify all the manifests now so that the user is notified of failure
# as soon as possible.
root_config = x.root_config
portdb = root_config.trees["porttree"].dbapi
quiet_config = quiet_settings[root_config.root]
- ebuild_path = portdb.findname(x.cpv)
+ ebuild_path = portdb.findname(x.cpv, myrepo=x.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % x.cpv)
quiet_config["O"] = os.path.dirname(ebuild_path)
if pkg.root == self._running_root.root and \
portage.match_from_list(
portage.const.PORTAGE_PACKAGE_ATOM, [pkg]):
- if self._running_portage:
- return pkg.cpv != self._running_portage.cpv
- return True
+ if self._running_portage is None:
+ return True
+ elif pkg.cpv != self._running_portage.cpv or \
+ '9999' in pkg.cpv or \
+ 'git' in pkg.inherited or \
+ 'git-2' in pkg.inherited:
+ return True
return False
def _restart_if_necessary(self, pkg):
if myopt not in bad_resume_opts:
if myarg is True:
mynewargv.append(myopt)
+ elif isinstance(myarg, list):
+ # arguments like --exclude that use 'append' action
+ for x in myarg:
+ mynewargv.append("%s=%s" % (myopt, x))
else:
- mynewargv.append(myopt +"="+ str(myarg))
+ mynewargv.append("%s=%s" % (myopt, myarg))
# priority only needs to be adjusted on the first run
os.environ["PORTAGE_NICENESS"] = "0"
os.execv(mynewargv[0], mynewargv)
- def merge(self):
+ def _run_pkg_pretend(self):
+ """
+ Since pkg_pretend output may be important, this method sends all
+ output directly to stdout (regardless of options like --quiet or
+ --jobs).
+ """
+
+ failures = 0
+
+ # Use a local PollScheduler instance here, since we don't
+ # want tasks here to trigger the usual Scheduler callbacks
+ # that handle job scheduling and status display.
+ sched_iface = PollScheduler().sched_iface
+
+ for x in self._mergelist:
+ if not isinstance(x, Package):
+ continue
+
+ if x.operation == "uninstall":
+ continue
+
+ if x.metadata["EAPI"] in ("0", "1", "2", "3"):
+ continue
+
+ if "pretend" not in x.metadata.defined_phases:
+ continue
+
+ out_str =">>> Running pre-merge checks for " + colorize("INFORM", x.cpv) + "\n"
+ portage.util.writemsg_stdout(out_str, noiselevel=-1)
+
+ root_config = x.root_config
+ settings = self.pkgsettings[root_config.root]
+ settings.setcpv(x)
+ tmpdir = tempfile.mkdtemp()
+ tmpdir_orig = settings["PORTAGE_TMPDIR"]
+ settings["PORTAGE_TMPDIR"] = tmpdir
+
+ try:
+ if x.built:
+ tree = "bintree"
+ bintree = root_config.trees["bintree"].dbapi.bintree
+ fetched = False
+
+ # Display fetch on stdout, so that it's always clear what
+ # is consuming time here.
+ if bintree.isremote(x.cpv):
+ fetcher = BinpkgFetcher(pkg=x,
+ scheduler=sched_iface)
+ fetcher.start()
+ if fetcher.wait() != os.EX_OK:
+ failures += 1
+ continue
+ fetched = fetcher.pkg_path
+
+ verifier = BinpkgVerifier(pkg=x,
+ scheduler=sched_iface)
+ verifier.start()
+ if verifier.wait() != os.EX_OK:
+ failures += 1
+ continue
+
+ if fetched:
+ bintree.inject(x.cpv, filename=fetched)
+ tbz2_file = bintree.getname(x.cpv)
+ infloc = os.path.join(tmpdir, x.category, x.pf, "build-info")
+ os.makedirs(infloc)
+ portage.xpak.tbz2(tbz2_file).unpackinfo(infloc)
+ ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
+ settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
+ settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
+
+ else:
+ tree = "porttree"
+ portdb = root_config.trees["porttree"].dbapi
+ ebuild_path = portdb.findname(x.cpv, myrepo=x.repo)
+ if ebuild_path is None:
+ raise AssertionError("ebuild not found for '%s'" % x.cpv)
+ settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
+ if self._build_opts.buildpkgonly:
+ settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
+ else:
+ settings.configdict["pkg"]["MERGE_TYPE"] = "source"
+
+ portage.package.ebuild.doebuild.doebuild_environment(ebuild_path,
+ "pretend", settings=settings,
+ db=self.trees[settings["ROOT"]][tree].dbapi)
+ prepare_build_dirs(root_config.root, settings, cleanup=0)
+
+ vardb = root_config.trees['vartree'].dbapi
+ settings["REPLACING_VERSIONS"] = " ".join(
+ set(portage.versions.cpv_getversion(match) \
+ for match in vardb.match(x.slot_atom) + \
+ vardb.match('='+x.cpv)))
+ pretend_phase = EbuildPhase(
+ phase="pretend", scheduler=sched_iface,
+ settings=settings)
+
+ pretend_phase.start()
+ ret = pretend_phase.wait()
+ if ret != os.EX_OK:
+ failures += 1
+ portage.elog.elog_process(x.cpv, settings)
+ finally:
+ shutil.rmtree(tmpdir)
+ settings["PORTAGE_TMPDIR"] = tmpdir_orig
+ if failures:
+ return 1
+ return os.EX_OK
+
+ def merge(self):
if "--resume" in self.myopts:
# We're resuming.
portage.writemsg_stdout(
if rval != os.EX_OK:
return rval
+ rval = self._env_sanity_check()
+ if rval != os.EX_OK:
+ return rval
+
# TODO: Immediately recalculate deps here if --keep-going
# is enabled and corrupt manifests are detected.
rval = self._check_manifests()
if rval != os.EX_OK and not keep_going:
return rval
+ rval = self._run_pkg_pretend()
+ if rval != os.EX_OK:
+ return rval
+
while True:
- rval = self._merge()
+
+ received_signal = []
+
+ def sighandler(signum, frame):
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+ portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % \
+ {"signal":signum})
+ self.terminate()
+ received_signal.append(128 + signum)
+
+ earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler)
+ earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler)
+
+ try:
+ rval = self._merge()
+ finally:
+ # Restore previous handlers
+ if earlier_sigint_handler is not None:
+ signal.signal(signal.SIGINT, earlier_sigint_handler)
+ else:
+ signal.signal(signal.SIGINT, signal.SIG_DFL)
+ if earlier_sigterm_handler is not None:
+ signal.signal(signal.SIGTERM, earlier_sigterm_handler)
+ else:
+ signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+ if received_signal:
+ sys.exit(received_signal[0])
+
if rval == os.EX_OK or fetchonly or not keep_going:
break
if "resume" not in mtimedb:
log_path = self._locate_failure_log(failed_pkg)
if log_path is not None:
try:
- log_file = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='r', encoding=_encodings['content'], errors='replace')
+ log_file = open(_unicode_encode(log_path,
+ encoding=_encodings['fs'], errors='strict'), mode='rb')
except IOError:
pass
+ else:
+ if log_path.endswith('.gz'):
+ log_file = gzip.GzipFile(filename='',
+ mode='rb', fileobj=log_file)
if log_file is not None:
try:
for line in log_file:
writemsg_level(line, noiselevel=-1)
+ except zlib.error as e:
+ writemsg_level("%s\n" % (e,), level=logging.ERROR,
+ noiselevel=-1)
finally:
log_file.close()
failure_log_shown = True
msg()
if len(self._failed_pkgs_all) > 1 or \
- (self._failed_pkgs_all and "--keep-going" in self.myopts):
+ (self._failed_pkgs_all and keep_going):
if len(self._failed_pkgs_all) > 1:
msg = "The following %d packages have " % \
len(self._failed_pkgs_all) + \
printer.eerror(line)
printer.eerror("")
for failed_pkg in self._failed_pkgs_all:
- msg = " %s" % (colorize('INFORM', failed_pkg.pkg.__str__()),)
+ # Use _unicode_decode() to force unicode format string so
+ # that Package.__unicode__() is called in python2.
+ msg = _unicode_decode(" %s") % (failed_pkg.pkg,)
log_path = self._locate_failure_log(failed_pkg)
if log_path is not None:
msg += ", Log file:"
def _do_merge_exit(self, merge):
pkg = merge.merge.pkg
+ self._running_tasks.remove(pkg)
if merge.returncode != os.EX_OK:
settings = merge.merge.settings
build_dir = settings.get("PORTAGE_BUILDDIR")
build_dir=build_dir, build_log=build_log,
pkg=pkg,
returncode=merge.returncode))
- self._failed_pkg_msg(self._failed_pkgs[-1], "install", "to")
-
- self._status_display.failed = len(self._failed_pkgs)
+ if not self._terminated_tasks:
+ self._failed_pkg_msg(self._failed_pkgs[-1], "install", "to")
+ self._status_display.failed = len(self._failed_pkgs)
return
self._task_complete(pkg)
if pkg_to_replace is not None:
# When a package is replaced, mark it's uninstall
# task complete (if any).
- uninst_hash_key = \
- ("installed", pkg.root, pkg_to_replace.cpv, "uninstall")
- self._task_complete(uninst_hash_key)
+ if self._digraph is not None and \
+ pkg_to_replace in self._digraph:
+ try:
+ self._pkg_queue.remove(pkg_to_replace)
+ except ValueError:
+ pass
+ self._task_complete(pkg_to_replace)
+ else:
+ self._pkg_cache.pop(pkg_to_replace, None)
if pkg.installed:
return
mtimedb.commit()
def _build_exit(self, build):
- if build.returncode == os.EX_OK:
+ if build.returncode == os.EX_OK and self._terminated_tasks:
+ # We've been interrupted, so we won't
+ # add this to the merge queue.
+ self.curval += 1
+ self._running_tasks.remove(build.pkg)
+ self._deallocate_config(build.settings)
+ elif build.returncode == os.EX_OK:
self.curval += 1
merge = PackageMerge(merge=build)
if not build.build_opts.buildpkgonly and \
self._task_queues.merge.add(merge)
self._status_display.merges = len(self._task_queues.merge)
else:
+ self._running_tasks.remove(build.pkg)
settings = build.settings
build_dir = settings.get("PORTAGE_BUILDDIR")
build_log = settings.get("PORTAGE_LOG_FILE")
build_dir=build_dir, build_log=build_log,
pkg=build.pkg,
returncode=build.returncode))
- self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for")
-
- self._status_display.failed = len(self._failed_pkgs)
+ if not self._terminated_tasks:
+ self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for")
+ self._status_display.failed = len(self._failed_pkgs)
self._deallocate_config(build.settings)
self._jobs -= 1
self._status_display.running = self._jobs
self._status_display.reset()
self._digraph = None
self._task_queues.fetch.clear()
+ self._prefetchers.clear()
def _choose_pkg(self):
"""
return None
if self._digraph is None:
- if (self._jobs or self._task_queues.merge) and \
+ if self._is_work_scheduled() and \
not ("--nodeps" in self.myopts and \
(self._max_jobs is True or self._max_jobs > 1)):
self._choose_pkg_return_early = True
return None
return self._pkg_queue.pop(0)
- if not (self._jobs or self._task_queues.merge):
+ if not self._is_work_scheduled():
return self._pkg_queue.pop(0)
self._prune_digraph()
node in later):
dependent = True
break
- node_stack.extend(graph.child_nodes(node))
+
+ # Don't traverse children of uninstall nodes since
+ # those aren't dependencies in the usual sense.
+ if node.operation != "uninstall":
+ node_stack.extend(graph.child_nodes(node))
return dependent
self._opts_no_background.intersection(self.myopts):
self._set_max_jobs(1)
- merge_queue = self._task_queues.merge
-
while self._schedule():
if self._poll_event_handlers:
self._poll_loop()
while True:
self._schedule()
- if not (self._jobs or merge_queue):
+ if not self._is_work_scheduled():
break
if self._poll_event_handlers:
self._poll_loop()
def _keep_scheduling(self):
- return bool(self._pkg_queue and \
+ return bool(not self._terminated_tasks and self._pkg_queue and \
not (self._failed_pkgs and not self._build_opts.fetchonly))
- def _schedule_tasks(self):
-
- # When the number of jobs drops to zero, process all waiting merges.
- if not self._jobs and self._merge_wait_queue:
- for task in self._merge_wait_queue:
- task.addExitListener(self._merge_wait_exit_handler)
- self._task_queues.merge.add(task)
- self._status_display.merges = len(self._task_queues.merge)
- self._merge_wait_scheduled.extend(self._merge_wait_queue)
- del self._merge_wait_queue[:]
+ def _is_work_scheduled(self):
+ return bool(self._running_tasks)
- self._schedule_tasks_imp()
- self._status_display.display()
+ def _schedule_tasks(self):
- state_change = 0
- for q in self._task_queues.values():
- if q.schedule():
- state_change += 1
+ while True:
- # Cancel prefetchers if they're the only reason
- # the main poll loop is still running.
- if self._failed_pkgs and not self._build_opts.fetchonly and \
- not (self._jobs or self._task_queues.merge) and \
- self._task_queues.fetch:
- self._task_queues.fetch.clear()
- state_change += 1
+ # When the number of jobs drops to zero, process all waiting merges.
+ if not self._jobs and self._merge_wait_queue:
+ for task in self._merge_wait_queue:
+ task.addExitListener(self._merge_wait_exit_handler)
+ self._task_queues.merge.add(task)
+ self._status_display.merges = len(self._task_queues.merge)
+ self._merge_wait_scheduled.extend(self._merge_wait_queue)
+ del self._merge_wait_queue[:]
- if state_change:
self._schedule_tasks_imp()
self._status_display.display()
+ state_change = 0
+ for q in self._task_queues.values():
+ if q.schedule():
+ state_change += 1
+
+ # Cancel prefetchers if they're the only reason
+ # the main poll loop is still running.
+ if self._failed_pkgs and not self._build_opts.fetchonly and \
+ not self._is_work_scheduled() and \
+ self._task_queues.fetch:
+ self._task_queues.fetch.clear()
+ state_change += 1
+
+ if not (state_change or \
+ (not self._jobs and self._merge_wait_queue)):
+ break
+
return self._keep_scheduling()
def _job_delay(self):
self._pkg_count.curval += 1
task = self._task(pkg)
+ self._running_tasks.add(pkg)
if pkg.installed:
merge = PackageMerge(merge=task)
if pkg.operation != "uninstall":
vardb = pkg.root_config.trees["vartree"].dbapi
previous_cpv = vardb.match(pkg.slot_atom)
+ if not previous_cpv and vardb.cpv_exists(pkg.cpv):
+ # same cpv, different SLOT
+ previous_cpv = [pkg.cpv]
if previous_cpv:
previous_cpv = previous_cpv.pop()
pkg_to_replace = self._pkg(previous_cpv,
- "installed", pkg.root_config, installed=True)
+ "installed", pkg.root_config, installed=True,
+ operation="uninstall")
task = MergeListItem(args_set=self._args_set,
background=self._background, binpkg_opts=self._binpkg_opts,
"""
print(colorize("GOOD", "*** Resuming merge..."))
+ # free some memory before creating
+ # the resume depgraph
+ self._destroy_graph()
+
myparams = create_depgraph_params(self.myopts, None)
success = False
e = None
self._post_mod_echo_msgs.append(mydepgraph.display_problems)
return False
mydepgraph.display_problems()
-
- mylist = mydepgraph.altlist()
- mydepgraph.break_refs(mylist)
- mydepgraph.break_refs(dropped_tasks)
- self._mergelist = mylist
- self._set_digraph(mydepgraph.schedulerGraph())
+ self._init_graph(mydepgraph.schedulerGraph())
msg_width = 75
for task in dropped_tasks:
if world_locked:
world_set.unlock()
- def _pkg(self, cpv, type_name, root_config, installed=False):
+ def _pkg(self, cpv, type_name, root_config, installed=False,
+ operation=None, myrepo=None):
"""
Get a package instance from the cache, or create a new
one if necessary. Raises KeyError from aux_get if it
failures for some reason (package does not exist or is
corrupt).
"""
- operation = "merge"
- if installed:
- operation = "nomerge"
- if self._digraph is not None:
- # Reuse existing instance when available.
- pkg = self._digraph.get(
- (type_name, root_config.root, cpv, operation))
- if pkg is not None:
- return pkg
+ if type_name != "ebuild":
+ # For installed (and binary) packages we don't care for the repo
+ # when it comes to hashing, because there can only be one cpv.
+ # So overwrite the repo_key with type_name.
+ repo_key = type_name
+ myrepo = None
+ elif myrepo is None:
+ raise AssertionError(
+ "Scheduler._pkg() called without 'myrepo' argument")
+ else:
+ repo_key = myrepo
+
+ if operation is None:
+ if installed:
+ operation = "nomerge"
+ else:
+ operation = "merge"
+
+ # Reuse existing instance when available.
+ pkg = self._pkg_cache.get(
+ (type_name, root_config.root, cpv, operation, repo_key))
+ if pkg is not None:
+ return pkg
tree_type = depgraph.pkg_tree_map[type_name]
db = root_config.trees[tree_type].dbapi
db_keys = list(self.trees[root_config.root][
tree_type].dbapi._aux_cache_keys)
- metadata = zip(db_keys, db.aux_get(cpv, db_keys))
- return Package(built=(type_name != 'ebuild'),
- cpv=cpv, metadata=metadata,
- root_config=root_config, installed=installed)
+ metadata = zip(db_keys, db.aux_get(cpv, db_keys, myrepo=myrepo))
+ pkg = Package(built=(type_name != "ebuild"),
+ cpv=cpv, installed=installed, metadata=metadata,
+ root_config=root_config, type_name=type_name)
+ self._pkg_cache[pkg] = pkg
+ return pkg
projects / portage.git / blobdiff