From 785fe0b42a66b1d137955a4b816b3c254a1b785d Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 16 Jul 2011 16:30:14 -0700 Subject: [PATCH] EbuildBuild: skip the fetch queue when possible Since commit f07f8386e945b48358c11c121960e4833c539752, it was possible for EbuildBuild to wait on the fetch queue even in cases in which all required files had been previously fetched. Now this case is optimized to skip the fetch queue, as discribed in bug #375331, comment #2. --- pym/_emerge/EbuildBuild.py | 25 +++++++- pym/_emerge/EbuildFetcher.py | 120 ++++++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 16 deletions(-) diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index b6beb495b..1776d1eac 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -166,12 +166,34 @@ class EbuildBuild(CompositeTask): portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, + ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) + try: + already_fetched = fetcher.already_fetched(self.settings) + except portage.exception.InvalidDependString as e: + msg_lines = [] + msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ + (self.pkg.cpv, e) + msg_lines.append(msg) + fetcher._eerror(msg_lines) + portage.elog.elog_process(self.pkg.cpv, self.settings) + self.returncode = 1 + self._current_task = None + self._unlock_builddir() + self.wait() + return + + if already_fetched: + # This case is optimized to skip the fetch queue. + fetcher = None + self._fetch_exit(fetcher) + return + # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) @@ -180,7 +202,8 @@ class EbuildBuild(CompositeTask): def _fetch_exit(self, fetcher): - if self._default_exit(fetcher) != os.EX_OK: + if fetcher is not None and \ + self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index c076288cc..066678748 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -14,24 +14,97 @@ from portage import _encodings from portage import _unicode_encode from portage import _unicode_decode from portage.elog.messages import eerror -from portage.package.ebuild.fetch import fetch +from portage.package.ebuild.fetch import _check_distfile, fetch from portage.util._pty import _create_pty_or_pipe class EbuildFetcher(SpawnProcess): - __slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \ + __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall", + "pkg", "prefetch") + \ ("_digests", "_settings", "_uri_map") + def already_fetched(self, settings): + """ + Returns True if all files are already exist locally and have correct + digests, otherwise return False. When returning True, appropriate + digest checking messages are produced for display and/or logging. + When returning False, no messages are produced, since we assume + that a fetcher process will later be executed in order to produce + such messages. This will raise InvalidDependString if SRC_URI is + invalid. + """ + + uri_map = self._get_uri_map() + if not uri_map: + return True + + digests = self._get_digests() + distdir = settings["DISTDIR"] + allow_missing = "allow-missing-manifests" in settings.features + + for filename in uri_map: + # Use stat rather than lstat since fetch() creates + # symlinks when PORTAGE_RO_DISTDIRS is used. + try: + st = os.stat(os.path.join(distdir, filename)) + except OSError: + return False + if st.st_size == 0: + return False + expected_size = digests.get(filename, {}).get('size') + if expected_size is None: + continue + if st.st_size != expected_size: + return False + + stdout_orig = sys.stdout + stderr_orig = sys.stderr + global_havecolor = portage.output.havecolor + out = io.StringIO() + eout = portage.output.EOutput() + eout.quiet = settings.get("PORTAGE_QUIET") == "1" + success = True + try: + sys.stdout = out + sys.stderr = out + if portage.output.havecolor: + portage.output.havecolor = not self.background + + for filename in uri_map: + mydigests = digests.get(filename) + if mydigests is None: + if not allow_missing: + success = False + break + continue + ok, st = _check_distfile(os.path.join(distdir, filename), + mydigests, eout, show_errors=False) + if not ok: + success = False + break + finally: + sys.stdout = stdout_orig + sys.stderr = stderr_orig + portage.output.havecolor = global_havecolor + + if success: + # When returning uncessessfully, no messages are produced, since + # we assume that a fetcher process will later be executed in order + # to produce such messages. + msg = out.getvalue() + if msg: + self.scheduler.output(msg, log_path=self.logfile) + + return success + def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi - ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo) - if ebuild_path is None: - raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) + ebuild_path = self._get_ebuild_path() try: - uri_map = self._get_uri_map(portdb, ebuild_path) + uri_map = self._get_uri_map() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ @@ -48,9 +121,6 @@ class EbuildFetcher(SpawnProcess): self.wait() return - self._digests = portage.Manifest( - os.path.dirname(ebuild_path), None).getTypeDigests("DIST") - settings = self.config_pool.allocate() settings.setcpv(self.pkg) portage.doebuild_environment(ebuild_path, 'fetch', @@ -75,7 +145,6 @@ class EbuildFetcher(SpawnProcess): settings["NOCOLOR"] = nocolor self._settings = settings - self._uri_map = uri_map SpawnProcess._start(self) # Free settings now since it's no longer needed in @@ -110,7 +179,7 @@ class EbuildFetcher(SpawnProcess): allow_missing = 'allow-missing-manifests' in self._settings.features try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, - digests=copy.deepcopy(self._digests), + digests=copy.deepcopy(self._get_digests()), allow_missing_digests=allow_missing): rval = os.EX_OK except SystemExit: @@ -122,16 +191,37 @@ class EbuildFetcher(SpawnProcess): # finally blocks from earlier in the call stack. See bug #345289. os._exit(rval) - def _get_uri_map(self, portdb, ebuild_path): + def _get_ebuild_path(self): + if self.ebuild_path is not None: + return self.ebuild_path + portdb = self.pkg.root_config.trees["porttree"].dbapi + self.ebuild_path = portdb.findname(self.pkg.cpv, myrepo=self.pkg.repo) + if self.ebuild_path is None: + raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) + return self.ebuild_path + + def _get_digests(self): + if self._digests is not None: + return self._digests + self._digests = portage.Manifest(os.path.dirname( + self._get_ebuild_path()), None).getTypeDigests("DIST") + return self._digests + + def _get_uri_map(self): """ This can raise InvalidDependString from portdbapi.getFetchMap(). """ - pkgdir = os.path.dirname(ebuild_path) + if self._uri_map is not None: + return self._uri_map + pkgdir = os.path.dirname(self._get_ebuild_path()) mytree = os.path.dirname(os.path.dirname(pkgdir)) use = None if not self.fetchall: use = self.pkg.use.enabled - return portdb.getFetchMap(self.pkg.cpv, useflags=use, mytree=mytree) + portdb = self.pkg.root_config.trees["porttree"].dbapi + self._uri_map = portdb.getFetchMap(self.pkg.cpv, + useflags=use, mytree=mytree) + return self._uri_map def _prefetch_size_ok(self, uri_map, settings, ebuild_path): distdir = settings["DISTDIR"] @@ -148,7 +238,7 @@ class EbuildFetcher(SpawnProcess): return False sizes[filename] = st.st_size - digests = self._digests + digests = self._get_digests() for filename, actual_size in sizes.items(): size = digests.get(filename, {}).get('size') if size is None: -- 2.26.2