From a28f2dd07cafbbb136712cc1c75b89f494fa702f Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 25 Feb 2010 07:58:15 +0000 Subject: [PATCH] Move digestgen to portage.util.digestgen. svn path=/main/trunk/; revision=15454 --- bin/repoman | 9 +- pym/_emerge/Scheduler.py | 3 +- pym/portage/__init__.py | 180 +--------------------- pym/portage/package/ebuild/doebuild.py | 3 +- pym/portage/util/digestgen.py | 204 +++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 185 deletions(-) create mode 100644 pym/portage/util/digestgen.py diff --git a/bin/repoman b/bin/repoman index c1f1481d0..9f2d88c59 100755 --- a/bin/repoman +++ b/bin/repoman @@ -77,6 +77,7 @@ from portage.output import bold, create_color_func, darkgreen, \ green, nocolor, red, turquoise, yellow from portage.output import ConsoleStyleFile, StyleWriter from portage.util import cmp_sort_key, writemsg_level +from portage.util.digestgen import digestgen if sys.hexversion >= 0x3000000: basestring = str @@ -1011,7 +1012,7 @@ for x in scanlist: portage._doebuild_manifest_exempt_depend -= 1 repoman_settings["O"] = checkdir - if not portage.digestgen(mysettings=repoman_settings, myportdb=portdb): + if not digestgen(mysettings=repoman_settings, myportdb=portdb): print("Unable to generate manifest.") dofail = 1 if options.mode == "manifest": @@ -2335,7 +2336,7 @@ else: mydone=[] if repolevel==3: # In a package dir repoman_settings["O"] = startdir - portage.digestgen(mysettings=repoman_settings, myportdb=portdb) + digestgen(mysettings=repoman_settings, myportdb=portdb) elif repolevel==2: # In a category dir for x in myfiles: xs=x.split("/") @@ -2349,7 +2350,7 @@ else: repoman_settings["O"] = os.path.join(startdir, xs[0]) if not os.path.isdir(repoman_settings["O"]): continue - portage.digestgen(mysettings=repoman_settings, myportdb=portdb) + digestgen(mysettings=repoman_settings, myportdb=portdb) elif repolevel==1: # repo-cvsroot print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n") for x in myfiles: @@ -2364,7 +2365,7 @@ else: repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1]) if not os.path.isdir(repoman_settings["O"]): continue - portage.digestgen(mysettings=repoman_settings, myportdb=portdb) + digestgen(mysettings=repoman_settings, myportdb=portdb) else: print(red("I'm confused... I don't know where I am!")) sys.exit(1) diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index e508e233f..071b8e8d7 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -24,6 +24,7 @@ bad = create_color_func("BAD") from portage.sets import SETPREFIX from portage.sets.base import InternalPackageSet from portage.util import writemsg, writemsg_level +from portage.util.digestgen import digestgen from _emerge.BinpkgPrefetcher import BinpkgPrefetcher from _emerge.Blocker import Blocker @@ -591,7 +592,7 @@ class Scheduler(PollScheduler): if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % x.cpv) pkgsettings['O'] = os.path.dirname(ebuild_path) - if not portage.digestgen(mysettings=pkgsettings, myportdb=portdb): + if not digestgen(mysettings=pkgsettings, myportdb=portdb): writemsg_level( "!!! Unable to generate manifest for '%s'.\n" \ % x.cpv, level=logging.ERROR, noiselevel=-1) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index c459c1b31..46988f3ad 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -119,6 +119,7 @@ try: 'pickle_read,pickle_write,stack_dictlist,stack_dicts,' + \ 'stack_lists,unique_array,varexpand,writedict,writemsg,' + \ 'writemsg_stdout,write_atomic', + 'portage.util.digestgen:digestgen', 'portage.util.digraph:digraph', 'portage.util.env_update:env_update', 'portage.util.ExtractKernelVersion:ExtractKernelVersion', @@ -536,185 +537,6 @@ def abssymlink(symlink): _doebuild_manifest_exempt_depend = 0 -def digestgen(myarchives=None, mysettings=None, - overwrite=None, manifestonly=None, myportdb=None): - """ - Generates a digest file if missing. Fetches files if necessary. - NOTE: myarchives and mysettings used to be positional arguments, - so their order must be preserved for backward compatibility. - @param mysettings: the ebuild config (mysettings["O"] must correspond - to the ebuild's parent directory) - @type mysettings: config - @param myportdb: a portdbapi instance - @type myportdb: portdbapi - @rtype: int - @returns: 1 on success and 0 on failure - """ - if mysettings is None: - raise TypeError("portage.digestgen(): missing" + \ - " required 'mysettings' parameter") - if myportdb is None: - warnings.warn("portage.digestgen() called without 'myportdb' parameter", - DeprecationWarning, stacklevel=2) - global portdb - myportdb = portdb - if overwrite is not None: - warnings.warn("portage.digestgen() called with " + \ - "deprecated 'overwrite' parameter", - DeprecationWarning, stacklevel=2) - if manifestonly is not None: - warnings.warn("portage.digestgen() called with " + \ - "deprecated 'manifestonly' parameter", - DeprecationWarning, stacklevel=2) - global _doebuild_manifest_exempt_depend - try: - _doebuild_manifest_exempt_depend += 1 - distfiles_map = {} - fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb) - for cpv in fetchlist_dict: - try: - for myfile in fetchlist_dict[cpv]: - distfiles_map.setdefault(myfile, []).append(cpv) - except portage.exception.InvalidDependString as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e - return 0 - mytree = os.path.dirname(os.path.dirname(mysettings["O"])) - manifest1_compat = False - mf = Manifest(mysettings["O"], mysettings["DISTDIR"], - fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) - # Don't require all hashes since that can trigger excessive - # fetches when sufficient digests already exist. To ease transition - # while Manifest 1 is being removed, only require hashes that will - # exist before and after the transition. - required_hash_types = set() - required_hash_types.add("size") - required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) - dist_hashes = mf.fhashdict.get("DIST", {}) - - # To avoid accidental regeneration of digests with the incorrect - # files (such as partially downloaded files), trigger the fetch - # code if the file exists and it's size doesn't match the current - # manifest entry. If there really is a legitimate reason for the - # digest to change, `ebuild --force digest` can be used to avoid - # triggering this code (or else the old digests can be manually - # removed from the Manifest). - missing_files = [] - for myfile in distfiles_map: - myhashes = dist_hashes.get(myfile) - if not myhashes: - try: - st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) - except OSError: - st = None - if st is None or st.st_size == 0: - missing_files.append(myfile) - continue - size = myhashes.get("size") - - try: - st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - if size == 0: - missing_files.append(myfile) - continue - if required_hash_types.difference(myhashes): - missing_files.append(myfile) - continue - else: - if st.st_size == 0 or size is not None and size != st.st_size: - missing_files.append(myfile) - continue - - if missing_files: - mytree = os.path.realpath(os.path.dirname( - os.path.dirname(mysettings["O"]))) - fetch_settings = config(clone=mysettings) - debug = mysettings.get("PORTAGE_DEBUG") == "1" - for myfile in missing_files: - uris = set() - for cpv in distfiles_map[myfile]: - myebuild = os.path.join(mysettings["O"], - catsplit(cpv)[1] + ".ebuild") - # for RESTRICT=fetch, mirror, etc... - doebuild_environment(myebuild, "fetch", - mysettings["ROOT"], fetch_settings, - debug, 1, myportdb) - uris.update(myportdb.getFetchMap( - cpv, mytree=mytree)[myfile]) - - fetch_settings["A"] = myfile # for use by pkg_nofetch() - - try: - st = os.stat(os.path.join( - mysettings["DISTDIR"],myfile)) - except OSError: - st = None - - if not fetch({myfile : uris}, fetch_settings): - writemsg(_("!!! Fetch failed for %s, can't update " - "Manifest\n") % myfile, noiselevel=-1) - if myfile in dist_hashes and \ - st is not None and st.st_size > 0: - # stat result is obtained before calling fetch(), - # since fetch may rename the existing file if the - # digest does not match. - writemsg(_("!!! If you would like to " - "forcefully replace the existing " - "Manifest entry\n!!! for %s, use " - "the following command:\n") % myfile + \ - "!!! " + colorize("INFORM", - "ebuild --force %s manifest" % \ - os.path.basename(myebuild)) + "\n", - noiselevel=-1) - return 0 - writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) - try: - mf.create(assumeDistHashesSometimes=True, - assumeDistHashesAlways=( - "assume-digests" in mysettings.features)) - except portage.exception.FileNotFound as e: - writemsg(_("!!! File %s doesn't exist, can't update " - "Manifest\n") % e, noiselevel=-1) - return 0 - except portage.exception.PortagePackageException as e: - writemsg(("!!! %s\n") % (e,), noiselevel=-1) - return 0 - try: - mf.write(sign=False) - except portage.exception.PermissionDenied as e: - writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1) - return 0 - if "assume-digests" not in mysettings.features: - distlist = list(mf.fhashdict.get("DIST", {})) - distlist.sort() - auto_assumed = [] - for filename in distlist: - if not os.path.exists( - os.path.join(mysettings["DISTDIR"], filename)): - auto_assumed.append(filename) - if auto_assumed: - mytree = os.path.realpath( - os.path.dirname(os.path.dirname(mysettings["O"]))) - cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:]) - pkgs = myportdb.cp_list(cp, mytree=mytree) - pkgs.sort() - writemsg_stdout(" digest.assumed" + portage.output.colorize("WARN", - str(len(auto_assumed)).rjust(18)) + "\n") - for pkg_key in pkgs: - fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree) - pv = pkg_key.split("/")[1] - for filename in auto_assumed: - if filename in fetchlist: - writemsg_stdout( - " %s::%s\n" % (pv, filename)) - return 1 - finally: - _doebuild_manifest_exempt_depend -= 1 - def digestParseFile(myfilename, mysettings=None): """(filename) -- Parses a given file for entries matching: diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index edeb5aaa6..6a00a1e61 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -21,11 +21,12 @@ import time import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.config:check_config_instance', + 'portage.util.digestgen:digestgen', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) from portage import auxdbkeys, bsd_chflags, dep_check, digestcheck, \ - digestgen, eapi_is_supported, merge, os, selinux, StringIO, \ + eapi_is_supported, merge, os, selinux, StringIO, \ unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \ _shell_quote, _split_ebuild_name_glep55, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_BINARY, \ diff --git a/pym/portage/util/digestgen.py b/pym/portage/util/digestgen.py new file mode 100644 index 000000000..2dd0caef4 --- /dev/null +++ b/pym/portage/util/digestgen.py @@ -0,0 +1,204 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['digestgen'] + +import errno +import warnings + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.package.ebuild.doebuild:doebuild_environment', +) + +from portage import os +from portage.const import MANIFEST2_REQUIRED_HASH +from portage.dbapi.porttree import FetchlistDict +from portage.exception import InvalidDependString, FileNotFound, \ + PermissionDenied, PortagePackageException +from portage.localization import _ +from portage.manifest import Manifest +from portage.output import colorize +from portage.package.ebuild.config import config +from portage.package.ebuild.fetch import fetch +from portage.util import writemsg, writemsg_stdout +from portage.versions import catsplit + +def digestgen(myarchives=None, mysettings=None, + overwrite=None, manifestonly=None, myportdb=None): + """ + Generates a digest file if missing. Fetches files if necessary. + NOTE: myarchives and mysettings used to be positional arguments, + so their order must be preserved for backward compatibility. + @param mysettings: the ebuild config (mysettings["O"] must correspond + to the ebuild's parent directory) + @type mysettings: config + @param myportdb: a portdbapi instance + @type myportdb: portdbapi + @rtype: int + @returns: 1 on success and 0 on failure + """ + if mysettings is None: + raise TypeError("portage.digestgen(): missing" + \ + " required 'mysettings' parameter") + if myportdb is None: + warnings.warn("portage.digestgen() called without 'myportdb' parameter", + DeprecationWarning, stacklevel=2) + myportdb = portage.portdb + if overwrite is not None: + warnings.warn("portage.digestgen() called with " + \ + "deprecated 'overwrite' parameter", + DeprecationWarning, stacklevel=2) + if manifestonly is not None: + warnings.warn("portage.digestgen() called with " + \ + "deprecated 'manifestonly' parameter", + DeprecationWarning, stacklevel=2) + + try: + portage._doebuild_manifest_exempt_depend += 1 + distfiles_map = {} + fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb) + for cpv in fetchlist_dict: + try: + for myfile in fetchlist_dict[cpv]: + distfiles_map.setdefault(myfile, []).append(cpv) + except InvalidDependString as e: + writemsg("!!! %s\n" % str(e), noiselevel=-1) + del e + return 0 + mytree = os.path.dirname(os.path.dirname(mysettings["O"])) + manifest1_compat = False + mf = Manifest(mysettings["O"], mysettings["DISTDIR"], + fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) + # Don't require all hashes since that can trigger excessive + # fetches when sufficient digests already exist. To ease transition + # while Manifest 1 is being removed, only require hashes that will + # exist before and after the transition. + required_hash_types = set() + required_hash_types.add("size") + required_hash_types.add(MANIFEST2_REQUIRED_HASH) + dist_hashes = mf.fhashdict.get("DIST", {}) + + # To avoid accidental regeneration of digests with the incorrect + # files (such as partially downloaded files), trigger the fetch + # code if the file exists and it's size doesn't match the current + # manifest entry. If there really is a legitimate reason for the + # digest to change, `ebuild --force digest` can be used to avoid + # triggering this code (or else the old digests can be manually + # removed from the Manifest). + missing_files = [] + for myfile in distfiles_map: + myhashes = dist_hashes.get(myfile) + if not myhashes: + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError: + st = None + if st is None or st.st_size == 0: + missing_files.append(myfile) + continue + size = myhashes.get("size") + + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError as e: + if e.errno != errno.ENOENT: + raise + del e + if size == 0: + missing_files.append(myfile) + continue + if required_hash_types.difference(myhashes): + missing_files.append(myfile) + continue + else: + if st.st_size == 0 or size is not None and size != st.st_size: + missing_files.append(myfile) + continue + + if missing_files: + mytree = os.path.realpath(os.path.dirname( + os.path.dirname(mysettings["O"]))) + fetch_settings = config(clone=mysettings) + debug = mysettings.get("PORTAGE_DEBUG") == "1" + for myfile in missing_files: + uris = set() + for cpv in distfiles_map[myfile]: + myebuild = os.path.join(mysettings["O"], + catsplit(cpv)[1] + ".ebuild") + # for RESTRICT=fetch, mirror, etc... + doebuild_environment(myebuild, "fetch", + mysettings["ROOT"], fetch_settings, + debug, 1, myportdb) + uris.update(myportdb.getFetchMap( + cpv, mytree=mytree)[myfile]) + + fetch_settings["A"] = myfile # for use by pkg_nofetch() + + try: + st = os.stat(os.path.join( + mysettings["DISTDIR"],myfile)) + except OSError: + st = None + + if not fetch({myfile : uris}, fetch_settings): + writemsg(_("!!! Fetch failed for %s, can't update " + "Manifest\n") % myfile, noiselevel=-1) + if myfile in dist_hashes and \ + st is not None and st.st_size > 0: + # stat result is obtained before calling fetch(), + # since fetch may rename the existing file if the + # digest does not match. + writemsg(_("!!! If you would like to " + "forcefully replace the existing " + "Manifest entry\n!!! for %s, use " + "the following command:\n") % myfile + \ + "!!! " + colorize("INFORM", + "ebuild --force %s manifest" % \ + os.path.basename(myebuild)) + "\n", + noiselevel=-1) + return 0 + writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) + try: + mf.create(assumeDistHashesSometimes=True, + assumeDistHashesAlways=( + "assume-digests" in mysettings.features)) + except FileNotFound as e: + writemsg(_("!!! File %s doesn't exist, can't update " + "Manifest\n") % e, noiselevel=-1) + return 0 + except PortagePackageException as e: + writemsg(("!!! %s\n") % (e,), noiselevel=-1) + return 0 + try: + mf.write(sign=False) + except PermissionDenied as e: + writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1) + return 0 + if "assume-digests" not in mysettings.features: + distlist = list(mf.fhashdict.get("DIST", {})) + distlist.sort() + auto_assumed = [] + for filename in distlist: + if not os.path.exists( + os.path.join(mysettings["DISTDIR"], filename)): + auto_assumed.append(filename) + if auto_assumed: + mytree = os.path.realpath( + os.path.dirname(os.path.dirname(mysettings["O"]))) + cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:]) + pkgs = myportdb.cp_list(cp, mytree=mytree) + pkgs.sort() + writemsg_stdout(" digest.assumed" + colorize("WARN", + str(len(auto_assumed)).rjust(18)) + "\n") + for pkg_key in pkgs: + fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree) + pv = pkg_key.split("/")[1] + for filename in auto_assumed: + if filename in fetchlist: + writemsg_stdout( + " %s::%s\n" % (pv, filename)) + return 1 + finally: + portage._doebuild_manifest_exempt_depend -= 1 -- 2.26.2