From f3101b3adce6731790f80f83fafece54b7bd8a63 Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Fri, 23 Sep 2011 16:43:28 -0700 Subject: [PATCH] manifest: controllable per repo This adds three states to layout.conf key use-manifest; false, true, and strict. false means "don't use manifests at all" true means "use and generate manifests, but allow them to be missing" strict means "manifests must be used everywhere in this repo" BUG=chromium-os:11308 TEST=repoman manifest usage. --- man/make.conf.5 | 4 ---- man/portage.5 | 3 +++ pym/_emerge/EbuildFetcher.py | 21 ++++++++++++--------- pym/portage/const.py | 1 - pym/portage/manifest.py | 11 +++++++++-- pym/portage/package/ebuild/digestcheck.py | 3 +-- pym/portage/package/ebuild/digestgen.py | 6 ++++++ pym/portage/package/ebuild/doebuild.py | 7 +++---- pym/portage/repository/config.py | 15 ++++++++++++++- 9 files changed, 48 insertions(+), 23 deletions(-) diff --git a/man/make.conf.5 b/man/make.conf.5 index 3f3efae8b..6c64686c7 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -204,10 +204,6 @@ non-developers as well. The \fBsandbox\fR feature is very important and should not be disabled by default. .RS .TP -.B allow\-missing\-manifests -Allow missing manifest entries. This is primarily useful for temporary -trees or instances where manifests aren't used. -.TP .B assume\-digests When commiting work to cvs with \fBrepoman\fR(1), assume that all existing SRC_URI digests are correct. This feature also affects digest generation via diff --git a/man/portage.5 b/man/portage.5 index 2af667621..1d9545ab6 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -781,6 +781,9 @@ aliases = foo-overlay sign\-manifests = false # thin\-manifests only contain DIST entries thin\-manifests = true +# indicate that this repo requires manifests for each package, and is +# considered a failure if a manifest file is missing/incorrect +use\-manifests = strict .fi .RE .TP diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index 4389f8408..61c7848ad 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -21,7 +21,7 @@ class EbuildFetcher(SpawnProcess): __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall", "pkg", "prefetch") + \ - ("_digests", "_settings", "_uri_map") + ("_digests", "_manifest", "_settings", "_uri_map") def already_fetched(self, settings): """ @@ -40,7 +40,7 @@ class EbuildFetcher(SpawnProcess): digests = self._get_digests() distdir = settings["DISTDIR"] - allow_missing = "allow-missing-manifests" in settings.features + allow_missing = self._get_manifest().allow_missing for filename in uri_map: # Use stat rather than lstat since fetch() creates @@ -179,7 +179,7 @@ class EbuildFetcher(SpawnProcess): not in ('yes', 'true') rval = 1 - allow_missing = 'allow-missing-manifests' in self._settings.features + allow_missing = self._get_manifest().allow_missing try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, digests=copy.deepcopy(self._get_digests()), @@ -203,13 +203,16 @@ class EbuildFetcher(SpawnProcess): raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) return self.ebuild_path + def _get_manifest(self): + if self._manifest is None: + pkgdir = os.path.dirname(self._get_ebuild_path()) + self._manifest = self.pkg.root_config.settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))).load_manifest(pkgdir, None) + return self._manifest + def _get_digests(self): - if self._digests is not None: - return self._digests - pkgdir = os.path.dirname(self._get_ebuild_path()) - mf = self.pkg.root_config.settings.repositories.get_repo_for_location( - os.path.dirname(os.path.dirname(pkgdir))) - self._digests = mf.load_manifest(pkgdir, None).getTypeDigests("DIST") + if self._digests is None: + self._digests = self._get_manifest().getTypeDigests("DIST") return self._digests def _get_uri_map(self): diff --git a/pym/portage/const.py b/pym/portage/const.py index f24a1a912..8b5f4acab 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -86,7 +86,6 @@ EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure" "package", "preinst", "postinst","prerm", "postrm", "nofetch", "config", "info", "other") SUPPORTED_FEATURES = frozenset([ - "allow-missing-manifests", "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", "ccache", "chflags", "clean-logs", "collision-protect", "compress-build-logs", diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 449f9fdf4..7cac09cde 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -100,7 +100,8 @@ class Manifest2Entry(ManifestEntry): class Manifest(object): parsers = (parseManifest2,) def __init__(self, pkgdir, distdir, fetchlist_dict=None, - manifest1_compat=DeprecationWarning, from_scratch=False, thin=False): + manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, + allow_missing=False, allow_create=True): """ Create new Manifest instance for package in pkgdir. Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of @@ -135,6 +136,8 @@ class Manifest(object): self.guessType = guessThinManifestFileType else: self.guessType = guessManifestFileType + self.allow_missing = allow_missing + self.allow_create = allow_create def getFullname(self): """ Returns the absolute path to the Manifest file for this instance """ @@ -237,6 +240,8 @@ class Manifest(object): def write(self, sign=False, force=False): """ Write Manifest instance to disk, optionally signing it """ + if not self.allow_create: + return self.checkIntegrity() try: myentries = list(self._createManifestEntries()) @@ -265,7 +270,7 @@ class Manifest(object): if myentries: write_atomic(self.getFullname(), "".join("%s\n" % str(myentry) for myentry in myentries)) - else: + elif self.thin: # With thin manifest, there's no need to have # a Manifest file if there are no DIST entries. try: @@ -330,6 +335,8 @@ class Manifest(object): distfiles to raise a FileNotFound exception for (if no file or existing checksums are available), and defaults to all distfiles when not specified.""" + if not self.allow_create: + return if checkExisting: self.checkAllHashes() if assumeDistHashesSometimes or assumeDistHashesAlways: diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py index 6cbaad97a..067aacca9 100644 --- a/pym/portage/package/ebuild/digestcheck.py +++ b/pym/portage/package/ebuild/digestcheck.py @@ -27,7 +27,6 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": return 1 - allow_missing = "allow-missing-manifests" in mysettings.features pkgdir = mysettings["O"] if mf is None: mf = mysettings.repositories.get_repo_for_location( @@ -72,7 +71,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1) writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1) return 0 - if allow_missing or mf.thin: + if mf.thin or mf.allow_missing: # In this case we ignore any missing digests that # would otherwise be detected below. return 1 diff --git a/pym/portage/package/ebuild/digestgen.py b/pym/portage/package/ebuild/digestgen.py index d4146dd38..f14368d23 100644 --- a/pym/portage/package/ebuild/digestgen.py +++ b/pym/portage/package/ebuild/digestgen.py @@ -60,6 +60,12 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"], fetchlist_dict=fetchlist_dict) + + if not mf.allow_create: + writemsg_stdout(_(">>> Skipping creating Manifest for %s; " + "repository is configured to not use them\n") % mysettings["O"]) + return 1 + # Don't require all hashes since that can trigger excessive # fetches when sufficient digests already exist. To ease transition # while Manifest 1 is being removed, only require hashes that will diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index e89d77d17..bd33d917c 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -484,7 +484,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, global _doebuild_manifest_cache pkgdir = os.path.dirname(myebuild) manifest_path = os.path.join(pkgdir, "Manifest") - allow_missing_manifests = "allow-missing-manifests" in mysettings.features if tree == "porttree": repo_config = mysettings.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(pkgdir))) @@ -497,7 +496,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, not repo_config.thin_manifest and \ mydo not in ("digest", "manifest", "help") and \ not portage._doebuild_manifest_exempt_depend and \ - not (allow_missing_manifests and not os.path.exists(manifest_path)): + not (repo_config.allow_missing_manifests and not os.path.exists(manifest_path)): # Always verify the ebuild checksums before executing it. global _doebuild_broken_ebuilds @@ -522,7 +521,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, try: mf.checkFileHashes("EBUILD", os.path.basename(myebuild)) except KeyError: - if not (allow_missing_manifests and + if not (mf.allow_missing and os.path.basename(myebuild) not in mf.fhashdict["EBUILD"]): out = portage.output.EOutput() out.eerror(_("Missing digest for '%s'") % (myebuild,)) @@ -547,7 +546,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mf.getFullname() in _doebuild_broken_manifests: return 1 - if mf is not _doebuild_manifest_cache and not allow_missing_manifests: + if mf is not _doebuild_manifest_cache and not mf.allow_missing: # Make sure that all of the ebuilds are # actually listed in the Manifest. diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 8b1e641b2..3cb550152 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -42,7 +42,8 @@ class RepoConfig(object): """Stores config of one repository""" __slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo', - 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'sign_manifest', 'thin_manifest'] + 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'sign_manifest', 'thin_manifest', + 'allow_missing_manifest', 'create_manifest', 'disable_manifest'] def __init__(self, name, repo_opts): """Build a RepoConfig with options in repo_opts @@ -113,9 +114,16 @@ class RepoConfig(object): self.missing_repo_name = missing self.sign_manifest = True self.thin_manifest = False + self.allow_missing_manifest = False + self.create_manifest = True + self.disable_manifest = False def load_manifest(self, *args, **kwds): kwds['thin'] = self.thin_manifest + kwds['allow_missing'] = self.allow_missing_manifest + kwds['allow_create'] = self.create_manifest + if self.disable_manifest: + kwds['from_scratch'] = True return manifest.Manifest(*args, **kwds) def update(self, new_repo): @@ -345,6 +353,11 @@ class RepoConfigLoader(object): if layout_data.get('thin-manifests', '').lower() == 'true': repo.thin_manifest = True + manifest_policy = layout_data.get('use-manifests', 'strict').lower() + repo.allow_missing_manifest = manifest_policy != 'strict' + repo.create_manifest = manifest_policy != 'false' + repo.disable_manifest = manifest_policy == 'false' + #Take aliases into account. new_prepos = {} for repo_name, repo in prepos.items(): -- 2.26.2