From: Fabian Groffen Date: Mon, 26 Feb 2007 21:52:36 +0000 (-0000) Subject: Merged from trunk 6039:6048 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=02c57b76626cd0b8b3688be24cfbf75638f88a91;p=portage.git Merged from trunk 6039:6048 For bug #167667, use the existence of ${PORTDIR}/manifest1_obsolete to trigger the disabling of manifest1/digest file generation. This toggle is repo/overlay specific, allowing a specific overlay to drop manifest1 buy adding the manifest1_obsolete file. For bug #167667, add support for ${PORTDIR}/manifest1_obsolete and automatically remove ${FILESDIR}/digest-* from cvs when appropriate. (prefix: do the same thing for SVN) Don't bother to parse ${FILESDIR}/digest-* when manifest1_compat is disabled. For bug #167668, use an rsync "hide" filter rule to exclude **/files/digest-* and delete them on the receiver. Remove quotes since rsync doesn't accept them (and there is no shell to remove them since we use spawn() for rsync now). svn path=/main/branches/prefix/; revision=6083 --- diff --git a/bin/repoman b/bin/repoman index 328485a6f..3d02f3dea 100755 --- a/bin/repoman +++ b/bin/repoman @@ -629,9 +629,13 @@ kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list"))) if not kwlist: err("Couldn't read KEYWORDS from arch.list") +manifest1_compat = not os.path.exists( + os.path.join(portdir, "manifest1_obsolete")) if portdir_overlay != portdir: kwlist.update(portage.grabfile( os.path.join(portdir_overlay, "profiles", "arch.list"))) + manifest1_compat = not os.path.exists( + os.path.join(portdir_overlay, "manifest1_obsolete")) scanlist=[] if repolevel==2: @@ -938,8 +942,10 @@ for x in scanlist: if os.path.exists(checkdir+"/files"): filesdirlist=os.listdir(checkdir+"/files") - for y in filesdirlist: - if y[:7]=="digest-": + if manifest1_compat: + for y in filesdirlist: + if not y.startswith("digest-"): + continue if y[7:] not in dadded: #digest not added to cvs or svn stats["digest.notadded"]=stats["digest.notadded"]+1 @@ -1072,7 +1078,8 @@ for x in scanlist: if y in dadded: stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1 fails["ebuild.disjointed"].append(x+"/"+y+".ebuild") - if not os.path.exists(checkdir+"/files/digest-"+y): + if manifest1_compat and \ + not os.path.exists(os.path.join(checkdir, "files", "digest-"+y)): if mymode=="fix": if "--pretend" in myoptions: print "You will need to run:" @@ -1757,6 +1764,54 @@ elif "--pretend" not in myoptions: except AttributeError: print "Failed to insert portage version in message!" commitmessage+="\n(Portage version: Unknown)" + + if not manifest1_compat: + myfiles = myupdates + myremoved + mymanifests + filesdirs = set() + if repolevel == 3: + filesdirs.add(os.path.join(".", "files")) + elif repolevel in (1, 2): + for x in myfiles: + xs = x.split(os.path.sep) + if len(xs) < 4-repolevel: + continue + xs = xs[0:4-repolevel] + xs.append("files") + filesdirs.add(os.path.join(*xs)) + else: + raise AssertionError("repolevel=%s" % str(repolevel)) + digest_files = [] + for x in filesdirs: + dir_path = os.path.join(startdir, x) + if not os.path.isdir(dir_path): + continue + for y in os.listdir(dir_path): + if y.startswith("digest-"): + digest_files.append(os.path.join(x, y)) + if digest_files: + digest_files.sort() + if "--pretend" in myoptions: + print "(rm %s)" % " ".join(digest_files) + if isCvs: + print "(cvs remove %s)" % " ".join(digest_files) + elif isSvn: + print "(svn remove %s)" % " ".join(digest_files) + else: + for x in digest_files: + os.unlink(os.path.join(startdir, x)) + if isCvs: + retval = spawn(["cvs", "remove"] + digest_files, + env=os.environ) + elif isSvn: + retval = spawn(["svn", "remove"] + digest_files, + env=os.environ) + if retval != os.EX_OK: + print "!!! Exiting on vcs (shell) error code:",retval + sys.exit(retval) + myremoved.extend(digest_files) + myremoved = list(set(myremoved)) + myremoved.sort() + if myupdates or myremoved: myfiles = myupdates + myremoved fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") diff --git a/cnf/make.globals b/cnf/make.globals index c4ea2b572..e88550de9 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -54,7 +54,7 @@ PORTAGE_RSYNC_RETRIES="3" # Number of seconds rsync will wait before timing out. #RSYNC_TIMEOUT="180" -PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --compress --force --whole-file --delete --delete-after --stats --timeout=180 --exclude='/distfiles' --exclude='/local' --exclude='/packages'" +PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --compress --force --whole-file --delete --delete-after --stats --timeout=180 --exclude=/distfiles --exclude=/local --exclude=/packages --filter=H_**/files/digest-*" # Minimal CONFIG_PROTECT CONFIG_PROTECT="/etc" diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index d70a788c2..42412730e 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -3796,9 +3796,10 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): "--delete-after", # Delete only after everything else is done "--stats", # Show final statistics about what was transfered "--timeout="+str(mytimeout), # IO timeout if not done in X seconds - "--exclude='/distfiles'", # Exclude distfiles from consideration - "--exclude='/local'", # Exclude local from consideration - "--exclude='/packages'", # Exclude packages from consideration + "--exclude=/distfiles", # Exclude distfiles from consideration + "--exclude=/local", # Exclude local from consideration + "--exclude=/packages", # Exclude packages from consideration + "--filter=H_**/files/digest-*" # Exclude manifest1 digests and delete on the receiving side ]) else: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 54181b746..48ae2f437 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2619,8 +2619,11 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None writemsg("!!! Invalid SRC_URI for '%s'.\n" % cpv, noiselevel=-1) del e return 0 + mytree = os.path.dirname(os.path.dirname(mysettings["O"])) + manifest1_compat = not os.path.exists( + os.path.join(mytree, "manifest1_obsolete")) mf = Manifest(mysettings["O"], mysettings["DISTDIR"], - fetchlist_dict=fetchlist_dict) + fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) # Don't require all hashes since that can trigger excessive # fetches when sufficient digests already exist. To ease transition # while Manifest 1 is being removed, only require hashes that will diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 732d403f8..3405d7926 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -117,9 +117,9 @@ class Manifest(object): self.hashes = sets.Set(self.hashes) for t in portage.const.MANIFEST2_IDENTIFIERS: self.fhashdict[t] = {} + self.compat = manifest1_compat if not from_scratch: self._read() - self.compat = manifest1_compat if fetchlist_dict != None: self.fetchlist_dict = fetchlist_dict else: @@ -181,8 +181,8 @@ class Manifest(object): self._readManifest(self.getFullname(), myhashdict=self.fhashdict) except FileNotFound: pass - self._readDigests(myhashdict=self.fhashdict) - + if self.compat: + self._readDigests(myhashdict=self.fhashdict) def _parseManifestLines(self, mylines): """Parse manifest lines and return a list of manifest entries.""" @@ -439,7 +439,8 @@ class Manifest(object): else: distfilehashes = {} self.__init__(self.pkgdir, self.distdir, - fetchlist_dict=self.fetchlist_dict, from_scratch=True) + fetchlist_dict=self.fetchlist_dict, from_scratch=True, + manifest1_compat=self.compat) for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(self.pkgdir): break for f in pkgdir_files: