Merged from trunk 6039:6048

For bug #167667, use the existence of ${PORTDIR}/manifest1_obsolete to
trigger the disabling of manifest1/digest file generation.  This toggle
is repo/overlay specific, allowing a specific overlay to drop manifest1
buy adding the manifest1_obsolete file.

For bug #167667, add support for ${PORTDIR}/manifest1_obsolete and
automatically remove ${FILESDIR}/digest-* from cvs when appropriate.
(prefix: do the same thing for SVN)

Don't bother to parse ${FILESDIR}/digest-* when manifest1_compat is
disabled.

For bug #167668, use an rsync "hide" filter rule to exclude
**/files/digest-* and delete them on the receiver.

Remove quotes since rsync doesn't accept them (and there is no shell to
remove them since we use spawn() for rsync now).

svn path=/main/branches/prefix/; revision=6083

diff --git a/bin/repoman b/bin/repoman
index 328485a6f59c0c898762dd0709a11bf4920bbef1..3d02f3dea12b561930e6b8bb79ef40991d922530 100755 (executable)
--- a/bin/repoman
+++ b/bin/repoman
@@ -629,9 +629,13 @@ kwlist = set(portage.grabfile(os.path.join(portdir, "profiles", "arch.list")))
 if not kwlist:
        err("Couldn't read KEYWORDS from arch.list")
 
+manifest1_compat = not os.path.exists(
+       os.path.join(portdir, "manifest1_obsolete"))
 if portdir_overlay != portdir:
        kwlist.update(portage.grabfile(
                os.path.join(portdir_overlay, "profiles", "arch.list")))
+       manifest1_compat = not os.path.exists(
+               os.path.join(portdir_overlay, "manifest1_obsolete"))
 
 scanlist=[]
 if repolevel==2:
@@ -938,8 +942,10 @@ for x in scanlist:
 
        if os.path.exists(checkdir+"/files"):
                filesdirlist=os.listdir(checkdir+"/files")
-               for y in filesdirlist:
-                       if y[:7]=="digest-":
+               if manifest1_compat:
+                       for y in filesdirlist:
+                               if not y.startswith("digest-"):
+                                       continue
                                if y[7:] not in dadded:
                                        #digest not added to cvs or svn
                                        stats["digest.notadded"]=stats["digest.notadded"]+1
@@ -1072,7 +1078,8 @@ for x in scanlist:
                        if y in dadded:
                                stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1
                                fails["ebuild.disjointed"].append(x+"/"+y+".ebuild")
-               if not os.path.exists(checkdir+"/files/digest-"+y):
+               if manifest1_compat and \
+                       not os.path.exists(os.path.join(checkdir, "files", "digest-"+y)):
                        if mymode=="fix":
                                if "--pretend" in myoptions:
                                        print "You will need to run:"
@@ -1757,6 +1764,54 @@ elif "--pretend" not in myoptions:
        except AttributeError:
                print "Failed to insert portage version in message!"
                commitmessage+="\n(Portage version: Unknown)"
+
+       if not manifest1_compat:
+               myfiles = myupdates + myremoved + mymanifests
+               filesdirs = set()
+               if repolevel == 3:
+                       filesdirs.add(os.path.join(".", "files"))
+               elif repolevel in (1, 2):
+                       for x in myfiles:
+                               xs = x.split(os.path.sep)
+                               if len(xs) < 4-repolevel:
+                                       continue
+                               xs = xs[0:4-repolevel]
+                               xs.append("files")
+                               filesdirs.add(os.path.join(*xs))
+               else:
+                       raise AssertionError("repolevel=%s" % str(repolevel))
+               digest_files = []
+               for x in filesdirs:
+                       dir_path = os.path.join(startdir, x)
+                       if not os.path.isdir(dir_path):
+                               continue
+                       for y in os.listdir(dir_path):
+                               if y.startswith("digest-"):
+                                       digest_files.append(os.path.join(x, y))
+               if digest_files:
+                       digest_files.sort()
+                       if "--pretend" in myoptions:
+                               print "(rm %s)" % " ".join(digest_files)
+                               if isCvs:
+                                       print "(cvs remove %s)" % " ".join(digest_files)
+                               elif isSvn:
+                                       print "(svn remove %s)" % " ".join(digest_files)
+                       else:
+                               for x in digest_files:
+                                       os.unlink(os.path.join(startdir, x))
+                               if isCvs:
+                                       retval = spawn(["cvs", "remove"] + digest_files,
+                                                       env=os.environ)
+                               elif isSvn:
+                                       retval = spawn(["svn", "remove"] + digest_files,
+                                                       env=os.environ)
+                               if retval != os.EX_OK:
+                                       print "!!! Exiting on vcs (shell) error code:",retval
+                                       sys.exit(retval)
+                       myremoved.extend(digest_files)
+                       myremoved = list(set(myremoved))
+                       myremoved.sort()
+
        if myupdates or myremoved:
                myfiles = myupdates + myremoved
                fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")

diff --git a/cnf/make.globals b/cnf/make.globals
index c4ea2b5726915b74a749dddd4517671915b7a23e..e88550de92de9d09a3663bf8a63cbb2fd8e12313 100644 (file)
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -54,7 +54,7 @@ PORTAGE_RSYNC_RETRIES="3"
 # Number of seconds rsync will wait before timing out.
 #RSYNC_TIMEOUT="180"
 
-PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --compress --force --whole-file --delete --delete-after --stats --timeout=180 --exclude='/distfiles' --exclude='/local' --exclude='/packages'"
+PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --compress --force --whole-file --delete --delete-after --stats --timeout=180 --exclude=/distfiles --exclude=/local --exclude=/packages --filter=H_**/files/digest-*"
 
 # Minimal CONFIG_PROTECT
 CONFIG_PROTECT="/etc"

diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py
index d70a788c26dffeb23e8687e6a029b7db477dc4e8..42412730e45653543f732259a7a0b3dd4759ed86 100644 (file)
--- a/pym/emerge/__init__.py
+++ b/pym/emerge/__init__.py
@@ -3796,9 +3796,10 @@ def action_sync(settings, trees, mtimedb, myopts, myaction):
                                "--delete-after", # Delete only after everything else is done
                                "--stats",        # Show final statistics about what was transfered
                                "--timeout="+str(mytimeout), # IO timeout if not done in X seconds
-                               "--exclude='/distfiles'",   # Exclude distfiles from consideration
-                               "--exclude='/local'",       # Exclude local     from consideration
-                               "--exclude='/packages'",    # Exclude packages  from consideration
+                               "--exclude=/distfiles",   # Exclude distfiles from consideration
+                               "--exclude=/local",       # Exclude local     from consideration
+                               "--exclude=/packages",    # Exclude packages  from consideration
+                               "--filter=H_**/files/digest-*" # Exclude manifest1 digests and delete on the receiving side
                        ])
 
                else:

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 54181b74605381ab2d90caa3b4f48cdf4af3d0fc..48ae2f4370643b9a193f1ba42a47cc3c502ab698 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -2619,8 +2619,11 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None
                                writemsg("!!! Invalid SRC_URI for '%s'.\n" % cpv, noiselevel=-1)
                                del e
                                return 0
+               mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
+               manifest1_compat = not os.path.exists(
+                       os.path.join(mytree, "manifest1_obsolete"))
                mf = Manifest(mysettings["O"], mysettings["DISTDIR"],
-                       fetchlist_dict=fetchlist_dict)
+                       fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat)
                # Don't require all hashes since that can trigger excessive
                # fetches when sufficient digests already exist.  To ease transition
                # while Manifest 1 is being removed, only require hashes that will

diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 732d403f8ef92dfd6672ac0ac9742647d47b44e2..3405d7926d63ce19c97d4ccbed0a4556771516fc 100644 (file)
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -117,9 +117,9 @@ class Manifest(object):
                self.hashes = sets.Set(self.hashes)
                for t in portage.const.MANIFEST2_IDENTIFIERS:
                        self.fhashdict[t] = {}
+               self.compat = manifest1_compat
                if not from_scratch:
                        self._read()
-               self.compat = manifest1_compat
                if fetchlist_dict != None:
                        self.fetchlist_dict = fetchlist_dict
                else:
@@ -181,8 +181,8 @@ class Manifest(object):
                        self._readManifest(self.getFullname(), myhashdict=self.fhashdict)
                except FileNotFound:
                        pass
-               self._readDigests(myhashdict=self.fhashdict)
-               
+               if self.compat:
+                       self._readDigests(myhashdict=self.fhashdict)
 
        def _parseManifestLines(self, mylines):
                """Parse manifest lines and return a list of manifest entries."""
@@ -439,7 +439,8 @@ class Manifest(object):
                else:
                        distfilehashes = {}
                self.__init__(self.pkgdir, self.distdir,
-                       fetchlist_dict=self.fetchlist_dict, from_scratch=True)
+                       fetchlist_dict=self.fetchlist_dict, from_scratch=True,
+                       manifest1_compat=self.compat)
                for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(self.pkgdir):
                        break
                for f in pkgdir_files: