Merged from trunk 8150:8174

   | 8151    | In dblink.getcontents(), use a regular expression to detect  |
   | zmedico | when path normalization is required. Also, only join with    |
   |         | ${ROOT} when necessary. This allows unnecessary normpath and |
   |         | join calls to be optimized away in the general case,         |
   |         | reducing the cpu time for `equery belongs ` by about 50%. M  |
   |         | pym/portage/dbapi/vartree.py                                 |

   | 8152    | Only re.compile() the normalize_needed pattern once rather   |
   | zmedico | than for each dblink constructor call.                       |

   | 8153    | Optimize dblink.isowner() to use fewer stat calls by only    |
   | zmedico | collecting stat results for parent directories. This         |
   |         | provides equivalent accuracy to the previous approach but    |
   |         | will perform much better when used to scan all installed     |
   |         | packages for owners in the event of a file collision.        |

   | 8154    | Bug #196043 - Implement a `portageq owners []+` command that |
   | zmedico | is suitable for identifying all packages that own one or     |
   |         | more files when a file collision has occurred. This uses     |
   |         | dblink.isowner() so that the query works properly even when  |
   |         | paths are ambiguous due to symlinked directories.            |

   | 8155    | Bug #196043 - Update the file collision ewarn notice to try  |
   | zmedico | and clarify the cases when a bug should NOT be filed. Also,  |
   |         | recommend the new `portageq owners / ` command since it      |
   |         | works properly even when paths are ambiguous due to          |
   |         | symlinked directories.                                       |

   | 8161    | Add missing "to" to fix grammer.                             |
   | zmedico |                                                              |

   | 8163    | Bug #196043 - Unify the ouput handling for file collisions   |
   | zmedico | so that similar eerror messages are generated whether or not |
   |         | collision-protect is enabled.                                |

   | 8166    | Fix the normalize_needed regex to properly match // anywhere |
   | zmedico | in the path. Also add support for detection of . or .. where |
   |         | appropriate.                                                 |

   | 8168    | Fix collision-protect so that it properly cancels the        |
   | zmedico | preinst phase like it used to.                               |

   | 8170    | Adjust quote usage in collision-protect eerror output.       |
   | zmedico |                                                              |

   | 8172    | Bug #195375 - Make dblink.treewalk() read inforoot/SLOT      |
   | zmedico | since it differs from the expected SLOT value when when      |
   |         | USE=multislot is enabled. A warning message will be shown if |
   |         | the slot differs and --quiet mode is not enabled.            |

   | 8174    | Handle EnvironmentError instead of OSError since open()      |
   | zmedico | actually raises IOError. Also, treat a missing SLOT file as  |
   |         | SLOT="" since it is currently possible to install an ebuild  |
   |         | with an undefined SLOT even though repoman generates a       |
   |         | SLOT.missing error with such an ebuild.                      |

svn path=/main/branches/prefix/; revision=8210

diff --git a/bin/portageq b/bin/portageq
index e9009c3fa1688a2da6e75b134f013b67c8fcfc48..a0642b03fc1f9f6e68c9094720421be4d8ff1942 100755 (executable)
--- a/bin/portageq
+++ b/bin/portageq
@@ -123,6 +123,69 @@ def metadata(argv):
 
 metadata.uses_root = True
 
+def owners(argv):
+       """<root> [<filename>]+
+       Given a list of files, print the packages that own the files and which
+       files belong to each package. Files owned by a package are listed on
+       the lines below it, indented by a single tab character (\\t). All file
+       paths must start with <root>. Returns 1 if no owners could be found,
+       and 0 otherwise.
+       """
+       if len(argv) < 2:
+               sys.stderr.write("ERROR: insufficient parameters!\n")
+               sys.stderr.flush()
+               return 2
+
+       from portage import catsplit, dblink
+       settings = portage.settings
+       root = settings["ROOT"]
+       vardb = portage.db[root]["vartree"].dbapi
+
+       cwd = None
+       try:
+               cwd = os.getcwd()
+       except OSError:
+               pass
+
+       files = []
+       for f in argv[1:]:
+               f = portage.normalize_path(f)
+               if not f.startswith(os.path.sep):
+                       if cwd is None:
+                               sys.stderr.write("ERROR: cwd does not exist!\n")
+                               sys.stderr.flush()
+                               return 2
+                       f = os.path.join(cwd, f)
+                       f = portage.normalize_path(f)
+               if not f.startswith(root):
+                       sys.stderr.write("ERROR: file paths must begin with <root>!\n")
+                       sys.stderr.flush()
+                       return 2
+               files.append(f[len(root):])
+
+       found_owner = False
+       for cpv in vardb.cpv_all():
+               cat, pkg = catsplit(cpv)
+               mylink = dblink(cat, pkg, root, settings, vartree=vardb.vartree)
+               myfiles = []
+               for f in files:
+                       if mylink.isowner(f, root):
+                               myfiles.append(f)
+               if myfiles:
+                       found_owner = True
+                       sys.stdout.write("%s\n" % cpv)
+                       for f in myfiles:
+                               sys.stdout.write("\t%s\n" % \
+                                       os.path.join(root, f.lstrip(os.path.sep)))
+                       sys.stdout.flush()
+       if not found_owner:
+               sys.stderr.write("None of the installed packages claim the file(s).\n")
+               sys.stderr.flush()
+               return 1
+       return 0
+
+owners.uses_root = True
+
 def best_visible(argv):
        """<root> [<category/package>]+
        Returns category/package-version (without .ebuild).
@@ -373,7 +436,9 @@ def main():
                        import portage
                if uses_root:
                        sys.argv[2] = portage.root
-               function(sys.argv[2:])
+               retval = function(sys.argv[2:])
+               if retval:
+                       sys.exit(retval)
        except KeyError:
                usage(sys.argv)
                sys.exit(os.EX_USAGE)

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a298f48b38b893f7430eefb560b9a4f88342aeb9..e66e76672f6e1a931ab471a19e430098b09bde03 100644 (file)
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -20,7 +20,7 @@ from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs,
 from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp
 
 from portage import listdir, dep_expand, flatten, key_expand, \
-       doebuild_environment, doebuild, env_update, \
+       doebuild_environment, doebuild, env_update, prepare_build_dirs, \
        abssymlink, movefile, _movefile, bsd_chflags
 
 from portage.elog import elog_process
@@ -821,6 +821,10 @@ class dblink(object):
        This class provides an interface to the installed package database
        At present this is implemented as a text backend in /var/db/pkg.
        """
+
+       import re
+       _normalize_needed = re.compile(r'.*//.*|^[^/]|.+/$|(^|.*/)\.\.?(/.*|$)')
+
        def __init__(self, cat, pkg, myroot, mysettings, treetype=None,
                vartree=None):
                """
@@ -944,6 +948,10 @@ class dblink(object):
                mylines = myc.readlines()
                myc.close()
                null_byte = "\0"
+               normalize_needed = self._normalize_needed
+               myroot = self.myroot
+               if myroot == os.path.sep:
+                       myroot = None
                pos = 0
                for line in mylines:
                        pos += 1
@@ -957,8 +965,12 @@ class dblink(object):
                        # we do this so we can remove from non-root filesystems
                        # (use the ROOT var to allow maintenance on other partitions)
                        try:
-                               mydat[1] = normalize_path(os.path.join(
-                                       self.myroot, mydat[1].lstrip(os.path.sep)))
+                               if normalize_needed.match(mydat[1]):
+                                       mydat[1] = normalize_path(mydat[1])
+                                       if not mydat[1].startswith(os.path.sep):
+                                               mydat[1] = os.path.sep + mydat[1]
+                               if myroot:
+                                       mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep))
                                if mydat[0] == "obj":
                                        #format: type, mtime, md5sum
                                        pkgfiles[" ".join(mydat[1:-2])] = [mydat[0], mydat[-1], mydat[-2]]
@@ -1368,9 +1380,14 @@ class dblink(object):
 
        def isowner(self,filename, destroot):
                """ 
-               Check if filename is a new file or belongs to this package
-               (for this or a previous version)
-               
+               Check if a file belongs to this package. This may
+               result in a stat call for the parent directory of
+               every installed file, since the inode numbers are
+               used to work around the problem of ambiguous paths
+               caused by symlinked directories. The results of
+               stat calls are cached to optimize multiple calls
+               to this method.
+
                @param filename:
                @type filename:
                @param destroot:
@@ -1387,23 +1404,46 @@ class dblink(object):
                if pkgfiles and destfile in pkgfiles:
                        return True
                if pkgfiles:
+                       # Use stat rather than lstat since we want to follow
+                       # any symlinks to the real parent directory.
+                       parent_path = os.path.dirname(destfile)
                        try:
-                               mylstat = os.lstat(destfile)
+                               parent_stat = os.stat(parent_path)
                        except EnvironmentError, e:
                                if e.errno != errno.ENOENT:
                                        raise
                                del e
-                               return True
+                               return False
                        if self._contents_inodes is None:
-                               self._contents_inodes = set()
+                               self._contents_inodes = {}
+                               parent_paths = set()
                                for x in pkgfiles:
+                                       p_path = os.path.dirname(x)
+                                       if p_path in parent_paths:
+                                               continue
+                                       parent_paths.add(p_path)
                                        try:
-                                               lstat = os.lstat(x)
-                                               self._contents_inodes.add((lstat.st_dev, lstat.st_ino))
+                                               s = os.stat(p_path)
                                        except OSError:
                                                pass
-                       if (mylstat.st_dev, mylstat.st_ino) in self._contents_inodes:
-                                return True
+                                       else:
+                                               inode_key = (s.st_dev, s.st_ino)
+                                               # Use lists of paths in case multiple
+                                               # paths reference the same inode.
+                                               p_path_list = self._contents_inodes.get(inode_key)
+                                               if p_path_list is None:
+                                                       p_path_list = []
+                                                       self._contents_inodes[inode_key] = p_path_list
+                                               if p_path not in p_path_list:
+                                                       p_path_list.append(p_path)
+                       p_path_list = self._contents_inodes.get(
+                               (parent_stat.st_dev, parent_stat.st_ino))
+                       if p_path_list:
+                               basename = os.path.basename(destfile)
+                               for p_path in p_path_list:
+                                       x = os.path.join(p_path, basename)
+                                       if x in pkgfiles:
+                                               return True
 
                return False
 
@@ -1540,50 +1580,6 @@ class dblink(object):
                                                                        break
                                        if stopmerge:
                                                collisions.append(f)
-                       if stopmerge and "collision-protect" in self.settings.features:
-                               print red("*")+" This package is blocked because it wants to overwrite"
-                               print red("*")+" files belonging to other packages (see list below)."
-                               print red("*")+" If you have no clue what this is all about report it "
-                               print red("*")+" as a bug for this package on http://bugs.gentoo.org"
-                               print
-                               print red("package "+self.cat+"/"+self.pkg+" NOT merged")
-                               print
-                               print "Detected file collision(s):"
-                               print
-                               from portage.output import colorize
-                               for f in collisions:
-                                       print "     '%s'" % colorize("INFORM",
-                                               os.path.join(destroot, f.lstrip(os.path.sep)))
-                               print
-                               print "Searching all installed packages for file collisions..."
-                               print "Press Ctrl-C to Stop"
-                               print
-                               """ Note: The isowner calls result in a stat call for *every*
-                               single installed file, since the inode numbers are used to work
-                               around the problem of ambiguous paths caused by symlinked files
-                               and/or directories.  Though it is slow, it is as accurate as
-                               possible."""
-                               found_owner = False
-                               for cpv in self.vartree.dbapi.cpv_all():
-                                       cat, pkg = catsplit(cpv)
-                                       mylink = dblink(cat, pkg, destroot, self.settings,
-                                               vartree=self.vartree)
-                                       mycollisions = []
-                                       for f in collisions:
-                                               if mylink.isowner(f, destroot):
-                                                       mycollisions.append(f)
-                                       if mycollisions:
-                                               found_owner = True
-                                               print " * %s:" % cpv
-                                               print
-                                               for f in mycollisions:
-                                                       print "     '%s'" % \
-                                                               os.path.join(destroot, f.lstrip(os.path.sep))
-                                               print
-                               if not found_owner:
-                                       print "None of the installed packages claim the above file(s)."
-                                       print
-                               sys.exit(1)
                        return collisions
 
        def _security_check(self, installed_instances):
@@ -1675,6 +1671,26 @@ class dblink(object):
                                noiselevel=-1)
                        return 1
 
+               inforoot_slot_file = os.path.join(inforoot, "SLOT")
+               slot = None
+               try:
+                       f = open(inforoot_slot_file)
+                       try:
+                               slot = f.read().strip()
+                       finally:
+                               f.close()
+               except EnvironmentError, e:
+                       if e.errno != errno.ENOENT:
+                               raise
+                       del e
+
+               if slot is None:
+                       slot = ""
+
+               if slot != self.settings["SLOT"]:
+                       writemsg("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \
+                               (self.settings["SLOT"], slot))
+
                if not os.path.exists(self.dbcatdir):
                        os.makedirs(self.dbcatdir)
 
@@ -1683,7 +1699,7 @@ class dblink(object):
                        otherversions.append(v.split("/")[1])
 
                slot_matches = self.vartree.dbapi.match(
-                       "%s:%s" % (self.mysplit[0], self.settings["SLOT"]))
+                       "%s:%s" % (self.mysplit[0], slot))
                if self.mycpv not in slot_matches and \
                        self.vartree.dbapi.cpv_exists(self.mycpv):
                        # handle multislot or unapplied slotmove
@@ -1744,64 +1760,116 @@ class dblink(object):
                collisions = self._collision_protect(srcroot, destroot, others_in_slot,
                        myfilelist+mylinklist)
 
-               if True:
-                       """ The merge process may move files out of the image directory,
-                       which causes invalidation of the .installed flag."""
-                       try:
-                               os.unlink(os.path.join(
-                                       os.path.dirname(normalize_path(srcroot)), ".installed"))
-                       except OSError, e:
-                               if e.errno != errno.ENOENT:
-                                       raise
-                               del e
-
-               self.dbdir = self.dbtmpdir
-               self.delete()
-               if not os.path.exists(self.dbtmpdir):
-                       os.makedirs(self.dbtmpdir)
-
-               writemsg_stdout(">>> Merging %s %s %s\n" % (self.mycpv,"to",destroot))
-
-               # run preinst script
+               # Make sure the ebuild environment is initialized and that ${T}/elog
+               # exists for logging of collision-protect eerror messages.
                if myebuild is None:
                        myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
-               a = doebuild(myebuild, "preinst", destroot, self.settings, cleanup=cleanup,
-                       use_cache=0, tree=self.treetype, mydbapi=mydbapi,
-                       vartree=self.vartree)
+               doebuild_environment(myebuild, "preinst", destroot,
+                       self.settings, 0, 0, mydbapi)
+               prepare_build_dirs(destroot, self.settings, cleanup)
+
+               def eerror(lines):
+                       cmd = "source '%s/isolated-functions.sh' ; " % PORTAGE_BIN_PATH
+                       for line in lines:
+                               cmd += "eerror '%s' ; " % line
+                       from portage import process
+                       process.spawn(["bash", "-c", cmd],
+                               env=self.settings.environ())
 
                if collisions:
+                       collision_protect = "collision-protect" in self.settings.features
                        msg = "This package will overwrite one or more files that" + \
-                       " may belong to other packages (see list below)." + \
-                       " Add \"collision-protect\" to FEATURES in make.conf" + \
-                       " if you would like the merge to abort in cases like this."
+                       " may belong to other packages (see list below)."
+                       if not collision_protect:
+                               msg += " Add \"collision-protect\" to FEATURES in" + \
+                               " make.conf if you would like the merge to abort" + \
+                               " in cases like this."
                        if self.settings.get("PORTAGE_QUIET") != "1":
-                               msg += " If you have determined that one or more of the" + \
-                               " files actually belong to another installed package then" + \
-                               " go to http://bugs.gentoo.org and report it as a bug." + \
-                               " Be sure to identify both this package and the other" + \
-                               " installed package in the bug report. Use a command such" + \
-                               " as \\`equery belongs <filename>\\` to identify the" + \
-                               " installed package that owns a file. Do NOT file a bug" + \
-                               " without reporting exactly which two packages install" + \
-                               " the same file(s)."
+                               msg += " You can use a command such as" + \
+                               " `portageq owners / <filename>` to identify the" + \
+                               " installed package that owns a file. If portageq" + \
+                               " reports that only one package owns a file then do NOT" + \
+                               " file a bug report. A bug report is only useful if it" + \
+                               " identifies at least two or more packages that are known" + \
+                               " to install the same file(s)." + \
+                               " If a collision occurs and you" + \
+                               " can not explain where the file came from then you" + \
+                               " should simply ignore the collision since there is not" + \
+                               " enough information to determine if a real problem" + \
+                               " exists. Please do NOT file a bug report at" + \
+                               " http://bugs.gentoo.org unless you report exactly which" + \
+                               " two packages install the same file(s). Once again," + \
+                               " please do NOT file a bug report unless you have" + \
+                               " completely understood the above message."
 
                        self.settings["EBUILD_PHASE"] = "preinst"
-                       cmd = "source '%s/isolated-functions.sh' ; " % PORTAGE_BIN_PATH
                        from textwrap import wrap
                        msg = wrap(msg, 70)
-                       for line in msg:
-                               cmd += "eerror \"%s\" ; " % line
-                       cmd += "eerror ; "
-                       cmd += "eerror \"Detected file collision(s):\" ; "
-                       cmd += "eerror ; "
+                       if collision_protect:
+                               msg.append("")
+                               msg.append("package %s NOT merged" % self.settings.mycpv)
+                       msg.append("")
+                       msg.append("Detected file collision(s):")
+                       msg.append("")
 
                        for f in collisions:
-                               cmd += "eerror \"     '%s'\" ; " % \
-                                       os.path.join(destroot, f.lstrip(os.path.sep))
+                               msg.append("\t%s" % \
+                                       os.path.join(destroot, f.lstrip(os.path.sep)))
 
-                       from portage import process
-                       process.spawn(["bash", "-c", cmd],
-                               env=self.settings.environ())
+                       eerror(msg)
+
+                       if collision_protect:
+                               msg = []
+                               msg.append("")
+                               msg.append("Searching all installed" + \
+                                       " packages for file collisions...")
+                               msg.append("")
+                               msg.append("Press Ctrl-C to Stop")
+                               msg.append("")
+                               eerror(msg)
+
+                               found_owner = False
+                               for cpv in self.vartree.dbapi.cpv_all():
+                                       cat, pkg = catsplit(cpv)
+                                       mylink = dblink(cat, pkg, destroot, self.settings,
+                                               vartree=self.vartree)
+                                       mycollisions = []
+                                       for f in collisions:
+                                               if mylink.isowner(f, destroot):
+                                                       mycollisions.append(f)
+                                       if mycollisions:
+                                               found_owner = True
+                                               msg = []
+                                               msg.append("%s" % cpv)
+                                               for f in mycollisions:
+                                                       msg.append("\t%s" % os.path.join(destroot,
+                                                               f.lstrip(os.path.sep)))
+                                               eerror(msg)
+                               if not found_owner:
+                                       eerror(["None of the installed" + \
+                                               " packages claim the file(s)."])
+                               return 1
+
+               writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot))
+
+               # The merge process may move files out of the image directory,
+               # which causes invalidation of the .installed flag.
+               try:
+                       os.unlink(os.path.join(
+                               os.path.dirname(normalize_path(srcroot)), ".installed"))
+               except OSError, e:
+                       if e.errno != errno.ENOENT:
+                               raise
+                       del e
+
+               self.dbdir = self.dbtmpdir
+               self.delete()
+               ensure_dirs(self.dbtmpdir)
+
+               # run preinst script
+               a = doebuild(myebuild, "preinst", destroot, self.settings,
+                       use_cache=0, tree=self.treetype, mydbapi=mydbapi,
+                       vartree=self.vartree)
 
                # XXX: Decide how to handle failures here.
                if a != os.EX_OK: