From: Fabian Groffen Date: Sun, 21 Oct 2007 12:30:10 +0000 (-0000) Subject: Merged from trunk 8150:8174 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=6f630c86ad1691f83d4c8827dc31570c8b768fd6;p=portage.git Merged from trunk 8150:8174 | 8151 | In dblink.getcontents(), use a regular expression to detect | | zmedico | when path normalization is required. Also, only join with | | | ${ROOT} when necessary. This allows unnecessary normpath and | | | join calls to be optimized away in the general case, | | | reducing the cpu time for `equery belongs ` by about 50%. M | | | pym/portage/dbapi/vartree.py | | 8152 | Only re.compile() the normalize_needed pattern once rather | | zmedico | than for each dblink constructor call. | | 8153 | Optimize dblink.isowner() to use fewer stat calls by only | | zmedico | collecting stat results for parent directories. This | | | provides equivalent accuracy to the previous approach but | | | will perform much better when used to scan all installed | | | packages for owners in the event of a file collision. | | 8154 | Bug #196043 - Implement a `portageq owners []+` command that | | zmedico | is suitable for identifying all packages that own one or | | | more files when a file collision has occurred. This uses | | | dblink.isowner() so that the query works properly even when | | | paths are ambiguous due to symlinked directories. | | 8155 | Bug #196043 - Update the file collision ewarn notice to try | | zmedico | and clarify the cases when a bug should NOT be filed. Also, | | | recommend the new `portageq owners / ` command since it | | | works properly even when paths are ambiguous due to | | | symlinked directories. | | 8161 | Add missing "to" to fix grammer. | | zmedico | | | 8163 | Bug #196043 - Unify the ouput handling for file collisions | | zmedico | so that similar eerror messages are generated whether or not | | | collision-protect is enabled. | | 8166 | Fix the normalize_needed regex to properly match // anywhere | | zmedico | in the path. Also add support for detection of . or .. where | | | appropriate. | | 8168 | Fix collision-protect so that it properly cancels the | | zmedico | preinst phase like it used to. | | 8170 | Adjust quote usage in collision-protect eerror output. | | zmedico | | | 8172 | Bug #195375 - Make dblink.treewalk() read inforoot/SLOT | | zmedico | since it differs from the expected SLOT value when when | | | USE=multislot is enabled. A warning message will be shown if | | | the slot differs and --quiet mode is not enabled. | | 8174 | Handle EnvironmentError instead of OSError since open() | | zmedico | actually raises IOError. Also, treat a missing SLOT file as | | | SLOT="" since it is currently possible to install an ebuild | | | with an undefined SLOT even though repoman generates a | | | SLOT.missing error with such an ebuild. | svn path=/main/branches/prefix/; revision=8210 --- diff --git a/bin/portageq b/bin/portageq index e9009c3fa..a0642b03f 100755 --- a/bin/portageq +++ b/bin/portageq @@ -123,6 +123,69 @@ def metadata(argv): metadata.uses_root = True +def owners(argv): + """ []+ + Given a list of files, print the packages that own the files and which + files belong to each package. Files owned by a package are listed on + the lines below it, indented by a single tab character (\\t). All file + paths must start with . Returns 1 if no owners could be found, + and 0 otherwise. + """ + if len(argv) < 2: + sys.stderr.write("ERROR: insufficient parameters!\n") + sys.stderr.flush() + return 2 + + from portage import catsplit, dblink + settings = portage.settings + root = settings["ROOT"] + vardb = portage.db[root]["vartree"].dbapi + + cwd = None + try: + cwd = os.getcwd() + except OSError: + pass + + files = [] + for f in argv[1:]: + f = portage.normalize_path(f) + if not f.startswith(os.path.sep): + if cwd is None: + sys.stderr.write("ERROR: cwd does not exist!\n") + sys.stderr.flush() + return 2 + f = os.path.join(cwd, f) + f = portage.normalize_path(f) + if not f.startswith(root): + sys.stderr.write("ERROR: file paths must begin with !\n") + sys.stderr.flush() + return 2 + files.append(f[len(root):]) + + found_owner = False + for cpv in vardb.cpv_all(): + cat, pkg = catsplit(cpv) + mylink = dblink(cat, pkg, root, settings, vartree=vardb.vartree) + myfiles = [] + for f in files: + if mylink.isowner(f, root): + myfiles.append(f) + if myfiles: + found_owner = True + sys.stdout.write("%s\n" % cpv) + for f in myfiles: + sys.stdout.write("\t%s\n" % \ + os.path.join(root, f.lstrip(os.path.sep))) + sys.stdout.flush() + if not found_owner: + sys.stderr.write("None of the installed packages claim the file(s).\n") + sys.stderr.flush() + return 1 + return 0 + +owners.uses_root = True + def best_visible(argv): """ []+ Returns category/package-version (without .ebuild). @@ -373,7 +436,9 @@ def main(): import portage if uses_root: sys.argv[2] = portage.root - function(sys.argv[2:]) + retval = function(sys.argv[2:]) + if retval: + sys.exit(retval) except KeyError: usage(sys.argv) sys.exit(os.EX_USAGE) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index a298f48b3..e66e76672 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -20,7 +20,7 @@ from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp from portage import listdir, dep_expand, flatten, key_expand, \ - doebuild_environment, doebuild, env_update, \ + doebuild_environment, doebuild, env_update, prepare_build_dirs, \ abssymlink, movefile, _movefile, bsd_chflags from portage.elog import elog_process @@ -821,6 +821,10 @@ class dblink(object): This class provides an interface to the installed package database At present this is implemented as a text backend in /var/db/pkg. """ + + import re + _normalize_needed = re.compile(r'.*//.*|^[^/]|.+/$|(^|.*/)\.\.?(/.*|$)') + def __init__(self, cat, pkg, myroot, mysettings, treetype=None, vartree=None): """ @@ -944,6 +948,10 @@ class dblink(object): mylines = myc.readlines() myc.close() null_byte = "\0" + normalize_needed = self._normalize_needed + myroot = self.myroot + if myroot == os.path.sep: + myroot = None pos = 0 for line in mylines: pos += 1 @@ -957,8 +965,12 @@ class dblink(object): # we do this so we can remove from non-root filesystems # (use the ROOT var to allow maintenance on other partitions) try: - mydat[1] = normalize_path(os.path.join( - self.myroot, mydat[1].lstrip(os.path.sep))) + if normalize_needed.match(mydat[1]): + mydat[1] = normalize_path(mydat[1]) + if not mydat[1].startswith(os.path.sep): + mydat[1] = os.path.sep + mydat[1] + if myroot: + mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep)) if mydat[0] == "obj": #format: type, mtime, md5sum pkgfiles[" ".join(mydat[1:-2])] = [mydat[0], mydat[-1], mydat[-2]] @@ -1368,9 +1380,14 @@ class dblink(object): def isowner(self,filename, destroot): """ - Check if filename is a new file or belongs to this package - (for this or a previous version) - + Check if a file belongs to this package. This may + result in a stat call for the parent directory of + every installed file, since the inode numbers are + used to work around the problem of ambiguous paths + caused by symlinked directories. The results of + stat calls are cached to optimize multiple calls + to this method. + @param filename: @type filename: @param destroot: @@ -1387,23 +1404,46 @@ class dblink(object): if pkgfiles and destfile in pkgfiles: return True if pkgfiles: + # Use stat rather than lstat since we want to follow + # any symlinks to the real parent directory. + parent_path = os.path.dirname(destfile) try: - mylstat = os.lstat(destfile) + parent_stat = os.stat(parent_path) except EnvironmentError, e: if e.errno != errno.ENOENT: raise del e - return True + return False if self._contents_inodes is None: - self._contents_inodes = set() + self._contents_inodes = {} + parent_paths = set() for x in pkgfiles: + p_path = os.path.dirname(x) + if p_path in parent_paths: + continue + parent_paths.add(p_path) try: - lstat = os.lstat(x) - self._contents_inodes.add((lstat.st_dev, lstat.st_ino)) + s = os.stat(p_path) except OSError: pass - if (mylstat.st_dev, mylstat.st_ino) in self._contents_inodes: - return True + else: + inode_key = (s.st_dev, s.st_ino) + # Use lists of paths in case multiple + # paths reference the same inode. + p_path_list = self._contents_inodes.get(inode_key) + if p_path_list is None: + p_path_list = [] + self._contents_inodes[inode_key] = p_path_list + if p_path not in p_path_list: + p_path_list.append(p_path) + p_path_list = self._contents_inodes.get( + (parent_stat.st_dev, parent_stat.st_ino)) + if p_path_list: + basename = os.path.basename(destfile) + for p_path in p_path_list: + x = os.path.join(p_path, basename) + if x in pkgfiles: + return True return False @@ -1540,50 +1580,6 @@ class dblink(object): break if stopmerge: collisions.append(f) - if stopmerge and "collision-protect" in self.settings.features: - print red("*")+" This package is blocked because it wants to overwrite" - print red("*")+" files belonging to other packages (see list below)." - print red("*")+" If you have no clue what this is all about report it " - print red("*")+" as a bug for this package on http://bugs.gentoo.org" - print - print red("package "+self.cat+"/"+self.pkg+" NOT merged") - print - print "Detected file collision(s):" - print - from portage.output import colorize - for f in collisions: - print " '%s'" % colorize("INFORM", - os.path.join(destroot, f.lstrip(os.path.sep))) - print - print "Searching all installed packages for file collisions..." - print "Press Ctrl-C to Stop" - print - """ Note: The isowner calls result in a stat call for *every* - single installed file, since the inode numbers are used to work - around the problem of ambiguous paths caused by symlinked files - and/or directories. Though it is slow, it is as accurate as - possible.""" - found_owner = False - for cpv in self.vartree.dbapi.cpv_all(): - cat, pkg = catsplit(cpv) - mylink = dblink(cat, pkg, destroot, self.settings, - vartree=self.vartree) - mycollisions = [] - for f in collisions: - if mylink.isowner(f, destroot): - mycollisions.append(f) - if mycollisions: - found_owner = True - print " * %s:" % cpv - print - for f in mycollisions: - print " '%s'" % \ - os.path.join(destroot, f.lstrip(os.path.sep)) - print - if not found_owner: - print "None of the installed packages claim the above file(s)." - print - sys.exit(1) return collisions def _security_check(self, installed_instances): @@ -1675,6 +1671,26 @@ class dblink(object): noiselevel=-1) return 1 + inforoot_slot_file = os.path.join(inforoot, "SLOT") + slot = None + try: + f = open(inforoot_slot_file) + try: + slot = f.read().strip() + finally: + f.close() + except EnvironmentError, e: + if e.errno != errno.ENOENT: + raise + del e + + if slot is None: + slot = "" + + if slot != self.settings["SLOT"]: + writemsg("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \ + (self.settings["SLOT"], slot)) + if not os.path.exists(self.dbcatdir): os.makedirs(self.dbcatdir) @@ -1683,7 +1699,7 @@ class dblink(object): otherversions.append(v.split("/")[1]) slot_matches = self.vartree.dbapi.match( - "%s:%s" % (self.mysplit[0], self.settings["SLOT"])) + "%s:%s" % (self.mysplit[0], slot)) if self.mycpv not in slot_matches and \ self.vartree.dbapi.cpv_exists(self.mycpv): # handle multislot or unapplied slotmove @@ -1744,64 +1760,116 @@ class dblink(object): collisions = self._collision_protect(srcroot, destroot, others_in_slot, myfilelist+mylinklist) - if True: - """ The merge process may move files out of the image directory, - which causes invalidation of the .installed flag.""" - try: - os.unlink(os.path.join( - os.path.dirname(normalize_path(srcroot)), ".installed")) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e - - self.dbdir = self.dbtmpdir - self.delete() - if not os.path.exists(self.dbtmpdir): - os.makedirs(self.dbtmpdir) - - writemsg_stdout(">>> Merging %s %s %s\n" % (self.mycpv,"to",destroot)) - - # run preinst script + # Make sure the ebuild environment is initialized and that ${T}/elog + # exists for logging of collision-protect eerror messages. if myebuild is None: myebuild = os.path.join(inforoot, self.pkg + ".ebuild") - a = doebuild(myebuild, "preinst", destroot, self.settings, cleanup=cleanup, - use_cache=0, tree=self.treetype, mydbapi=mydbapi, - vartree=self.vartree) + doebuild_environment(myebuild, "preinst", destroot, + self.settings, 0, 0, mydbapi) + prepare_build_dirs(destroot, self.settings, cleanup) + + def eerror(lines): + cmd = "source '%s/isolated-functions.sh' ; " % PORTAGE_BIN_PATH + for line in lines: + cmd += "eerror '%s' ; " % line + from portage import process + process.spawn(["bash", "-c", cmd], + env=self.settings.environ()) if collisions: + collision_protect = "collision-protect" in self.settings.features msg = "This package will overwrite one or more files that" + \ - " may belong to other packages (see list below)." + \ - " Add \"collision-protect\" to FEATURES in make.conf" + \ - " if you would like the merge to abort in cases like this." + " may belong to other packages (see list below)." + if not collision_protect: + msg += " Add \"collision-protect\" to FEATURES in" + \ + " make.conf if you would like the merge to abort" + \ + " in cases like this." if self.settings.get("PORTAGE_QUIET") != "1": - msg += " If you have determined that one or more of the" + \ - " files actually belong to another installed package then" + \ - " go to http://bugs.gentoo.org and report it as a bug." + \ - " Be sure to identify both this package and the other" + \ - " installed package in the bug report. Use a command such" + \ - " as \\`equery belongs \\` to identify the" + \ - " installed package that owns a file. Do NOT file a bug" + \ - " without reporting exactly which two packages install" + \ - " the same file(s)." + msg += " You can use a command such as" + \ + " `portageq owners / ` to identify the" + \ + " installed package that owns a file. If portageq" + \ + " reports that only one package owns a file then do NOT" + \ + " file a bug report. A bug report is only useful if it" + \ + " identifies at least two or more packages that are known" + \ + " to install the same file(s)." + \ + " If a collision occurs and you" + \ + " can not explain where the file came from then you" + \ + " should simply ignore the collision since there is not" + \ + " enough information to determine if a real problem" + \ + " exists. Please do NOT file a bug report at" + \ + " http://bugs.gentoo.org unless you report exactly which" + \ + " two packages install the same file(s). Once again," + \ + " please do NOT file a bug report unless you have" + \ + " completely understood the above message." self.settings["EBUILD_PHASE"] = "preinst" - cmd = "source '%s/isolated-functions.sh' ; " % PORTAGE_BIN_PATH from textwrap import wrap msg = wrap(msg, 70) - for line in msg: - cmd += "eerror \"%s\" ; " % line - cmd += "eerror ; " - cmd += "eerror \"Detected file collision(s):\" ; " - cmd += "eerror ; " + if collision_protect: + msg.append("") + msg.append("package %s NOT merged" % self.settings.mycpv) + msg.append("") + msg.append("Detected file collision(s):") + msg.append("") for f in collisions: - cmd += "eerror \" '%s'\" ; " % \ - os.path.join(destroot, f.lstrip(os.path.sep)) + msg.append("\t%s" % \ + os.path.join(destroot, f.lstrip(os.path.sep))) - from portage import process - process.spawn(["bash", "-c", cmd], - env=self.settings.environ()) + eerror(msg) + + if collision_protect: + msg = [] + msg.append("") + msg.append("Searching all installed" + \ + " packages for file collisions...") + msg.append("") + msg.append("Press Ctrl-C to Stop") + msg.append("") + eerror(msg) + + found_owner = False + for cpv in self.vartree.dbapi.cpv_all(): + cat, pkg = catsplit(cpv) + mylink = dblink(cat, pkg, destroot, self.settings, + vartree=self.vartree) + mycollisions = [] + for f in collisions: + if mylink.isowner(f, destroot): + mycollisions.append(f) + if mycollisions: + found_owner = True + msg = [] + msg.append("%s" % cpv) + for f in mycollisions: + msg.append("\t%s" % os.path.join(destroot, + f.lstrip(os.path.sep))) + eerror(msg) + if not found_owner: + eerror(["None of the installed" + \ + " packages claim the file(s)."]) + return 1 + + writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot)) + + # The merge process may move files out of the image directory, + # which causes invalidation of the .installed flag. + try: + os.unlink(os.path.join( + os.path.dirname(normalize_path(srcroot)), ".installed")) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + + self.dbdir = self.dbtmpdir + self.delete() + ensure_dirs(self.dbtmpdir) + + # run preinst script + a = doebuild(myebuild, "preinst", destroot, self.settings, + use_cache=0, tree=self.treetype, mydbapi=mydbapi, + vartree=self.vartree) # XXX: Decide how to handle failures here. if a != os.EX_OK: