From de1fca36ad6700559bb85b00f82cbdf2c08ea421 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Mon, 1 Dec 2008 21:10:53 +0000 Subject: [PATCH] Merged from trunk -r12086:12117 | 12090 | Note that file names in /etc/portage/package.* directories | | zmedico | are sorted ascending alphabetical order before being summed | | | together. | | 12092 | Bug #248782 - Handle permission error in | | zmedico | EbuildFetchonly.execute() if PORTAGE_TMPDIR is not writable. | | 12094 | Bug #248464 - With git, there's never any keyword expansion, | | zmedico | so there's no need to regenerate manifests and all files | | | will be committed in one big commit at the end. | | 12096 | Remove redundant reference to 'mynew' since 'myupdates' | | zmedico | already contains those files. | | 12098 | Eliminate redundant Manifest separation code by combining | | zmedico | mychanged + mynew sooner. | | 12101 | Add some more clarification to the 'ebuild phase exited | | zmedico | unexpectedly' message. | | 12103 | Make Scheduler.merge() bail out early if PORTAGE_TMPDIR is | | zmedico | not properly set. | | 12105 | Bug #205044 - When creating $EBUILD_EXIT_STATUS_FILE, don't | | zmedico | direct output to /dev/null since it should never fail and if | | | it does then the error message might be useful. | | 12107 | Don't direct to /dev/null when creating | | zmedico | $EBUILD_EXIT_STATUS_FILE inside die. | | 12109 | Bug #235642 - Create hardlinks when merging identical files. | | zmedico | This works by using a tuple of (md5, st_size) as a key to a | | | list of hardlink candidates. Multiple candidates are used in | | | case some happen to be merged to separate devices. | | 12111 | In movefile(), ignore the hardlink_candidates parameter when | | zmedico | it's an empty list. | | 12113 | For bug #235642, include the stat mode, uid, and gid bits in | | zmedico | the hardlink key. | | 12115 | Don't call prepare_build_dirs() inside doebuild() when | | zmedico | called for parallel fetching. | | 12117 | Use stat st_dev attributes instead of the older approach. | | zmedico | | svn path=/main/branches/prefix/; revision=12132 --- bin/ebuild.sh | 12 ++++-- bin/isolated-functions.sh | 16 +------- bin/repoman | 45 ++++++++++++---------- man/portage.5 | 5 ++- pym/_emerge/__init__.py | 23 ++++++++++- pym/portage/__init__.py | 74 +++++++++++++++++++++++++++++++----- pym/portage/dbapi/vartree.py | 13 ++++++- 7 files changed, 136 insertions(+), 52 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 0deb938c4..d614d3b8e 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -275,9 +275,13 @@ register_die_hook() { # Ensure that $PWD is sane whenever possible, to protect against # exploitation of insecure search path for python -c in ebuilds. # See bug #239560. -if ! hasq "$EBUILD_PHASE" clean depend help ; then +if ! hasq "$EBUILD_PHASE" clean cleanrm depend help ; then cd "$PORTAGE_BUILDDIR" || \ die "PORTAGE_BUILDDIR does not exist: '$PORTAGE_BUILDDIR'" +else + # Don't try to create this when it's parent + # directory doesn't necessarily exist. + unset EBUILD_EXIT_STATUS_FILE fi #if no perms are specified, dirs/files will have decent defaults @@ -2087,8 +2091,10 @@ ebuild_main() { exit 1 ;; esac - [ -n "${EBUILD_EXIT_STATUS_FILE}" ] && \ - touch "${EBUILD_EXIT_STATUS_FILE}" &>/dev/null + if [ -n "$EBUILD_EXIT_STATUS_FILE" ] ; then + > "$EBUILD_EXIT_STATUS_FILE" || \ + die "failed to create '$EBUILD_EXIT_STATUS_FILE'" + fi } [[ -n $EBUILD_SH_ARGS ]] && ebuild_main diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index d984a11f8..1e3079cb2 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -128,21 +128,7 @@ die() { done fi - [[ -n ${PORTAGE_LOG_FILE} ]] \ - && eerror "build log: '${PORTAGE_LOG_FILE}'" - if [ -f "${T}/environment" ] ; then - eerror "ebuild environment: '${T}/environment'" - elif [ -d "${T}" ] ; then - { - set - export - } > "${T}/die.env" - eerror "ebuild environment: '${T}/die.env'" - fi - eerror "S: '${S}'" - - [ -n "${EBUILD_EXIT_STATUS_FILE}" ] && \ - touch "${EBUILD_EXIT_STATUS_FILE}" &>/dev/null + [ -n "$EBUILD_EXIT_STATUS_FILE" ] && > "$EBUILD_EXIT_STATUS_FILE" # subshell die support kill -s SIGTERM ${EBUILD_MASTER_PID} diff --git a/bin/repoman b/bin/repoman index 43fc033da..4f2af4087 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1758,23 +1758,14 @@ else: # Manifests need to be regenerated after all other commits, so don't commit # them now even if they have changed. mymanifests = set() - changed_set = set() - new_set = set() - for f in mychanged: + myupdates = set() + for f in mychanged + mynew: if "Manifest" == os.path.basename(f): mymanifests.add(f) else: - changed_set.add(f) - for f in mynew: - if "Manifest" == os.path.basename(f): - mymanifests.add(f) - else: - new_set.add(f) - mychanged = list(changed_set) - mynew = list(new_set) + myupdates.add(f) + myupdates = list(myupdates) mymanifests = list(mymanifests) - del changed_set, new_set - myupdates = mychanged + mynew myheaders = [] mydirty = [] headerstring = "'\$(Header|Id)" @@ -1795,8 +1786,16 @@ else: if myout[0] == 0: myheaders.append(myfile) - print "*",green(str(len(myupdates))),"files being committed...",green(str(len(myheaders))),"have headers that will change." - print "*","Files with headers will cause the manifests to be made and recommited." + print "* %s files being committed..." % green(str(len(myupdates))), + if vcs == 'git': + # With git, there's never any keyword expansion, so there's + # no need to regenerate manifests and all files will be + # committed in one big commit at the end. + print + else: + print "%s have headers that will change." % green(str(len(myheaders))) + print "* Files with headers will cause the " + \ + "manifests to be made and recommited." logging.info("myupdates:", str(myupdates)) logging.info("myheaders:", str(myheaders)) @@ -1844,7 +1843,7 @@ else: commitmessage += ", RepoMan options: --force" commitmessage += ")" - if myupdates or myremoved: + if vcs != 'git' and (myupdates or myremoved): myfiles = myupdates + myremoved if not myheaders and "sign" not in repoman_settings.features: myfiles += mymanifests @@ -1937,8 +1936,8 @@ else: write_atomic(x, "".join(mylines)) manifest_commit_required = True - if myupdates or myremoved or mynew: - myfiles=myupdates+myremoved+mynew + if vcs != 'git' and (myupdates or myremoved): + myfiles = myupdates + myremoved for x in range(len(myfiles)-1, -1, -1): if myfiles[x].count("/") < 4-repolevel: del myfiles[x] @@ -2059,7 +2058,13 @@ else: portage.writemsg("!!! Disabled FEATURES='sign'\n") signed = False - if manifest_commit_required or signed: + if vcs == 'git' or manifest_commit_required or signed: + + myfiles = mymanifests[:] + if vcs == 'git': + myfiles += myupdates + myfiles += myremoved + myfiles.sort() fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") mymsg = os.fdopen(fd, "w") @@ -2075,7 +2080,7 @@ else: commit_cmd.append("commit") commit_cmd.extend(vcs_local_opts) commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in mymanifests) + commit_cmd.extend(f.lstrip("./") for f in myfiles) try: if options.pretend: diff --git a/man/portage.5 b/man/portage.5 index c6fc61437..d95d28d83 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -371,13 +371,14 @@ virtual/aspell\-dict app\-dicts/aspell\-en .BR /etc/portage/ Any file in this directory that begins with "package." can be more than just a flat file. If it is a directory, then all the files in that directory will be -summed together as if it were a single file. +sorted in ascending alphabetical order by file name and summed together as if +it were a single file. .I Example: .nf -/etc/portage/package.keywords/kde /etc/portage/package.keywords/common /etc/portage/package.keywords/e17 +/etc/portage/package.keywords/kde .fi .RS .TP diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 1fb1c15fe..bf81246e8 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1632,7 +1632,12 @@ class EbuildFetchonly(SlotObject): settings = self.settings global_tmpdir = settings["PORTAGE_TMPDIR"] from tempfile import mkdtemp - private_tmpdir = mkdtemp("", "._portage_fetch_.", global_tmpdir) + try: + private_tmpdir = mkdtemp("", "._portage_fetch_.", global_tmpdir) + except OSError, e: + if e.errno != portage.exception.PermissionDenied.errno: + raise + raise portage.exception.PermissionDenied(global_tmpdir) settings["PORTAGE_TMPDIR"] = private_tmpdir settings.backup_changes("PORTAGE_TMPDIR") try: @@ -9734,6 +9739,22 @@ class Scheduler(PollScheduler): for root in self.trees: root_config = self.trees[root]["root_config"] + + # Even for --pretend --fetch mode, PORTAGE_TMPDIR is required + # since it might spawn pkg_nofetch which requires PORTAGE_BUILDDIR + # for ensuring sane $PWD (bug #239560) and storing elog messages. + tmpdir = root_config.settings.get("PORTAGE_TMPDIR", "") + if not tmpdir or not os.path.isdir(tmpdir): + msg = "The directory specified in your " + \ + "PORTAGE_TMPDIR variable, '%s', " % tmpdir + \ + "does not exist. Please create this " + \ + "directory or correct your PORTAGE_TMPDIR setting." + msg = textwrap.wrap(msg, 70) + out = portage.output.EOutput() + for l in msg: + out.eerror(l) + return 1 + if self._background: root_config.settings.unlock() root_config.settings["PORTAGE_BACKGROUND"] = "1" diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 3288e17af..3c9239165 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -5190,9 +5190,20 @@ def _doebuild_exit_status_check(mydo, settings): "is known to be triggered " + \ "by things such as failed variable " + \ "assignments (bug #190128) or bad substitution " + \ - "errors (bug #200313). This behavior may also be " + \ - "triggered by a corrupt bash binary or a hardware " + \ - "problem such as memory or cpu malfunction." + "errors (bug #200313). Normally, before exiting, bash should " + \ + "have displayed an error message above. If bash did not " + \ + "produce an error message above, it's possible " + \ + "that the ebuild has called `exit` when it " + \ + "should have called `die` instead. This behavior may also " + \ + "be triggered by a corrupt bash binary or a hardware " + \ + "problem such as memory or cpu malfunction. If the problem is not " + \ + "reproducible or it appears to occur randomly, then it is likely " + \ + "to be triggered by a hardware problem. " + \ + "If you suspect a hardware problem then you should " + \ + "try some basic hardware diagnostics such as memtest. " + \ + "Please do not report this as a bug unless it is consistently " + \ + "reproducible and you are sure that your bash binary and hardware " + \ + "are functioning properly." return msg def _doebuild_exit_status_check_and_log(settings, mydo, retval): @@ -5331,6 +5342,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, fetchall = 1 mydo = "fetch" + parallel_fetchonly = mydo in ("fetch", "fetchall") and \ + "PORTAGE_PARALLEL_FETCHONLY" in mysettings + if mydo not in clean_phases and not os.path.exists(myebuild): writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo), noiselevel=-1) @@ -5564,7 +5578,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Build directory creation isn't required for any of these. have_build_dirs = False - if not mydo in ("digest", "help", "manifest"): + if not parallel_fetchonly and mydo not in ("digest", "help", "manifest"): mystatus = prepare_build_dirs(myroot, mysettings, cleanup) if mystatus: return mystatus @@ -6005,7 +6019,8 @@ def _movefile(src, dest, **kwargs): raise portage.exception.PortageException( "mv '%s' '%s'" % (src, dest)) -def movefile(src,dest,newmtime=None,sstat=None,mysettings=None): +def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, + hardlink_candidates=None): """moves a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. Move is atomic.""" @@ -6077,8 +6092,44 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None): print "!!!",e return None + hardlinked = False + # Since identical files might be merged to multiple filesystems, + # so os.link() calls might fail for some paths, so try them all. + # For atomic replacement, first create the link as a temp file + # and them use os.rename() to replace the destination. + if hardlink_candidates: + head, tail = os.path.split(dest) + hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \ + (tail, os.getpid())) + try: + os.unlink(hardlink_tmp) + except OSError, e: + if e.errno != errno.ENOENT: + writemsg("!!! Failed to remove hardlink temp file: %s\n" % \ + (hardlink_tmp,), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) + return None + del e + for hardlink_src in hardlink_candidates: + try: + os.link(hardlink_src, hardlink_tmp) + except OSError: + continue + else: + try: + os.rename(hardlink_tmp, dest) + except OSError, e: + writemsg("!!! Failed to rename %s to %s\n" % \ + (hardlink_tmp, dest), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) + return None + hardlinked = True + break + renamefailed=1 - if sstat[stat.ST_DEV]==dstat[stat.ST_DEV] or selinux_enabled: + if hardlinked: + renamefailed = False + if not hardlinked and (selinux_enabled or sstat.st_dev == dstat.st_dev): try: if selinux_enabled: ret=selinux.secure_rename(src,dest) @@ -6139,11 +6190,14 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None): return None try: - if newmtime is not None: - os.utime(dest, (newmtime, newmtime)) + if hardlinked: + newmtime = long(os.stat(dest).st_mtime) else: - os.utime(dest, (sstat.st_atime, sstat.st_mtime)) - newmtime = long(sstat.st_mtime) + if newmtime is not None: + os.utime(dest, (newmtime, newmtime)) + else: + os.utime(dest, (sstat.st_atime, sstat.st_mtime)) + newmtime = long(sstat.st_mtime) except OSError: # The utime can fail here with EPERM even though the move succeeded. # Instead of failing, use stat to return the mtime if possible. diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index b5e6005dc..5e695ed07 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2282,6 +2282,7 @@ class dblink(object): self._contents_inodes = None self._contents_basenames = None self._linkmap_broken = False + self._md5_merge_map = {} def lockdb(self): if self._lock_vdb: @@ -3895,6 +3896,7 @@ class dblink(object): if self.mergeme(srcroot, destroot, outfile, None, secondhand, cfgfiledict, mymtime): return 1 + self._md5_merge_map.clear() #restore umask os.umask(prevmask) @@ -4296,9 +4298,18 @@ class dblink(object): # whether config protection or not, we merge the new file the # same way. Unless moveme=0 (blocking directory) if moveme: - mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) + hardlink_key = (mymd5, mystat.st_size, + mystat.st_mode, mystat.st_uid, mystat.st_gid) + hardlink_candidates = self._md5_merge_map.get(hardlink_key) + if hardlink_candidates is None: + hardlink_candidates = [] + self._md5_merge_map[hardlink_key] = hardlink_candidates + mymtime = movefile(mysrc, mydest, newmtime=thismtime, + sstat=mystat, mysettings=self.settings, + hardlink_candidates=hardlink_candidates) if mymtime is None: return 1 + hardlink_candidates.append(mydest) zing = ">>>" if mymtime != None: -- 2.26.2