From: Fabian Groffen Date: Sun, 2 Dec 2007 21:50:41 +0000 (-0000) Subject: Merged from trunk 8781:8808 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=f376f5147ff64c31fbe65061e17a9209185a9ca6;p=portage.git Merged from trunk 8781:8808 | 8783 | When die has been called and ${T}/environment does not | | zmedico | exist, dump the current environment to ${T}/die.env in case | | | it helps for debugging. | | 8785 | In create_trees(), when isolating the environment of the | | zmedico | ROOT=/ config, use it's env.d keys as a blacklist to make | | | env.d override the calling environment. | | 8787 | Add missing newline after message. | | zmedico | | | 8789 | Move duplicate module loading code into a function. | | zmedico | | | 8791 | After each merge, collect status from child processes in | | zmedico | order to clean up zombies (such as the parallel-fetch | | | process). | | 8792 | When os.waitid() raises OSError, discard the pid since | | zmedico | there's no need to wait on it again. | | 8794 | Show the pid in the "starting parallel fetching" message. | | zmedico | | | 8796 | * whitelist some misc variables from the calling environment | | zmedico | * blacklist the same variables in save_ebuild_env() so that | | | the latest values from the calling environment always | | | override those from the ebuild environment | | 8798 | For portage.spawn() calls, use a _shell_quote() function to | | zmedico | quote the path of the binary since. | | 8801 | In dblink.treewalk(), tweak the "Safely unmerging | | zmedico | already-installed instance" logic to make sure that it works | | | in all possible cases. | | 8803 | Synchronize self-reinstallation logic so that emerge will | | zmedico | always restart itself when dblink.merge() creates temporary | | | copies of PORTAGE_{BIN,PYM}_PATH. | | 8805 | Fix useq() so that it won't generate a bogus IUSE QA Notice | | zmedico | if called during the "depend" phase with a flag that uses | | | IUSE defaults. | | 8807 | Add back missing err() function. | | zmedico | | | 8808 | Convert caterror() to call warn() instead of err() since we | | zmedico | don't want to exit here. | svn path=/main/branches/prefix/; revision=8814 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 7e3c21b1a..d5964a568 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -139,6 +139,8 @@ useq() { # Make sure we have this USE flag in IUSE if ! hasq "${u}" ${IUSE} ${E_IUSE} && \ + ! hasq "+${u}" ${IUSE} ${E_IUSE} && \ + ! hasq "-${u}" ${IUSE} ${E_IUSE} && \ ! hasq "${u}" ${PORTAGE_ARCHLIST} selinux && \ [[ ${u} != prefix ]] && \ [[ ${u} != arch_* ]] && \ diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 911178286..2d2280516 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -98,8 +98,15 @@ diefunc() { eerror "If you need support, post the topmost build error, and the call stack if relevant." [[ -n ${PORTAGE_LOG_FILE} ]] \ && eerror "A complete build log is located at '${PORTAGE_LOG_FILE}'." - [ -f "${T}/environment" ] && \ + if [ -f "${T}/environment" ] ; then eerror "The ebuild environment file is located at '${T}/environment'." + elif [ -d "${T}" ] ; then + { + set + export + } > "${T}/die.env" + eerror "The ebuild environment file is located at '${T}/die.env'." + fi if [[ -n ${EBUILD_OVERLAY_ECLASSES} ]] ; then eerror "This ebuild used the following eclasses from overlays:" local x @@ -400,6 +407,10 @@ save_ebuild_env() { unset COLORTERM DISPLAY EDITOR LESS LESSOPEN LOGNAME LS_COLORS PAGER \ TERM TERMCAP USER + # other variables inherited from the calling environment + unset CVS_RSH ECHANGELOG_USER GPG_AGENT_INFO \ + SSH_AGENT_PID SSH_AUTH_SOCK STY WINDOW XAUTHORITY + # There's no need to bloat environment.bz2 with internally defined # functions and variables, so filter them out if possible. diff --git a/bin/repoman b/bin/repoman index 4705dfd2c..a95491362 100755 --- a/bin/repoman +++ b/bin/repoman @@ -83,6 +83,13 @@ if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \ not sys.stdout.isatty(): nocolor() +def warn(txt): + print "repoman: " + txt + +def err(txt): + warn(txt) + sys.exit(1) + def exithandler(signum=None, frame=None): logging.fatal("Interrupted; exiting...") sys.exit(1) @@ -528,7 +535,10 @@ for x in range(0,repolevel-1): repodir = os.path.dirname(repodir) def caterror(mycat): - err(mycat+" is not an official category. Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.") + warn(mycat + " is not an official category. " + \ + "Skipping QA checks in this directory.\n" + \ + "Please ensure that you add " + catdir + \ + " to " + repodir + "/profiles/categories\nif it is a new category.") def parse_use_local_desc(mylines, usedict=None): """returns a dict of the form {cpv:set(flags)}""" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 3133e9c56..1b162b3dd 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -4087,8 +4087,37 @@ class MergeTask(object): self.pkgsettings[root] = portage.config( clone=trees[root]["vartree"].settings) self.curval = 0 + self._spawned_pids = [] def merge(self, mylist, favorites, mtimedb): + try: + return self._merge(mylist, favorites, mtimedb) + finally: + if self._spawned_pids: + from portage import process + process.spawned_pids.extend(self._spawned_pids) + self._spawned_pids = [] + + def _poll_child_processes(self): + """ + After each merge, collect status from child processes + in order to clean up zombies (such as the parallel-fetch + process). + """ + spawned_pids = self._spawned_pids + if not spawned_pids: + return + for pid in list(spawned_pids): + try: + if os.waitpid(pid, os.WNOHANG) == (0, 0): + continue + except OSError: + # This pid has been cleaned up elsewhere, + # so remove it from our list. + pass + spawned_pids.remove(pid) + + def _merge(self, mylist, favorites, mtimedb): from portage.elog import elog_process from portage.elog.filtering import filter_mergephases failed_fetches = [] @@ -4173,7 +4202,6 @@ class MergeTask(object): print red("!!!")+" you have it disabled, thus parallel-fetching is being disabled" print red("!!!") elif len(mymergelist) > 1: - print ">>> starting parallel fetching" fetch_log = EPREFIX+"/var/log/emerge-fetch.log" logfile = open(fetch_log, "w") fd_pipes = {1:logfile.fileno(), 2:logfile.fileno()} @@ -4195,11 +4223,15 @@ class MergeTask(object): fetch_args.append(myopt) else: fetch_args.append(myopt +"="+ myarg) - portage.process.spawn(fetch_args, env=fetch_env, - fd_pipes=fd_pipes, returnpid=True) + self._spawned_pids.extend( + portage.process.spawn( + fetch_args, env=fetch_env, + fd_pipes=fd_pipes, returnpid=True)) logfile.close() # belongs to the spawned process del fetch_log, logfile, fd_pipes, fetch_env, fetch_args, \ resume_opts + print ">>> starting parallel fetching pid %d" % \ + self._spawned_pids[-1] metadata_keys = [k for k in portage.auxdbkeys \ if not k.startswith("UNUSED_")] + ["USE"] @@ -4477,11 +4509,7 @@ class MergeTask(object): # Figure out if we need a restart. mysplit=portage.pkgsplit(x[2]) if mysplit[0] == "sys-apps/portage" and x[1] == "/": - myver=mysplit[1]+"-"+mysplit[2] - if myver[-3:]=='-r0': - myver=myver[:-3] - if (myver != portage.VERSION) and \ - "livecvsportage" not in self.settings.features and \ + if "livecvsportage" not in self.settings.features and \: EPREFIX == BPREFIX: if len(mymergelist) > mergecount: emergelog(xterm_titles, @@ -4532,6 +4560,7 @@ class MergeTask(object): # due to power failure, SIGKILL, etc... mtimedb.commit() self.curval += 1 + self._poll_child_processes() if "--pretend" not in self.myopts: emergelog(xterm_titles, " *** Finished. Cleaning up...") diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 22128a949..93bf7a328 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -885,6 +885,14 @@ class config(object): "TERM", "TERMCAP", "USER", ] + # other variables inherited from the calling environment + _environ_whitelist += [ + "CVS_RSH", "ECHANGELOG_USER", + "GPG_AGENT_INFO", + "SSH_AGENT_PID", "SSH_AUTH_SOCK", + "STY", "WINDOW", "XAUTHORITY", + ] + _environ_whitelist = frozenset(_environ_whitelist) # Filter selected variables in the config.environ() method so that @@ -893,10 +901,7 @@ class config(object): # misc variables inherited from the calling environment _environ_filter += [ - "CVS_RSH", "ECHANGELOG_USER", - "GPG_AGENT_INFO", "INFOPATH", "MANPATH", - "SSH_AGENT_PID", "SSH_AUTH_SOCK", - "STY", "WINDOW", "XAUTHORITY", + "INFOPATH", "MANPATH", "HOST", "GROUP", "LOGNAME", "MAIL", "REMOTEHOST", "SECURITYSESSIONID", "TERMINFO", "TERM_PROGRAM", "TERM_PROGRAM_VERSION", @@ -2645,6 +2650,17 @@ class config(object): pass return self._selinux_enabled +def _shell_quote(s): + """ + Quote a string in double-quotes and use backslashes to + escape any backslashes, double-quotes, dollar signs, or + backquotes in the string. + """ + for letter in "\\\"$`": + if letter in s: + s = s.replace(letter, "\\" + letter) + return "\"%s\"" % s + # In some cases, openpty can be slow when it fails. Therefore, # stop trying to use it after the first failure. _disable_openpty = False @@ -3701,7 +3717,7 @@ def spawnebuild(mydo,actionmap,mysettings,debug,alwaysdep=0,logfile=None): portage_bin_path = mysettings["PORTAGE_BIN_PATH"] misc_sh_binary = os.path.join(portage_bin_path, os.path.basename(MISC_SH_BINARY)) - mycommand = " ".join([misc_sh_binary, + mycommand = " ".join([_shell_quote(misc_sh_binary), "install_qa_check", "install_symlink_html_docs"]) qa_retval = spawn(mycommand, mysettings, debug=debug, logfile=logfile, **kwargs) if qa_retval: @@ -4315,7 +4331,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, mysettings["dbkey"] = "" pr, pw = os.pipe() fd_pipes = {0:0, 1:1, 2:2, 9:pw} - mypids = spawn(ebuild_sh_binary + " depend", mysettings, + mypids = spawn(_shell_quote(ebuild_sh_binary) + " depend", + mysettings, fd_pipes=fd_pipes, returnpid=True, droppriv=droppriv) os.close(pw) # belongs exclusively to the child process now maxbytes = 1024 @@ -4344,7 +4361,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, mysettings["dbkey"] = \ os.path.join(mysettings.depcachedir, "aux_db_key_temp") - return spawn(ebuild_sh_binary + " depend", mysettings, + return spawn(_shell_quote(ebuild_sh_binary) + " depend", + mysettings, droppriv=droppriv) # Validate dependency metadata here to ensure that ebuilds with invalid @@ -4426,7 +4444,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, saved_env = None if saved_env: retval = os.system( - "bzip2 -dc '%s' > '%s'" % (saved_env, env_file)) + "bzip2 -dc %s > %s" % \ + (_shell_quote(saved_env), + _shell_quote(env_file))) try: env_stat = os.stat(env_file) except OSError, e: @@ -4438,7 +4458,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, env_stat and env_stat.st_size > 0: pass else: - writemsg("!!! Error extracting saved environment: '%s'" % \ + writemsg(("!!! Error extracting saved " + \ + "environment: '%s'\n") % \ saved_env, noiselevel=-1) try: os.unlink(env_file) @@ -4458,11 +4479,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # if any of these are being called, handle them -- running them out of # the sandbox -- and stop now. if mydo in ["clean","cleanrm"]: - return spawn(ebuild_sh_binary + " clean", mysettings, + return spawn(_shell_quote(ebuild_sh_binary) + " clean", mysettings, debug=debug, free=1, logfile=None) elif mydo == "help": - return spawn(ebuild_sh_binary + " " + mydo, mysettings, - debug=debug, free=1, logfile=logfile) + return spawn(_shell_quote(ebuild_sh_binary) + " " + mydo, + mysettings, debug=debug, free=1, logfile=logfile) elif mydo == "setup": infodir = os.path.join( mysettings["PORTAGE_BUILDDIR"], "build-info") @@ -4470,7 +4491,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, """Load USE flags for setup phase of a binary package. Ideally, the environment.bz2 would be used instead.""" mysettings.load_infodir(infodir) - retval = spawn(ebuild_sh_binary + " " + mydo, mysettings, + retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, debug=debug, free=1, logfile=logfile) retval = exit_status_check(retval) if secpass >= 2: @@ -4481,13 +4503,15 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, filemode=060, filemask=0) return retval elif mydo == "preinst": - phase_retval = spawn(" ".join((ebuild_sh_binary, mydo)), + phase_retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, debug=debug, free=1, logfile=logfile) phase_retval = exit_status_check(phase_retval) if phase_retval == os.EX_OK: # Post phase logic and tasks that have been factored out of # ebuild.sh. - myargs = [misc_sh_binary, "preinst_bsdflags", "preinst_mask", + myargs = [_shell_quote(misc_sh_binary), + "preinst_bsdflags", "preinst_mask", "preinst_sfperms", "preinst_selinux_labels", "preinst_suid_scan"] _doebuild_exit_status_unlink( @@ -4502,13 +4526,14 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return phase_retval elif mydo == "postinst": mysettings.load_infodir(mysettings["O"]) - phase_retval = spawn(" ".join((ebuild_sh_binary, mydo)), + phase_retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, debug=debug, free=1, logfile=logfile) phase_retval = exit_status_check(phase_retval) if phase_retval == os.EX_OK: # Post phase logic and tasks that have been factored out of # ebuild.sh. - myargs = [misc_sh_binary, "postinst_bsdflags"] + myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"] _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) mysettings["EBUILD_PHASE"] = "" @@ -4521,7 +4546,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return phase_retval elif mydo in ("prerm", "postrm", "config", "info"): mysettings.load_infodir(mysettings["O"]) - retval = spawn(ebuild_sh_binary + " " + mydo, + retval = spawn( + _shell_quote(ebuild_sh_binary) + " " + mydo, mysettings, debug=debug, free=1, logfile=logfile) retval = exit_status_check(retval) return retval @@ -4657,8 +4683,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, fakeroot = "fakeroot" in mysettings.features - ebuild_sh = ebuild_sh_binary + " %s" - misc_sh = misc_sh_binary + " dyn_%s" + ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s" + misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s" # args are for the to spawn function actionmap = { @@ -6122,11 +6148,13 @@ def create_trees(config_root=None, target_root=None, trees=None): backupenv_whitelist = settings._environ_whitelist backupenv = settings.configdict["backupenv"] + env_d = settings.configdict["env.d"] for k, v in os.environ.iteritems(): if k in backupenv_whitelist: continue - if v == backupenv.get(k): - del backupenv[k] + if k in env_d or \ + v == backupenv.get(k): + backupenv.pop(k, None) settings.regenerate() settings.lock() settings.validate() diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 1fc9b213d..1253fa58b 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2004,6 +2004,16 @@ class dblink(object): outfile.flush() outfile.close() + # These caches are populated during collision-protect and the data + # they contain is now invalid. It's very important to invalidate + # the contents_inodes cache so that FEATURES=unmerge-orphans + # doesn't unmerge anything that belongs to this package that has + # just been merged. + others_in_slot.append(self) # self has just been merged + for dblnk in others_in_slot: + dblnk.contentscache = None + dblnk._contents_inodes = None + # If portage is reinstalling itself, remove the old # version now since we want to use the temporary # PORTAGE_BIN_PATH that will be removed when we return. @@ -2013,19 +2023,13 @@ class dblink(object): "portage" == pkgsplit(self.pkg)[0]: reinstall_self = True - for dblnk in others_in_slot: + for dblnk in list(others_in_slot): + if dblnk is self: + continue if dblnk.mycpv != self.mycpv and \ not reinstall_self: continue writemsg_stdout(">>> Safely unmerging already-installed instance...\n") - # These caches are populated during collision-protect and the data - # they contain is now invalid. It's very important to invalidate - # the contents_inodes cache so that FEATURES=unmerge-orphans - # doesn't unmerge anything that belongs to this package that has - # just been merged. - self.contentscache = None - self._contents_inodes = None - others_in_slot.append(self) # self has just been merged others_in_slot.remove(dblnk) # dblnk will unmerge itself now dblnk.unmerge(trimworld=0, ldpath_mtimes=prev_mtimes, others_in_slot=others_in_slot) @@ -2361,7 +2365,8 @@ class dblink(object): """ if self.myroot == "/" and \ "sys-apps" == self.cat and \ - "portage" == pkgsplit(self.pkg)[0]: + "portage" == pkgsplit(self.pkg)[0] and \ + "livecvsportage" not in self.settings.features: settings = self.settings base_path_orig = os.path.dirname(settings["PORTAGE_BIN_PATH"]) from tempfile import mkdtemp diff --git a/pym/portage/elog/__init__.py b/pym/portage/elog/__init__.py index d3e333534..5aa6930c7 100644 --- a/pym/portage/elog/__init__.py +++ b/pym/portage/elog/__init__.py @@ -43,11 +43,21 @@ def _combine_logentries(logentries): return "".join(rValue) _elog_mod_imports = {} +def _load_mod(name): + global _elog_mod_imports + m = _elog_mod_imports.get(name) + if m is None: + m = __import__(name) + for comp in name.split(".")[1:]: + m = getattr(m, comp) + _elog_mod_imports[name] = m + return m + _elog_atexit_handlers = [] _preserve_logentries = {} def elog_process(cpv, mysettings, phasefilter=None): - global _elog_mod_imports + logsystems = mysettings.get("PORTAGE_ELOG_SYSTEM","").split() for s in logsystems: # allow per module overrides of PORTAGE_ELOG_CLASSES @@ -57,13 +67,7 @@ def elog_process(cpv, mysettings, phasefilter=None): # - is nicer than _ for module names, so allow people to use it. s = s.replace("-", "_") try: - name = "portage.elog.mod_" + s - m = _elog_mod_imports.get(name) - if m is None: - m = __import__(name) - for comp in name.split(".")[1:]: - m = getattr(m, comp) - _elog_mod_imports[name] = m + _load_mod("portage.elog.mod_" + s) except ImportError: pass @@ -113,15 +117,7 @@ def elog_process(cpv, mysettings, phasefilter=None): # - is nicer than _ for module names, so allow people to use it. s = s.replace("-", "_") try: - # FIXME: ugly ad.hoc import code - # TODO: implement a common portage module loader - name = "portage.elog.mod_" + s - m = _elog_mod_imports.get(name) - if m is None: - m = __import__(name) - for comp in name.split(".")[1:]: - m = getattr(m, comp) - _elog_mod_imports[name] = m + m = _load_mod("portage.elog.mod_" + s) def timeout_handler(signum, frame): raise PortageException("Timeout in elog_process for system '%s'" % s) import signal