From e59b33583ee9b8c3b299fa129e4809d969cb37fc Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 20 Aug 2008 20:52:00 +0000 Subject: [PATCH] Merged from trunk 11422:11443 | 11423 | Document /var/lib/portage/world_sets. | | zmedico | | | 11432 | Fix atom -> str breakage in StaticFileSet.write(). | | zmedico | | | 11433 | Don't assume that the "world" set is an instance of | | zmedico | WorldSet, allowing the user to substitute some other set | | | type if desired. | | 11434 | * Truncate status display output when necessary to avoid | | zmedico | line overflow and unwanted terminal scroll. * When the load | | | average goes above 10, save space by showing fewer decimal | | | places in the output. | | 11435 | Update for @everything -> @installed rename. Thanks to | | zmedico | Arfrever. | | 11436 | The following error is known to occur with Linux kernel | | zmedico | versions less than 2.6.24: select.error: (4, 'Interrupted | | | system call') This error has been observed after a SIGSTOP, | | | followed by SIGCONT. Treat it similar to EAGAIN if timeout | | | is None, otherwise just return without any events. Thanks to | | | Jeremy Olexa for reporting and testing this | | | patch. | | 11437 | Remove reference to use.default. Thanks to ulm for | | zmedico | reporting. | | 11438 | Add an 'IUSE defaults' section. | | zmedico | | | 11439 | When in background mode, redirect "adjusting permissions" | | zmedico | messages for features directories like CCACHE_DIR and | | | DISTCC_DIR. Thanks to Arfrever for reporting. | | 11440 | Make sure EBUILD_PHASE is properly unset when calling | | zmedico | misc-functions.sh. | | 11441 | Make dblink call back to the scheduler for "clean" and | | zmedico | "cleanrm" phases, to handle output redirection. Thanks to | | | Arfrever for reporting. | | 11442 | Fix dep_wordreduce() so that it doesn't cancel out blocker | | zmedico | atoms that happen to be matched by package.provided. | | 11443 | Fix spelling of overriding for bug #235250. | | zmedico | | svn path=/main/branches/prefix/; revision=11445 --- doc/config/sets.docbook | 2 +- doc/package/ebuild/eapi/1.docbook | 8 ++ man/emerge.1 | 4 - man/portage.5 | 10 +++ pym/_emerge/__init__.py | 139 ++++++++++++++++++++++-------- pym/portage/__init__.py | 58 +++++++++++-- pym/portage/dbapi/vartree.py | 23 +++-- pym/portage/sets/files.py | 6 +- 8 files changed, 190 insertions(+), 60 deletions(-) diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook index 14e171fb2..cfce6ecf6 100644 --- a/doc/config/sets.docbook +++ b/doc/config/sets.docbook @@ -533,7 +533,7 @@ system: uses PackagesSystemSet world: uses WorldSet security: uses NewAffectedSet with default options - everything: uses EverythingSet + installed: uses EverythingSet preserved-rebuild: uses PreservedLibraryConsumerSet live-rebuild: uses VariableSet module-rebuild: uses OwnerSet diff --git a/doc/package/ebuild/eapi/1.docbook b/doc/package/ebuild/eapi/1.docbook index ca421352f..a17887875 100644 --- a/doc/package/ebuild/eapi/1.docbook +++ b/doc/package/ebuild/eapi/1.docbook @@ -1,5 +1,13 @@
EAPI 1 +
+ IUSE defaults + + A + or - prefix added to the beginning of a flag in IUSE creates + a default USE setting that respectively enables or disables the + corresponding USE flag. + +
Phases
diff --git a/man/emerge.1 b/man/emerge.1 index 65a893e9c..e26c4c46f 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -697,10 +697,6 @@ files. Contains profile\-specific variables for the build process. \fBDo not edit this file\fR. .TP -.B /etc/make.profile/use.defaults -Contains a list of packages which, if installed, cause the respective USE -flag to be enabled by default. \fBDo not edit this file\fR. -.TP .B /usr/portage/profiles/use.desc Contains the master list of USE flags with descriptions of their functions. \fBDo not edit this file\fR. diff --git a/man/portage.5 b/man/portage.5 index 3a1237f59..2901faa84 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -80,6 +80,7 @@ database to track installed packages .nf config world +world_sets .fi .SH "GLOSSARY" In the following sections, some terminology may be foreign to you or used @@ -736,6 +737,15 @@ games\-misc/fortune\-mod\-gentoo\-dev dev\-libs/uclibc app\-cdr/cdemu .fi +.TP +.BR world_sets +This is like the world file but instead of package atoms it contains +packages sets which always begin with the @ character. + +.I Example: +.nf +@system +.fi .RE .SH "REPORTING BUGS" Please report bugs via http://bugs.gentoo.org/ diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 595cabb67..953728d1d 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2229,6 +2229,7 @@ class MiscFunctionsProcess(SpawnProcess): def _start(self): settings = self.settings + settings.pop("EBUILD_PHASE", None) portage_bin_path = settings["PORTAGE_BIN_PATH"] misc_sh_binary = os.path.join(portage_bin_path, os.path.basename(portage.const.MISC_SH_BINARY)) @@ -2817,7 +2818,7 @@ class EbuildProcess(SpawnProcess): def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) - if self.phase != "clean": + if self.phase not in ("clean", "cleanrm"): self.returncode = portage._doebuild_exit_status_check_and_log( self.settings, self.phase, self.returncode) @@ -7574,8 +7575,15 @@ class depgraph(object): return root_config = self.roots[self.target_root] world_set = root_config.sets["world"] - world_set.lock() - world_set.load() # maybe it's changed on disk + + world_locked = False + if hasattr(world_set, "lock"): + world_set.lock() + world_locked = True + + if hasattr(world_set, "load"): + world_set.load() # maybe it's changed on disk + args_set = self._sets["args"] portdb = self.trees[self.target_root]["porttree"].dbapi added_favorites = set() @@ -7611,7 +7619,9 @@ class depgraph(object): colorize("INFORM", str(a)) if all_added: world_set.update(all_added) - world_set.unlock() + + if world_locked: + world_set.unlock() def loadResumeCommand(self, resume_data, skip_masked=False): """ @@ -8375,7 +8385,25 @@ class PollScheduler(object): not self._poll_event_handlers: raise StopIteration( "timeout is None and there are no poll() event handlers") - self._poll_event_queue.extend(self._poll_obj.poll(timeout)) + + # The following error is known to occur with Linux kernel versions + # less than 2.6.24: + # + # select.error: (4, 'Interrupted system call') + # + # This error has been observed after a SIGSTOP, followed by SIGCONT. + # Treat it similar to EAGAIN if timeout is None, otherwise just return + # without any events. + while True: + try: + self._poll_event_queue.extend(self._poll_obj.poll(timeout)) + break + except select.error, e: + writemsg_level("\n!!! select error: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + del e + if timeout is not None: + break def _next_poll_event(self, timeout=None): """ @@ -8606,6 +8634,7 @@ class JobStatusDisplay(object): object.__setattr__(self, "_changed", False) object.__setattr__(self, "_displayed", False) object.__setattr__(self, "_last_display_time", 0) + object.__setattr__(self, "width", 80) self.reset() isatty = hasattr(out, "isatty") and out.isatty() @@ -8710,11 +8739,21 @@ class JobStatusDisplay(object): self._changed = True self.display() - def _load_avg_str(self, digits=2): + def _load_avg_str(self): try: avg = os.getloadavg() except OSError, e: return str(e) + + max_avg = max(avg) + + if max_avg < 10: + digits = 2 + elif max_avg < 100: + digits = 1 + else: + digits = 0 + return ", ".join(("%%.%df" % digits ) % x for x in avg) def display(self): @@ -8788,8 +8827,18 @@ class JobStatusDisplay(object): f.add_literal_data("Load avg: ") f.add_literal_data(load_avg_str) - self._update(color_output.getvalue()) - xtermTitle(" ".join(plain_output.getvalue().split())) + # Truncate to fit width, to avoid making the terminal scroll if the + # line overflows (happens when the load average is large). + plain_output = plain_output.getvalue() + if self._isatty and len(plain_output) > self.width: + # Use plain_output here since it's easier to truncate + # properly than the color output which contains console + # color codes. + self._update(plain_output[:self.width]) + else: + self._update(color_output.getvalue()) + + xtermTitle(" ".join(plain_output.split())) class Scheduler(PollScheduler): @@ -8864,6 +8913,21 @@ class Scheduler(PollScheduler): for k in self._binpkg_opts.__slots__: setattr(self._binpkg_opts, k, "--" + k.replace("_", "-") in myopts) + self.curval = 0 + self._logger = self._emerge_log_class() + self._task_queues = self._task_queues_class() + for k in self._task_queues.allowed_keys: + setattr(self._task_queues, k, + SequentialTaskQueue()) + self._status_display = JobStatusDisplay() + self._max_load = myopts.get("--load-average") + max_jobs = myopts.get("--jobs") + if max_jobs is None: + max_jobs = 1 + self._set_max_jobs(max_jobs) + background = self._background_mode() + self._background = background + # The root where the currently running # portage instance is installed. self._running_root = trees["/"]["root_config"] @@ -8874,12 +8938,18 @@ class Scheduler(PollScheduler): self._config_pool = {} self._blocker_db = {} for root in trees: + root_config = trees[root]["root_config"] + if background: + root_config.settings.unlock() + root_config.settings["PORTAGE_BACKGROUND"] = "1" + root_config.settings.backup_changes("PORTAGE_BACKGROUND") + root_config.settings.lock() + self.pkgsettings[root] = portage.config( clone=trees[root]["vartree"].settings) self._config_pool[root] = [] self._blocker_db[root] = BlockerDB(trees[root]["root_config"]) - self.curval = 0 - self._logger = self._emerge_log_class() + fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, schedule=self._schedule_fetch) self._sched_iface = self._iface_class( @@ -8893,17 +8963,10 @@ class Scheduler(PollScheduler): scheduleYield=self._schedule_yield, unregister=self._unregister) - self._task_queues = self._task_queues_class() - for k in self._task_queues.allowed_keys: - setattr(self._task_queues, k, - SequentialTaskQueue()) - self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] self._completed_tasks = set() - self._status_display = JobStatusDisplay() - self._failed_pkgs = [] self._failed_pkgs_all = [] self._failed_pkgs_die_msgs = [] @@ -8915,15 +8978,6 @@ class Scheduler(PollScheduler): curval=0, maxval=merge_count) self._status_display.maxval = self._pkg_count.maxval - max_jobs = myopts.get("--jobs") - if max_jobs is None: - max_jobs = 1 - self._set_max_jobs(max_jobs) - background = self._background_mode() - self._background = background - - self._max_load = myopts.get("--load-average") - # The load average takes some time to respond when new # jobs are added, so we need to limit the rate of adding # new jobs. @@ -10069,18 +10123,29 @@ class Scheduler(PollScheduler): pkg_count = self._pkg_count root_config = pkg.root_config world_set = root_config.sets["world"] - world_set.lock() + world_locked = False + if hasattr(world_set, "lock"): + world_set.lock() + world_locked = True + try: - world_set.load() # maybe it's changed on disk + if hasattr(world_set, "load"): + world_set.load() # maybe it's changed on disk + atom = create_world_atom(pkg, args_set, root_config) if atom: - self._status_msg(('Recording %s in "world" ' + \ - 'favorites file...') % atom) - logger.log(" === (%s of %s) Updating world file (%s)" % \ - (pkg_count.curval, pkg_count.maxval, pkg.cpv)) - world_set.add(atom) + if hasattr(world_set, "add"): + self._status_msg(('Recording %s in "world" ' + \ + 'favorites file...') % atom) + logger.log(" === (%s of %s) Updating world file (%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv)) + world_set.add(atom) + else: + writemsg_level('\n!!! Unable to record %s in "world"\n' % \ + (atom,), level=logging.WARN, noiselevel=-1) finally: - world_set.unlock() + if world_locked: + world_set.unlock() def _pkg(self, cpv, type_name, root_config, installed=False): """ @@ -10672,10 +10737,10 @@ def unmerge(root_config, myopts, unmerge_action, raise UninstallFailure(retval) sys.exit(retval) else: - if clean_world: + if clean_world and hasattr(sets["world"], "cleanPackage"): sets["world"].cleanPackage(vartree.dbapi, y) emergelog(xterm_titles, " >>> unmerge success: "+y) - if clean_world: + if clean_world and hasattr(sets["world"], "remove"): for s in root_config.setconfig.active: sets["world"].remove(SETPREFIX+s) return 1 @@ -13463,7 +13528,7 @@ def emerge_main(): (eclasses_overridden[eclass_name], eclass_name), noiselevel=-1) writemsg(prefix + "\n", noiselevel=-1) - msg = "It is best to avoid overridding eclasses from PORTDIR " + \ + msg = "It is best to avoid overriding eclasses from PORTDIR " + \ "because it will trigger invalidation of cached ebuild metadata " + \ "that is distributed with the portage tree. If you must " + \ "override eclasses from PORTDIR then you are advised to add " + \ diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index bc805115d..6da037e19 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -14,6 +14,7 @@ try: import sys import copy import errno + import logging import os import re import shutil @@ -997,6 +998,7 @@ class config(object): "EMERGE_WARNING_DELAY", "FETCHCOMMAND", "FETCHCOMMAND_FTP", "FETCHCOMMAND_HTTP", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", + "PORTAGE_BACKGROUND", "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER", "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", @@ -4854,6 +4856,37 @@ def prepare_build_dirs(myroot, mysettings, cleanup): writemsg("File Not Found: '%s'\n" % str(e), noiselevel=-1) return 1 + _prepare_workdir(mysettings) + _prepare_features_dirs(mysettings) + +def _adjust_perms_msg(settings, msg): + + def write(msg): + writemsg(msg, noiselevel=-1) + + background = settings.get("PORTAGE_BACKGROUND") == "1" + log_path = settings.get("PORTAGE_LOG_FILE") + log_file = None + + if background and log_path is not None: + try: + log_file = open(log_path, 'a') + except IOError: + def write(msg): + pass + else: + def write(msg): + log_file.write(msg) + log_file.flush() + + try: + write(msg) + finally: + if log_file is not None: + log_file.close() + +def _prepare_features_dirs(mysettings): + features_dirs = { "ccache":{ "basedir_var":"CCACHE_DIR", @@ -4912,16 +4945,18 @@ def prepare_build_dirs(myroot, mysettings, cleanup): not dirmode == (stat.S_IMODE(subdir_st.st_mode) & dirmode))): droppriv_fix = True break + if droppriv_fix: - writemsg(colorize("WARN", " * ") + \ - "Adjusting permissions " + \ - "for FEATURES=userpriv: '%s'\n" % mydir, - noiselevel=-1) + _adjust_perms_msg(mysettings, + colorize("WARN", " * ") + \ + "Adjusting permissions " + \ + "for FEATURES=userpriv: '%s'\n" % mydir) elif modified: - writemsg(colorize("WARN", " * ") + \ - "Adjusting permissions " + \ - "for FEATURES=%s: '%s'\n" % (myfeature, mydir), - noiselevel=-1) + _adjust_perms_msg(mysettings, + colorize("WARN", " * ") + \ + "Adjusting permissions " + \ + "for FEATURES=%s: '%s'\n" % (myfeature, mydir)) + if modified or kwargs["always_recurse"] or droppriv_fix: def onerror(e): raise # The feature is disabled if a single error @@ -4941,6 +4976,7 @@ def prepare_build_dirs(myroot, mysettings, cleanup): noiselevel=-1) time.sleep(5) +def _prepare_workdir(mysettings): workdir_mode = 0700 try: mode = mysettings["PORTAGE_WORKDIR_MODE"] @@ -5539,6 +5575,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if phase_retval == os.EX_OK: _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) + mysettings.pop("EBUILD_PHASE", None) phase_retval = spawn( " ".join(_post_pkg_preinst_cmd(mysettings)), mysettings, debug=debug, free=1, logfile=logfile) @@ -5560,6 +5597,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if phase_retval == os.EX_OK: _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) + mysettings.pop("EBUILD_PHASE", None) phase_retval = spawn(" ".join(_post_pkg_postinst_cmd(mysettings)), mysettings, debug=debug, free=1, logfile=logfile) phase_retval = exit_status_check(phase_retval) @@ -6495,12 +6533,14 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, def dep_wordreduce(mydeplist,mysettings,mydbapi,mode,use_cache=1): "Reduces the deplist to ones and zeros" deplist=mydeplist[:] - for mypos in xrange(len(deplist)): + for mypos, token in enumerate(deplist): if isinstance(deplist[mypos], list): #recurse deplist[mypos]=dep_wordreduce(deplist[mypos],mysettings,mydbapi,mode,use_cache=use_cache) elif deplist[mypos]=="||": pass + elif token[:1] == "!": + deplist[mypos] = False else: mykey = dep_getkey(deplist[mypos]) if mysettings and mykey in mysettings.pprovideddict and \ diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 50e73c355..963f1cf3f 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2230,10 +2230,15 @@ class dblink(object): # process logs created during pre/postrm elog_process(self.mycpv, self.settings, phasefilter=filter_unmergephases) if retval == os.EX_OK: - doebuild(myebuildpath, "cleanrm", self.myroot, - self.settings, tree="vartree", - mydbapi=self.vartree.dbapi, - vartree=self.vartree) + if scheduler is None: + doebuild(myebuildpath, "cleanrm", self.myroot, + self.settings, tree="vartree", + mydbapi=self.vartree.dbapi, + vartree=self.vartree) + else: + scheduler.dblinkEbuildPhase( + self, self.vartree.dbapi, + myebuildpath, "cleanrm") finally: unlockdir(builddir_lock) try: @@ -3710,8 +3715,14 @@ class dblink(object): if retval == os.EX_OK and "noclean" not in self.settings.features: if myebuild is None: myebuild = os.path.join(inforoot, self.pkg + ".ebuild") - doebuild(myebuild, "clean", myroot, self.settings, - tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree) + + if self._scheduler is None: + doebuild(myebuild, "clean", myroot, + self.settings, tree=self.treetype, + mydbapi=mydbapi, vartree=self.vartree) + else: + self._scheduler.dblinkEbuildPhase( + self, mydbapi, myebuild, "clean") finally: self.unlockdb() return retval diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index d4d888eae..947b371a8 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -58,9 +58,9 @@ class StaticFileSet(EditablePackageSet): return ValidAtomValidator(atom) def write(self): - write_atomic(self._filename, "\n".join(sorted( - chain(self._atoms, self._nonatoms)))+"\n") - + write_atomic(self._filename, "".join("%s\n" % (atom,) \ + for atom in sorted(chain(self._atoms, self._nonatoms)))) + def load(self): try: mtime = os.stat(self._filename).st_mtime -- 2.26.2