From aa5c7ffe15522075edb598b4514cd4477af76bec Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 2 Apr 2009 13:06:41 +0000 Subject: [PATCH] Merged from trunk -r13262:13275 | 13263 | Add parsing code for --jobs and --load-average. | | zmedico | | | 13264 | Fix typo in comment. | | zmedico | | | 13265 | Automatically enable FEATURES=metadata-transfer if it's not | | zmedico | enabled, instead of bailing out. | | 13266 | Add support for a EGENCACHE_DEFAULT_OPTS variable in | | zmedico | make.conf. | | 13267 | Combine the --rdeps-only and --root-deps options into a | | zmedico | single --root-deps option which takes an optional 'rdeps' | | | argument. | | 13268 | Clarify docs for --root-deps. | | zmedico | | | 13269 | Document new prepare and configure commands for EAPI 2. | | zmedico | | | 13270 | Add a man page for egencache(1). | | zmedico | | | 13271 | Allow --root-deps for ROOT=/, so people can do weird things | | zmedico | like use it with -e to remerge deep runtime deps without | | | pulling in build-time deps. It's not something we really | | | want to advertise, but it's good to leave the option open in | | | case it happens to be useful. Thanks to Ned Ludd | | | for the suggestion. | | 13272 | Remove the ATOMS section and move --update to a new ACTIONS | | zmedico | section. | | 13273 | Bug #264494 - In case the ebuild has enabled set -e during | | zmedico | an ebuild phase, proactively call set +e inside qa_source(), | | | qa_call(), and die(). | | 13274 | - Add more info to the DESCRIPTION section. - Add a BUGS | | zmedico | section to explain issues related to limitations in the | | | cache format and its validation mechanism. | | 13275 | Note that only updates to PORTDIR are currently supported. | | zmedico | | svn path=/main/branches/prefix/; revision=13278 --- bin/ebuild.sh | 2 + bin/egencache | 79 ++++++++++++++++++++++++++----- bin/isolated-functions.sh | 1 + man/Makefile.am | 1 + man/ebuild.1 | 12 +++++ man/egencache.1 | 98 +++++++++++++++++++++++++++++++++++++++ man/emerge.1 | 21 ++++----- pym/_emerge/__init__.py | 33 ++++++++++--- pym/_emerge/help.py | 23 ++++----- pym/portage/__init__.py | 2 +- 10 files changed, 226 insertions(+), 46 deletions(-) create mode 100644 man/egencache.1 diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 7b938b913..17b602b87 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -35,6 +35,7 @@ qa_source() { local retval source "$@" retval=$? + set +e [[ $shopts != $(shopt) ]] && eqawarn "QA Notice: Global shell options changed and were not restored while sourcing '$*'" [[ "$IFS" != "$OLDIFS" ]] && @@ -47,6 +48,7 @@ qa_call() { local retval "$@" retval=$? + set +e [[ $shopts != $(shopt) ]] && eqawarn "QA Notice: Global shell options changed and were not restored while calling '$*'" [[ "$IFS" != "$OLDIFS" ]] && diff --git a/bin/egencache b/bin/egencache index e8981deba..66af2a75a 100755 --- a/bin/egencache +++ b/bin/egencache @@ -50,11 +50,41 @@ def parse_args(args): action="store_true", help="enable rsync stat collision workaround " + \ "for bug 139134 (use with --update)") + parser.add_option("--ignore-default-opts", + action="store_true", + help="do not use the EGENCACHE_DEFAULT_OPTS environment variable") options, args = parser.parse_args(args) - if not options.update: - parser.error('No action specified (--update ' + \ - 'is the only available action)') + if options.jobs: + jobs = None + try: + jobs = int(options.jobs) + except ValueError: + jobs = -1 + + if jobs < 1: + parser.error("Invalid: --jobs='%s'" % \ + (options.jobs,)) + + options.jobs = jobs + + else: + options.jobs = None + + if options.load_average: + try: + load_average = float(options.load_average) + except ValueError: + load_average = 0.0 + + if load_average <= 0.0: + parser.error("Invalid: --load-average='%s'" % \ + (options.load_average,)) + + options.load_average = load_average + + else: + options.load_average = None if options.config_root is not None and \ not os.path.isdir(options.config_root): @@ -74,7 +104,7 @@ def parse_args(args): if str(atom) != atom.cp: parser.error('Atom is too specific: %s' % (atom,)) - return options, args + return parser, options, args class GenCache(object): def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, @@ -112,7 +142,7 @@ class GenCache(object): # file mtime nor size changes, it will prevent rsync from # detecting changes. Cache backends may raise this # exception from _setitem() if they detect this type of stat - # collision. These exceptions are be handled by bumping the + # collision. These exceptions are handled by bumping the # mtime on the ebuild (and the corresponding cache entry). # See bug #139134. max_mtime = sc.mtime @@ -192,7 +222,7 @@ class GenCache(object): level=logging.ERROR, noiselevel=-1) def egencache_main(args): - options, args = parse_args(args) + parser, options, atoms = parse_args(args) config_root = options.config_root if config_root is None: @@ -211,17 +241,42 @@ def egencache_main(args): settings = portage.config(config_root=config_root, target_root='/', env=env) - if 'metadata-transfer' not in settings.features: - writemsg_level("ecachegen: error: " + \ - "FEATURES=metadata-transfer is not enabled\n", - level=logging.ERROR, noiselevel=-1) + default_opts = None + if not options.ignore_default_opts: + default_opts = settings.get('EGENCACHE_DEFAULT_OPTS', '').split() + + if default_opts: + parser, options, args = parse_args(default_opts + args) + + if options.config_root is not None: + config_root = options.config_root + + if options.cache_dir is not None: + env['PORTAGE_DEPCACHEDIR'] = options.cache_dir + + settings = portage.config(config_root=config_root, + target_root='/', env=env) + + if not options.update: + parser.error('No action specified (--update ' + \ + 'is the only available action)') return 1 + if 'metadata-transfer' not in settings.features: + writemsg_level("ecachegen: warning: " + \ + "automatically enabling FEATURES=metadata-transfer\n", + level=logging.WARNING, noiselevel=-1) + settings.features.add('metadata-transfer') + settings['FEATURES'] = ' '.join(sorted(settings.features)) + settings.backup_changes('FEATURES') + + settings.lock() + portdb = portage.portdbapi(settings["PORTDIR"], mysettings=settings) cp_iter = None - if args: - cp_iter = iter(args) + if atoms: + cp_iter = iter(atoms) gen_cache = GenCache(portdb, cp_iter=cp_iter, max_jobs=options.jobs, diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 4755d25ea..3f3ef6f4d 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -54,6 +54,7 @@ dump_trace() { } die() { + set +e if [ -n "${QA_INTERCEPTORS}" ] ; then # die was called from inside inherit. We need to clean up # QA_INTERCEPTORS since sed is called below. diff --git a/man/Makefile.am b/man/Makefile.am index 5898419fe..ad663fc41 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -4,6 +4,7 @@ man_MANS = \ dispatch-conf.1 \ ebuild.1 \ ebuild.5 \ + egencache.1 \ emaint.1 \ emerge.1 \ env-update.1 \ diff --git a/man/ebuild.1 b/man/ebuild.1 index 52ab7b735..6c134c7cb 100644 --- a/man/ebuild.1 +++ b/man/ebuild.1 @@ -93,6 +93,18 @@ often required. The src_unpack() function is also responsible for making the appropriate patches to the sources so that they're ready for compilation. .TP +.BR prepare +Prepares the extracted sources by running the \fIsrc_prepare()\fR +function specified in the ebuild file. When src_prepare() starts, the +current working directory will be set to ${S}. This function is supported +beginning with \fBEAPI 2\fR. +.TP +.BR configure +Configures the extracted sources by running the \fIsrc_configure()\fR +function specified in the ebuild file. When src_configure() starts, the +current working directory will be set to ${S}. This function is supported +beginning with \fBEAPI 2\fR. +.TP .BR compile Compiles the extracted sources by running the \fIsrc_compile()\fR function specified in the ebuild file. When src_compile() starts, the diff --git a/man/egencache.1 b/man/egencache.1 new file mode 100644 index 000000000..67b166f9d --- /dev/null +++ b/man/egencache.1 @@ -0,0 +1,98 @@ +.TH "EGENCACHE" "1" "March 2009" "Portage 2.2" "Portage" +.SH "NAME" +egencache \- generate metadata cache for ebuild repositories +.SH "SYNOPSIS" +.B egencache +.I [options] --update [ATOM]\fR... +.SH "DESCRIPTION" +The egencache program generates metadata cache for ebuild repositories and +stores it in the \fImetadata/cache/\fR directory within the repository itself, +for distribution. Currenly, only the repository located at \fBPORTDIR\fR +(configured in \fBmake.conf(5)\fR) can be updated by this program, but a +future version will have the ability to select a different repository. +.SH ACTIONS +.TP +.BR "\-\-update [ATOM] ... " +Update the \fImetadata/cache/\fR directory (generate metadata as necessary). +If no package atoms are specified then all will be updated. See ebuild(5) +for the details on package atom syntax. +.SH OPTIONS +.TP +.BR "\-\-cache\-dir=CACHE_DIR" +Location of the intermediate metadata cache which is stored in a different +format that includes eclass state. See the \fBBUGS\fR section for +information about why this is necessary. +.br +Defaults to /var/cache/edb/dep. +.TP +.BR "\-\-config\-root=CONFIG_ROOT" +Location of portage config files. +.br +Defaults to /. +.TP +.BR "\-\-ignore-default-opts" +Causes \fIEGENCACHE_DEFAULT_OPTS\fR to be ignored. +.TP +.BR "\-\-jobs=JOBS" +Specifies the maximum number of ebuild processes to spawn simultaneously. +Also see the related \fB\-\-load\-average\fR option. +.TP +.BR \-\-load\-average=LOAD +Specifies that maximum load allowed when spawning multiple jobs. +.TP +.BR "\-\-rsync" +When used together with the \fB\-\-update\fR action, this enables a workaround +for cases in which the content of a cache entry changes and neither the file +mtime nor size changes, preventing rsync from detecting changes. Such cases are +handled by bumping the mtime on the ebuild (and the corresponding cache entry). +This option should only be needed for distribution via something like +\fBrsync\fR(1), which relies on timestamps and file sizes to detect changes +(see \fBbug 139134\fR). It's not needed with \fBgit\fR(1) since that uses a +more thorough mechanism which allows it to detect changed inode numbers +(described in \fIracy-git.txt\fR in the git technical docs). +.SH "ENVIRONMENT OPTIONS" +.TP +\fBEGENCACHE_DEFAULT_OPTS\fR +If this variable is set in \fBmake.conf\fR(5) then any options that it +contains will be added to the beginning of the command line on every +invocation. These options will not be added if the +\fB\-\-ignore-default\-opts\fR option is specified. +.SH "BUGS" +There are significant limitations associated with the metadata +cache format that is distributed in the \fImetadata/cache/\fR directory +of the repository. These limitations are related to the cache validation +mechanism. Currently, the validation mechanism involves comparison of +a cache entry mtime to the mtime of the corresponding \fBebuild(5)\fR. This +mechanism is unreliable in cases when eclass changes result in metadata +changes, since no information about eclass state is available in the cache. +Also, since the mtime of the cache entry must correspond to that of the +ebuild, the cache format is only suitable for distribution via protocols +that preserve timestamps (such as \fBrsync(1))\fR. For cache that is +distributed via \fBgit(1)\fR repositories, there is currently a workaround +implemented in \fBemerge\fR(1) \fB\-\-sync\fR which updates ebuild mtimes +to match their corresponding cache entries (except for ebuilds that are +modified relative to HEAD). + +In order to solve the above problems, a future extension +to the cache format will include additional +validation data in the form of digests for both the ebuild +and its inherited eclasses. Until the +cache format has been extended in this way, it is necessary to enable +\fBmetadata-transfer\fR in \fBFEATURES\fR (see \fBmake.conf(5)\fR). +This causes intermediate cache (in a different format that includes +eclass state) to be generated inside the directory which is configurable +via the \fB\-\-cache\-dir\fR option. +.SH "REPORTING BUGS" +Please report bugs via http://bugs.gentoo.org/ +.SH "AUTHORS" +.nf +Zac Medico +.fi +.SH "FILES" +.TP +.B /etc/make.conf +Contains variables. +.SH "SEE ALSO" +.BR emerge (1), +.BR make.conf (5), +.BR portage (5) diff --git a/man/emerge.1 b/man/emerge.1 index 4af770716..3e6063e6e 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -414,13 +414,6 @@ b blocked by another package (automatically resolved conflict) Results may vary, but the general outcome is a reduced or condensed output from portage's displays. .TP -.BR "\-\-rdeps\-only" -Discard all build\-time dependencies. This option is commonly used together -with \fBROOT\fR and it should not be enabled under normal circumstances. For -currently supported \fBEAPI\fR values, the dependencies specified in the -\fBDEPEND\fR variable are discarded. However, behavior may change for new -\fBEAPI\fRs when related extensions are added in the future. -.TP .BR "\-\-reinstall changed\-use" Tells emerge to include installed packages where USE flags have changed since installation. Unlike \fB\-\-newuse\fR, this option does @@ -430,11 +423,15 @@ enabled are added or removed. .BR \-\-root=DIR Set the \fBROOT\fR environment variable. .TP -.BR "\-\-root\-deps" -Install build\-time dependencies to \fBROOT\fR instead of /. This option -should not be enabled under normal circumstances. For currently supported -\fBEAPI\fR values, the dependencies specified in the \fBDEPEND\fR variable -are used. However, behavior may change for new +.BR "\-\-root\-deps[=rdeps]" +If no argument is given then build\-time dependencies of packages for +\fBROOT\fR are installed to +\fBROOT\fR instead of /. If the \fBrdeps\fR argument is given then discard +all build\-time dependencies of packages for \fBROOT\fR. This option is +only meaningful when used together with \fBROOT\fR and it should not +be enabled under normal circumstances. For currently supported +\fBEAPI\fR values, the build-time dependencies are specified in the +\fBDEPEND\fR variable. However, behavior may change for new \fBEAPI\fRs when related extensions are added in the future. .TP .BR "\-\-skipfirst" diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 44ec88b9a..500a2a5df 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -222,7 +222,6 @@ options=[ "--nospinner", "--oneshot", "--onlydeps", "--pretend", "--quiet", "--resume", -"--rdeps-only", "--root-deps", "--searchdesc", "--selective", "--skipfirst", "--tree", @@ -5303,12 +5302,12 @@ class depgraph(object): edepend["DEPEND"] = "" bdeps_root = "/" - if self.target_root != "/": - if "--root-deps" in self.myopts: - bdeps_root = myroot - if "--rdeps-only" in self.myopts: - bdeps_root = "/" - edepend["DEPEND"] = "" + root_deps = self.myopts.get("--root-deps") + if root_deps is not None: + if root_deps is True: + bdeps_root = myroot + elif root_deps == "rdeps": + edepend["DEPEND"] = "" deps = ( (bdeps_root, edepend["DEPEND"], @@ -14854,11 +14853,22 @@ def insert_optional_args(args): new_args = [] jobs_opts = ("-j", "--jobs") + root_deps_opt = '--root-deps' + root_deps_choices = ('True', 'rdeps') arg_stack = args[:] arg_stack.reverse() while arg_stack: arg = arg_stack.pop() + if arg == root_deps_opt: + new_args.append(arg) + if arg_stack and arg_stack[-1] in root_deps_choices: + new_args.append(arg_stack.pop()) + else: + # insert default argument + new_args.append('True') + continue + short_job_opt = bool("j" in arg and arg[:1] == "-" and arg[:2] != "--") if not (short_job_opt or arg in jobs_opts): new_args.append(arg) @@ -14951,6 +14961,12 @@ def parse_opts(tmpcmdline, silent=False): "help" : "specify the target root filesystem for merging packages", "action" : "store" }, + + "--root-deps": { + "help" : "modify interpretation of depedencies", + "type" : "choice", + "choices" :("True", "rdeps") + }, } from optparse import OptionParser @@ -14979,6 +14995,9 @@ def parse_opts(tmpcmdline, silent=False): myoptions, myargs = parser.parse_args(args=tmpcmdline) + if myoptions.root_deps == "True": + myoptions.root_deps = True + if myoptions.jobs: jobs = None if myoptions.jobs == "True": diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 15aa88a02..4e379429e 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -410,15 +410,6 @@ def help(myaction,myopts,havecolor=1): print " Effects vary, but the general outcome is a reduced or condensed" print " output from portage's displays." print - print " "+green("--rdeps-only") - desc = "Discard all build-time dependencies. This option is commonly used together " + \ - "with ROOT and it should not be enabled under normal circumstances. For " + \ - "currently supported EAPI values, the dependencies specified in the " + \ - "DEPEND variable are discarded. However, behavior may change for new " + \ - "EAPIs when related extensions are added in the future." - for line in wrap(desc, desc_width): - print desc_indent + line - print print " "+green("--reinstall ") + turquoise("changed-use") print " Tells emerge to include installed packages where USE flags have" print " changed since installation. Unlike --newuse, this option does" @@ -431,11 +422,15 @@ def help(myaction,myopts,havecolor=1): for line in wrap(desc, desc_width): print desc_indent + line print - print " "+green("--root-deps") - desc = "Install build-time dependencies to ROOT instead of /. This option " + \ - "should not be enabled under normal circumstances. For currently supported " + \ - "EAPI values, the dependencies specified in the DEPEND variable " + \ - "are used. However, behavior may change for new " + \ + print " "+green("--root-deps[=rdeps]") + desc = "If no argument is given then build-time dependencies of packages for " + \ + "ROOT are installed to " + \ + "ROOT instead of /. If the rdeps argument is given then discard " + \ + "all build-time dependencies of packages for ROOT. This option is " + \ + "only meaningful when used together with ROOT and it should not " + \ + "be enabled under normal circumstances. For currently supported " + \ + "EAPI values, the build-time dependencies are specified in the " + \ + "DEPEND variable. However, behavior may change for new " + \ "EAPIs when related extensions are added in the future." for line in wrap(desc, desc_width): print desc_indent + line diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index eedb1389d..b91dd006d 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1098,7 +1098,7 @@ class config(object): _environ_filter += [ "ACCEPT_KEYWORDS", "AUTOCLEAN", "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", - "CONFIG_PROTECT_MASK", "EMERGE_DEFAULT_OPTS", + "CONFIG_PROTECT_MASK", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_WARNING_DELAY", "FETCHCOMMAND", "FETCHCOMMAND_FTP", "FETCHCOMMAND_HTTP", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", -- 2.26.2