From 1a2e39aa49d2f121626bf9d281d2b98177ffad75 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 22 Nov 2008 11:31:14 +0000 Subject: [PATCH] Merged from trunk -r11982:12004 | 11984 | Bug #139134 - Make the flat_hash and metadata modules write | | zmedico | the _mtime_ field inside the file instead of mangling the | | | mtime of the cache entry file. Also, fix | | | FsBased._ensure_access() to properly skip the utime() call | | | when no mtime is passed in. Theses cache changes are | | | compatible with current stable portage (2.1.4.x), which | | | uses the _mtime_ field contained in the file when | | | available. | | 11986 | Store the _eclasses_ field instead of INHERITED. | | zmedico | | | 11988 | include quotes when showing make output | | SpankMan | | | 11990 | For emerge --metadata runs, update _eclasses_ metadata to | | zmedico | insert local eclass paths. | | 11992 | Make serialize_eclasses() sort eclass names. | | zmedico | | | 11994 | When warning about 'missing repo_name', also give the exact | | zmedico | path where the entry should exist, and explain that it | | | should be a plain text file containing a unique name of the | | | first line. This should give the users enough information | | | to correct the problem without needing to seek help. | | 11995 | Move the repo_name check/warning into a function. | | zmedico | | | 11997 | Bug #245779 - Document the new category subdirectory | | zmedico | behavior for PKGDIR. | | 11999 | Bug #245716 - When generating QA notices for 'command not | | zmedico | found' messages in the build log, exclude messages that are | | | emitted from configure scripts since those are normal in | | | some cases. | | 12001 | Remove --fetchonly --resume time.sleep() call which is an | | zmedico | artifact from the previous parallel-fetch implementation. | | 12003 | Bug #247625 - Update PORTAGE_IONICE_COMMAND docs to reflect | | zmedico | that it is unset by default. | | 12004 | Remove PORTAGE_IONICE_COMMAND setting since it's commented | | zmedico | out. | svn path=/main/branches/prefix/; revision=12037 --- bin/emake | 11 +++++++- cnf/make.globals | 4 --- man/ebuild.1 | 4 +-- man/emerge.1 | 4 +-- man/make.conf.5 | 16 +++++++---- pym/_emerge/__init__.py | 47 ++++++++++++++++++-------------- pym/portage/__init__.py | 4 ++- pym/portage/cache/flat_hash.py | 3 +- pym/portage/cache/fs_template.py | 2 +- pym/portage/cache/metadata.py | 8 ------ pym/portage/cache/template.py | 4 +-- pym/portage/cache/util.py | 39 ++++++++++++++++---------- pym/portage/eclass_cache.py | 20 ++++++++++---- 13 files changed, 98 insertions(+), 68 deletions(-) diff --git a/bin/emake b/bin/emake index e18c3604b..46343ac4b 100755 --- a/bin/emake +++ b/bin/emake @@ -11,6 +11,15 @@ # /etc/make.globals here because emake is only called from an # ebuild. -[[ $PORTAGE_QUIET = 1 ]] || echo ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" >&2 +if [[ $PORTAGE_QUIET != 1 ]] ; then + ( + for arg in ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" ; do + [[ ${arg} == *" "* ]] \ + && printf "'%s' " "${arg}" \ + || printf "%s " "${arg}" + done + printf "\n" + ) >&2 +fi exec ${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE} "$@" diff --git a/cnf/make.globals b/cnf/make.globals index 5580090cc..d251f4e6c 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -71,10 +71,6 @@ PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" # Minimum size of existing file for RESUMECOMMAND to be called. PORTAGE_FETCH_RESUME_MIN_SIZE="350K" -# Command called to adjust the io priority of portage and it's subprocesses. -# Note: should be wrapped inside a uid check -#PORTAGE_IONICE_COMMAND="ionice -c 3 -p \${PID}" - # Number of times 'emerge --sync' will run before giving up. PORTAGE_RSYNC_RETRIES="3" diff --git a/man/ebuild.1 b/man/ebuild.1 index 1384bb157..52ab7b735 100644 --- a/man/ebuild.1 +++ b/man/ebuild.1 @@ -161,9 +161,7 @@ or other similar setups that the user may wish to run. .BR package This command is a lot like the \fImerge\fR command, except that after fetching, unpacking, compiling and installing, a .tbz2 binary package -tarball is created and stored in ${PKGDIR}/All (${PKGDIR} defaults to -/usr/portage/packages). A symbolic link is created in -${PKGDIR}/${CATEGORY} that points to the package in ${PKGDIR}/All. +tarball is created and stored in \fBPKGDIR\fR (see \fBmake.conf\fR(5)). .TP .BR rpm Builds a RedHat RPM package from the files in the temporary diff --git a/man/emerge.1 b/man/emerge.1 index 3da39a620..e79dd1e08 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -246,8 +246,8 @@ Tells emerge to build binary packages for all ebuilds processed in addition to actually merging the packages. Useful for maintainers or if you administrate multiple Gentoo Linux systems (build once, emerge tbz2s everywhere) as well as disaster recovery. The package -will be created in the -\fI${PKGDIR}/All\fR directory. An alternative for already\-merged +will be created in the \fBPKGDIR\fR directory (see \fBmake.conf\fR(5)). +An alternative for already\-merged packages is to use \fBquickpkg\fR(1) which creates a tbz2 from the live filesystem. .TP diff --git a/man/make.conf.5 b/man/make.conf.5 index a8f93ca1c..530ef6aef 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -401,7 +401,13 @@ Defines if color should be disabled by default. Defaults to false. .TP \fBPKGDIR\fR = \fI[path]\fR -Defines the location where created .tbz2 binary packages will be stored. Note +Defines the location where created .tbz2 binary packages will be +stored when the \fBemerge\fR(1) \fB\-\-buildpkg\fR option is enabled. +By default, a given package is stored in a subdirectory corresponding +to it's category. However, for backward compatibility with the layout +used by older versions of portage, if the \fI${PKGDIR}/All\fR directory +exists then all packages will be stored inside of it and symlinks to +the packages will be created in the category subdirectories. Note that locations under /usr/portage are not necessarily safe for data storage. See the \fBPORTDIR\fR documentation for more information. .br @@ -482,11 +488,9 @@ as K, M, or G. This variable should contain a command for portage to call in order to adjust the io priority of portage and it's subprocesses. The command string should contain a \\${PID} place-holder that will be substituted -with an integer pid. For more information about ionice, see \fBionice\fR(1). -Set this variable to an empty string if you do not want portage to call -ionice. -.br -Defaults to "ionice -c 3 -p \\${PID}". +with an integer pid. For example, a value of "ionice \-c 3 \-p \\${PID}" +will set idle io priority. For more information about ionice, see +\fBionice\fR(1). This variable is unset by default. .TP \fBPORTAGE_NICENESS\fR = \fI[number]\fR The value of this variable will be added to the current nice level that diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index c8415db65..d7f82ab6a 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -13338,11 +13338,6 @@ def action_build(settings, trees, mtimedb, if ("--resume" in myopts): favorites=mtimedb["resume"]["favorites"] - if "PORTAGE_PARALLEL_FETCHONLY" in settings: - """ parallel-fetch uses --resume --fetchonly and we don't want - it to write the mtimedb""" - mtimedb.filename = None - time.sleep(3) # allow the parent to have first fetch mymergelist = mydepgraph.altlist() mydepgraph.break_refs(mymergelist) mergetask = Scheduler(settings, trees, mtimedb, myopts, @@ -13934,6 +13929,32 @@ def expand_set_arguments(myfiles, myaction, root_config): newargs.append(a) return (newargs, retval) +def repo_name_check(trees): + missing_repo_names = set() + for root, root_trees in trees.iteritems(): + if "porttree" in root_trees: + portdb = root_trees["porttree"].dbapi + missing_repo_names.update(portdb.porttrees) + repos = portdb.getRepositories() + for r in repos: + missing_repo_names.discard(portdb.getRepositoryPath(r)) + + if missing_repo_names: + msg = [] + msg.append("WARNING: One or more repositories " + \ + "have missing repo_name entries:") + msg.append("") + for p in missing_repo_names: + msg.append("\t%s/profiles/repo_name" % (p,)) + msg.append("") + msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \ + "should be a plain text file containing a unique " + \ + "name for the repository on the first line.", 70)) + writemsg_level("".join("%s\n" % l for l in msg), + level=logging.WARNING, noiselevel=-1) + + return bool(missing_repo_names) + def emerge_main(): global portage # NFC why this is necessary now - genone portage._disable_legacy_globals() @@ -13994,21 +14015,7 @@ def emerge_main(): if "--quiet" not in myopts: portage.deprecated_profile_check() - for root in trees: - if "porttree" in trees[root]: - db = trees[root]["porttree"].dbapi - paths = (db.mysettings["PORTDIR"]+" "+db.mysettings["PORTDIR_OVERLAY"]).split() - paths = [os.path.realpath(p) for p in paths] - repos = db.getRepositories() - for r in repos: - p = db.getRepositoryPath(r) - try: - paths.remove(p) - except ValueError: - pass - for p in paths: - writemsg("WARNING: repository at %s is missing a repo_name entry\n" % p) - + repo_name_check(trees) eclasses_overridden = {} for mytrees in trees.itervalues(): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 47fa7e59b..c491ef6e8 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4529,6 +4529,7 @@ def _check_build_log(mysettings, out=None): bash_command_not_found = [] bash_command_not_found_re = re.compile( r'(.*): line (\d*): (.*): command not found$') + command_not_found_exclude_re = re.compile(r'/configure: line ') helper_missing_file = [] helper_missing_file_re = re.compile( r'^!!! (do|new).*: .* does not exist$') @@ -4550,7 +4551,8 @@ def _check_build_log(mysettings, out=None): am_maintainer_mode_exclude_re.search(line) is None: am_maintainer_mode.append(line.rstrip("\n")) - if bash_command_not_found_re.match(line) is not None: + if bash_command_not_found_re.match(line) is not None and \ + command_not_found_exclude_re.search(line) is None: bash_command_not_found.append(line.rstrip("\n")) if helper_missing_file_re.match(line) is not None: diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py index 219a4a22d..b9781b0c7 100644 --- a/pym/portage/cache/flat_hash.py +++ b/pym/portage/cache/flat_hash.py @@ -18,6 +18,7 @@ class database(fs_template.FsBased): self.label.lstrip(os.path.sep).rstrip(os.path.sep)) write_keys = set(self._known_keys) write_keys.add("_eclasses_") + write_keys.add("_mtime_") self._write_keys = sorted(write_keys) if not self.readonly and not os.path.exists(self.location): self._ensure_dirs() @@ -83,7 +84,7 @@ class database(fs_template.FsBased): myf.write("%s=%s\n" % (k, v)) finally: myf.close() - self._ensure_access(fp, mtime=values["_mtime_"]) + self._ensure_access(fp) #update written. now we move it. diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py index e619db0ac..f3dc0ce44 100644 --- a/pym/portage/cache/fs_template.py +++ b/pym/portage/cache/fs_template.py @@ -36,7 +36,7 @@ class FsBased(template.database): try: os.chown(path, -1, self._gid) os.chmod(path, self._perms) - if mtime: + if mtime != -1: mtime=long(mtime) os.utime(path, (mtime, mtime)) except (OSError, IOError): diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py index 521d71ac1..635eeec69 100644 --- a/pym/portage/cache/metadata.py +++ b/pym/portage/cache/metadata.py @@ -78,12 +78,4 @@ class database(flat_hash.database): def _setitem(self, cpv, values): - values = ProtectedDict(values) - - # hack. proper solution is to make this a __setitem__ override, since template.__setitem__ - # serializes _eclasses_, then we reconstruct it. - if "_eclasses_" in values: - values["INHERITED"] = ' '.join(reconstruct_eclasses(cpv, values["_eclasses_"]).keys()) - del values["_eclasses_"] - flat_hash.database._setitem(self, cpv, values) diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py index 3cf8309dd..238f9dc40 100644 --- a/pym/portage/cache/template.py +++ b/pym/portage/cache/template.py @@ -189,8 +189,8 @@ def serialize_eclasses(eclass_dict): """ if not eclass_dict: return "" - return "\t".join(["%s\t%s\t%s" % (k, v[0], str(v[1])) \ - for k, v in eclass_dict.iteritems()]) + return "\t".join(k + "\t%s\t%s" % eclass_dict[k] \ + for k in sorted(eclass_dict)) def reconstruct_eclasses(cpv, eclass_string): """returns a dict when handed a string generated by serialize_eclasses""" diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py index dafaed094..dc3881b19 100644 --- a/pym/portage/cache/util.py +++ b/pym/portage/cache/util.py @@ -3,6 +3,8 @@ # License: GPL2 # $Id$ +__all__ = ["mirror_cache", "non_quiet_mirroring", "quiet_mirroring"] + from itertools import chain from portage.cache import cache_errors @@ -65,25 +67,34 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, if write_it: try: - inherited = entry.get("INHERITED", None) + inherited = entry.get("INHERITED", "") + eclasses = entry.get("_eclasses_") except cache_errors.CacheError, ce: noise.exception(x, ce) del ce continue + + if eclasses is not None: + if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]): + noise.eclass_stale(x) + continue + inherited = eclasses + else: + inherited = inherited.split() + if inherited: - if src_cache.complete_eclass_entries: - if not "_eclasses_" in entry: - noise.corruption(x,"missing _eclasses_ field") - continue - if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]): - noise.eclass_stale(x) - continue - else: - entry["_eclasses_"] = eclass_cache.get_eclass_data(entry["INHERITED"].split(), \ - from_master_only=True) - if not entry["_eclasses_"]: - noise.eclass_stale(x) - continue + if src_cache.complete_eclass_entries and eclasses is None: + noise.corruption(x, "missing _eclasses_ field") + continue + + # Even if _eclasses_ already exists, replace it with data from + # eclass_cache, in order to insert local eclass paths. + eclasses = eclass_cache.get_eclass_data(inherited, + from_master_only=True) + if eclasses is None: + noise.eclass_stale(x) + continue + entry["_eclasses_"] = eclasses # by this time, if it reaches here, the eclass has been validated, and the entry has # been updated/translated (if needs be, for metadata/cache mainly) diff --git a/pym/portage/eclass_cache.py b/pym/portage/eclass_cache.py index de20d307c..93c956f8a 100644 --- a/pym/portage/eclass_cache.py +++ b/pym/portage/eclass_cache.py @@ -3,6 +3,8 @@ # License: GPL2 # $Id$ +__all__ = ["cache"] + from portage.util import normalize_path, writemsg import errno, os, sys from portage.data import portage_gid @@ -63,14 +65,22 @@ class cache(object): except OSError: continue ys=y[:-eclass_len] - self.eclasses[ys] = (x, long(mtime)) - self._eclass_locations[ys] = x if x == self._master_eclass_root: master_eclasses[ys] = mtime - else: - master_mtime = master_eclasses.get(ys) - if master_mtime and master_mtime != mtime: + self.eclasses[ys] = (x, mtime) + self._eclass_locations[ys] = x + continue + + master_mtime = master_eclasses.get(ys) + if master_mtime is not None: + if master_mtime == mtime: + # It appears to be identical to the master, + # so prefer the master entry. + continue + else: self._master_eclasses_overridden[ys] = x + self.eclasses[ys] = (x, mtime) + self._eclass_locations[ys] = x def is_eclass_data_valid(self, ec_dict): if not isinstance(ec_dict, dict): -- 2.26.2