From fa630aac049bab17ca94bf5fc9ff98f786defb4e Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 16 Aug 2008 08:14:06 +0000 Subject: [PATCH] Merged from trunk 11409:11419 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 11410 | Convert exceptions from parse_metadata_use() into | | zmedico | "metadata.bad" errors instead of exiting. | | 11411 | When a build fails, generate a status message showing which | | zmedico | package failed and the path of the relevant log file if | | | available. Thanks to _neuron_ for the suggestion. Here is | | | some sample output: >>> Emerging (1 of 1) foo-bar/baz-1.0 | | | >>> Failed to emerge foo-bar/baz-1.0, Log file: >>> | | | '/var/log/portage/foo-bar:baz-1.0:20080814-202327.log' | | 11412 | * Replace vartree.dep_match() call with normal dbapi.match() | | zmedico | call. * Don't silently swallow unexpected KeyError | | | exceptions from dbapi.match() inside unmerge(). | | 11413 | In order to avoid an unhandled InvalidAtom exception, | | zmedico | validate atoms from command line arguments before passing | | | them to unmerge(). Thanks to Cardoe for reporting. | | 11414 | Validate atoms from command line args before calling | | zmedico | action_depclean(). | | 11415 | Pass atom instances into action_depclean() instead of the | | zmedico | plain strings. | | 11416 | For py3k compatibility, replace map() call with a normal for | | zmedico | loop. Thanks to Ali Polatel . | | 11417 | Use a list comprehension instead of strange map() usage | | zmedico | which yields odd results when fed to the py3k converter. | | | Thanks to René 'Necoro' Neumann. | | 11418 | For consistency, make get_open_fds() return a plain iterator | | zmedico | in any case, rather than return a list in some cases. | | 11419 | Bug #172496 - Add a PORTAGE_COMPRESS_EXCLUDE_SUFFIXES config | | zmedico | variable which is used to filter ecompress arguments. This | | | allows the user to control compression of files installed | | | dodoc, excluding files with specific suffix patterns. The | | | current default value includes all of the suffixes that are | | | recognized by dohtml, as well as pdf. | svn path=/main/branches/prefix/; revision=11420 --- bin/ecompress | 16 +++++ bin/repoman | 24 ++++--- cnf/make.globals | 3 + man/make.conf.5 | 7 +++ pym/_emerge/__init__.py | 114 +++++++++++++++++++++++++++------- pym/portage/dbapi/porttree.py | 3 +- pym/portage/process.py | 3 +- 7 files changed, 137 insertions(+), 33 deletions(-) diff --git a/bin/ecompress b/bin/ecompress index 4c73689f0..c5e82c021 100755 --- a/bin/ecompress +++ b/bin/ecompress @@ -63,6 +63,22 @@ case $1 in exit 1 ;; *) + mask_ext_re="" + set -f + for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do + mask_ext_re+="|$x" + done + set +f + mask_ext_re="^(${mask_ext_re:1})\$" + declare -a filtered_args=() + i=0 + for x in "$@" ; do + [[ ${x##*.} =~ $mask_ext_re ]] && continue + filtered_args[$i]=$x + (( i++ )) + done + set "${filtered_args[@]}" + # If a compressed version of the file already exists, simply # delete it so that the compressor doesn't whine (bzip2 will # complain and skip, gzip will prompt for input) diff --git a/bin/repoman b/bin/repoman index 92f316497..57ffb6ad5 100755 --- a/bin/repoman +++ b/bin/repoman @@ -997,8 +997,21 @@ for x in scanlist: fails["metadata.missing"].append(x+"/metadata.xml") #metadata.xml parse check else: + metadata_bad = False + + # load USE flags from metadata.xml + try: + f = open(os.path.join(checkdir, "metadata.xml")) + utilities.parse_metadata_use(f, muselist) + f.close() + except (EnvironmentError, ParseError), e: + metadata_bad = True + stats["metadata.bad"] += 1 + fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e)) + del e + #Only carry out if in package directory or check forced - if xmllint_capable: + if xmllint_capable and not metadata_bad: # xmlint can produce garbage output even on success, so only dump # the ouput when it fails. st, out = commands.getstatusoutput( @@ -1011,14 +1024,7 @@ for x in scanlist: stats["metadata.bad"]+=1 fails["metadata.bad"].append(x+"/metadata.xml") - #load USE flags from metadata.xml - try: - f = open(os.path.join(checkdir, "metadata.xml")) - utilities.parse_metadata_use(f, muselist) - f.close() - except (EnvironmentError, ParseError), e: - logging.exception("Couldn't read from metadata.xml") - sys.exit(1) + del metadata_bad allmasked = True diff --git a/cnf/make.globals b/cnf/make.globals index 30932b054..fcd7f9c25 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -56,6 +56,9 @@ EMERGE_WARNING_DELAY="10" # This option will be removed and forced to yes. AUTOCLEAN="yes" +# Don't compress files with these suffixes. +PORTAGE_COMPRESS_EXCLUDE_SUFFIXES="css gif htm[l]? jp[e]?g js pdf png" + # Number of mirrors to try when a downloaded file has an incorrect checksum. PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" diff --git a/man/make.conf.5 b/man/make.conf.5 index c60203038..6eab4b2dd 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -430,6 +430,13 @@ install phase. \fBPORTAGE_COMPRESS_FLAGS\fR = \fI"\-9"\fR This variable contains flags for the \fBPORTAGE_COMPRESS\fR command. .TP +.TP +\fBPORTAGE_COMPRESS_EXCLUDE_SUFFIXES\fR = \fI"gif htm[l]? jp[e]?g pdf png"\fR +This variable contains a space delimited list of file suffixes for which +matching files are excluded when the \fBPORTAGE_COMPRESS\fR command is +called. Regular expressions are supported and the match is performed only +against the portion of the file name which follows the last period character. +.TP .B PORTAGE_ELOG_CLASSES .TP .B PORTAGE_ELOG_SYSTEM diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index f6fa77b4b..595cabb67 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -9471,27 +9471,12 @@ class Scheduler(PollScheduler): log_paths = [failed_pkg.build_log] - if not (build_dir and os.path.isdir(build_dir)): - log_paths.append(failed_pkg.fetch_log) - - for log_path in log_paths: - if not log_path: - continue - - try: - log_size = os.stat(log_path).st_size - except OSError: - continue - - if log_size == 0: - continue - + log_path = self._locate_failure_log(failed_pkg) + if log_path is not None: try: log_file = open(log_path, 'rb') except IOError: - continue - - break + pass if log_file is not None: try: @@ -9548,6 +9533,32 @@ class Scheduler(PollScheduler): self._failed_pkgs_die_msgs.append( (mysettings, key, errors)) + def _locate_failure_log(self, failed_pkg): + + build_dir = failed_pkg.build_dir + log_file = None + + log_paths = [failed_pkg.build_log] + + if not (build_dir and os.path.isdir(build_dir)): + log_paths.append(failed_pkg.fetch_log) + + for log_path in log_paths: + if not log_path: + continue + + try: + log_size = os.stat(log_path).st_size + except OSError: + continue + + if log_size == 0: + continue + + return log_path + + return None + def _add_packages(self): pkg_queue = self._pkg_queue for pkg in self._mergelist: @@ -9577,6 +9588,7 @@ class Scheduler(PollScheduler): build_dir=build_dir, build_log=build_log, fetch_log=fetch_log, pkg=pkg, returncode=merge.returncode)) + self._failed_pkg_msg(self._failed_pkgs[-1], "install", "to") self._status_display.failed = len(self._failed_pkgs) return @@ -9621,6 +9633,7 @@ class Scheduler(PollScheduler): build_dir=build_dir, build_log=build_log, fetch_log=fetch_log, pkg=build.pkg, returncode=build.returncode)) + self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for") self._status_display.failed = len(self._failed_pkgs) self._deallocate_config(build.settings) @@ -9894,6 +9907,21 @@ class Scheduler(PollScheduler): return task + def _failed_pkg_msg(self, failed_pkg, action, preposition): + pkg = failed_pkg.pkg + msg = "%s to %s %s" % \ + (bad("Failed"), action, colorize("INFORM", pkg.cpv)) + if pkg.root != "/": + msg += " %s %s" % (preposition, pkg.root) + + log_path = self._locate_failure_log(failed_pkg) + if log_path is not None: + msg += ", Log file:" + self._status_msg(msg) + + if log_path is not None: + self._status_msg(" '%s'" % (colorize("INFORM", log_path),)) + def _status_msg(self, msg): """ Display a brief status message (no newlines) in the status display. @@ -10344,9 +10372,7 @@ def unmerge(root_config, myopts, unmerge_action, # cycle through all our candidate deps and determine # what will and will not get unmerged try: - mymatch=localtree.dep_match(x) - except KeyError: - mymatch=None + mymatch = vartree.dbapi.match(x) except ValueError, errpkgs: print "\n\n!!! The short ebuild name \"" + \ x + "\" is ambiguous. Please specify" @@ -13743,6 +13769,20 @@ def emerge_main(): elif myaction in ("clean", "unmerge") or \ (myaction == "prune" and "--nodeps" in myopts): validate_ebuild_environment(trees) + + # Ensure atoms are valid before calling unmerge(). + # For backward compat, leading '=' is not required. + for x in myfiles: + if is_valid_package_atom(x) or \ + is_valid_package_atom("=" + x): + continue + msg = [] + msg.append("'%s' is not a valid package atom." % (x,)) + msg.append("Please check ebuild(5) for full details.") + writemsg_level("".join("!!! %s\n" % line for line in msg), + level=logging.ERROR, noiselevel=-1) + return 1 + # When given a list of atoms, unmerge # them in the order given. ordered = myaction == "unmerge" @@ -13752,9 +13792,39 @@ def emerge_main(): post_emerge(root_config, myopts, mtimedb, os.EX_OK) elif myaction in ("depclean", "prune"): + + # Ensure atoms are valid before calling unmerge(). + vardb = trees[settings["ROOT"]]["vartree"].dbapi + valid_atoms = [] + for x in myfiles: + if is_valid_package_atom(x): + try: + valid_atoms.append( + portage.dep_expand(x, mydb=vardb, settings=settings)) + except ValueError, e: + msg = "The short ebuild name \"" + x + \ + "\" is ambiguous. Please specify " + \ + "one of the following " + \ + "fully-qualified ebuild names instead:" + for line in textwrap.wrap(msg, 70): + writemsg_level("!!! %s\n" % (line,), + level=logging.ERROR, noiselevel=-1) + for i in e[0]: + writemsg_level(" %s\n" % colorize("INFORM", i), + level=logging.ERROR, noiselevel=-1) + writemsg_level("\n", level=logging.ERROR, noiselevel=-1) + return 1 + continue + msg = [] + msg.append("'%s' is not a valid package atom." % (x,)) + msg.append("Please check ebuild(5) for full details.") + writemsg_level("".join("!!! %s\n" % line for line in msg), + level=logging.ERROR, noiselevel=-1) + return 1 + validate_ebuild_environment(trees) action_depclean(settings, trees, mtimedb["ldpath"], - myopts, myaction, myfiles, spinner) + myopts, myaction, valid_atoms, spinner) if not (buildpkgonly or fetchonly or pretend): post_emerge(root_config, myopts, mtimedb, os.EX_OK) # "update", "system", or just process files: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 2948ba6a4..614f6dec3 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -263,7 +263,8 @@ class portdbapi(dbapi): # if newer version, wipe everything and negate eapi eapi = metadata["EAPI"] metadata = {} - map(lambda x: metadata.setdefault(x, ""), auxdbkeys) + for x in self._known_keys: + metadata.setdefault(x, "") metadata["EAPI"] = "-" + eapi if metadata.get("INHERITED", False): diff --git a/pym/portage/process.py b/pym/portage/process.py index e9a1fab70..3bc6bd1a8 100644 --- a/pym/portage/process.py +++ b/pym/portage/process.py @@ -21,7 +21,8 @@ except ImportError: if os.path.isdir("/proc/%i/fd" % os.getpid()): def get_open_fds(): - return map(int, [fd for fd in os.listdir("/proc/%i/fd" % os.getpid()) if fd.isdigit()]) + return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \ + if fd.isdigit()) else: def get_open_fds(): return xrange(max_fd_limit) -- 2.26.2