Merged from trunk 11409:11419
authorFabian Groffen <grobian@gentoo.org>
Sat, 16 Aug 2008 08:14:06 +0000 (08:14 -0000)
committerFabian Groffen <grobian@gentoo.org>
Sat, 16 Aug 2008 08:14:06 +0000 (08:14 -0000)
   | 11410   | Convert exceptions from parse_metadata_use() into            |
   | zmedico | "metadata.bad" errors instead of exiting.                    |

   | 11411   | When a build fails, generate a status message showing which  |
   | zmedico | package failed and the path of the relevant log file if      |
   |         | available. Thanks to _neuron_ for the suggestion. Here is    |
   |         | some sample output: >>> Emerging (1 of 1) foo-bar/baz-1.0    |
   |         | >>> Failed to emerge foo-bar/baz-1.0, Log file: >>>          |
   |         | '/var/log/portage/foo-bar:baz-1.0:20080814-202327.log'       |

   | 11412   | * Replace vartree.dep_match() call with normal dbapi.match() |
   | zmedico | call. * Don't silently swallow unexpected KeyError           |
   |         | exceptions from dbapi.match() inside unmerge().              |

   | 11413   | In order to avoid an unhandled InvalidAtom exception,        |
   | zmedico | validate atoms from command line arguments before passing    |
   |         | them to unmerge(). Thanks to Cardoe for reporting.           |

   | 11414   | Validate atoms from command line args before calling         |
   | zmedico | action_depclean().                                           |

   | 11415   | Pass atom instances into action_depclean() instead of the    |
   | zmedico | plain strings.                                               |

   | 11416   | For py3k compatibility, replace map() call with a normal for |
   | zmedico | loop. Thanks to Ali Polatel <hawking@g.o>.                   |

   | 11417   | Use a list comprehension instead of strange map() usage      |
   | zmedico | which yields odd results when fed to the py3k converter.     |
   |         | Thanks to RenĂ© 'Necoro' Neumann.                            |

   | 11418   | For consistency, make get_open_fds() return a plain iterator |
   | zmedico | in any case, rather than return a list in some cases.        |

   | 11419   | Bug #172496 - Add a PORTAGE_COMPRESS_EXCLUDE_SUFFIXES config |
   | zmedico | variable which is used to filter ecompress arguments. This   |
   |         | allows the user to control compression of files installed    |
   |         | dodoc, excluding files with specific suffix patterns. The    |
   |         | current default value includes all of the suffixes that are  |
   |         | recognized by dohtml, as well as pdf.                        |

svn path=/main/branches/prefix/; revision=11420

bin/ecompress
bin/repoman
cnf/make.globals
man/make.conf.5
pym/_emerge/__init__.py
pym/portage/dbapi/porttree.py
pym/portage/process.py

index 4c73689f0511c9df8fe090a5c07c87671230400f..c5e82c021e79df8d23d5d11961f79e090e06d3e4 100755 (executable)
@@ -63,6 +63,22 @@ case $1 in
                exit 1
                ;;
        *)
+               mask_ext_re=""
+               set -f
+               for x in $PORTAGE_COMPRESS_EXCLUDE_SUFFIXES ; do
+                       mask_ext_re+="|$x"
+               done
+               set +f
+               mask_ext_re="^(${mask_ext_re:1})\$"
+               declare -a filtered_args=()
+               i=0
+               for x in "$@" ; do
+                       [[ ${x##*.} =~ $mask_ext_re ]] && continue
+                       filtered_args[$i]=$x
+                       (( i++ ))
+               done
+               set "${filtered_args[@]}"
+
                # If a compressed version of the file already exists, simply
                # delete it so that the compressor doesn't whine (bzip2 will
                # complain and skip, gzip will prompt for input)
index 92f316497dd586018895c39bef40ca534a360c08..57ffb6ad5326e6ced4e6284a8066427c55fb267c 100755 (executable)
@@ -997,8 +997,21 @@ for x in scanlist:
                fails["metadata.missing"].append(x+"/metadata.xml")
        #metadata.xml parse check
        else:
+               metadata_bad = False
+
+               # load USE flags from metadata.xml
+               try:
+                       f = open(os.path.join(checkdir, "metadata.xml"))
+                       utilities.parse_metadata_use(f, muselist)
+                       f.close()
+               except (EnvironmentError, ParseError), e:
+                       metadata_bad = True
+                       stats["metadata.bad"] += 1
+                       fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
+                       del e
+
                #Only carry out if in package directory or check forced
-               if xmllint_capable:
+               if xmllint_capable and not metadata_bad:
                        # xmlint can produce garbage output even on success, so only dump
                        # the ouput when it fails.
                        st, out = commands.getstatusoutput(
@@ -1011,14 +1024,7 @@ for x in scanlist:
                                stats["metadata.bad"]+=1
                                fails["metadata.bad"].append(x+"/metadata.xml")
 
-               #load USE flags from metadata.xml
-               try:
-                       f = open(os.path.join(checkdir, "metadata.xml"))
-                       utilities.parse_metadata_use(f, muselist)
-                       f.close()
-               except (EnvironmentError, ParseError), e:
-                       logging.exception("Couldn't read from metadata.xml")
-                       sys.exit(1)
+               del metadata_bad
 
        allmasked = True
 
index 30932b0549e51afb52db97e51a7d5772223c834d..fcd7f9c252a77cdfafda6b5d3121b65ee8468d82 100644 (file)
@@ -56,6 +56,9 @@ EMERGE_WARNING_DELAY="10"
 # This option will be removed and forced to yes.
 AUTOCLEAN="yes"
 
+# Don't compress files with these suffixes.
+PORTAGE_COMPRESS_EXCLUDE_SUFFIXES="css gif htm[l]? jp[e]?g js pdf png"
+
 # Number of mirrors to try when a downloaded file has an incorrect checksum.
 PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5"
 
index c6020303896a4570890eb8cf4898b4f9cc185dae..6eab4b2ddd8da6c5c392e21b969a52554c90eff3 100644 (file)
@@ -430,6 +430,13 @@ install phase.
 \fBPORTAGE_COMPRESS_FLAGS\fR = \fI"\-9"\fR
 This variable contains flags for the \fBPORTAGE_COMPRESS\fR command.
 .TP
+.TP
+\fBPORTAGE_COMPRESS_EXCLUDE_SUFFIXES\fR = \fI"gif htm[l]? jp[e]?g pdf png"\fR
+This variable contains a space delimited list of file suffixes for which
+matching files are excluded when the \fBPORTAGE_COMPRESS\fR command is
+called. Regular expressions are supported and the match is performed only
+against the portion of the file name which follows the last period character.
+.TP
 .B PORTAGE_ELOG_CLASSES
 .TP
 .B PORTAGE_ELOG_SYSTEM
index f6fa77b4b2036e5ac7451c14da0c8bbf6428e935..595cabb67f0eb65028c8bc98ac9c63925a005ab4 100644 (file)
@@ -9471,27 +9471,12 @@ class Scheduler(PollScheduler):
 
                        log_paths = [failed_pkg.build_log]
 
-                       if not (build_dir and os.path.isdir(build_dir)):
-                               log_paths.append(failed_pkg.fetch_log)
-
-                       for log_path in log_paths:
-                               if not log_path:
-                                       continue
-
-                               try:
-                                       log_size = os.stat(log_path).st_size
-                               except OSError:
-                                       continue
-
-                               if log_size == 0:
-                                       continue
-
+                       log_path = self._locate_failure_log(failed_pkg)
+                       if log_path is not None:
                                try:
                                        log_file = open(log_path, 'rb')
                                except IOError:
-                                       continue
-
-                               break
+                                       pass
 
                        if log_file is not None:
                                try:
@@ -9548,6 +9533,32 @@ class Scheduler(PollScheduler):
                        self._failed_pkgs_die_msgs.append(
                                (mysettings, key, errors))
 
+       def _locate_failure_log(self, failed_pkg):
+
+               build_dir = failed_pkg.build_dir
+               log_file = None
+
+               log_paths = [failed_pkg.build_log]
+
+               if not (build_dir and os.path.isdir(build_dir)):
+                       log_paths.append(failed_pkg.fetch_log)
+
+               for log_path in log_paths:
+                       if not log_path:
+                               continue
+
+                       try:
+                               log_size = os.stat(log_path).st_size
+                       except OSError:
+                               continue
+
+                       if log_size == 0:
+                               continue
+
+                       return log_path
+
+               return None
+
        def _add_packages(self):
                pkg_queue = self._pkg_queue
                for pkg in self._mergelist:
@@ -9577,6 +9588,7 @@ class Scheduler(PollScheduler):
                                build_dir=build_dir, build_log=build_log,
                                fetch_log=fetch_log, pkg=pkg,
                                returncode=merge.returncode))
+                       self._failed_pkg_msg(self._failed_pkgs[-1], "install", "to")
 
                        self._status_display.failed = len(self._failed_pkgs)
                        return
@@ -9621,6 +9633,7 @@ class Scheduler(PollScheduler):
                                build_dir=build_dir, build_log=build_log,
                                fetch_log=fetch_log, pkg=build.pkg,
                                returncode=build.returncode))
+                       self._failed_pkg_msg(self._failed_pkgs[-1], "emerge", "for")
 
                        self._status_display.failed = len(self._failed_pkgs)
                        self._deallocate_config(build.settings)
@@ -9894,6 +9907,21 @@ class Scheduler(PollScheduler):
 
                return task
 
+       def _failed_pkg_msg(self, failed_pkg, action, preposition):
+               pkg = failed_pkg.pkg
+               msg = "%s to %s %s" % \
+                       (bad("Failed"), action, colorize("INFORM", pkg.cpv))
+               if pkg.root != "/":
+                       msg += " %s %s" % (preposition, pkg.root)
+
+               log_path = self._locate_failure_log(failed_pkg)
+               if log_path is not None:
+                       msg += ", Log file:"
+               self._status_msg(msg)
+
+               if log_path is not None:
+                       self._status_msg(" '%s'" % (colorize("INFORM", log_path),))
+
        def _status_msg(self, msg):
                """
                Display a brief status message (no newlines) in the status display.
@@ -10344,9 +10372,7 @@ def unmerge(root_config, myopts, unmerge_action,
                        # cycle through all our candidate deps and determine
                        # what will and will not get unmerged
                        try:
-                               mymatch=localtree.dep_match(x)
-                       except KeyError:
-                               mymatch=None
+                               mymatch = vartree.dbapi.match(x)
                        except ValueError, errpkgs:
                                print "\n\n!!! The short ebuild name \"" + \
                                        x + "\" is ambiguous.  Please specify"
@@ -13743,6 +13769,20 @@ def emerge_main():
        elif myaction in ("clean", "unmerge") or \
                (myaction == "prune" and "--nodeps" in myopts):
                validate_ebuild_environment(trees)
+
+               # Ensure atoms are valid before calling unmerge().
+               # For backward compat, leading '=' is not required.
+               for x in myfiles:
+                       if is_valid_package_atom(x) or \
+                               is_valid_package_atom("=" + x):
+                               continue
+                       msg = []
+                       msg.append("'%s' is not a valid package atom." % (x,))
+                       msg.append("Please check ebuild(5) for full details.")
+                       writemsg_level("".join("!!! %s\n" % line for line in msg),
+                               level=logging.ERROR, noiselevel=-1)
+                       return 1
+
                # When given a list of atoms, unmerge
                # them in the order given.
                ordered = myaction == "unmerge"
@@ -13752,9 +13792,39 @@ def emerge_main():
                                post_emerge(root_config, myopts, mtimedb, os.EX_OK)
 
        elif myaction in ("depclean", "prune"):
+
+               # Ensure atoms are valid before calling unmerge().
+               vardb = trees[settings["ROOT"]]["vartree"].dbapi
+               valid_atoms = []
+               for x in myfiles:
+                       if is_valid_package_atom(x):
+                               try:
+                                       valid_atoms.append(
+                                               portage.dep_expand(x, mydb=vardb, settings=settings))
+                               except ValueError, e:
+                                       msg = "The short ebuild name \"" + x + \
+                                               "\" is ambiguous.  Please specify " + \
+                                               "one of the following " + \
+                                               "fully-qualified ebuild names instead:"
+                                       for line in textwrap.wrap(msg, 70):
+                                               writemsg_level("!!! %s\n" % (line,),
+                                                       level=logging.ERROR, noiselevel=-1)
+                                       for i in e[0]:
+                                               writemsg_level("    %s\n" % colorize("INFORM", i),
+                                                       level=logging.ERROR, noiselevel=-1)
+                                       writemsg_level("\n", level=logging.ERROR, noiselevel=-1)
+                                       return 1
+                               continue
+                       msg = []
+                       msg.append("'%s' is not a valid package atom." % (x,))
+                       msg.append("Please check ebuild(5) for full details.")
+                       writemsg_level("".join("!!! %s\n" % line for line in msg),
+                               level=logging.ERROR, noiselevel=-1)
+                       return 1
+
                validate_ebuild_environment(trees)
                action_depclean(settings, trees, mtimedb["ldpath"],
-                       myopts, myaction, myfiles, spinner)
+                       myopts, myaction, valid_atoms, spinner)
                if not (buildpkgonly or fetchonly or pretend):
                        post_emerge(root_config, myopts, mtimedb, os.EX_OK)
        # "update", "system", or just process files:
index 2948ba6a446a3a8668dcb194fbd7f93517575dae..614f6dec3c5bb994aa8030f88d08c0bdd17afbc6 100644 (file)
@@ -263,7 +263,8 @@ class portdbapi(dbapi):
                        # if newer version, wipe everything and negate eapi
                        eapi = metadata["EAPI"]
                        metadata = {}
-                       map(lambda x: metadata.setdefault(x, ""), auxdbkeys)
+                       for x in self._known_keys:
+                               metadata.setdefault(x, "")
                        metadata["EAPI"] = "-" + eapi
 
                if metadata.get("INHERITED", False):
index e9a1fab703d20ed491f26940efa501836d91e824..3bc6bd1a8cbdc7df38450ae1f04ac7b93065c2b8 100644 (file)
@@ -21,7 +21,8 @@ except ImportError:
 
 if os.path.isdir("/proc/%i/fd" % os.getpid()):
        def get_open_fds():
-               return map(int, [fd for fd in os.listdir("/proc/%i/fd" % os.getpid()) if fd.isdigit()])
+               return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \
+                       if fd.isdigit())
 else:
        def get_open_fds():
                return xrange(max_fd_limit)