Merged from trunk -r14422:14432
authorFabian Groffen <grobian@gentoo.org>
Sat, 26 Sep 2009 18:46:04 +0000 (18:46 -0000)
committerFabian Groffen <grobian@gentoo.org>
Sat, 26 Sep 2009 18:46:04 +0000 (18:46 -0000)
   | 14423    | Add a test case for this issue:                             |
   | zmedico  | http://bugs.python.org/issue5334                            |

   | 14424    | Fix length calculation so it doesn't assume the length of   |
   | zmedico  | the encoded string is the same as the unicode string.       |

   | 14425    | Remove unused imports.                                      |
   | zmedico  |                                                             |

   | 14426    | Use imp.reload() instead of reload() for compatibility with |
   | arfrever | Python 3.                                                   |

   | 14427    | Define __bool__() for compatibility with Python 3.          |
   | arfrever |                                                             |

   | 14428    | Remove ugly code from __len__, since defining __bool__      |
   | zmedico  | fixes the same issue.                                       |

   | 14429    | Fix formatting of sizes with Python 3.                      |
   | arfrever |                                                             |

   | 14430    | Make sure self._allowed_keys are sorted in the table create |
   | zmedico  | statement. They already are sorted, but do it explicitly    |
   |          | here for clarity.                                           |

   | 14431    | Fix UnboundLocalError triggered by atom without category    |
   | zmedico  | for nonexistent package. Thanks to Jeremy Olexa             |
   |          | <darkside@gentoo.org> for reporting.                        |

   | 14432    | Bug #285767 - Add support to to identify and eliminate      |
   | zmedico  | redundant package selections when multiple atoms happen to  |
   |          | specify a version range.                                    |

svn path=/main/branches/prefix/; revision=14438

12 files changed:
bin/ebuild
bin/repoman
pym/_emerge/Dependency.py
pym/_emerge/SequentialTaskQueue.py
pym/_emerge/depgraph.py
pym/_emerge/search.py
pym/portage/cache/sqlite.py
pym/portage/dep.py
pym/portage/proxy/objectproxy.py
pym/portage/sets/base.py
pym/portage/tests/ebuild/test_array_fromfile_eof.py [new file with mode: 0644]
pym/portage/tests/ebuild/test_pty_eof.py

index ac5781ef8555ab673c01310cb1ae929bc29f3a14..bcf513cc340ef851ce07de31f895501487a240ab 100755 (executable)
@@ -26,6 +26,7 @@ def debug_signal(signum, frame):
        pdb.set_trace()
 signal.signal(signal.SIGUSR1, debug_signal)
 
+import imp
 import optparse
 import os
 
@@ -123,7 +124,7 @@ if ebuild_portdir not in portage.portdb.porttrees:
                os.environ.get("PORTDIR_OVERLAY","") + " " + ebuild_portdir
        print("Appending %s to PORTDIR_OVERLAY..." % ebuild_portdir)
        portage.close_portdbapi_caches()
-       reload(portage)
+       imp.reload(portage)
 del portage.portdb.porttrees[1:]
 if ebuild_portdir != portage.portdb.porttree_root:
        portage.portdb.porttrees.append(ebuild_portdir)
index 1ce1556d5733756aca7e8c6b4de2a28be404aebb..981998b057b7b5758330535d88949928de7cb906 100755 (executable)
@@ -1127,7 +1127,7 @@ for x in scanlist:
                        # current policy is no files over 20k, this is the check.
                        elif mystat.st_size > 20480:
                                stats["file.size"] += 1
-                               fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y)
+                               fails["file.size"].append("("+ str(mystat.st_size//1024) + "K) "+x+"/files/"+y)
 
                        m = disallowed_filename_chars_re.search(
                                os.path.basename(y.rstrip(os.sep)))
index 6f52744c0752184ce5a8e9026fd346a3c5bd67f8..16cb41f70def76ddf2fe12b77aeef508e25c10df 100644 (file)
@@ -5,7 +5,7 @@
 from _emerge.DepPriority import DepPriority
 from _emerge.SlotObject import SlotObject
 class Dependency(SlotObject):
-       __slots__ = ("atom", "blocker", "depth",
+       __slots__ = ("atom", "blocker", "child", "depth",
                "parent", "onlydeps", "priority", "root")
        def __init__(self, **kwargs):
                SlotObject.__init__(self, **kwargs)
index 7dcc6e516750259e895ac7784fdd506c915352ef..4909290c06a69a58af3fb51367778b08ff9f0c6c 100644 (file)
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import sys
 from _emerge.SlotObject import SlotObject
 from collections import deque
 class SequentialTaskQueue(SlotObject):
@@ -79,9 +80,11 @@ class SequentialTaskQueue(SlotObject):
                        task.cancel()
                self._dirty = False
 
-       def __nonzero__(self):
+       def __bool__(self):
                return bool(self._task_queue or self.running_tasks)
 
+       if sys.hexversion < 0x3000000:
+               __nonzero__ = __bool__
+
        def __len__(self):
                return len(self._task_queue) + len(self.running_tasks)
-
index e8d1600af4845acded5a40acb55f8a84afa7fff4..30277f94829a1fc18a4f54a3129716582133d826 100644 (file)
@@ -687,8 +687,19 @@ class depgraph(object):
                                        root=dep.parent.root)
                                self._dynamic_config._blocker_parents.add(blocker, dep.parent)
                        return 1
-               dep_pkg, existing_node = self._select_package(dep.root, dep.atom,
-                       onlydeps=dep.onlydeps)
+
+               if dep.child is None:
+                       dep_pkg, existing_node = self._select_package(dep.root, dep.atom,
+                               onlydeps=dep.onlydeps)
+               else:
+                       # The caller has selected a specific package
+                       # via self._minimize_packages().
+                       dep_pkg = dep.child
+                       existing_node = self._dynamic_config._slot_pkg_map[
+                               dep.root].get(dep_pkg.slot_atom)
+                       if existing_node is not dep_pkg:
+                               existing_node = None 
+
                if not dep_pkg:
                        if dep.priority.optional:
                                # This could be an unecessary build-time dep
@@ -1151,16 +1162,18 @@ class depgraph(object):
                if debug:
                        print("Candidates:", selected_atoms)
 
-               vardb = self._frozen_config.roots[dep_root].trees["vartree"].dbapi
+               root_config = self._frozen_config.roots[dep_root]
+               vardb = root_config.trees["vartree"].dbapi
 
-               for atom in selected_atoms[pkg]:
+               for atom, child in self._minimize_children(
+                       pkg, dep_priority, root_config, selected_atoms[pkg]):
 
                        mypriority = dep_priority.copy()
                        if not atom.blocker and vardb.match(atom):
                                mypriority.satisfied = True
 
                        if not self._add_dep(Dependency(atom=atom,
-                               blocker=atom.blocker, depth=depth, parent=pkg,
+                               blocker=atom.blocker, child=child, depth=depth, parent=pkg,
                                priority=mypriority, root=dep_root),
                                allow_unsatisfied=allow_unsatisfied):
                                return 0
@@ -1184,14 +1197,15 @@ class depgraph(object):
                                root=dep_root)):
                                return 0
 
-                       for atom in atoms:
+                       for atom, child in self._minimize_children(
+                               pkg, self._priority(runtime=True), root_config, atoms):
                                # This is a GLEP 37 virtual, so its deps are all runtime.
                                mypriority = self._priority(runtime=True)
                                if not atom.blocker and vardb.match(atom):
                                        mypriority.satisfied = True
 
                                if not self._add_dep(Dependency(atom=atom,
-                                       blocker=atom.blocker, depth=virt_pkg.depth,
+                                       blocker=atom.blocker, child=child, depth=virt_pkg.depth,
                                        parent=virt_pkg, priority=mypriority, root=dep_root),
                                        allow_unsatisfied=allow_unsatisfied):
                                        return 0
@@ -1201,6 +1215,73 @@ class depgraph(object):
 
                return 1
 
+       def _minimize_children(self, parent, priority, root_config, atoms):
+               """
+               Selects packages to satisfy the given atoms, and minimizes the
+               number of selected packages. This serves to identify and eliminate
+               redundant package selections when multiple atoms happen to specify
+               a version range.
+               """
+
+               atom_pkg_map = {}
+
+               for atom in atoms:
+                       if atom.blocker:
+                               yield (atom, None)
+                               continue
+                       dep_pkg, existing_node = self._select_package(
+                               root_config.root, atom)
+                       if dep_pkg is None:
+                               yield (atom, None)
+                               continue
+                       atom_pkg_map[atom] = dep_pkg
+
+               if len(atom_pkg_map) < 2:
+                       for item in atom_pkg_map.items():
+                               yield item
+                       return
+
+               cp_pkg_map = {}
+               pkg_atom_map = {}
+               for atom, pkg in atom_pkg_map.items():
+                       pkg_atom_map.setdefault(pkg, set()).add(atom)
+                       cp_pkg_map.setdefault(pkg.cp, set()).add(pkg)
+
+               for cp, pkgs in cp_pkg_map.items():
+                       if len(pkgs) < 2:
+                               for pkg in pkgs:
+                                       for atom in pkg_atom_map[pkg]:
+                                               yield (atom, pkg)
+                               continue
+
+                       # Use a digraph to identify and eliminate any
+                       # redundant package selections.
+                       atom_pkg_graph = digraph()
+                       cp_atoms = set()
+                       for pkg1 in pkgs:
+                               for atom in pkg_atom_map[pkg1]:
+                                       cp_atoms.add(atom)
+                                       atom_pkg_graph.add(pkg1, atom)
+                                       atom_set = InternalPackageSet(initial_atoms=(atom,))
+                                       for pkg2 in pkgs:
+                                               if pkg2 is pkg1:
+                                                       continue
+                                               if atom_set.findAtomForPackage(pkg2):
+                                                       atom_pkg_graph.add(pkg2, atom)
+
+                       for pkg in pkgs:
+                               eliminate_pkg = True
+                               for atom in atom_pkg_graph.parent_nodes(pkg):
+                                       if len(atom_pkg_graph.child_nodes(atom)) < 2:
+                                               eliminate_pkg = False
+                                               break
+                               if eliminate_pkg:
+                                       atom_pkg_graph.remove(pkg)
+
+                       for atom in cp_atoms:
+                               child_pkgs = atom_pkg_graph.child_nodes(atom)
+                               yield (atom, child_pkgs[0])
+
        def _queue_disjunctive_deps(self, pkg, dep_root, dep_priority, dep_struct):
                """
                Queue disjunctive (virtual and ||) deps in self._dynamic_config._dep_disjunctive_stack.
@@ -1483,7 +1564,7 @@ class depgraph(object):
                                                # Allow the depgraph to choose which virtual.
                                                atom = Atom(null_atom.replace('null/', 'virtual/', 1))
                                        else:
-                                               null_atom
+                                               atom = null_atom
 
                                args.append(AtomArg(arg=x, atom=atom,
                                        root_config=root_config))
index 355307972920e3e60b6aa18034b381df1504653e..f0611f28a946143c3a9b83df56cbb9624bf838fe 100644 (file)
@@ -341,7 +341,7 @@ class search(object):
                                                        break
 
                                        if myebuild and file_size_str is None:
-                                               mystr = str(mysum[0] / 1024)
+                                               mystr = str(mysum[0] // 1024)
                                                mycount = len(mystr)
                                                while (mycount > 3):
                                                        mycount -= 3
index 1ac6f1ffee49606d4089374042ae4d851ddfbf1a..0c23a78ae37239e8278fc39a94ec7bad6f6ec2b5 100644 (file)
@@ -103,7 +103,7 @@ class database(fs_template.FsBased):
                self._db_table["packages"]["create"] = " ".join(create_statement)
                self._db_table["packages"]["columns"] = \
                        self._db_table["packages"]["internal_columns"] + \
-                       self._allowed_keys
+                       sorted(self._allowed_keys)
 
                cursor = self._db_cursor
                for k, v in self._db_table.items():
index 98448d9593f76bc59019d3dec5afe0ff0d8cdde0..1fe4bc905fba7e73b1fd03dde43516b2020168e5 100644 (file)
@@ -412,9 +412,12 @@ class _use_dep(object):
                        raise InvalidAtom(_("Invalid use dep: '%s'") % (token,))
                return flag
 
-       def __nonzero__(self):
+       def __bool__(self):
                return bool(self.tokens)
 
+       if sys.hexversion < 0x3000000:
+               __nonzero__ = __bool__
+
        def __str__(self):
                if not self.tokens:
                        return ""
index 6dffff7d5ed57434b297eae91d1943eedcbf8dd1..a064d705668739482fa25325a45f92e6c07eedc0 100644 (file)
@@ -2,6 +2,8 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
+import sys
+
 __all__ = ['ObjectProxy']
 
 class ObjectProxy(object):
@@ -45,15 +47,7 @@ class ObjectProxy(object):
                return iter(object.__getattribute__(self, '_get_target')())
 
        def __len__(self):
-               try:
-                       return len(object.__getattribute__(self, '_get_target')())
-               except TypeError:
-                       # For python 3.x with boolean target, len() results in
-                       # TypeError, so return 0 or 1.
-                       if bool(object.__getattribute__(self, '_get_target')()):
-                               return 1
-                       else:
-                               return 0
+               return len(object.__getattribute__(self, '_get_target')())
 
        def __repr__(self):
                return repr(object.__getattribute__(self, '_get_target')())
@@ -70,8 +64,11 @@ class ObjectProxy(object):
        def __ne__(self, other):
                return object.__getattribute__(self, '_get_target')() != other
 
-       def __nonzero__(self):
+       def __bool__(self):
                return bool(object.__getattribute__(self, '_get_target')())
 
+       if sys.hexversion < 0x3000000:
+               __nonzero__ = __bool__
+
        def __int__(self):
                return int(object.__getattribute__(self, '_get_target')())
index d58b5f297be960cfca6c1c0a76d685f1a7d76be5..216c8cc1653f7f05944ca2c20cfc5fa1c9f01217 100644 (file)
@@ -41,10 +41,13 @@ class PackageSet(object):
                for x in self._nonatoms:
                        yield x
 
-       def __nonzero__(self):
+       def __bool__(self):
                self._load()
                return bool(self._atoms or self._nonatoms)
 
+       if sys.hexversion < 0x3000000:
+               __nonzero__ = __bool__
+
        def supportsOperation(self, op):
                if not op in OPERATIONS:
                        raise ValueError(op)
diff --git a/pym/portage/tests/ebuild/test_array_fromfile_eof.py b/pym/portage/tests/ebuild/test_array_fromfile_eof.py
new file mode 100644 (file)
index 0000000..efb292f
--- /dev/null
@@ -0,0 +1,44 @@
+# Copyright 2009 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+import array
+import tempfile
+
+from portage import _unicode_decode
+from portage import _unicode_encode
+from portage.tests import TestCase
+
+class ArrayFromfileEofTestCase(TestCase):
+
+       def testArrayFromfileEof(self):
+               # This tests if the following python issue is fixed
+               # in the currently running version of python:
+               #   http://bugs.python.org/issue5334
+
+               input_data = "an arbitrary string"
+               input_bytes = _unicode_encode(input_data,
+                       encoding='utf_8', errors='strict')
+               f = tempfile.TemporaryFile()
+               f.write(input_bytes)
+
+               f.seek(0)
+               data = []
+               eof = False
+               while not eof:
+                       a = array.array('B')
+                       try:
+                               a.fromfile(f, len(input_bytes) + 1)
+                       except EOFError:
+                               # python-3.0 lost data here
+                               eof = True
+
+                       if not a:
+                               eof = True
+                       else:
+                               data.append(_unicode_decode(a.tostring(),
+                                       encoding='utf_8', errors='strict'))
+
+               f.close()
+
+               self.assertEqual(input_data, ''.join(data))
index 8723372b11f85b4423a082a449ba36a7bcee63b6..0dd1e85069800a343d8164c52f09b15b475807ea 100644 (file)
@@ -1,11 +1,8 @@
-# Copyright 1998-2007 Gentoo Foundation
+# Copyright 2009 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import pty
-
 import portage
-from portage import os
 from portage.tests import TestCase
 
 class PtyEofTestCase(TestCase):