From: Fabian Groffen Date: Sat, 26 Sep 2009 18:46:04 +0000 (-0000) Subject: Merged from trunk -r14422:14432 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=8249bd0e9293fecc5c5b3f1dd1b78e7f6ced3ad1;p=portage.git Merged from trunk -r14422:14432 | 14423 | Add a test case for this issue: | | zmedico | http://bugs.python.org/issue5334 | | 14424 | Fix length calculation so it doesn't assume the length of | | zmedico | the encoded string is the same as the unicode string. | | 14425 | Remove unused imports. | | zmedico | | | 14426 | Use imp.reload() instead of reload() for compatibility with | | arfrever | Python 3. | | 14427 | Define __bool__() for compatibility with Python 3. | | arfrever | | | 14428 | Remove ugly code from __len__, since defining __bool__ | | zmedico | fixes the same issue. | | 14429 | Fix formatting of sizes with Python 3. | | arfrever | | | 14430 | Make sure self._allowed_keys are sorted in the table create | | zmedico | statement. They already are sorted, but do it explicitly | | | here for clarity. | | 14431 | Fix UnboundLocalError triggered by atom without category | | zmedico | for nonexistent package. Thanks to Jeremy Olexa | | | for reporting. | | 14432 | Bug #285767 - Add support to to identify and eliminate | | zmedico | redundant package selections when multiple atoms happen to | | | specify a version range. | svn path=/main/branches/prefix/; revision=14438 --- diff --git a/bin/ebuild b/bin/ebuild index ac5781ef8..bcf513cc3 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -26,6 +26,7 @@ def debug_signal(signum, frame): pdb.set_trace() signal.signal(signal.SIGUSR1, debug_signal) +import imp import optparse import os @@ -123,7 +124,7 @@ if ebuild_portdir not in portage.portdb.porttrees: os.environ.get("PORTDIR_OVERLAY","") + " " + ebuild_portdir print("Appending %s to PORTDIR_OVERLAY..." % ebuild_portdir) portage.close_portdbapi_caches() - reload(portage) + imp.reload(portage) del portage.portdb.porttrees[1:] if ebuild_portdir != portage.portdb.porttree_root: portage.portdb.porttrees.append(ebuild_portdir) diff --git a/bin/repoman b/bin/repoman index 1ce1556d5..981998b05 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1127,7 +1127,7 @@ for x in scanlist: # current policy is no files over 20k, this is the check. elif mystat.st_size > 20480: stats["file.size"] += 1 - fails["file.size"].append("("+ str(mystat.st_size/1024) + "K) "+x+"/files/"+y) + fails["file.size"].append("("+ str(mystat.st_size//1024) + "K) "+x+"/files/"+y) m = disallowed_filename_chars_re.search( os.path.basename(y.rstrip(os.sep))) diff --git a/pym/_emerge/Dependency.py b/pym/_emerge/Dependency.py index 6f52744c0..16cb41f70 100644 --- a/pym/_emerge/Dependency.py +++ b/pym/_emerge/Dependency.py @@ -5,7 +5,7 @@ from _emerge.DepPriority import DepPriority from _emerge.SlotObject import SlotObject class Dependency(SlotObject): - __slots__ = ("atom", "blocker", "depth", + __slots__ = ("atom", "blocker", "child", "depth", "parent", "onlydeps", "priority", "root") def __init__(self, **kwargs): SlotObject.__init__(self, **kwargs) diff --git a/pym/_emerge/SequentialTaskQueue.py b/pym/_emerge/SequentialTaskQueue.py index 7dcc6e516..4909290c0 100644 --- a/pym/_emerge/SequentialTaskQueue.py +++ b/pym/_emerge/SequentialTaskQueue.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import sys from _emerge.SlotObject import SlotObject from collections import deque class SequentialTaskQueue(SlotObject): @@ -79,9 +80,11 @@ class SequentialTaskQueue(SlotObject): task.cancel() self._dirty = False - def __nonzero__(self): + def __bool__(self): return bool(self._task_queue or self.running_tasks) + if sys.hexversion < 0x3000000: + __nonzero__ = __bool__ + def __len__(self): return len(self._task_queue) + len(self.running_tasks) - diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index e8d1600af..30277f948 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -687,8 +687,19 @@ class depgraph(object): root=dep.parent.root) self._dynamic_config._blocker_parents.add(blocker, dep.parent) return 1 - dep_pkg, existing_node = self._select_package(dep.root, dep.atom, - onlydeps=dep.onlydeps) + + if dep.child is None: + dep_pkg, existing_node = self._select_package(dep.root, dep.atom, + onlydeps=dep.onlydeps) + else: + # The caller has selected a specific package + # via self._minimize_packages(). + dep_pkg = dep.child + existing_node = self._dynamic_config._slot_pkg_map[ + dep.root].get(dep_pkg.slot_atom) + if existing_node is not dep_pkg: + existing_node = None + if not dep_pkg: if dep.priority.optional: # This could be an unecessary build-time dep @@ -1151,16 +1162,18 @@ class depgraph(object): if debug: print("Candidates:", selected_atoms) - vardb = self._frozen_config.roots[dep_root].trees["vartree"].dbapi + root_config = self._frozen_config.roots[dep_root] + vardb = root_config.trees["vartree"].dbapi - for atom in selected_atoms[pkg]: + for atom, child in self._minimize_children( + pkg, dep_priority, root_config, selected_atoms[pkg]): mypriority = dep_priority.copy() if not atom.blocker and vardb.match(atom): mypriority.satisfied = True if not self._add_dep(Dependency(atom=atom, - blocker=atom.blocker, depth=depth, parent=pkg, + blocker=atom.blocker, child=child, depth=depth, parent=pkg, priority=mypriority, root=dep_root), allow_unsatisfied=allow_unsatisfied): return 0 @@ -1184,14 +1197,15 @@ class depgraph(object): root=dep_root)): return 0 - for atom in atoms: + for atom, child in self._minimize_children( + pkg, self._priority(runtime=True), root_config, atoms): # This is a GLEP 37 virtual, so its deps are all runtime. mypriority = self._priority(runtime=True) if not atom.blocker and vardb.match(atom): mypriority.satisfied = True if not self._add_dep(Dependency(atom=atom, - blocker=atom.blocker, depth=virt_pkg.depth, + blocker=atom.blocker, child=child, depth=virt_pkg.depth, parent=virt_pkg, priority=mypriority, root=dep_root), allow_unsatisfied=allow_unsatisfied): return 0 @@ -1201,6 +1215,73 @@ class depgraph(object): return 1 + def _minimize_children(self, parent, priority, root_config, atoms): + """ + Selects packages to satisfy the given atoms, and minimizes the + number of selected packages. This serves to identify and eliminate + redundant package selections when multiple atoms happen to specify + a version range. + """ + + atom_pkg_map = {} + + for atom in atoms: + if atom.blocker: + yield (atom, None) + continue + dep_pkg, existing_node = self._select_package( + root_config.root, atom) + if dep_pkg is None: + yield (atom, None) + continue + atom_pkg_map[atom] = dep_pkg + + if len(atom_pkg_map) < 2: + for item in atom_pkg_map.items(): + yield item + return + + cp_pkg_map = {} + pkg_atom_map = {} + for atom, pkg in atom_pkg_map.items(): + pkg_atom_map.setdefault(pkg, set()).add(atom) + cp_pkg_map.setdefault(pkg.cp, set()).add(pkg) + + for cp, pkgs in cp_pkg_map.items(): + if len(pkgs) < 2: + for pkg in pkgs: + for atom in pkg_atom_map[pkg]: + yield (atom, pkg) + continue + + # Use a digraph to identify and eliminate any + # redundant package selections. + atom_pkg_graph = digraph() + cp_atoms = set() + for pkg1 in pkgs: + for atom in pkg_atom_map[pkg1]: + cp_atoms.add(atom) + atom_pkg_graph.add(pkg1, atom) + atom_set = InternalPackageSet(initial_atoms=(atom,)) + for pkg2 in pkgs: + if pkg2 is pkg1: + continue + if atom_set.findAtomForPackage(pkg2): + atom_pkg_graph.add(pkg2, atom) + + for pkg in pkgs: + eliminate_pkg = True + for atom in atom_pkg_graph.parent_nodes(pkg): + if len(atom_pkg_graph.child_nodes(atom)) < 2: + eliminate_pkg = False + break + if eliminate_pkg: + atom_pkg_graph.remove(pkg) + + for atom in cp_atoms: + child_pkgs = atom_pkg_graph.child_nodes(atom) + yield (atom, child_pkgs[0]) + def _queue_disjunctive_deps(self, pkg, dep_root, dep_priority, dep_struct): """ Queue disjunctive (virtual and ||) deps in self._dynamic_config._dep_disjunctive_stack. @@ -1483,7 +1564,7 @@ class depgraph(object): # Allow the depgraph to choose which virtual. atom = Atom(null_atom.replace('null/', 'virtual/', 1)) else: - null_atom + atom = null_atom args.append(AtomArg(arg=x, atom=atom, root_config=root_config)) diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py index 355307972..f0611f28a 100644 --- a/pym/_emerge/search.py +++ b/pym/_emerge/search.py @@ -341,7 +341,7 @@ class search(object): break if myebuild and file_size_str is None: - mystr = str(mysum[0] / 1024) + mystr = str(mysum[0] // 1024) mycount = len(mystr) while (mycount > 3): mycount -= 3 diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index 1ac6f1ffe..0c23a78ae 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -103,7 +103,7 @@ class database(fs_template.FsBased): self._db_table["packages"]["create"] = " ".join(create_statement) self._db_table["packages"]["columns"] = \ self._db_table["packages"]["internal_columns"] + \ - self._allowed_keys + sorted(self._allowed_keys) cursor = self._db_cursor for k, v in self._db_table.items(): diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 98448d959..1fe4bc905 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -412,9 +412,12 @@ class _use_dep(object): raise InvalidAtom(_("Invalid use dep: '%s'") % (token,)) return flag - def __nonzero__(self): + def __bool__(self): return bool(self.tokens) + if sys.hexversion < 0x3000000: + __nonzero__ = __bool__ + def __str__(self): if not self.tokens: return "" diff --git a/pym/portage/proxy/objectproxy.py b/pym/portage/proxy/objectproxy.py index 6dffff7d5..a064d7056 100644 --- a/pym/portage/proxy/objectproxy.py +++ b/pym/portage/proxy/objectproxy.py @@ -2,6 +2,8 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import sys + __all__ = ['ObjectProxy'] class ObjectProxy(object): @@ -45,15 +47,7 @@ class ObjectProxy(object): return iter(object.__getattribute__(self, '_get_target')()) def __len__(self): - try: - return len(object.__getattribute__(self, '_get_target')()) - except TypeError: - # For python 3.x with boolean target, len() results in - # TypeError, so return 0 or 1. - if bool(object.__getattribute__(self, '_get_target')()): - return 1 - else: - return 0 + return len(object.__getattribute__(self, '_get_target')()) def __repr__(self): return repr(object.__getattribute__(self, '_get_target')()) @@ -70,8 +64,11 @@ class ObjectProxy(object): def __ne__(self, other): return object.__getattribute__(self, '_get_target')() != other - def __nonzero__(self): + def __bool__(self): return bool(object.__getattribute__(self, '_get_target')()) + if sys.hexversion < 0x3000000: + __nonzero__ = __bool__ + def __int__(self): return int(object.__getattribute__(self, '_get_target')()) diff --git a/pym/portage/sets/base.py b/pym/portage/sets/base.py index d58b5f297..216c8cc16 100644 --- a/pym/portage/sets/base.py +++ b/pym/portage/sets/base.py @@ -41,10 +41,13 @@ class PackageSet(object): for x in self._nonatoms: yield x - def __nonzero__(self): + def __bool__(self): self._load() return bool(self._atoms or self._nonatoms) + if sys.hexversion < 0x3000000: + __nonzero__ = __bool__ + def supportsOperation(self, op): if not op in OPERATIONS: raise ValueError(op) diff --git a/pym/portage/tests/ebuild/test_array_fromfile_eof.py b/pym/portage/tests/ebuild/test_array_fromfile_eof.py new file mode 100644 index 000000000..efb292f8a --- /dev/null +++ b/pym/portage/tests/ebuild/test_array_fromfile_eof.py @@ -0,0 +1,44 @@ +# Copyright 2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +import array +import tempfile + +from portage import _unicode_decode +from portage import _unicode_encode +from portage.tests import TestCase + +class ArrayFromfileEofTestCase(TestCase): + + def testArrayFromfileEof(self): + # This tests if the following python issue is fixed + # in the currently running version of python: + # http://bugs.python.org/issue5334 + + input_data = "an arbitrary string" + input_bytes = _unicode_encode(input_data, + encoding='utf_8', errors='strict') + f = tempfile.TemporaryFile() + f.write(input_bytes) + + f.seek(0) + data = [] + eof = False + while not eof: + a = array.array('B') + try: + a.fromfile(f, len(input_bytes) + 1) + except EOFError: + # python-3.0 lost data here + eof = True + + if not a: + eof = True + else: + data.append(_unicode_decode(a.tostring(), + encoding='utf_8', errors='strict')) + + f.close() + + self.assertEqual(input_data, ''.join(data)) diff --git a/pym/portage/tests/ebuild/test_pty_eof.py b/pym/portage/tests/ebuild/test_pty_eof.py index 8723372b1..0dd1e8506 100644 --- a/pym/portage/tests/ebuild/test_pty_eof.py +++ b/pym/portage/tests/ebuild/test_pty_eof.py @@ -1,11 +1,8 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ -import pty - import portage -from portage import os from portage.tests import TestCase class PtyEofTestCase(TestCase):