From: Fabian Groffen Date: Wed, 25 Nov 2009 15:13:31 +0000 (-0000) Subject: Merged from trunk -r14852:14876 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=85f8f0036aa7d0c8d199f7105b43b0e09000f147;p=portage.git Merged from trunk -r14852:14876 | 14853 | Disable backtracking in the depgraph constructor call when | | zmedico | appropriate. | | 14854 | Bug #293730 - Fix bad || choices triggered in some cases | | zmedico | inside _dep_check_composite_db._visible() when --update is | | | not enabled. | | 14855 | Fix depgraph._prune_tree_display() to preserve uninstall | | zmedico | tasks. Thanks to Jonathan Callen for reporting. | | 14856 | Fix blocker display to work properly with --tree | | zmedico | --unordered-display. | | 14857 | In depgraph._serialize_tasks(), ensure that unsolvable | | zmedico | blockers can't be accidentally marked as satisfied. | | 14858 | Tweak logic from previous commit. | | zmedico | | | 14859 | Optimized the code for bug #288083 and make it handle more | | zmedico | cases. Now Package instances have 'visible' and 'masks' | | | attributes, since this information needs to be accessed in | | | more places now. | | 14873 | Bug #293998 - Use OSError.errno instead of trying to | | zmedico | subscript the exception when handling errno.EXDEV in | | | movefile(), since exceptions do not support subscripting in | | | python3. | | 14874 | Add "repository" attribute to CategorySet (revert r11597). | | zmedico | Thanks to TGL for this patch, posted on bug | | | #291414. | | 14875 | Bug #293614 - When the backtracking limit is reached and | | zmedico | there is a SLOT conflict to display, hint that a larger | | | --backtrack value might solve the conflict automatically. | | 14876 | Add a test case which simply compiles all modules, useful | | zmedico | for testing syntax compatibility with the current version of | | | python. | svn path=/main/branches/prefix/; revision=14880 --- diff --git a/doc/config/sets.docbook b/doc/config/sets.docbook index ec8ff9a2a..35beef41d 100644 --- a/doc/config/sets.docbook +++ b/doc/config/sets.docbook @@ -124,6 +124,7 @@ [installed category packages] class = portage.sets.dbapi.CategorySet multiset = true + repository = vartree name_pattern = $category/* @@ -438,6 +439,13 @@ category: Required. The name of an existing ebuild category which should be used to create the package set. + repository: Optional, defaults to + porttree. It determines which repository class should + be used to create the package set. Valid values for this option are: + porttree (normal ebuild repository), + vartree (installed package repository) + and bintree (local binary package repository). + only_visible: Optional, defaults to true. When set to true the set will only include visible packages, when set to false it will also include masked packages. diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py index be678185d..f9844ce45 100644 --- a/pym/_emerge/Package.py +++ b/pym/_emerge/Package.py @@ -9,6 +9,7 @@ import portage from portage.cache.mappings import slot_dict_class from portage.dep import paren_reduce, use_reduce, \ paren_normalize, paren_enclose, _slot_re +from portage.const import EPREFIX from _emerge.Task import Task if sys.hexversion >= 0x3000000: @@ -22,8 +23,8 @@ class Package(Task): "installed", "metadata", "onlydeps", "operation", "root_config", "type_name", "category", "counter", "cp", "cpv_split", - "inherited", "invalid", "iuse", "mtime", - "pf", "pv_split", "root", "slot", "slot_atom",) + \ + "inherited", "invalid", "iuse", "masks", "mtime", + "pf", "pv_split", "root", "slot", "slot_atom", "visible",) + \ ("_use",) metadata_keys = [ @@ -51,6 +52,95 @@ class Package(Task): self.category, self.pf = portage.catsplit(self.cpv) self.cpv_split = portage.catpkgsplit(self.cpv) self.pv_split = self.cpv_split[1:] + self.masks = self._masks() + self.visible = self._visible(self.masks) + + def _masks(self): + masks = {} + settings = self.root_config.settings + + if self.invalid is not None: + masks['invalid'] = self.invalid + + if not settings._accept_chost(self.cpv, self.metadata): + masks['CHOST'] = self.metadata['CHOST'] + + eapi = self.metadata["EAPI"] + if not portage.eapi_is_supported(eapi): + masks['EAPI.unsupported'] = eapi + if portage._eapi_is_deprecated(eapi): + masks['EAPI.deprecated'] = eapi + + missing_keywords = settings._getMissingKeywords( + self.cpv, self.metadata) + if missing_keywords: + masks['KEYWORDS'] = missing_keywords + + if self.built and not self.installed: + # we can have an old binary which has no EPREFIX information + if "EPREFIX" not in self.metadata or not self.metadata["EPREFIX"]: + masks['EPREFIX.missing'] = '' + if len(self.metadata["EPREFIX"].strip()) < len(EPREFIX): + masks['EPREFIX.tooshort'] = self.metadata["EPREFIX"].strip() + + try: + missing_properties = settings._getMissingProperties( + self.cpv, self.metadata) + if missing_properties: + masks['PROPERTIES'] = missing_properties + except portage.exception.InvalidDependString: + # already recorded as 'invalid' + pass + + mask_atom = settings._getMaskAtom(self.cpv, self.metadata) + if mask_atom is not None: + masks['package.mask'] = mask_atom + + system_mask = settings._getProfileMaskAtom( + self.cpv, self.metadata) + if system_mask is not None: + masks['profile.system'] = system_mask + + try: + missing_licenses = settings._getMissingLicenses( + self.cpv, self.metadata) + if missing_licenses: + masks['LICENSE'] = missing_licenses + except portage.exception.InvalidDependString: + # already recorded as 'invalid' + pass + + if not masks: + masks = None + + return masks + + def _visible(self, masks): + + if masks is not None: + + if 'EAPI.unsupported' in masks: + return False + + if self.built and not self.installed and ( \ + 'EPREFIX.missing' in masks or \ + 'EPREFIX.tooshort' in masks): + return False + + if not self.installed and ( \ + 'invalid' in masks or \ + 'CHOST' in masks or \ + 'EAPI.deprecated' in masks or \ + 'KEYWORDS' in masks or \ + 'PROPERTIES' in masks): + return False + + if 'package.mask' in masks or \ + 'profile.system' in masks or \ + 'LICENSE' in masks: + return False + + return True def _invalid_metadata(self, msg_type, msg): if self.invalid is None: diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index 5228e0c55..93beb83b2 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -49,7 +49,6 @@ from _emerge.search import search from _emerge.SetArg import SetArg from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice from _emerge.UnmergeDepPriority import UnmergeDepPriority -from _emerge.visible import visible if sys.hexversion >= 0x3000000: basestring = str @@ -74,6 +73,7 @@ class _frozen_depgraph_config(object): self.roots = {} # All Package instances self._pkg_cache = {} + self._highest_license_masked = {} for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -538,6 +538,14 @@ class depgraph(object): msg.append("satisfy simultaneously. If such a conflict exists in ") msg.append("the dependencies of two different packages, then those ") msg.append("packages can not be installed simultaneously.") + backtrack_opt = self._frozen_config.myopts.get('--backtrack') + if not self._dynamic_config._allow_backtracking and \ + (backtrack_opt is None or \ + (backtrack_opt > 0 and backtrack_opt < 30)): + msg.append(" You may want to try a larger value of the ") + msg.append("--backtrack option, such as --backtrack=30, ") + msg.append("in order to see if that will solve this conflict ") + msg.append("automatically.") from formatter import AbstractFormatter, DumbWriter f = AbstractFormatter(DumbWriter(sys.stderr, maxcol=72)) @@ -989,6 +997,7 @@ class depgraph(object): self._dynamic_config._slot_pkg_map[pkg.root][pkg.slot_atom] = pkg self._dynamic_config.mydbapi[pkg.root].cpv_inject(pkg) self._dynamic_config._filtered_trees[pkg.root]["porttree"].dbapi._clear_cache() + self._check_masks(pkg) if not pkg.installed: # Allow this package to satisfy old-style virtuals in case it @@ -1047,6 +1056,17 @@ class depgraph(object): dep_stack.append(pkg) return 1 + def _check_masks(self, pkg): + + slot_key = (pkg.root, pkg.slot_atom) + + # Check for upgrades in the same slot that are + # masked due to a LICENSE change in a newer + # version that is not masked for any other reason. + other_pkg = self._frozen_config._highest_license_masked.get(slot_key) + if other_pkg is not None and pkg < other_pkg: + self._dynamic_config._masked_license_updates.add(other_pkg) + def _add_parent_atom(self, pkg, parent_atom): parent_atoms = self._dynamic_config._parent_atoms.get(pkg) if parent_atoms is None: @@ -2350,7 +2370,7 @@ class depgraph(object): pkg, existing = ret if pkg is not None: settings = pkg.root_config.settings - if visible(settings, pkg) and not (pkg.installed and \ + if pkg.visible and not (pkg.installed and \ settings._getMissingKeywords(pkg.cpv, pkg.metadata)): self._dynamic_config._visible_pkgs[pkg.root].cpv_inject(pkg) return ret @@ -2425,12 +2445,9 @@ class depgraph(object): # here, packages that have been masked since they # were installed can be automatically downgraded # to an unmasked version. - try: - if not visible(pkgsettings, pkg): - continue - except portage.exception.InvalidDependString: - if not installed: - continue + + if not pkg.visible: + continue # Enable upgrade or downgrade to a version # with visible KEYWORDS when the installed @@ -2463,7 +2480,7 @@ class depgraph(object): except portage.exception.PackageNotFound: continue else: - if not visible(pkgsettings, pkg_eb): + if not pkg_eb.visible: continue # Calculation of USE for unbuilt ebuilds is relatively @@ -2762,6 +2779,14 @@ class depgraph(object): installed=installed, metadata=metadata, onlydeps=onlydeps, root_config=root_config, type_name=type_name) self._frozen_config._pkg_cache[pkg] = pkg + + if not pkg.visible and \ + 'LICENSE' in pkg.masks and len(pkg.masks) == 1: + slot_key = (pkg.root, pkg.slot_atom) + other_pkg = self._frozen_config._highest_license_masked.get(slot_key) + if other_pkg is None or pkg > other_pkg: + self._frozen_config._highest_license_masked[slot_key] = pkg + return pkg def _validate_blockers(self): @@ -2806,35 +2831,11 @@ class depgraph(object): # packages masked by license, since the user likely wants # to adjust ACCEPT_LICENSE. if pkg in final_db: - if pkg_in_graph and not visible(pkgsettings, pkg): - self._dynamic_config._masked_installed.add(pkg) - elif pkgsettings._getMissingLicenses(pkg.cpv, pkg.metadata): + if not pkg.visible and \ + (pkg_in_graph or 'LICENSE' in pkg.masks): self._dynamic_config._masked_installed.add(pkg) - elif pkg_in_graph or complete or deep: - # Check for upgrades in the same slot that are - # masked due to a LICENSE change in a newer - # version that is not masked for any other reason. - # Only do this for packages that are already in - # the graph, or complete or deep graphs, since - # otherwise it is likely a waste of time. - got_mask = False - for db, pkg_type, built, installed, db_keys in dbs: - if installed: - continue - if got_mask: - break - for upgrade_pkg in self._iter_match_pkgs( - root_config, pkg_type, pkg.slot_atom): - if upgrade_pkg <= pkg: - break - if not visible(pkgsettings, - upgrade_pkg, ignore=('LICENSE',)): - continue - if pkgsettings._getMissingLicenses( - upgrade_pkg.cpv, upgrade_pkg.metadata): - self._dynamic_config._masked_license_updates.add(upgrade_pkg) - got_mask = True - break + else: + self._check_masks(pkg) blocker_atoms = None blockers = None @@ -3759,7 +3760,9 @@ class depgraph(object): for blocker in blocker_nodes: if not myblocker_uninstalls.child_nodes(blocker): myblocker_uninstalls.remove(blocker) - solved_blockers.add(blocker) + if blocker not in \ + self._dynamic_config._unsolvable_blockers: + solved_blockers.add(blocker) retlist.append(node) @@ -3772,9 +3775,8 @@ class depgraph(object): # it serves as an indicator that blocking packages # will be temporarily installed simultaneously. for blocker in solved_blockers: - retlist.append(Blocker(atom=blocker.atom, - root=blocker.root, eapi=blocker.eapi, - satisfied=True)) + blocker.satisfied = True + retlist.append(blocker) unsolvable_blockers = set(self._dynamic_config._unsolvable_blockers.leaf_nodes()) for node in myblocker_uninstalls.root_nodes(): @@ -4053,8 +4055,13 @@ class depgraph(object): unsatisfied_blockers = [] ordered_nodes = [] for x in mylist: - if isinstance(x, Blocker) and not x.satisfied: - unsatisfied_blockers.append(x) + if isinstance(x, Blocker): + counters.blocks += 1 + if x.satisfied: + ordered_nodes.append(x) + counters.blocks_satisfied += 1 + else: + unsatisfied_blockers.append(x) else: ordered_nodes.append(x) @@ -4096,10 +4103,6 @@ class depgraph(object): else: blocker_style = "PKG_BLOCKER" addl = "%s %s " % (colorize(blocker_style, "B"), fetch) - if ordered: - counters.blocks += 1 - if x.satisfied: - counters.blocks_satisfied += 1 resolved = portage.dep_expand( str(x.atom).lstrip("!"), mydb=vardb, settings=pkgsettings) if "--columns" in self._frozen_config.myopts and "--quiet" in self._frozen_config.myopts: @@ -4650,7 +4653,7 @@ class depgraph(object): else: seen_nodes.add(node) - if isinstance(node, Package): + if isinstance(node, (Blocker, Package)): display_list.append((node, depth, True)) else: depth = -1 @@ -4738,7 +4741,7 @@ class depgraph(object): del display_list[i] continue if ordered and isinstance(node, Package) \ - and node.operation == 'merge': + and node.operation in ('merge', 'uninstall'): last_merge_depth = depth continue if depth >= last_merge_depth or \ @@ -4990,8 +4993,7 @@ class depgraph(object): continue raise - if "merge" == pkg.operation and \ - not visible(root_config.settings, pkg): + if "merge" == pkg.operation and not pkg.visible: if skip_masked: masked_tasks.append(Dependency(root=pkg.root, parent=pkg)) else: @@ -5263,13 +5265,8 @@ class _dep_check_composite_db(portage.dbapi): arg = None if arg: return False - if pkg.installed: - try: - if not visible( - self._depgraph._frozen_config.pkgsettings[pkg.root], pkg): - return False - except portage.exception.InvalidDependString: - pass + if pkg.installed and not pkg.visible: + return False in_graph = self._depgraph._dynamic_config._slot_pkg_map[ self._root].get(pkg.slot_atom) if in_graph is None: @@ -5278,7 +5275,10 @@ class _dep_check_composite_db(portage.dbapi): # conflicts). highest_visible, in_graph = self._depgraph._select_package( self._root, pkg.slot_atom) - if pkg != highest_visible: + # Note: highest_visible is not necessarily the real highest + # visible, especially when --update is not enabled, so use + # < operator instead of !=. + if pkg < highest_visible: return False elif in_graph != pkg: # Mask choices for packages that would trigger a slot @@ -5419,7 +5419,7 @@ def _backtrack_depgraph(settings, trees, myopts, myparams, backtrack_max = myopts.get('--backtrack', 5) runtime_pkg_mask = None - allow_backtracking = True + allow_backtracking = backtrack_max > 0 backtracked = 0 frozen_config = _frozen_depgraph_config(settings, trees, myopts, spinner) diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py index e5ad91da2..f36a249e0 100644 --- a/pym/_emerge/search.py +++ b/pym/_emerge/search.py @@ -11,7 +11,6 @@ from portage.output import bold, bold as white, darkgreen, green, red from portage.util import writemsg_stdout from _emerge.Package import Package -from _emerge.visible import visible class search(object): @@ -111,9 +110,10 @@ class search(object): pkg_type = "installed" elif built: pkg_type = "binary" - return visible(self.settings, - Package(type_name=pkg_type, root_config=self.root_config, - cpv=cpv, built=built, installed=installed, metadata=metadata)) + return Package(type_name=pkg_type, + root_config=self.root_config, + cpv=cpv, built=built, installed=installed, + metadata=metadata).visible def _xmatch(self, level, atom): """ diff --git a/pym/_emerge/visible.py b/pym/_emerge/visible.py deleted file mode 100644 index 04cfc00d8..000000000 --- a/pym/_emerge/visible.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 1999-2009 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id$ - -import portage - -def visible(pkgsettings, pkg, ignore=None): - """ - Check if a package is visible. This can raise an InvalidDependString - exception if LICENSE is invalid. - TODO: optionally generate a list of masking reasons - @rtype: Boolean - @returns: True if the package is visible, False otherwise. - """ - if not pkg.metadata["SLOT"]: - return False - if not pkg.installed: - if pkg.invalid: - return False - if not pkgsettings._accept_chost(pkg.cpv, pkg.metadata): - return False - if pkg.built and not pkg.installed: - # we can have an old binary which has no EPREFIX information - if "EPREFIX" not in pkg.metadata or not pkg.metadata["EPREFIX"]: - return False - if len(pkg.metadata["EPREFIX"].strip()) < len(pkgsettings["EPREFIX"]): - return False - eapi = pkg.metadata["EAPI"] - if not portage.eapi_is_supported(eapi): - return False - if not pkg.installed: - if portage._eapi_is_deprecated(eapi): - return False - if pkgsettings._getMissingKeywords(pkg.cpv, pkg.metadata): - return False - try: - if pkgsettings._getMissingProperties(pkg.cpv, pkg.metadata): - return False - except portage.exception.InvalidDependString: - return False - if pkgsettings._getMaskAtom(pkg.cpv, pkg.metadata): - return False - if pkgsettings._getProfileMaskAtom(pkg.cpv, pkg.metadata): - return False - try: - if pkgsettings._getMissingLicenses(pkg.cpv, pkg.metadata): - if ignore is None or 'LICENSE' not in ignore: - return False - except portage.exception.InvalidDependString: - return False - return True diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 2949b3a7b..03247b46e 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -7546,10 +7546,8 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, else: ret=os.rename(src,dest) renamefailed=0 - except SystemExit as e: - raise - except Exception as e: - if e[0]!=errno.EXDEV: + except OSError as e: + if e.errno != errno.EXDEV: # Some random error. print(_("!!! Failed to move %(src)s to %(dest)s") % {"src": src, "dest": dest}) print("!!!",e) diff --git a/pym/portage/sets/dbapi.py b/pym/portage/sets/dbapi.py index f2e7dd110..a5d7c7153 100644 --- a/pym/portage/sets/dbapi.py +++ b/pym/portage/sets/dbapi.py @@ -232,6 +232,13 @@ class CategorySet(PackageSet): myatoms.append(cp) self._setAtoms(myatoms) + def _builderGetRepository(cls, options, repositories): + repository = options.get("repository", "porttree") + if not repository in repositories: + raise SetConfigError(_("invalid repository class '%s'") % repository) + return repository + _builderGetRepository = classmethod(_builderGetRepository) + def _builderGetVisible(cls, options): return get_boolean(options, "only_visible", True) _builderGetVisible = classmethod(_builderGetVisible) @@ -244,9 +251,10 @@ class CategorySet(PackageSet): if not category in settings.categories: raise SetConfigError(_("invalid category name '%s'") % category) + repository = cls._builderGetRepository(options, trees.keys()) visible = cls._builderGetVisible(options) - return CategorySet(category, dbapi=trees["porttree"].dbapi, only_visible=visible) + return CategorySet(category, dbapi=trees[repository].dbapi, only_visible=visible) singleBuilder = classmethod(singleBuilder) def multiBuilder(cls, options, settings, trees): @@ -260,6 +268,7 @@ class CategorySet(PackageSet): else: categories = settings.categories + repository = cls._builderGetRepository(options, trees.keys()) visible = cls._builderGetVisible(options) name_pattern = options.get("name_pattern", "$category/*") @@ -267,7 +276,7 @@ class CategorySet(PackageSet): raise SetConfigError(_("name_pattern doesn't include $category placeholder")) for cat in categories: - myset = CategorySet(cat, trees["porttree"].dbapi, only_visible=visible) + myset = CategorySet(cat, trees[repository].dbapi, only_visible=visible) myname = name_pattern.replace("$category", cat) myname = myname.replace("${category}", cat) rValue[myname] = myset diff --git a/pym/portage/tests/lint/__init__.py b/pym/portage/tests/lint/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pym/portage/tests/lint/__test__ b/pym/portage/tests/lint/__test__ new file mode 100644 index 000000000..e69de29bb diff --git a/pym/portage/tests/lint/test_compile_modules.py b/pym/portage/tests/lint/test_compile_modules.py new file mode 100644 index 000000000..da039f4dd --- /dev/null +++ b/pym/portage/tests/lint/test_compile_modules.py @@ -0,0 +1,23 @@ +# Copyright 2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +from portage.const import PORTAGE_PYM_PATH +from portage.tests import TestCase +from portage import os +from portage import _encodings +from portage import _unicode_decode + +import py_compile + +class CompileModulesTestCase(TestCase): + + def testCompileModules(self): + for parent, dirs, files in os.walk(PORTAGE_PYM_PATH): + parent = _unicode_decode(parent, + encoding=_encodings['fs'], errors='strict') + for x in files: + x = _unicode_decode(x, + encoding=_encodings['fs'], errors='strict') + if x[-3:] == '.py': + py_compile.compile(os.path.join(parent, x), doraise=True)