1 # Copyright 2010-2012 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
4 from itertools import permutations
9 from portage import shutil
10 from portage.const import (GLOBAL_CONFIG_PATH, PORTAGE_BASE_PATH,
12 from portage.dep import Atom, _repo_separator
13 from portage.package.ebuild.config import config
14 from portage.package.ebuild.digestgen import digestgen
15 from portage._sets import load_default_config
16 from portage._sets.base import InternalPackageSet
17 from portage.util import ensure_dirs, normalize_path
18 from portage.versions import catsplit
21 from _emerge.actions import calc_depclean
22 from _emerge.Blocker import Blocker
23 from _emerge.create_depgraph_params import create_depgraph_params
24 from _emerge.depgraph import backtrack_depgraph
25 from _emerge.RootConfig import RootConfig
27 if sys.hexversion >= 0x3000000:
30 class ResolverPlayground(object):
32 This class helps to create the necessary files on disk and
33 the needed settings instances, etc. for the resolver to do
37 config_files = frozenset(("eapi", "layout.conf", "make.conf", "package.accept_keywords",
38 "package.keywords", "package.license", "package.mask", "package.properties",
39 "package.unmask", "package.use", "package.use.aliases", "package.use.stable.mask",
40 "use.aliases", "use.force", "use.mask", "layout.conf"))
42 metadata_xml_template = """<?xml version="1.0" encoding="UTF-8"?>
43 <!DOCTYPE pkgmetadata SYSTEM "http://www.gentoo.org/dtd/metadata.dtd">
47 <email>maintainer-needed@gentoo.org</email>
48 <description>Description of the maintainership</description>
50 <longdescription>Long description of the package</longdescription>
57 def __init__(self, ebuilds={}, binpkgs={}, installed={}, profile={}, repo_configs={}, \
58 user_config={}, sets={}, world=[], world_sets=[], distfiles={},
59 targetroot=False, debug=False):
61 ebuilds: cpv -> metadata mapping simulating available ebuilds.
62 installed: cpv -> metadata mapping simulating installed packages.
63 If a metadata key is missing, it gets a default value.
64 profile: settings defined by the profile.
67 self.eprefix = normalize_path(tempfile.mkdtemp())
68 self.eroot = self.eprefix + os.sep
70 self.target_root = os.path.join(self.eroot, 'target_root')
72 self.target_root = os.sep
73 self.distdir = os.path.join(self.eroot, "var", "portage", "distfiles")
74 self.pkgdir = os.path.join(self.eprefix, "pkgdir")
75 self.portdir = os.path.join(self.eroot, "usr/portage")
76 self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
77 os.makedirs(self.portdir)
78 os.makedirs(self.vdbdir)
81 portage.util.noiselimit = -2
84 #Make sure the main repo is always created
85 self._get_repo_dir("test_repo")
87 self._create_distfiles(distfiles)
88 self._create_ebuilds(ebuilds)
89 self._create_binpkgs(binpkgs)
90 self._create_installed(installed)
91 self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
92 self._create_world(world, world_sets)
94 self.settings, self.trees = self._load_config()
96 self._create_ebuild_manifests(ebuilds)
98 portage.util.noiselimit = 0
100 def _get_repo_dir(self, repo):
102 Create the repo directory if needed.
104 if repo not in self.repo_dirs:
105 if repo == "test_repo":
106 repo_path = self.portdir
108 repo_path = os.path.join(self.eroot, "usr", "local", repo)
110 self.repo_dirs[repo] = repo_path
111 profile_path = os.path.join(repo_path, "profiles")
114 os.makedirs(profile_path)
118 repo_name_file = os.path.join(profile_path, "repo_name")
119 f = open(repo_name_file, "w")
120 f.write("%s\n" % repo)
123 return self.repo_dirs[repo]
125 def _create_distfiles(self, distfiles):
126 os.makedirs(self.distdir)
127 for k, v in distfiles.items():
128 with open(os.path.join(self.distdir, k), 'wb') as f:
131 def _create_ebuilds(self, ebuilds):
133 a = Atom("=" + cpv, allow_repo=True)
138 metadata = ebuilds[cpv].copy()
139 copyright_header = metadata.pop("COPYRIGHT_HEADER", None)
140 eapi = metadata.pop("EAPI", "0")
141 misc_content = metadata.pop("MISC_CONTENT", None)
142 metadata.setdefault("DEPEND", "")
143 metadata.setdefault("SLOT", "0")
144 metadata.setdefault("KEYWORDS", "x86")
145 metadata.setdefault("IUSE", "")
147 unknown_keys = set(metadata).difference(
148 portage.dbapi.dbapi._known_keys)
150 raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" %
151 (cpv, sorted(unknown_keys)))
153 repo_dir = self._get_repo_dir(repo)
154 ebuild_dir = os.path.join(repo_dir, a.cp)
155 ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
157 os.makedirs(ebuild_dir)
161 f = open(ebuild_path, "w")
162 if copyright_header is not None:
163 f.write(copyright_header)
164 f.write('EAPI="%s"\n' % eapi)
165 for k, v in metadata.items():
166 f.write('%s="%s"\n' % (k, v))
167 if misc_content is not None:
168 f.write(misc_content)
171 def _create_ebuild_manifests(self, ebuilds):
172 tmpsettings = config(clone=self.settings)
173 tmpsettings['PORTAGE_QUIET'] = '1'
175 a = Atom("=" + cpv, allow_repo=True)
180 repo_dir = self._get_repo_dir(repo)
181 ebuild_dir = os.path.join(repo_dir, a.cp)
182 ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
184 portdb = self.trees[self.eroot]["porttree"].dbapi
185 tmpsettings['O'] = ebuild_dir
186 if not digestgen(mysettings=tmpsettings, myportdb=portdb):
187 raise AssertionError('digest creation failed for %s' % ebuild_path)
189 def _create_binpkgs(self, binpkgs):
190 for cpv, metadata in binpkgs.items():
191 a = Atom("=" + cpv, allow_repo=True)
196 cat, pf = catsplit(a.cpv)
197 metadata = metadata.copy()
198 metadata.setdefault("SLOT", "0")
199 metadata.setdefault("KEYWORDS", "x86")
200 metadata.setdefault("BUILD_TIME", "0")
201 metadata["repository"] = repo
202 metadata["CATEGORY"] = cat
205 repo_dir = self.pkgdir
206 category_dir = os.path.join(repo_dir, cat)
207 binpkg_path = os.path.join(category_dir, pf + ".tbz2")
208 ensure_dirs(category_dir)
209 t = portage.xpak.tbz2(binpkg_path)
210 t.recompose_mem(portage.xpak.xpak_mem(metadata))
212 def _create_installed(self, installed):
213 for cpv in installed:
214 a = Atom("=" + cpv, allow_repo=True)
219 vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
221 os.makedirs(vdb_pkg_dir)
225 metadata = installed[cpv].copy()
226 metadata.setdefault("SLOT", "0")
227 metadata.setdefault("BUILD_TIME", "0")
228 metadata.setdefault("COUNTER", "0")
229 metadata.setdefault("KEYWORDS", "~x86")
231 unknown_keys = set(metadata).difference(
232 portage.dbapi.dbapi._known_keys)
233 unknown_keys.discard("BUILD_TIME")
234 unknown_keys.discard("COUNTER")
235 unknown_keys.discard("repository")
236 unknown_keys.discard("USE")
238 raise ValueError("metadata of installed '%s' contains unknown keys: %s" %
239 (cpv, sorted(unknown_keys)))
241 metadata["repository"] = repo
242 for k, v in metadata.items():
243 with open(os.path.join(vdb_pkg_dir, k), "w") as f:
246 def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
248 user_config_dir = os.path.join(self.eroot, USER_CONFIG_PATH)
251 os.makedirs(user_config_dir)
255 for repo in self.repo_dirs:
256 repo_dir = self._get_repo_dir(repo)
257 profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
258 metadata_dir = os.path.join(repo_dir, "metadata")
259 os.makedirs(metadata_dir)
261 #Create $REPO/profiles/categories
264 ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
265 if ebuilds_repo is None:
266 ebuilds_repo = "test_repo"
267 if ebuilds_repo == repo:
268 categories.add(catsplit(cpv)[0])
270 categories_file = os.path.join(profile_dir, "categories")
271 f = open(categories_file, "w")
272 for cat in categories:
276 #Create $REPO/profiles/license_groups
277 license_file = os.path.join(profile_dir, "license_groups")
278 f = open(license_file, "w")
279 f.write("EULA TEST\n")
282 repo_config = repo_configs.get(repo)
284 for config_file, lines in repo_config.items():
285 if config_file not in self.config_files:
286 raise ValueError("Unknown config file: '%s'" % config_file)
288 if config_file in ("layout.conf",):
289 file_name = os.path.join(repo_dir, "metadata", config_file)
291 file_name = os.path.join(profile_dir, config_file)
292 f = open(file_name, "w")
294 f.write("%s\n" % line)
297 #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
298 os.makedirs(os.path.join(repo_dir, "eclass"))
300 if repo == "test_repo":
301 #Create a minimal profile in /usr/portage
302 sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
303 os.makedirs(sub_profile_dir)
305 if not (profile and "eapi" in profile):
306 eapi_file = os.path.join(sub_profile_dir, "eapi")
307 f = open(eapi_file, "w")
311 make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
312 f = open(make_defaults_file, "w")
313 f.write("ARCH=\"x86\"\n")
314 f.write("ACCEPT_KEYWORDS=\"x86\"\n")
317 use_force_file = os.path.join(sub_profile_dir, "use.force")
318 f = open(use_force_file, "w")
322 parent_file = os.path.join(sub_profile_dir, "parent")
323 f = open(parent_file, "w")
328 for config_file, lines in profile.items():
329 if config_file not in self.config_files:
330 raise ValueError("Unknown config file: '%s'" % config_file)
332 file_name = os.path.join(sub_profile_dir, config_file)
333 f = open(file_name, "w")
335 f.write("%s\n" % line)
338 #Create profile symlink
339 os.symlink(sub_profile_dir, os.path.join(user_config_dir, "make.profile"))
341 #Create minimal herds.xml
342 herds_xml = """<?xml version="1.0" encoding="UTF-8"?>
343 <!DOCTYPE herds SYSTEM "http://www.gentoo.org/dtd/herds.dtd">
344 <?xml-stylesheet href="/xsl/herds.xsl" type="text/xsl" ?>
345 <?xml-stylesheet href="/xsl/guide.xsl" type="text/xsl" ?>
348 <name>base-system</name>
349 <email>base-system@gentoo.org</email>
350 <description>Core system utilities and libraries.</description>
352 <email>base-system@gentoo.orgg</email>
353 <name>Base System</name>
354 <role>Base System Maintainer</role>
359 with open(os.path.join(metadata_dir, "metadata.xml"), 'w') as f:
362 # Write empty entries for each repository, in order to exercise
363 # RepoConfigLoader's repos.conf processing.
364 repos_conf_file = os.path.join(user_config_dir, "repos.conf")
365 f = open(repos_conf_file, "w")
366 for repo in sorted(self.repo_dirs.keys()):
367 f.write("[%s]\n" % repo)
372 for repo_name in sorted(self.repo_dirs):
373 path = self.repo_dirs[repo_name]
374 if path != self.portdir:
375 portdir_overlay.append(path)
378 "ACCEPT_KEYWORDS": "x86",
380 "DISTDIR" : self.distdir,
381 "EMERGE_WARNING_DELAY": "0",
382 "PKGDIR": self.pkgdir,
383 "PORTDIR": self.portdir,
384 "PORTAGE_INST_GID": str(portage.data.portage_gid),
385 "PORTAGE_INST_UID": str(portage.data.portage_uid),
386 "PORTDIR_OVERLAY": " ".join("'%s'" % x for x in portdir_overlay),
387 "PORTAGE_TMPDIR": os.path.join(self.eroot, 'var/tmp'),
390 if os.environ.get("NOCOLOR"):
391 make_conf["NOCOLOR"] = os.environ["NOCOLOR"]
393 # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
394 # need to be inherited by ebuild subprocesses.
395 if 'PORTAGE_USERNAME' in os.environ:
396 make_conf['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
397 if 'PORTAGE_GRPNAME' in os.environ:
398 make_conf['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
401 for k_v in make_conf.items():
402 make_conf_lines.append('%s="%s"' % k_v)
404 if "make.conf" in user_config:
405 make_conf_lines.extend(user_config["make.conf"])
407 if not portage.process.sandbox_capable or \
408 os.environ.get("SANDBOX_ON") == "1":
409 # avoid problems from nested sandbox instances
410 make_conf_lines.append('FEATURES="${FEATURES} -sandbox"')
412 configs = user_config.copy()
413 configs["make.conf"] = make_conf_lines
415 for config_file, lines in configs.items():
416 if config_file not in self.config_files:
417 raise ValueError("Unknown config file: '%s'" % config_file)
419 file_name = os.path.join(user_config_dir, config_file)
420 f = open(file_name, "w")
422 f.write("%s\n" % line)
425 #Create /usr/share/portage/config/make.globals
426 make_globals_path = os.path.join(self.eroot,
427 GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals")
428 ensure_dirs(os.path.dirname(make_globals_path))
429 os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"),
432 #Create /usr/share/portage/config/sets/portage.conf
433 default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
436 os.makedirs(default_sets_conf_dir)
440 provided_sets_portage_conf = \
441 os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
442 os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
444 set_config_dir = os.path.join(user_config_dir, "sets")
447 os.makedirs(set_config_dir)
451 for sets_file, lines in sets.items():
452 file_name = os.path.join(set_config_dir, sets_file)
453 f = open(file_name, "w")
455 f.write("%s\n" % line)
458 def _create_world(self, world, world_sets):
459 #Create /var/lib/portage/world
460 var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
461 os.makedirs(var_lib_portage)
463 world_file = os.path.join(var_lib_portage, "world")
464 world_set_file = os.path.join(var_lib_portage, "world_sets")
466 f = open(world_file, "w")
468 f.write("%s\n" % atom)
471 f = open(world_set_file, "w")
472 for atom in world_sets:
473 f.write("%s\n" % atom)
476 def _load_config(self):
478 create_trees_kwargs = {}
479 if self.target_root != os.sep:
480 create_trees_kwargs["target_root"] = self.target_root
482 trees = portage.create_trees(env={}, eprefix=self.eprefix,
483 **create_trees_kwargs)
485 for root, root_trees in trees.items():
486 settings = root_trees["vartree"].settings
487 settings._init_dirs()
488 setconfig = load_default_config(settings, root_trees)
489 root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
491 return trees[trees._target_eroot]["vartree"].settings, trees
493 def run(self, atoms, options={}, action=None):
494 options = options.copy()
495 options["--pretend"] = True
497 options["--debug"] = True
500 if options.get("--depclean"):
502 elif options.get("--prune"):
505 if "--usepkgonly" in options:
506 options["--usepkg"] = True
508 global_noiselimit = portage.util.noiselimit
509 global_emergelog_disable = _emerge.emergelog._disable
513 portage.util.noiselimit = -2
514 _emerge.emergelog._disable = True
516 if action in ("depclean", "prune"):
517 rval, cleanlist, ordered, req_pkg_count = \
518 calc_depclean(self.settings, self.trees, None,
519 options, action, InternalPackageSet(initial_atoms=atoms, allow_wildcard=True), None)
520 result = ResolverPlaygroundDepcleanResult( \
521 atoms, rval, cleanlist, ordered, req_pkg_count)
523 params = create_depgraph_params(options, action)
524 success, depgraph, favorites = backtrack_depgraph(
525 self.settings, self.trees, options, params, action, atoms, None)
526 depgraph._show_merge_list()
527 depgraph.display_problems()
528 result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
530 portage.util.noiselimit = global_noiselimit
531 _emerge.emergelog._disable = global_emergelog_disable
535 def run_TestCase(self, test_case):
536 if not isinstance(test_case, ResolverPlaygroundTestCase):
537 raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase")
538 for atoms in test_case.requests:
539 result = self.run(atoms, test_case.options, test_case.action)
540 if not test_case.compare_with_result(result):
544 for eroot in self.trees:
545 portdb = self.trees[eroot]["porttree"].dbapi
546 portdb.close_caches()
547 portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
549 print("\nEROOT=%s" % self.eroot)
551 shutil.rmtree(self.eroot)
553 class ResolverPlaygroundTestCase(object):
555 def __init__(self, request, **kwargs):
556 self.all_permutations = kwargs.pop("all_permutations", False)
557 self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False)
558 self.ambiguous_merge_order = kwargs.pop("ambiguous_merge_order", False)
559 self.check_repo_names = kwargs.pop("check_repo_names", False)
560 self.merge_order_assertions = kwargs.pop("merge_order_assertions", False)
562 if self.all_permutations:
563 self.requests = list(permutations(request))
565 self.requests = [request]
567 self.options = kwargs.pop("options", {})
568 self.action = kwargs.pop("action", None)
569 self.test_success = True
571 self._checks = kwargs.copy()
573 def compare_with_result(self, result):
574 checks = dict.fromkeys(result.checks)
575 for key, value in self._checks.items():
576 if not key in checks:
577 raise KeyError("Not an available check: '%s'" % key)
581 for key, value in checks.items():
582 got = getattr(result, key)
585 if key in result.optional_checks and expected is None:
588 if key == "mergelist":
589 if not self.check_repo_names:
590 #Strip repo names if we don't check them
597 new_got.append(cpv.split(_repo_separator)[0])
602 if isinstance(obj, basestring):
604 new_expected.append(obj)
607 obj.split(_repo_separator)[0])
609 new_expected.append(set())
612 cpv = cpv.split(_repo_separator)[0]
613 new_expected[-1].add(cpv)
614 expected = new_expected
615 if self.ignore_mergelist_order and got is not None:
617 expected = set(expected)
619 if self.ambiguous_merge_order and got:
620 expected_stack = list(reversed(expected))
621 got_stack = list(reversed(got))
624 while got_stack and expected_stack:
625 got_token = got_stack.pop()
626 expected_obj = expected_stack.pop()
627 if isinstance(expected_obj, basestring):
628 new_expected.append(expected_obj)
629 if got_token == expected_obj:
631 # result doesn't match, so stop early
634 expected_obj = set(expected_obj)
636 expected_obj.remove(got_token)
638 # result doesn't match, so stop early
641 new_expected.append(got_token)
642 while got_stack and expected_obj:
643 got_token = got_stack.pop()
645 expected_obj.remove(got_token)
649 new_expected.append(got_token)
651 # result doesn't match, so stop early
654 # result does not match, so stop early
656 new_expected.append(tuple(expected_obj))
659 # result does not match, add leftovers to new_expected
661 expected_stack.reverse()
662 new_expected.extend(expected_stack)
663 expected = new_expected
665 if match and self.merge_order_assertions:
666 for node1, node2 in self.merge_order_assertions:
667 if not (got.index(node1) < got.index(node2)):
668 fail_msgs.append("atoms: (" + \
669 ", ".join(result.atoms) + "), key: " + \
670 ("merge_order_assertions, expected: %s" % \
671 str((node1, node2))) + \
672 ", got: " + str(got))
674 elif key in ("unstable_keywords", "needed_p_mask_changes") and expected is not None:
675 expected = set(expected)
678 fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \
679 key + ", expected: " + str(expected) + ", got: " + str(got))
681 self.test_success = False
682 self.fail_msg = "\n".join(fail_msgs)
686 class ResolverPlaygroundResult(object):
689 "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions",
690 "circular_dependency_solutions", "needed_p_mask_changes",
695 def __init__(self, atoms, success, mydepgraph, favorites):
697 self.success = success
698 self.depgraph = mydepgraph
699 self.favorites = favorites
700 self.mergelist = None
701 self.use_changes = None
702 self.license_changes = None
703 self.unstable_keywords = None
704 self.needed_p_mask_changes = None
705 self.slot_collision_solutions = None
706 self.circular_dependency_solutions = None
708 if self.depgraph._dynamic_config._serialized_tasks_cache is not None:
710 host_root = self.depgraph._frozen_config._running_root.root
711 for x in self.depgraph._dynamic_config._serialized_tasks_cache:
712 if isinstance(x, Blocker):
713 self.mergelist.append(x.atom)
716 if x.metadata["repository"] != "test_repo":
717 repo_str = _repo_separator + x.metadata["repository"]
718 mergelist_str = x.cpv + repo_str
720 if x.operation == "merge":
724 mergelist_str = "[%s]%s" % (desc, mergelist_str)
725 if x.root != host_root:
726 mergelist_str += "{targetroot}"
727 self.mergelist.append(mergelist_str)
729 if self.depgraph._dynamic_config._needed_use_config_changes:
730 self.use_changes = {}
731 for pkg, needed_use_config_changes in \
732 self.depgraph._dynamic_config._needed_use_config_changes.items():
733 new_use, changes = needed_use_config_changes
734 self.use_changes[pkg.cpv] = changes
736 if self.depgraph._dynamic_config._needed_unstable_keywords:
737 self.unstable_keywords = set()
738 for pkg in self.depgraph._dynamic_config._needed_unstable_keywords:
739 self.unstable_keywords.add(pkg.cpv)
741 if self.depgraph._dynamic_config._needed_p_mask_changes:
742 self.needed_p_mask_changes = set()
743 for pkg in self.depgraph._dynamic_config._needed_p_mask_changes:
744 self.needed_p_mask_changes.add(pkg.cpv)
746 if self.depgraph._dynamic_config._needed_license_changes:
747 self.license_changes = {}
748 for pkg, missing_licenses in self.depgraph._dynamic_config._needed_license_changes.items():
749 self.license_changes[pkg.cpv] = missing_licenses
751 if self.depgraph._dynamic_config._slot_conflict_handler is not None:
752 self.slot_collision_solutions = []
753 handler = self.depgraph._dynamic_config._slot_conflict_handler
755 for change in handler.changes:
758 new_change[pkg.cpv] = change[pkg]
759 self.slot_collision_solutions.append(new_change)
761 if self.depgraph._dynamic_config._circular_dependency_handler is not None:
762 handler = self.depgraph._dynamic_config._circular_dependency_handler
763 sol = handler.solutions
764 self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) )
766 class ResolverPlaygroundDepcleanResult(object):
769 "success", "cleanlist", "ordered", "req_pkg_count",
772 "ordered", "req_pkg_count",
775 def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count):
777 self.success = rval == 0
778 self.cleanlist = cleanlist
779 self.ordered = ordered
780 self.req_pkg_count = req_pkg_count