1 # Copyright 2010-2011 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
4 from itertools import permutations
10 from portage.const import GLOBAL_CONFIG_PATH, PORTAGE_BASE_PATH
11 from portage.dbapi.vartree import vartree
12 from portage.dbapi.porttree import portagetree
13 from portage.dbapi.bintree import binarytree
14 from portage.dep import Atom, _repo_separator
15 from portage.package.ebuild.config import config
16 from portage.package.ebuild.digestgen import digestgen
17 from portage._sets import load_default_config
18 from portage._sets.base import InternalPackageSet
19 from portage.util import ensure_dirs
20 from portage.versions import catsplit
23 from _emerge.actions import calc_depclean
24 from _emerge.Blocker import Blocker
25 from _emerge.create_depgraph_params import create_depgraph_params
26 from _emerge.depgraph import backtrack_depgraph
27 from _emerge.RootConfig import RootConfig
29 if sys.hexversion >= 0x3000000:
32 class ResolverPlayground(object):
34 This class helps to create the necessary files on disk and
35 the needed settings instances, etc. for the resolver to do
39 config_files = frozenset(("package.use", "package.mask", "package.keywords", \
40 "package.unmask", "package.properties", "package.license", "use.mask", "use.force"))
42 def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \
43 user_config={}, sets={}, world=[], debug=False):
45 ebuilds: cpv -> metadata mapping simulating available ebuilds.
46 installed: cpv -> metadata mapping simulating installed packages.
47 If a metadata key is missing, it gets a default value.
48 profile: settings defined by the profile.
52 self.eprefix = tempfile.mkdtemp()
53 self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
54 self.portdir = os.path.join(self.eroot, "usr/portage")
55 self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
56 os.makedirs(self.portdir)
57 os.makedirs(self.vdbdir)
60 portage.util.noiselimit = -2
63 #Make sure the main repo is always created
64 self._get_repo_dir("test_repo")
66 self._create_ebuilds(ebuilds)
67 self._create_installed(installed)
68 self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
69 self._create_world(world)
71 self.settings, self.trees = self._load_config()
73 self._create_ebuild_manifests(ebuilds)
75 portage.util.noiselimit = 0
77 def _get_repo_dir(self, repo):
79 Create the repo directory if needed.
81 if repo not in self.repo_dirs:
82 if repo == "test_repo":
83 repo_path = self.portdir
85 repo_path = os.path.join(self.eroot, "usr", "local", repo)
87 self.repo_dirs[repo] = repo_path
88 profile_path = os.path.join(repo_path, "profiles")
91 os.makedirs(profile_path)
95 repo_name_file = os.path.join(profile_path, "repo_name")
96 f = open(repo_name_file, "w")
97 f.write("%s\n" % repo)
100 return self.repo_dirs[repo]
102 def _create_ebuilds(self, ebuilds):
104 a = Atom("=" + cpv, allow_repo=True)
109 metadata = ebuilds[cpv].copy()
110 copyright_header = metadata.pop("COPYRIGHT_HEADER", None)
111 desc = metadata.pop("DESCRIPTION", None)
112 eapi = metadata.pop("EAPI", 0)
113 lic = metadata.pop("LICENSE", "")
114 properties = metadata.pop("PROPERTIES", "")
115 slot = metadata.pop("SLOT", 0)
116 keywords = metadata.pop("KEYWORDS", "x86")
117 homepage = metadata.pop("HOMEPAGE", None)
118 iuse = metadata.pop("IUSE", "")
119 depend = metadata.pop("DEPEND", "")
120 rdepend = metadata.pop("RDEPEND", None)
121 pdepend = metadata.pop("PDEPEND", None)
122 required_use = metadata.pop("REQUIRED_USE", None)
123 misc_content = metadata.pop("MISC_CONTENT", None)
126 raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
128 repo_dir = self._get_repo_dir(repo)
129 ebuild_dir = os.path.join(repo_dir, a.cp)
130 ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
132 os.makedirs(ebuild_dir)
136 f = open(ebuild_path, "w")
137 if copyright_header is not None:
138 f.write(copyright_header)
139 f.write('EAPI="' + str(eapi) + '"\n')
141 f.write('DESCRIPTION="%s"\n' % desc)
142 if homepage is not None:
143 f.write('HOMEPAGE="%s"\n' % homepage)
144 f.write('LICENSE="' + str(lic) + '"\n')
145 f.write('PROPERTIES="' + str(properties) + '"\n')
146 f.write('SLOT="' + str(slot) + '"\n')
147 f.write('KEYWORDS="' + str(keywords) + '"\n')
148 f.write('IUSE="' + str(iuse) + '"\n')
149 f.write('DEPEND="' + str(depend) + '"\n')
150 if rdepend is not None:
151 f.write('RDEPEND="' + str(rdepend) + '"\n')
152 if pdepend is not None:
153 f.write('PDEPEND="' + str(pdepend) + '"\n')
154 if required_use is not None:
155 f.write('REQUIRED_USE="' + str(required_use) + '"\n')
156 if misc_content is not None:
157 f.write(misc_content)
160 def _create_ebuild_manifests(self, ebuilds):
161 tmpsettings = config(clone=self.settings)
162 tmpsettings['PORTAGE_QUIET'] = '1'
164 a = Atom("=" + cpv, allow_repo=True)
169 repo_dir = self._get_repo_dir(repo)
170 ebuild_dir = os.path.join(repo_dir, a.cp)
171 ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
173 portdb = self.trees[self.root]["porttree"].dbapi
174 tmpsettings['O'] = ebuild_dir
175 if not digestgen(mysettings=tmpsettings, myportdb=portdb):
176 raise AssertionError('digest creation failed for %s' % ebuild_path)
178 def _create_installed(self, installed):
179 for cpv in installed:
180 a = Atom("=" + cpv, allow_repo=True)
185 vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
187 os.makedirs(vdb_pkg_dir)
191 metadata = installed[cpv].copy()
192 eapi = metadata.pop("EAPI", 0)
193 lic = metadata.pop("LICENSE", "")
194 properties = metadata.pop("PROPERTIES", "")
195 slot = metadata.pop("SLOT", 0)
196 keywords = metadata.pop("KEYWORDS", "~x86")
197 iuse = metadata.pop("IUSE", "")
198 use = metadata.pop("USE", "")
199 depend = metadata.pop("DEPEND", "")
200 rdepend = metadata.pop("RDEPEND", None)
201 pdepend = metadata.pop("PDEPEND", None)
202 required_use = metadata.pop("REQUIRED_USE", None)
205 raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
207 def write_key(key, value):
208 f = open(os.path.join(vdb_pkg_dir, key), "w")
209 f.write(str(value) + "\n")
212 write_key("EAPI", eapi)
213 write_key("COUNTER", "0")
214 write_key("LICENSE", lic)
215 write_key("PROPERTIES", properties)
216 write_key("SLOT", slot)
217 write_key("LICENSE", lic)
218 write_key("PROPERTIES", properties)
219 write_key("repository", repo)
220 write_key("KEYWORDS", keywords)
221 write_key("IUSE", iuse)
222 write_key("USE", use)
223 write_key("DEPEND", depend)
224 if rdepend is not None:
225 write_key("RDEPEND", rdepend)
226 if pdepend is not None:
227 write_key("PDEPEND", pdepend)
228 if required_use is not None:
229 write_key("REQUIRED_USE", required_use)
231 def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
233 for repo in self.repo_dirs:
234 repo_dir = self._get_repo_dir(repo)
235 profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
237 #Create $REPO/profiles/categories
240 ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
241 if ebuilds_repo is None:
242 ebuilds_repo = "test_repo"
243 if ebuilds_repo == repo:
244 categories.add(catsplit(cpv)[0])
246 categories_file = os.path.join(profile_dir, "categories")
247 f = open(categories_file, "w")
248 for cat in categories:
252 #Create $REPO/profiles/license_groups
253 license_file = os.path.join(profile_dir, "license_groups")
254 f = open(license_file, "w")
255 f.write("EULA TEST\n")
258 repo_config = repo_configs.get(repo)
260 for config_file, lines in repo_config.items():
261 if config_file not in self.config_files:
262 raise ValueError("Unknown config file: '%s'" % config_file)
264 file_name = os.path.join(profile_dir, config_file)
265 f = open(file_name, "w")
267 f.write("%s\n" % line)
270 #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
271 os.makedirs(os.path.join(repo_dir, "eclass"))
273 if repo == "test_repo":
274 #Create a minimal profile in /usr/portage
275 sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
276 os.makedirs(sub_profile_dir)
278 eapi_file = os.path.join(sub_profile_dir, "eapi")
279 f = open(eapi_file, "w")
283 make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
284 f = open(make_defaults_file, "w")
285 f.write("ARCH=\"x86\"\n")
286 f.write("ACCEPT_KEYWORDS=\"x86\"\n")
289 use_force_file = os.path.join(sub_profile_dir, "use.force")
290 f = open(use_force_file, "w")
294 parent_file = os.path.join(sub_profile_dir, "parent")
295 f = open(parent_file, "w")
300 for config_file, lines in profile.items():
301 if config_file not in self.config_files:
302 raise ValueError("Unknown config file: '%s'" % config_file)
304 file_name = os.path.join(sub_profile_dir, config_file)
305 f = open(file_name, "w")
307 f.write("%s\n" % line)
310 #Create profile symlink
311 os.makedirs(os.path.join(self.eroot, "etc"))
312 os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile"))
314 user_config_dir = os.path.join(self.eroot, "etc", "portage")
317 os.makedirs(user_config_dir)
321 repos_conf_file = os.path.join(user_config_dir, "repos.conf")
322 f = open(repos_conf_file, "w")
324 for repo in sorted(self.repo_dirs.keys()):
325 f.write("[%s]\n" % repo)
326 f.write("LOCATION=%s\n" % self.repo_dirs[repo])
327 if repo == "test_repo":
328 f.write("PRIORITY=%s\n" % -1000)
330 f.write("PRIORITY=%s\n" % priority)
334 for config_file, lines in user_config.items():
335 if config_file not in self.config_files:
336 raise ValueError("Unknown config file: '%s'" % config_file)
338 file_name = os.path.join(user_config_dir, config_file)
339 f = open(file_name, "w")
341 f.write("%s\n" % line)
344 #Create /usr/share/portage/config/make.globals
345 make_globals_path = os.path.join(self.eroot,
346 GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals")
347 ensure_dirs(os.path.dirname(make_globals_path))
348 os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"),
351 #Create /usr/share/portage/config/sets/portage.conf
352 default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
355 os.makedirs(default_sets_conf_dir)
359 provided_sets_portage_conf = \
360 os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
361 os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
363 set_config_dir = os.path.join(user_config_dir, "sets")
366 os.makedirs(set_config_dir)
370 for sets_file, lines in sets.items():
371 file_name = os.path.join(set_config_dir, sets_file)
372 f = open(file_name, "w")
374 f.write("%s\n" % line)
377 user_config_dir = os.path.join(self.eroot, "etc", "portage")
380 os.makedirs(user_config_dir)
384 for config_file, lines in user_config.items():
385 if config_file not in self.config_files:
386 raise ValueError("Unknown config file: '%s'" % config_file)
388 file_name = os.path.join(user_config_dir, config_file)
389 f = open(file_name, "w")
391 f.write("%s\n" % line)
394 def _create_world(self, world):
395 #Create /var/lib/portage/world
396 var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
397 os.makedirs(var_lib_portage)
399 world_file = os.path.join(var_lib_portage, "world")
401 f = open(world_file, "w")
403 f.write("%s\n" % atom)
406 def _load_config(self):
408 for repo_name in sorted(self.repo_dirs):
409 path = self.repo_dirs[repo_name]
410 if path != self.portdir:
411 portdir_overlay.append(path)
414 "ACCEPT_KEYWORDS": "x86",
415 "PORTDIR": self.portdir,
416 "PORTDIR_OVERLAY": " ".join(portdir_overlay),
417 'PORTAGE_TMPDIR' : os.path.join(self.eroot, 'var/tmp'),
420 # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
421 # need to be inherited by ebuild subprocesses.
422 if 'PORTAGE_USERNAME' in os.environ:
423 env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
424 if 'PORTAGE_GRPNAME' in os.environ:
425 env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
427 settings = config(_eprefix=self.eprefix, env=env)
432 "vartree": vartree(settings=settings),
433 "porttree": portagetree(self.root, settings=settings),
434 "bintree": binarytree(self.root,
435 os.path.join(self.eroot, "usr/portage/packages"),
440 for root, root_trees in trees.items():
441 settings = root_trees["vartree"].settings
442 settings._init_dirs()
443 setconfig = load_default_config(settings, root_trees)
444 root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
446 return settings, trees
448 def run(self, atoms, options={}, action=None):
449 options = options.copy()
450 options["--pretend"] = True
452 options["--debug"] = True
454 global_noiselimit = portage.util.noiselimit
455 global_emergelog_disable = _emerge.emergelog._disable
459 portage.util.noiselimit = -2
460 _emerge.emergelog._disable = True
462 if options.get("--depclean"):
463 rval, cleanlist, ordered, req_pkg_count = \
464 calc_depclean(self.settings, self.trees, None,
465 options, "depclean", InternalPackageSet(initial_atoms=atoms, allow_wildcard=True), None)
466 result = ResolverPlaygroundDepcleanResult( \
467 atoms, rval, cleanlist, ordered, req_pkg_count)
469 params = create_depgraph_params(options, action)
470 success, depgraph, favorites = backtrack_depgraph(
471 self.settings, self.trees, options, params, action, atoms, None)
472 depgraph._show_merge_list()
473 depgraph.display_problems()
474 result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
476 portage.util.noiselimit = global_noiselimit
477 _emerge.emergelog._disable = global_emergelog_disable
481 def run_TestCase(self, test_case):
482 if not isinstance(test_case, ResolverPlaygroundTestCase):
483 raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase")
484 for atoms in test_case.requests:
485 result = self.run(atoms, test_case.options, test_case.action)
486 if not test_case.compare_with_result(result):
490 portdb = self.trees[self.root]["porttree"].dbapi
491 portdb.close_caches()
492 portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
494 print("\nEROOT=%s" % self.eroot)
496 shutil.rmtree(self.eroot)
498 class ResolverPlaygroundTestCase(object):
500 def __init__(self, request, **kwargs):
501 self.all_permutations = kwargs.pop("all_permutations", False)
502 self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False)
503 self.ambiguous_merge_order = kwargs.pop("ambiguous_merge_order", False)
504 self.check_repo_names = kwargs.pop("check_repo_names", False)
505 self.merge_order_assertions = kwargs.pop("merge_order_assertions", False)
507 if self.all_permutations:
508 self.requests = list(permutations(request))
510 self.requests = [request]
512 self.options = kwargs.pop("options", {})
513 self.action = kwargs.pop("action", None)
514 self.test_success = True
516 self._checks = kwargs.copy()
518 def compare_with_result(self, result):
519 checks = dict.fromkeys(result.checks)
520 for key, value in self._checks.items():
521 if not key in checks:
522 raise KeyError("Not an available check: '%s'" % key)
526 for key, value in checks.items():
527 got = getattr(result, key)
530 if key in result.optional_checks and expected is None:
533 if key == "mergelist":
534 if not self.check_repo_names:
535 #Strip repo names if we don't check them
542 a = Atom("="+cpv, allow_repo=True)
543 new_got.append(a.cpv)
548 if isinstance(obj, basestring):
550 new_expected.append(obj)
552 a = Atom("="+obj, allow_repo=True)
553 new_expected.append(a.cpv)
555 new_expected.append(set())
558 cpv = Atom("="+cpv, allow_repo=True).cpv
559 new_expected[-1].add(cpv)
560 expected = new_expected
561 if self.ignore_mergelist_order and got is not None:
563 expected = set(expected)
565 if self.ambiguous_merge_order and got:
566 expected_stack = list(reversed(expected))
567 got_stack = list(reversed(got))
570 while got_stack and expected_stack:
571 got_token = got_stack.pop()
572 expected_obj = expected_stack.pop()
573 if isinstance(expected_obj, basestring):
574 new_expected.append(expected_obj)
575 if got_token == expected_obj:
577 # result doesn't match, so stop early
580 expected_obj = set(expected_obj)
582 expected_obj.remove(got_token)
584 # result doesn't match, so stop early
587 new_expected.append(got_token)
588 while got_stack and expected_obj:
589 got_token = got_stack.pop()
591 expected_obj.remove(got_token)
595 new_expected.append(got_token)
597 # result doesn't match, so stop early
600 # result does not match, so stop early
602 new_expected.append(tuple(expected_obj))
605 # result does not match, add leftovers to new_expected
607 expected_stack.reverse()
608 new_expected.extend(expected_stack)
609 expected = new_expected
611 if match and self.merge_order_assertions:
612 for node1, node2 in self.merge_order_assertions:
613 if not (got.index(node1) < got.index(node2)):
614 fail_msgs.append("atoms: (" + \
615 ", ".join(result.atoms) + "), key: " + \
616 ("merge_order_assertions, expected: %s" % \
617 str((node1, node2))) + \
618 ", got: " + str(got))
620 elif key in ("unstable_keywords", "needed_p_mask_changes") and expected is not None:
621 expected = set(expected)
624 fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \
625 key + ", expected: " + str(expected) + ", got: " + str(got))
627 self.test_success = False
628 self.fail_msg = "\n".join(fail_msgs)
632 class ResolverPlaygroundResult(object):
635 "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions",
636 "circular_dependency_solutions", "needed_p_mask_changes",
641 def __init__(self, atoms, success, mydepgraph, favorites):
643 self.success = success
644 self.depgraph = mydepgraph
645 self.favorites = favorites
646 self.mergelist = None
647 self.use_changes = None
648 self.license_changes = None
649 self.unstable_keywords = None
650 self.needed_p_mask_changes = None
651 self.slot_collision_solutions = None
652 self.circular_dependency_solutions = None
654 if self.depgraph._dynamic_config._serialized_tasks_cache is not None:
656 for x in self.depgraph._dynamic_config._serialized_tasks_cache:
657 if isinstance(x, Blocker):
658 self.mergelist.append(x.atom)
661 if x.metadata["repository"] != "test_repo":
662 repo_str = _repo_separator + x.metadata["repository"]
663 self.mergelist.append(x.cpv + repo_str)
665 if self.depgraph._dynamic_config._needed_use_config_changes:
666 self.use_changes = {}
667 for pkg, needed_use_config_changes in \
668 self.depgraph._dynamic_config._needed_use_config_changes.items():
669 new_use, changes = needed_use_config_changes
670 self.use_changes[pkg.cpv] = changes
672 if self.depgraph._dynamic_config._needed_unstable_keywords:
673 self.unstable_keywords = set()
674 for pkg in self.depgraph._dynamic_config._needed_unstable_keywords:
675 self.unstable_keywords.add(pkg.cpv)
677 if self.depgraph._dynamic_config._needed_p_mask_changes:
678 self.needed_p_mask_changes = set()
679 for pkg in self.depgraph._dynamic_config._needed_p_mask_changes:
680 self.needed_p_mask_changes.add(pkg.cpv)
682 if self.depgraph._dynamic_config._needed_license_changes:
683 self.license_changes = {}
684 for pkg, missing_licenses in self.depgraph._dynamic_config._needed_license_changes.items():
685 self.license_changes[pkg.cpv] = missing_licenses
687 if self.depgraph._dynamic_config._slot_conflict_handler is not None:
688 self.slot_collision_solutions = []
689 handler = self.depgraph._dynamic_config._slot_conflict_handler
691 for change in handler.changes:
694 new_change[pkg.cpv] = change[pkg]
695 self.slot_collision_solutions.append(new_change)
697 if self.depgraph._dynamic_config._circular_dependency_handler is not None:
698 handler = self.depgraph._dynamic_config._circular_dependency_handler
699 sol = handler.solutions
700 self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) )
702 class ResolverPlaygroundDepcleanResult(object):
705 "success", "cleanlist", "ordered", "req_pkg_count",
708 "ordered", "req_pkg_count",
711 def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count):
713 self.success = rval == 0
714 self.cleanlist = cleanlist
715 self.ordered = ordered
716 self.req_pkg_count = req_pkg_count