reposyntax: Remove unneeded special case
[portage.git] / pym / portage / tests / resolver / ResolverPlayground.py
1 # Copyright 2010 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3
4 from itertools import permutations
5 import shutil
6 import tempfile
7 import portage
8 from portage import os
9 from portage.const import PORTAGE_BASE_PATH
10 from portage.dbapi.vartree import vartree
11 from portage.dbapi.porttree import portagetree
12 from portage.dbapi.bintree import binarytree
13 from portage.dep import Atom, _repo_separator
14 from portage.package.ebuild.config import config
15 from portage.package.ebuild.digestgen import digestgen
16 from portage._sets import load_default_config
17 from portage.versions import catsplit
18
19 import _emerge
20 from _emerge.actions import calc_depclean
21 from _emerge.Blocker import Blocker
22 from _emerge.create_depgraph_params import create_depgraph_params
23 from _emerge.depgraph import backtrack_depgraph
24 from _emerge.RootConfig import RootConfig
25
26 class ResolverPlayground(object):
27         """
28         This class help to create the necessary files on disk and
29         the needed settings instances, etc. for the resolver to do
30         it's work.
31         """
32
33         config_files = frozenset(("package.use", "package.mask", "package.keywords", \
34                 "package.unmask", "package.properties", "package.license"))
35
36         def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \
37                 user_config={}, sets={}, world=[], debug=False):
38                 """
39                 ebuilds: cpv -> metadata mapping simulating avaiable ebuilds. 
40                 installed: cpv -> metadata mapping simulating installed packages.
41                         If a metadata key is missing, it gets a default value.
42                 profile: settings defined by the profile.
43                 """
44                 self.debug = debug
45                 self.root = "/"
46                 self.eprefix = tempfile.mkdtemp()
47                 self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
48                 self.portdir = os.path.join(self.eroot, "usr/portage")
49                 self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
50                 os.makedirs(self.portdir)
51                 os.makedirs(self.vdbdir)
52
53                 if not debug:
54                         portage.util.noiselimit = -2
55
56                 self.repo_dirs = {}
57                 #Make sure the main repo is always created
58                 self._get_repo_dir("test_repo")
59
60                 self._create_ebuilds(ebuilds)
61                 self._create_installed(installed)
62                 self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
63                 self._create_world(world)
64
65                 self.settings, self.trees = self._load_config()
66
67                 self._create_ebuild_manifests(ebuilds)
68                 
69                 portage.util.noiselimit = 0
70
71         def _get_repo_dir(self, repo):
72                 """
73                 Create the repo directory if needed.
74                 """
75                 if repo not in self.repo_dirs:
76                         if repo == "test_repo":
77                                 repo_path = self.portdir
78                         else:
79                                 repo_path = os.path.join(self.eroot, "usr", "local", repo)
80
81                         self.repo_dirs[repo] = repo_path
82                         profile_path = os.path.join(repo_path, "profiles")
83
84                         try:
85                                 os.makedirs(profile_path)
86                         except os.error:
87                                 pass
88
89                         repo_name_file = os.path.join(profile_path, "repo_name")
90                         f = open(repo_name_file, "w")
91                         f.write("%s\n" % repo)
92                         f.close()
93
94                 return self.repo_dirs[repo]
95
96         def _create_ebuilds(self, ebuilds):
97                 for cpv in ebuilds:
98                         a = Atom("=" + cpv, allow_repo=True)
99                         repo = a.repo
100                         if repo is None:
101                                 repo = "test_repo"
102
103                         metadata = ebuilds[cpv].copy()
104                         eapi = metadata.pop("EAPI", 0)
105                         lic = metadata.pop("LICENSE", "")
106                         properties = metadata.pop("PROPERTIES", "")
107                         slot = metadata.pop("SLOT", 0)
108                         keywords = metadata.pop("KEYWORDS", "x86")
109                         iuse = metadata.pop("IUSE", "")
110                         depend = metadata.pop("DEPEND", "")
111                         rdepend = metadata.pop("RDEPEND", None)
112                         pdepend = metadata.pop("PDEPEND", None)
113                         required_use = metadata.pop("REQUIRED_USE", None)
114
115                         if metadata:
116                                 raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
117
118                         repo_dir = self._get_repo_dir(repo)
119                         ebuild_dir = os.path.join(repo_dir, a.cp)
120                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
121                         try:
122                                 os.makedirs(ebuild_dir)
123                         except os.error:
124                                 pass
125
126                         f = open(ebuild_path, "w")
127                         f.write('EAPI="' + str(eapi) + '"\n')
128                         f.write('LICENSE="' + str(lic) + '"\n')
129                         f.write('PROPERTIES="' + str(properties) + '"\n')
130                         f.write('SLOT="' + str(slot) + '"\n')
131                         f.write('KEYWORDS="' + str(keywords) + '"\n')
132                         f.write('IUSE="' + str(iuse) + '"\n')
133                         f.write('DEPEND="' + str(depend) + '"\n')
134                         if rdepend is not None:
135                                 f.write('RDEPEND="' + str(rdepend) + '"\n')
136                         if pdepend is not None:
137                                 f.write('PDEPEND="' + str(pdepend) + '"\n')
138                         if required_use is not None:
139                                 f.write('REQUIRED_USE="' + str(required_use) + '"\n')
140                         f.close()
141
142         def _create_ebuild_manifests(self, ebuilds):
143                 tmpsettings = config(clone=self.settings)
144                 tmpsettings['PORTAGE_QUIET'] = '1'
145                 for cpv in ebuilds:
146                         a = Atom("=" + cpv, allow_repo=True)
147                         repo = a.repo
148                         if repo is None:
149                                 repo = "test_repo"
150
151                         repo_dir = self._get_repo_dir(repo)
152                         ebuild_dir = os.path.join(repo_dir, a.cp)
153                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
154
155                         portdb = self.trees[self.root]["porttree"].dbapi
156                         tmpsettings['O'] = ebuild_dir
157                         if not digestgen(mysettings=tmpsettings, myportdb=portdb):
158                                 raise AssertionError('digest creation failed for %s' % ebuild_path)
159
160         def _create_installed(self, installed):
161                 for cpv in installed:
162                         a = Atom("=" + cpv, allow_repo=True)
163                         repo = a.repo
164                         if repo is None:
165                                 repo = "test_repo"
166
167                         vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
168                         try:
169                                 os.makedirs(vdb_pkg_dir)
170                         except os.error:
171                                 pass
172
173                         metadata = installed[cpv].copy()
174                         eapi = metadata.pop("EAPI", 0)
175                         lic = metadata.pop("LICENSE", "")
176                         properties = metadata.pop("PROPERTIES", "")
177                         slot = metadata.pop("SLOT", 0)
178                         keywords = metadata.pop("KEYWORDS", "~x86")
179                         iuse = metadata.pop("IUSE", "")
180                         use = metadata.pop("USE", "")
181                         depend = metadata.pop("DEPEND", "")
182                         rdepend = metadata.pop("RDEPEND", None)
183                         pdepend = metadata.pop("PDEPEND", None)
184                         required_use = metadata.pop("REQUIRED_USE", None)
185
186                         if metadata:
187                                 raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
188
189                         def write_key(key, value):
190                                 f = open(os.path.join(vdb_pkg_dir, key), "w")
191                                 f.write(str(value) + "\n")
192                                 f.close()
193                         
194                         write_key("EAPI", eapi)
195                         write_key("LICENSE", lic)
196                         write_key("PROPERTIES", properties)
197                         write_key("SLOT", slot)
198                         write_key("LICENSE", lic)
199                         write_key("PROPERTIES", properties)
200                         write_key("repository", repo)
201                         write_key("KEYWORDS", keywords)
202                         write_key("IUSE", iuse)
203                         write_key("USE", use)
204                         write_key("DEPEND", depend)
205                         if rdepend is not None:
206                                 write_key("RDEPEND", rdepend)
207                         if pdepend is not None:
208                                 write_key("PDEPEND", pdepend)
209                         if required_use is not None:
210                                 write_key("REQUIRED_USE", required_use)
211
212         def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
213
214                 for repo in self.repo_dirs:
215                         repo_dir = self._get_repo_dir(repo)
216                         profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
217
218                         #Create $REPO/profiles/categories
219                         categories = set()
220                         for cpv in ebuilds:
221                                 ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
222                                 if ebuilds_repo is None:
223                                         ebuilds_repo = "test_repo"
224                                 if ebuilds_repo == repo:
225                                         categories.add(catsplit(cpv)[0])
226
227                         categories_file = os.path.join(profile_dir, "categories")
228                         f = open(categories_file, "w")
229                         for cat in categories:
230                                 f.write(cat + "\n")
231                         f.close()
232                         
233                         #Create $REPO/profiles/license_groups
234                         license_file = os.path.join(profile_dir, "license_groups")
235                         f = open(license_file, "w")
236                         f.write("EULA TEST\n")
237                         f.close()
238
239                         repo_config = repo_configs.get(repo) 
240                         if repo_config:
241                                 for config_file, lines in repo_config.items():
242                                         if config_file not in self.config_files:
243                                                 raise ValueError("Unknown config file: '%s'" % config_file)
244                 
245                                         file_name = os.path.join(profile_dir, config_file)
246                                         f = open(file_name, "w")
247                                         for line in lines:
248                                                 f.write("%s\n" % line)
249                                         f.close()
250
251                         #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
252                         os.makedirs(os.path.join(repo_dir, "eclass"))
253
254                         if repo == "test_repo":
255                                 #Create a minimal profile in /usr/portage
256                                 sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
257                                 os.makedirs(sub_profile_dir)
258
259                                 eapi_file = os.path.join(sub_profile_dir, "eapi")
260                                 f = open(eapi_file, "w")
261                                 f.write("0\n")
262                                 f.close()
263
264                                 make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
265                                 f = open(make_defaults_file, "w")
266                                 f.write("ARCH=\"x86\"\n")
267                                 f.write("ACCEPT_KEYWORDS=\"x86\"\n")
268                                 f.close()
269
270                                 use_force_file = os.path.join(sub_profile_dir, "use.force")
271                                 f = open(use_force_file, "w")
272                                 f.write("x86\n")
273                                 f.close()
274
275                                 if profile:
276                                         for config_file, lines in profile.items():
277                                                 if config_file not in self.config_files:
278                                                         raise ValueError("Unknown config file: '%s'" % config_file)
279
280                                                 file_name = os.path.join(sub_profile_dir, config_file)
281                                                 f = open(file_name, "w")
282                                                 for line in lines:
283                                                         f.write("%s\n" % line)
284                                                 f.close()
285
286                                 #Create profile symlink
287                                 os.makedirs(os.path.join(self.eroot, "etc"))
288                                 os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile"))
289
290                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
291
292                 try:
293                         os.makedirs(user_config_dir)
294                 except os.error:
295                         pass
296
297                 repos_conf_file = os.path.join(user_config_dir, "repos.conf")           
298                 f = open(repos_conf_file, "w")
299                 priority = 999
300                 for repo in sorted(self.repo_dirs.keys()):
301                         f.write("[%s]\n" % repo)
302                         f.write("LOCATION=%s\n" % self.repo_dirs[repo])
303                         if repo == "test_repo":
304                                 f.write("PRIORITY=%s\n" % 1000)
305                         else:
306                                 f.write("PRIORITY=%s\n" % priority)
307                                 priority -= 1
308                 f.close()
309
310                 for config_file, lines in user_config.items():
311                         if config_file not in self.config_files:
312                                 raise ValueError("Unknown config file: '%s'" % config_file)
313
314                         file_name = os.path.join(user_config_dir, config_file)
315                         f = open(file_name, "w")
316                         for line in lines:
317                                 f.write("%s\n" % line)
318                         f.close()
319
320                 #Create /usr/share/portage/config/sets/portage.conf
321                 default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
322                 
323                 try:
324                         os.makedirs(default_sets_conf_dir)
325                 except os.error:
326                         pass
327
328                 provided_sets_portage_conf = \
329                         os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
330                 os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
331
332                 set_config_dir = os.path.join(user_config_dir, "sets")
333
334                 try:
335                         os.makedirs(set_config_dir)
336                 except os.error:
337                         pass
338
339                 for sets_file, lines in sets.items():
340                         file_name = os.path.join(set_config_dir, sets_file)
341                         f = open(file_name, "w")
342                         for line in lines:
343                                 f.write("%s\n" % line)
344                         f.close()
345
346                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
347
348                 try:
349                         os.makedirs(user_config_dir)
350                 except os.error:
351                         pass
352
353                 for config_file, lines in user_config.items():
354                         if config_file not in self.config_files:
355                                 raise ValueError("Unknown config file: '%s'" % config_file)
356
357                         file_name = os.path.join(user_config_dir, config_file)
358                         f = open(file_name, "w")
359                         for line in lines:
360                                 f.write("%s\n" % line)
361                         f.close()
362
363         def _create_world(self, world):
364                 #Create /var/lib/portage/world
365                 var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
366                 os.makedirs(var_lib_portage)
367
368                 world_file = os.path.join(var_lib_portage, "world")
369
370                 f = open(world_file, "w")
371                 for atom in world:
372                         f.write("%s\n" % atom)
373                 f.close()
374
375         def _load_config(self):
376                 portdir_overlay = []
377                 for repo_name in sorted(self.repo_dirs):
378                         path = self.repo_dirs[repo_name]
379                         if path != self.portdir:
380                                 portdir_overlay.append(path)
381
382                 env = {
383                         "ACCEPT_KEYWORDS": "x86",
384                         "PORTDIR": self.portdir,
385                         "PORTDIR_OVERLAY": " ".join(portdir_overlay),
386                         'PORTAGE_TMPDIR'       : os.path.join(self.eroot, 'var/tmp'),
387                 }
388
389                 # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
390                 # need to be inherited by ebuild subprocesses.
391                 if 'PORTAGE_USERNAME' in os.environ:
392                         env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
393                 if 'PORTAGE_GRPNAME' in os.environ:
394                         env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
395
396                 settings = config(_eprefix=self.eprefix, env=env)
397                 settings.lock()
398
399                 trees = {
400                         self.root: {
401                                         "vartree": vartree(settings=settings),
402                                         "porttree": portagetree(self.root, settings=settings),
403                                         "bintree": binarytree(self.root,
404                                                 os.path.join(self.eroot, "usr/portage/packages"),
405                                                 settings=settings)
406                                 }
407                         }
408
409                 for root, root_trees in trees.items():
410                         settings = root_trees["vartree"].settings
411                         settings._init_dirs()
412                         setconfig = load_default_config(settings, root_trees)
413                         root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
414                 
415                 return settings, trees
416
417         def run(self, atoms, options={}, action=None):
418                 options = options.copy()
419                 options["--pretend"] = True
420                 if self.debug:
421                         options["--debug"] = True
422
423                 global_noiselimit = portage.util.noiselimit
424                 global_emergelog_disable = _emerge.emergelog._disable
425                 try:
426
427                         if not self.debug:
428                                 portage.util.noiselimit = -2
429                         _emerge.emergelog._disable = True
430
431                         if options.get("--depclean"):
432                                 rval, cleanlist, ordered, req_pkg_count = \
433                                         calc_depclean(self.settings, self.trees, None,
434                                         options, "depclean", atoms, None)
435                                 result = ResolverPlaygroundDepcleanResult( \
436                                         atoms, rval, cleanlist, ordered, req_pkg_count)
437                         else:
438                                 params = create_depgraph_params(options, action)
439                                 success, depgraph, favorites = backtrack_depgraph(
440                                         self.settings, self.trees, options, params, action, atoms, None)
441                                 depgraph._show_merge_list()
442                                 depgraph.display_problems()
443                                 result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
444                 finally:
445                         portage.util.noiselimit = global_noiselimit
446                         _emerge.emergelog._disable = global_emergelog_disable
447
448                 return result
449
450         def run_TestCase(self, test_case):
451                 if not isinstance(test_case, ResolverPlaygroundTestCase):
452                         raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase")
453                 for atoms in test_case.requests:
454                         result = self.run(atoms, test_case.options, test_case.action)
455                         if not test_case.compare_with_result(result):
456                                 return
457
458         def cleanup(self):
459                 portdb = self.trees[self.root]["porttree"].dbapi
460                 portdb.close_caches()
461                 portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
462                 if self.debug:
463                         print("\nEROOT=%s" % self.eroot)
464                 else:
465                         shutil.rmtree(self.eroot)
466
467 class ResolverPlaygroundTestCase(object):
468
469         def __init__(self, request, **kwargs):
470                 self.all_permutations = kwargs.pop("all_permutations", False)
471                 self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False)
472                 self.check_repo_names = kwargs.pop("check_repo_names", False)
473
474                 if self.all_permutations:
475                         self.requests = list(permutations(request))
476                 else:
477                         self.requests = [request]
478
479                 self.options = kwargs.pop("options", {})
480                 self.action = kwargs.pop("action", None)
481                 self.test_success = True
482                 self.fail_msg = None
483                 self._checks = kwargs.copy()
484
485         def compare_with_result(self, result):
486                 checks = dict.fromkeys(result.checks)
487                 for key, value in self._checks.items():
488                         if not key in checks:
489                                 raise KeyError("Not an avaiable check: '%s'" % key)
490                         checks[key] = value
491
492                 fail_msgs = []
493                 for key, value in checks.items():
494                         got = getattr(result, key)
495                         expected = value
496
497                         if key in result.optional_checks and expected is None:
498                                 continue
499
500                         if key == "mergelist":
501                                 if not self.check_repo_names:
502                                         #Strip repo names if we don't check them
503                                         if got:
504                                                 new_got = []
505                                                 for cpv in got:
506                                                         a = Atom("="+cpv, allow_repo=True)
507                                                         new_got.append(a.cpv)
508                                                 got = new_got
509                                         if expected:
510                                                 new_expected = []
511                                                 for cpv in expected:
512                                                         a = Atom("="+cpv, allow_repo=True)
513                                                         new_expected.append(a.cpv)
514                                                 expected = new_expected
515                                 if self.ignore_mergelist_order and got is not None:
516                                         got = set(got)
517                                         expected = set(expected)
518                         elif key == "unstable_keywords" and expected is not None:
519                                 expected = set(expected)
520
521                         if got != expected:
522                                 fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \
523                                         key + ", expected: " + str(expected) + ", got: " + str(got))
524                 if fail_msgs:
525                         self.test_success = False
526                         self.fail_msg = "\n".join(fail_msgs)
527                         return False
528                 return True
529
530 class ResolverPlaygroundResult(object):
531
532         checks = (
533                 "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions",
534                 "circular_dependency_solutions",
535                 )
536         optional_checks = (
537                 )
538
539         def __init__(self, atoms, success, mydepgraph, favorites):
540                 self.atoms = atoms
541                 self.success = success
542                 self.depgraph = mydepgraph
543                 self.favorites = favorites
544                 self.mergelist = None
545                 self.use_changes = None
546                 self.license_changes = None
547                 self.unstable_keywords = None
548                 self.slot_collision_solutions = None
549                 self.circular_dependency_solutions = None
550
551                 if self.depgraph._dynamic_config._serialized_tasks_cache is not None:
552                         self.mergelist = []
553                         for x in self.depgraph._dynamic_config._serialized_tasks_cache:
554                                 if isinstance(x, Blocker):
555                                         self.mergelist.append(x.atom)
556                                 else:
557                                         repo_str = ""
558                                         if x.metadata["repository"] != "test_repo":
559                                                 repo_str = _repo_separator + x.metadata["repository"]
560                                         self.mergelist.append(x.cpv + repo_str)
561
562                 if self.depgraph._dynamic_config._needed_use_config_changes:
563                         self.use_changes = {}
564                         for pkg, needed_use_config_changes in \
565                                 self.depgraph._dynamic_config._needed_use_config_changes.items():
566                                 new_use, changes = needed_use_config_changes
567                                 self.use_changes[pkg.cpv] = changes
568
569                 if self.depgraph._dynamic_config._needed_unstable_keywords:
570                         self.unstable_keywords = set()
571                         for pkg in self.depgraph._dynamic_config._needed_unstable_keywords:
572                                 self.unstable_keywords.add(pkg.cpv)
573
574                 if self.depgraph._dynamic_config._needed_license_changes:
575                         self.license_changes = {}
576                         for pkg, missing_licenses in self.depgraph._dynamic_config._needed_license_changes.items():
577                                 self.license_changes[pkg.cpv] = missing_licenses
578
579                 if self.depgraph._dynamic_config._slot_conflict_handler is not None:
580                         self.slot_collision_solutions  = []
581                         handler = self.depgraph._dynamic_config._slot_conflict_handler
582
583                         for solution in handler.solutions:
584                                 s = {}
585                                 for pkg in solution:
586                                         changes = {}
587                                         for flag, state in solution[pkg].items():
588                                                 if state == "enabled":
589                                                         changes[flag] = True
590                                                 else:
591                                                         changes[flag] = False
592                                         s[pkg.cpv] = changes
593                                 self.slot_collision_solutions.append(s)
594
595                 if self.depgraph._dynamic_config._circular_dependency_handler is not None:
596                         handler = self.depgraph._dynamic_config._circular_dependency_handler
597                         sol = handler.solutions
598                         self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) )
599
600 class ResolverPlaygroundDepcleanResult(object):
601
602         checks = (
603                 "success", "cleanlist", "ordered", "req_pkg_count",
604                 )
605         optional_checks = (
606                 "ordered", "req_pkg_count",
607                 )
608
609         def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count):
610                 self.atoms = atoms
611                 self.success = rval == 0
612                 self.cleanlist = cleanlist
613                 self.ordered = ordered
614                 self.req_pkg_count = req_pkg_count