Update timestamps in headers of modified files.
[portage.git] / pym / portage / tests / resolver / ResolverPlayground.py
1 # Copyright 2010-2011 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3
4 from itertools import permutations
5 import shutil
6 import tempfile
7 import portage
8 from portage import os
9 from portage.const import PORTAGE_BASE_PATH
10 from portage.dbapi.vartree import vartree
11 from portage.dbapi.porttree import portagetree
12 from portage.dbapi.bintree import binarytree
13 from portage.dep import Atom, _repo_separator
14 from portage.package.ebuild.config import config
15 from portage.package.ebuild.digestgen import digestgen
16 from portage._sets import load_default_config
17 from portage._sets.base import InternalPackageSet
18 from portage.versions import catsplit
19
20 import _emerge
21 from _emerge.actions import calc_depclean
22 from _emerge.Blocker import Blocker
23 from _emerge.create_depgraph_params import create_depgraph_params
24 from _emerge.depgraph import backtrack_depgraph
25 from _emerge.RootConfig import RootConfig
26
27 class ResolverPlayground(object):
28         """
29         This class help to create the necessary files on disk and
30         the needed settings instances, etc. for the resolver to do
31         it's work.
32         """
33
34         config_files = frozenset(("package.use", "package.mask", "package.keywords", \
35                 "package.unmask", "package.properties", "package.license", "use.mask", "use.force"))
36
37         def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \
38                 user_config={}, sets={}, world=[], debug=False):
39                 """
40                 ebuilds: cpv -> metadata mapping simulating avaiable ebuilds. 
41                 installed: cpv -> metadata mapping simulating installed packages.
42                         If a metadata key is missing, it gets a default value.
43                 profile: settings defined by the profile.
44                 """
45                 self.debug = debug
46                 self.root = "/"
47                 self.eprefix = tempfile.mkdtemp()
48                 self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
49                 self.portdir = os.path.join(self.eroot, "usr/portage")
50                 self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
51                 os.makedirs(self.portdir)
52                 os.makedirs(self.vdbdir)
53
54                 if not debug:
55                         portage.util.noiselimit = -2
56
57                 self.repo_dirs = {}
58                 #Make sure the main repo is always created
59                 self._get_repo_dir("test_repo")
60
61                 self._create_ebuilds(ebuilds)
62                 self._create_installed(installed)
63                 self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
64                 self._create_world(world)
65
66                 self.settings, self.trees = self._load_config()
67
68                 self._create_ebuild_manifests(ebuilds)
69                 
70                 portage.util.noiselimit = 0
71
72         def _get_repo_dir(self, repo):
73                 """
74                 Create the repo directory if needed.
75                 """
76                 if repo not in self.repo_dirs:
77                         if repo == "test_repo":
78                                 repo_path = self.portdir
79                         else:
80                                 repo_path = os.path.join(self.eroot, "usr", "local", repo)
81
82                         self.repo_dirs[repo] = repo_path
83                         profile_path = os.path.join(repo_path, "profiles")
84
85                         try:
86                                 os.makedirs(profile_path)
87                         except os.error:
88                                 pass
89
90                         repo_name_file = os.path.join(profile_path, "repo_name")
91                         f = open(repo_name_file, "w")
92                         f.write("%s\n" % repo)
93                         f.close()
94
95                 return self.repo_dirs[repo]
96
97         def _create_ebuilds(self, ebuilds):
98                 for cpv in ebuilds:
99                         a = Atom("=" + cpv, allow_repo=True)
100                         repo = a.repo
101                         if repo is None:
102                                 repo = "test_repo"
103
104                         metadata = ebuilds[cpv].copy()
105                         eapi = metadata.pop("EAPI", 0)
106                         lic = metadata.pop("LICENSE", "")
107                         properties = metadata.pop("PROPERTIES", "")
108                         slot = metadata.pop("SLOT", 0)
109                         keywords = metadata.pop("KEYWORDS", "x86")
110                         iuse = metadata.pop("IUSE", "")
111                         depend = metadata.pop("DEPEND", "")
112                         rdepend = metadata.pop("RDEPEND", None)
113                         pdepend = metadata.pop("PDEPEND", None)
114                         required_use = metadata.pop("REQUIRED_USE", None)
115
116                         if metadata:
117                                 raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
118
119                         repo_dir = self._get_repo_dir(repo)
120                         ebuild_dir = os.path.join(repo_dir, a.cp)
121                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
122                         try:
123                                 os.makedirs(ebuild_dir)
124                         except os.error:
125                                 pass
126
127                         f = open(ebuild_path, "w")
128                         f.write('EAPI="' + str(eapi) + '"\n')
129                         f.write('LICENSE="' + str(lic) + '"\n')
130                         f.write('PROPERTIES="' + str(properties) + '"\n')
131                         f.write('SLOT="' + str(slot) + '"\n')
132                         f.write('KEYWORDS="' + str(keywords) + '"\n')
133                         f.write('IUSE="' + str(iuse) + '"\n')
134                         f.write('DEPEND="' + str(depend) + '"\n')
135                         if rdepend is not None:
136                                 f.write('RDEPEND="' + str(rdepend) + '"\n')
137                         if pdepend is not None:
138                                 f.write('PDEPEND="' + str(pdepend) + '"\n')
139                         if required_use is not None:
140                                 f.write('REQUIRED_USE="' + str(required_use) + '"\n')
141                         f.close()
142
143         def _create_ebuild_manifests(self, ebuilds):
144                 tmpsettings = config(clone=self.settings)
145                 tmpsettings['PORTAGE_QUIET'] = '1'
146                 for cpv in ebuilds:
147                         a = Atom("=" + cpv, allow_repo=True)
148                         repo = a.repo
149                         if repo is None:
150                                 repo = "test_repo"
151
152                         repo_dir = self._get_repo_dir(repo)
153                         ebuild_dir = os.path.join(repo_dir, a.cp)
154                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
155
156                         portdb = self.trees[self.root]["porttree"].dbapi
157                         tmpsettings['O'] = ebuild_dir
158                         if not digestgen(mysettings=tmpsettings, myportdb=portdb):
159                                 raise AssertionError('digest creation failed for %s' % ebuild_path)
160
161         def _create_installed(self, installed):
162                 for cpv in installed:
163                         a = Atom("=" + cpv, allow_repo=True)
164                         repo = a.repo
165                         if repo is None:
166                                 repo = "test_repo"
167
168                         vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
169                         try:
170                                 os.makedirs(vdb_pkg_dir)
171                         except os.error:
172                                 pass
173
174                         metadata = installed[cpv].copy()
175                         eapi = metadata.pop("EAPI", 0)
176                         lic = metadata.pop("LICENSE", "")
177                         properties = metadata.pop("PROPERTIES", "")
178                         slot = metadata.pop("SLOT", 0)
179                         keywords = metadata.pop("KEYWORDS", "~x86")
180                         iuse = metadata.pop("IUSE", "")
181                         use = metadata.pop("USE", "")
182                         depend = metadata.pop("DEPEND", "")
183                         rdepend = metadata.pop("RDEPEND", None)
184                         pdepend = metadata.pop("PDEPEND", None)
185                         required_use = metadata.pop("REQUIRED_USE", None)
186
187                         if metadata:
188                                 raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
189
190                         def write_key(key, value):
191                                 f = open(os.path.join(vdb_pkg_dir, key), "w")
192                                 f.write(str(value) + "\n")
193                                 f.close()
194                         
195                         write_key("EAPI", eapi)
196                         write_key("LICENSE", lic)
197                         write_key("PROPERTIES", properties)
198                         write_key("SLOT", slot)
199                         write_key("LICENSE", lic)
200                         write_key("PROPERTIES", properties)
201                         write_key("repository", repo)
202                         write_key("KEYWORDS", keywords)
203                         write_key("IUSE", iuse)
204                         write_key("USE", use)
205                         write_key("DEPEND", depend)
206                         if rdepend is not None:
207                                 write_key("RDEPEND", rdepend)
208                         if pdepend is not None:
209                                 write_key("PDEPEND", pdepend)
210                         if required_use is not None:
211                                 write_key("REQUIRED_USE", required_use)
212
213         def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
214
215                 for repo in self.repo_dirs:
216                         repo_dir = self._get_repo_dir(repo)
217                         profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
218
219                         #Create $REPO/profiles/categories
220                         categories = set()
221                         for cpv in ebuilds:
222                                 ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
223                                 if ebuilds_repo is None:
224                                         ebuilds_repo = "test_repo"
225                                 if ebuilds_repo == repo:
226                                         categories.add(catsplit(cpv)[0])
227
228                         categories_file = os.path.join(profile_dir, "categories")
229                         f = open(categories_file, "w")
230                         for cat in categories:
231                                 f.write(cat + "\n")
232                         f.close()
233                         
234                         #Create $REPO/profiles/license_groups
235                         license_file = os.path.join(profile_dir, "license_groups")
236                         f = open(license_file, "w")
237                         f.write("EULA TEST\n")
238                         f.close()
239
240                         repo_config = repo_configs.get(repo) 
241                         if repo_config:
242                                 for config_file, lines in repo_config.items():
243                                         if config_file not in self.config_files:
244                                                 raise ValueError("Unknown config file: '%s'" % config_file)
245                 
246                                         file_name = os.path.join(profile_dir, config_file)
247                                         f = open(file_name, "w")
248                                         for line in lines:
249                                                 f.write("%s\n" % line)
250                                         f.close()
251
252                         #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
253                         os.makedirs(os.path.join(repo_dir, "eclass"))
254
255                         if repo == "test_repo":
256                                 #Create a minimal profile in /usr/portage
257                                 sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
258                                 os.makedirs(sub_profile_dir)
259
260                                 eapi_file = os.path.join(sub_profile_dir, "eapi")
261                                 f = open(eapi_file, "w")
262                                 f.write("0\n")
263                                 f.close()
264
265                                 make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
266                                 f = open(make_defaults_file, "w")
267                                 f.write("ARCH=\"x86\"\n")
268                                 f.write("ACCEPT_KEYWORDS=\"x86\"\n")
269                                 f.close()
270
271                                 use_force_file = os.path.join(sub_profile_dir, "use.force")
272                                 f = open(use_force_file, "w")
273                                 f.write("x86\n")
274                                 f.close()
275
276                                 if profile:
277                                         for config_file, lines in profile.items():
278                                                 if config_file not in self.config_files:
279                                                         raise ValueError("Unknown config file: '%s'" % config_file)
280
281                                                 file_name = os.path.join(sub_profile_dir, config_file)
282                                                 f = open(file_name, "w")
283                                                 for line in lines:
284                                                         f.write("%s\n" % line)
285                                                 f.close()
286
287                                 #Create profile symlink
288                                 os.makedirs(os.path.join(self.eroot, "etc"))
289                                 os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile"))
290
291                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
292
293                 try:
294                         os.makedirs(user_config_dir)
295                 except os.error:
296                         pass
297
298                 repos_conf_file = os.path.join(user_config_dir, "repos.conf")           
299                 f = open(repos_conf_file, "w")
300                 priority = 0
301                 for repo in sorted(self.repo_dirs.keys()):
302                         f.write("[%s]\n" % repo)
303                         f.write("LOCATION=%s\n" % self.repo_dirs[repo])
304                         if repo == "test_repo":
305                                 f.write("PRIORITY=%s\n" % -1000)
306                         else:
307                                 f.write("PRIORITY=%s\n" % priority)
308                                 priority += 1
309                 f.close()
310
311                 for config_file, lines in user_config.items():
312                         if config_file not in self.config_files:
313                                 raise ValueError("Unknown config file: '%s'" % config_file)
314
315                         file_name = os.path.join(user_config_dir, config_file)
316                         f = open(file_name, "w")
317                         for line in lines:
318                                 f.write("%s\n" % line)
319                         f.close()
320
321                 #Create /usr/share/portage/config/sets/portage.conf
322                 default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
323                 
324                 try:
325                         os.makedirs(default_sets_conf_dir)
326                 except os.error:
327                         pass
328
329                 provided_sets_portage_conf = \
330                         os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
331                 os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
332
333                 set_config_dir = os.path.join(user_config_dir, "sets")
334
335                 try:
336                         os.makedirs(set_config_dir)
337                 except os.error:
338                         pass
339
340                 for sets_file, lines in sets.items():
341                         file_name = os.path.join(set_config_dir, sets_file)
342                         f = open(file_name, "w")
343                         for line in lines:
344                                 f.write("%s\n" % line)
345                         f.close()
346
347                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
348
349                 try:
350                         os.makedirs(user_config_dir)
351                 except os.error:
352                         pass
353
354                 for config_file, lines in user_config.items():
355                         if config_file not in self.config_files:
356                                 raise ValueError("Unknown config file: '%s'" % config_file)
357
358                         file_name = os.path.join(user_config_dir, config_file)
359                         f = open(file_name, "w")
360                         for line in lines:
361                                 f.write("%s\n" % line)
362                         f.close()
363
364         def _create_world(self, world):
365                 #Create /var/lib/portage/world
366                 var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
367                 os.makedirs(var_lib_portage)
368
369                 world_file = os.path.join(var_lib_portage, "world")
370
371                 f = open(world_file, "w")
372                 for atom in world:
373                         f.write("%s\n" % atom)
374                 f.close()
375
376         def _load_config(self):
377                 portdir_overlay = []
378                 for repo_name in sorted(self.repo_dirs):
379                         path = self.repo_dirs[repo_name]
380                         if path != self.portdir:
381                                 portdir_overlay.append(path)
382
383                 env = {
384                         "ACCEPT_KEYWORDS": "x86",
385                         "PORTDIR": self.portdir,
386                         "PORTDIR_OVERLAY": " ".join(portdir_overlay),
387                         'PORTAGE_TMPDIR'       : os.path.join(self.eroot, 'var/tmp'),
388                 }
389
390                 # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
391                 # need to be inherited by ebuild subprocesses.
392                 if 'PORTAGE_USERNAME' in os.environ:
393                         env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
394                 if 'PORTAGE_GRPNAME' in os.environ:
395                         env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
396
397                 settings = config(_eprefix=self.eprefix, env=env)
398                 settings.lock()
399
400                 trees = {
401                         self.root: {
402                                         "vartree": vartree(settings=settings),
403                                         "porttree": portagetree(self.root, settings=settings),
404                                         "bintree": binarytree(self.root,
405                                                 os.path.join(self.eroot, "usr/portage/packages"),
406                                                 settings=settings)
407                                 }
408                         }
409
410                 for root, root_trees in trees.items():
411                         settings = root_trees["vartree"].settings
412                         settings._init_dirs()
413                         setconfig = load_default_config(settings, root_trees)
414                         root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
415                 
416                 return settings, trees
417
418         def run(self, atoms, options={}, action=None):
419                 options = options.copy()
420                 options["--pretend"] = True
421                 if self.debug:
422                         options["--debug"] = True
423
424                 global_noiselimit = portage.util.noiselimit
425                 global_emergelog_disable = _emerge.emergelog._disable
426                 try:
427
428                         if not self.debug:
429                                 portage.util.noiselimit = -2
430                         _emerge.emergelog._disable = True
431
432                         if options.get("--depclean"):
433                                 rval, cleanlist, ordered, req_pkg_count = \
434                                         calc_depclean(self.settings, self.trees, None,
435                                         options, "depclean", InternalPackageSet(initial_atoms=atoms, allow_wildcard=True), None)
436                                 result = ResolverPlaygroundDepcleanResult( \
437                                         atoms, rval, cleanlist, ordered, req_pkg_count)
438                         else:
439                                 params = create_depgraph_params(options, action)
440                                 success, depgraph, favorites = backtrack_depgraph(
441                                         self.settings, self.trees, options, params, action, atoms, None)
442                                 depgraph._show_merge_list()
443                                 depgraph.display_problems()
444                                 result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
445                 finally:
446                         portage.util.noiselimit = global_noiselimit
447                         _emerge.emergelog._disable = global_emergelog_disable
448
449                 return result
450
451         def run_TestCase(self, test_case):
452                 if not isinstance(test_case, ResolverPlaygroundTestCase):
453                         raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase")
454                 for atoms in test_case.requests:
455                         result = self.run(atoms, test_case.options, test_case.action)
456                         if not test_case.compare_with_result(result):
457                                 return
458
459         def cleanup(self):
460                 portdb = self.trees[self.root]["porttree"].dbapi
461                 portdb.close_caches()
462                 portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
463                 if self.debug:
464                         print("\nEROOT=%s" % self.eroot)
465                 else:
466                         shutil.rmtree(self.eroot)
467
468 class ResolverPlaygroundTestCase(object):
469
470         def __init__(self, request, **kwargs):
471                 self.all_permutations = kwargs.pop("all_permutations", False)
472                 self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False)
473                 self.check_repo_names = kwargs.pop("check_repo_names", False)
474
475                 if self.all_permutations:
476                         self.requests = list(permutations(request))
477                 else:
478                         self.requests = [request]
479
480                 self.options = kwargs.pop("options", {})
481                 self.action = kwargs.pop("action", None)
482                 self.test_success = True
483                 self.fail_msg = None
484                 self._checks = kwargs.copy()
485
486         def compare_with_result(self, result):
487                 checks = dict.fromkeys(result.checks)
488                 for key, value in self._checks.items():
489                         if not key in checks:
490                                 raise KeyError("Not an avaiable check: '%s'" % key)
491                         checks[key] = value
492
493                 fail_msgs = []
494                 for key, value in checks.items():
495                         got = getattr(result, key)
496                         expected = value
497
498                         if key in result.optional_checks and expected is None:
499                                 continue
500
501                         if key == "mergelist":
502                                 if not self.check_repo_names:
503                                         #Strip repo names if we don't check them
504                                         if got:
505                                                 new_got = []
506                                                 for cpv in got:
507                                                         a = Atom("="+cpv, allow_repo=True)
508                                                         new_got.append(a.cpv)
509                                                 got = new_got
510                                         if expected:
511                                                 new_expected = []
512                                                 for cpv in expected:
513                                                         a = Atom("="+cpv, allow_repo=True)
514                                                         new_expected.append(a.cpv)
515                                                 expected = new_expected
516                                 if self.ignore_mergelist_order and got is not None:
517                                         got = set(got)
518                                         expected = set(expected)
519                         elif key == "unstable_keywords" and expected is not None:
520                                 expected = set(expected)
521
522                         if got != expected:
523                                 fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \
524                                         key + ", expected: " + str(expected) + ", got: " + str(got))
525                 if fail_msgs:
526                         self.test_success = False
527                         self.fail_msg = "\n".join(fail_msgs)
528                         return False
529                 return True
530
531 class ResolverPlaygroundResult(object):
532
533         checks = (
534                 "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions",
535                 "circular_dependency_solutions",
536                 )
537         optional_checks = (
538                 )
539
540         def __init__(self, atoms, success, mydepgraph, favorites):
541                 self.atoms = atoms
542                 self.success = success
543                 self.depgraph = mydepgraph
544                 self.favorites = favorites
545                 self.mergelist = None
546                 self.use_changes = None
547                 self.license_changes = None
548                 self.unstable_keywords = None
549                 self.slot_collision_solutions = None
550                 self.circular_dependency_solutions = None
551
552                 if self.depgraph._dynamic_config._serialized_tasks_cache is not None:
553                         self.mergelist = []
554                         for x in self.depgraph._dynamic_config._serialized_tasks_cache:
555                                 if isinstance(x, Blocker):
556                                         self.mergelist.append(x.atom)
557                                 else:
558                                         repo_str = ""
559                                         if x.metadata["repository"] != "test_repo":
560                                                 repo_str = _repo_separator + x.metadata["repository"]
561                                         self.mergelist.append(x.cpv + repo_str)
562
563                 if self.depgraph._dynamic_config._needed_use_config_changes:
564                         self.use_changes = {}
565                         for pkg, needed_use_config_changes in \
566                                 self.depgraph._dynamic_config._needed_use_config_changes.items():
567                                 new_use, changes = needed_use_config_changes
568                                 self.use_changes[pkg.cpv] = changes
569
570                 if self.depgraph._dynamic_config._needed_unstable_keywords:
571                         self.unstable_keywords = set()
572                         for pkg in self.depgraph._dynamic_config._needed_unstable_keywords:
573                                 self.unstable_keywords.add(pkg.cpv)
574
575                 if self.depgraph._dynamic_config._needed_license_changes:
576                         self.license_changes = {}
577                         for pkg, missing_licenses in self.depgraph._dynamic_config._needed_license_changes.items():
578                                 self.license_changes[pkg.cpv] = missing_licenses
579
580                 if self.depgraph._dynamic_config._slot_conflict_handler is not None:
581                         self.slot_collision_solutions  = []
582                         handler = self.depgraph._dynamic_config._slot_conflict_handler
583
584                         for change in handler.changes:
585                                 new_change = {}
586                                 for pkg in change:
587                                         new_change[pkg.cpv] = change[pkg]
588                                 self.slot_collision_solutions.append(new_change)
589
590                 if self.depgraph._dynamic_config._circular_dependency_handler is not None:
591                         handler = self.depgraph._dynamic_config._circular_dependency_handler
592                         sol = handler.solutions
593                         self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) )
594
595 class ResolverPlaygroundDepcleanResult(object):
596
597         checks = (
598                 "success", "cleanlist", "ordered", "req_pkg_count",
599                 )
600         optional_checks = (
601                 "ordered", "req_pkg_count",
602                 )
603
604         def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count):
605                 self.atoms = atoms
606                 self.success = rval == 0
607                 self.cleanlist = cleanlist
608                 self.ordered = ordered
609                 self.req_pkg_count = req_pkg_count