ResolverPlayground: Add multi repo support
[portage.git] / pym / portage / tests / resolver / ResolverPlayground.py
1 # Copyright 2010 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3
4 from itertools import permutations
5 import shutil
6 import tempfile
7 import portage
8 from portage import os
9 from portage.const import PORTAGE_BASE_PATH
10 from portage.dbapi.vartree import vartree
11 from portage.dbapi.porttree import portagetree
12 from portage.dbapi.bintree import binarytree
13 from portage.dep import Atom, _repo_separator
14 from portage.package.ebuild.config import config
15 from portage._sets import load_default_config
16 from portage.versions import catsplit
17
18 import _emerge
19 from _emerge.actions import calc_depclean
20 from _emerge.Blocker import Blocker
21 from _emerge.create_depgraph_params import create_depgraph_params
22 from _emerge.depgraph import backtrack_depgraph
23 from _emerge.RootConfig import RootConfig
24
25 class ResolverPlayground(object):
26         """
27         This class help to create the necessary files on disk and
28         the needed settings instances, etc. for the resolver to do
29         it's work.
30         """
31
32         config_files = frozenset(("package.use", "package.mask", "package.keywords", \
33                 "package.unmask", "package.properties", "package.license"))
34
35         def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \
36                 user_config={}, sets={}, world=[], debug=False):
37                 """
38                 ebuilds: cpv -> metadata mapping simulating avaiable ebuilds. 
39                 installed: cpv -> metadata mapping simulating installed packages.
40                         If a metadata key is missing, it gets a default value.
41                 profile: settings defined by the profile.
42                 """
43                 self.debug = debug
44                 self.root = "/"
45                 self.eprefix = tempfile.mkdtemp()
46                 self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
47                 self.portdir = os.path.join(self.eroot, "usr/portage")
48                 self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
49                 os.makedirs(self.portdir)
50                 os.makedirs(self.vdbdir)
51
52                 if not debug:
53                         portage.util.noiselimit = -2
54
55                 self.repo_dirs = {}
56                 #Make sure the main repo is alaways created
57                 self._get_repo_dir("test_repo")
58
59                 self._create_ebuilds(ebuilds)
60                 self._create_installed(installed)
61                 self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets)
62                 self._create_world(world)
63
64                 self.settings, self.trees = self._load_config()
65
66                 self._create_ebuild_manifests(ebuilds)
67                 
68                 portage.util.noiselimit = 0
69
70         def _get_repo_dir(self, repo):
71                 """
72                 Create the repo directory if needed.
73                 """
74                 if repo not in self.repo_dirs:
75                         if repo == "test_repo":
76                                 repo_path = self.portdir
77                         else:
78                                 repo_path = os.path.join(self.eroot, "usr", "local", repo)
79
80                         self.repo_dirs[repo] = repo_path
81                         profile_path = os.path.join(repo_path, "profiles")
82
83                         try:
84                                 os.makedirs(profile_path)
85                         except os.error:
86                                 pass
87
88                         repo_name_file = os.path.join(profile_path, "repo_name")
89                         f = open(repo_name_file, "w")
90                         f.write("%s\n" % repo)
91                         f.close()
92
93                 return self.repo_dirs[repo]
94
95         def _create_ebuilds(self, ebuilds):
96                 for cpv in ebuilds:
97                         a = Atom("=" + cpv, allow_repo=True)
98                         repo = a.repo
99                         if repo is None:
100                                 repo = "test_repo"
101
102                         metadata = ebuilds[cpv].copy()
103                         eapi = metadata.pop("EAPI", 0)
104                         lic = metadata.pop("LICENSE", "")
105                         properties = metadata.pop("PROPERTIES", "")
106                         slot = metadata.pop("SLOT", 0)
107                         keywords = metadata.pop("KEYWORDS", "x86")
108                         iuse = metadata.pop("IUSE", "")
109                         depend = metadata.pop("DEPEND", "")
110                         rdepend = metadata.pop("RDEPEND", None)
111                         pdepend = metadata.pop("PDEPEND", None)
112                         required_use = metadata.pop("REQUIRED_USE", None)
113
114                         if metadata:
115                                 raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
116
117                         repo_dir = self._get_repo_dir(repo)
118                         ebuild_dir = os.path.join(repo_dir, a.cp)
119                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
120                         try:
121                                 os.makedirs(ebuild_dir)
122                         except os.error:
123                                 pass
124
125                         f = open(ebuild_path, "w")
126                         f.write('EAPI="' + str(eapi) + '"\n')
127                         f.write('LICENSE="' + str(lic) + '"\n')
128                         f.write('PROPERTIES="' + str(properties) + '"\n')
129                         f.write('SLOT="' + str(slot) + '"\n')
130                         f.write('KEYWORDS="' + str(keywords) + '"\n')
131                         f.write('IUSE="' + str(iuse) + '"\n')
132                         f.write('DEPEND="' + str(depend) + '"\n')
133                         if rdepend is not None:
134                                 f.write('RDEPEND="' + str(rdepend) + '"\n')
135                         if pdepend is not None:
136                                 f.write('PDEPEND="' + str(pdepend) + '"\n')
137                         if required_use is not None:
138                                 f.write('REQUIRED_USE="' + str(required_use) + '"\n')
139                         f.close()
140
141         def _create_ebuild_manifests(self, ebuilds):
142                 for cpv in ebuilds:
143                         a = Atom("=" + cpv, allow_repo=True)
144                         repo = a.repo
145                         if repo is None:
146                                 repo = "test_repo"
147
148                         repo_dir = self._get_repo_dir(repo)
149                         ebuild_dir = os.path.join(repo_dir, a.cp)
150                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
151
152                         portage.util.noiselimit = -1
153                         tmpsettings = config(clone=self.settings)
154                         portdb = self.trees[self.root]["porttree"].dbapi
155                         portage.doebuild(ebuild_path, "digest", self.root, tmpsettings,
156                                 tree="porttree", mydbapi=portdb)
157                         portage.util.noiselimit = 0
158
159         def _create_installed(self, installed):
160                 for cpv in installed:
161                         a = Atom("=" + cpv, allow_repo=True)
162                         repo = a.repo
163                         if repo is None:
164                                 repo = "test_repo"
165
166                         vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
167                         try:
168                                 os.makedirs(vdb_pkg_dir)
169                         except os.error:
170                                 pass
171
172                         metadata = installed[cpv].copy()
173                         eapi = metadata.pop("EAPI", 0)
174                         lic = metadata.pop("LICENSE", "")
175                         properties = metadata.pop("PROPERTIES", "")
176                         slot = metadata.pop("SLOT", 0)
177                         keywords = metadata.pop("KEYWORDS", "~x86")
178                         iuse = metadata.pop("IUSE", "")
179                         use = metadata.pop("USE", "")
180                         depend = metadata.pop("DEPEND", "")
181                         rdepend = metadata.pop("RDEPEND", None)
182                         pdepend = metadata.pop("PDEPEND", None)
183                         required_use = metadata.pop("REQUIRED_USE", None)
184
185                         if metadata:
186                                 raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
187
188                         def write_key(key, value):
189                                 f = open(os.path.join(vdb_pkg_dir, key), "w")
190                                 f.write(str(value) + "\n")
191                                 f.close()
192                         
193                         write_key("EAPI", eapi)
194                         write_key("LICENSE", lic)
195                         write_key("PROPERTIES", properties)
196                         write_key("SLOT", slot)
197                         write_key("LICENSE", lic)
198                         write_key("PROPERTIES", properties)
199                         write_key("repository", repo)
200                         write_key("KEYWORDS", keywords)
201                         write_key("IUSE", iuse)
202                         write_key("USE", use)
203                         write_key("DEPEND", depend)
204                         if rdepend is not None:
205                                 write_key("RDEPEND", rdepend)
206                         if pdepend is not None:
207                                 write_key("PDEPEND", pdepend)
208                         if required_use is not None:
209                                 write_key("REQUIRED_USE", required_use)
210
211         def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets):
212
213                 for repo in self.repo_dirs:
214                         repo_dir = self._get_repo_dir(repo)
215                         profile_dir = os.path.join(self._get_repo_dir(repo), "profiles")
216
217                         #Create $REPO/profiles/categories
218                         categories = set()
219                         for cpv in ebuilds:
220                                 ebuilds_repo = Atom("="+cpv, allow_repo=True).repo
221                                 if ebuilds_repo is None:
222                                         ebuilds_repo = "test_repo"
223                                 if ebuilds_repo == repo:
224                                         categories.add(catsplit(cpv)[0])
225
226                         categories_file = os.path.join(profile_dir, "categories")
227                         f = open(categories_file, "w")
228                         for cat in categories:
229                                 f.write(cat + "\n")
230                         f.close()
231                         
232                         #Create $REPO/profiles/license_groups
233                         license_file = os.path.join(profile_dir, "license_groups")
234                         f = open(license_file, "w")
235                         f.write("EULA TEST\n")
236                         f.close()
237
238                         repo_config = repo_configs.get(repo) 
239                         if repo_config:
240                                 for config_file, lines in repo_config.items():
241                                         if config_file not in self.config_files:
242                                                 raise ValueError("Unknown config file: '%s'" % config_file)
243                 
244                                         file_name = os.path.join(profile_dir, config_file)
245                                         f = open(file_name, "w")
246                                         for line in lines:
247                                                 f.write("%s\n" % line)
248                                         f.close()
249
250                         #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
251                         os.makedirs(os.path.join(repo_dir, "eclass"))
252
253                         if repo == "test_repo":
254                                 #Create a minimal profile in /usr/portage
255                                 sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
256                                 os.makedirs(sub_profile_dir)
257
258                                 eapi_file = os.path.join(sub_profile_dir, "eapi")
259                                 f = open(eapi_file, "w")
260                                 f.write("0\n")
261                                 f.close()
262
263                                 make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
264                                 f = open(make_defaults_file, "w")
265                                 f.write("ARCH=\"x86\"\n")
266                                 f.write("ACCEPT_KEYWORDS=\"x86\"\n")
267                                 f.close()
268
269                                 use_force_file = os.path.join(sub_profile_dir, "use.force")
270                                 f = open(use_force_file, "w")
271                                 f.write("x86\n")
272                                 f.close()
273
274                                 if profile:
275                                         for config_file, lines in profile.items():
276                                                 if config_file not in self.config_files:
277                                                         raise ValueError("Unknown config file: '%s'" % config_file)
278
279                                                 file_name = os.path.join(sub_profile_dir, config_file)
280                                                 f = open(file_name, "w")
281                                                 for line in lines:
282                                                         f.write("%s\n" % line)
283                                                 f.close()
284
285                                 #Create profile symlink
286                                 os.makedirs(os.path.join(self.eroot, "etc"))
287                                 os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile"))
288
289                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
290
291                 try:
292                         os.makedirs(user_config_dir)
293                 except os.error:
294                         pass
295
296                 repos_conf_file = os.path.join(user_config_dir, "repos.conf")           
297                 f = open(repos_conf_file, "w")
298                 priority = 1
299                 for repo in sorted(self.repo_dirs.keys()):
300                         f.write("[%s]\n" % repo)
301                         f.write("LOCATION=%s\n" % self.repo_dirs[repo])
302                         if repo == "test_repo":
303                                 f.write("PRIORITY=%s\n" % 1000)
304                         else:
305                                 f.write("PRIORITY=%s\n" % priority)
306                                 priority += 1
307                 f.close()
308
309                 for config_file, lines in user_config.items():
310                         if config_file not in self.config_files:
311                                 raise ValueError("Unknown config file: '%s'" % config_file)
312
313                         file_name = os.path.join(user_config_dir, config_file)
314                         f = open(file_name, "w")
315                         for line in lines:
316                                 f.write("%s\n" % line)
317                         f.close()
318
319                 #Create /usr/share/portage/config/sets/portage.conf
320                 default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
321                 
322                 try:
323                         os.makedirs(default_sets_conf_dir)
324                 except os.error:
325                         pass
326
327                 provided_sets_portage_conf = \
328                         os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
329                 os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
330
331                 set_config_dir = os.path.join(user_config_dir, "sets")
332
333                 try:
334                         os.makedirs(set_config_dir)
335                 except os.error:
336                         pass
337
338                 for sets_file, lines in sets.items():
339                         file_name = os.path.join(set_config_dir, sets_file)
340                         f = open(file_name, "w")
341                         for line in lines:
342                                 f.write("%s\n" % line)
343                         f.close()
344
345                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
346
347                 try:
348                         os.makedirs(user_config_dir)
349                 except os.error:
350                         pass
351
352                 for config_file, lines in user_config.items():
353                         if config_file not in self.config_files:
354                                 raise ValueError("Unknown config file: '%s'" % config_file)
355
356                         file_name = os.path.join(user_config_dir, config_file)
357                         f = open(file_name, "w")
358                         for line in lines:
359                                 f.write("%s\n" % line)
360                         f.close()
361
362         def _create_world(self, world):
363                 #Create /var/lib/portage/world
364                 var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
365                 os.makedirs(var_lib_portage)
366
367                 world_file = os.path.join(var_lib_portage, "world")
368
369                 f = open(world_file, "w")
370                 for atom in world:
371                         f.write("%s\n" % atom)
372                 f.close()
373
374         def _load_config(self):
375                 env = {
376                         "ACCEPT_KEYWORDS": "x86",
377                         "PORTDIR": self.portdir,
378                         'PORTAGE_TMPDIR'       : os.path.join(self.eroot, 'var/tmp'),
379                 }
380
381                 # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
382                 # need to be inherited by ebuild subprocesses.
383                 if 'PORTAGE_USERNAME' in os.environ:
384                         env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
385                 if 'PORTAGE_GRPNAME' in os.environ:
386                         env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
387
388                 settings = config(_eprefix=self.eprefix, env=env)
389                 settings.lock()
390
391                 trees = {
392                         self.root: {
393                                         "vartree": vartree(settings=settings),
394                                         "porttree": portagetree(self.root, settings=settings),
395                                         "bintree": binarytree(self.root,
396                                                 os.path.join(self.eroot, "usr/portage/packages"),
397                                                 settings=settings)
398                                 }
399                         }
400
401                 for root, root_trees in trees.items():
402                         settings = root_trees["vartree"].settings
403                         settings._init_dirs()
404                         setconfig = load_default_config(settings, root_trees)
405                         root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
406                 
407                 return settings, trees
408
409         def run(self, atoms, options={}, action=None):
410                 options = options.copy()
411                 options["--pretend"] = True
412                 options["--quiet"] = True
413                 if self.debug:
414                         options["--debug"] = True
415
416                 global_noiselimit = portage.util.noiselimit
417                 global_emergelog_disable = _emerge.emergelog._disable
418                 try:
419
420                         if not self.debug:
421                                 portage.util.noiselimit = -2
422                         _emerge.emergelog._disable = True
423
424                         if options.get("--depclean"):
425                                 rval, cleanlist, ordered, req_pkg_count = \
426                                         calc_depclean(self.settings, self.trees, None,
427                                         options, "depclean", atoms, None)
428                                 result = ResolverPlaygroundDepcleanResult( \
429                                         atoms, rval, cleanlist, ordered, req_pkg_count)
430                         else:
431                                 params = create_depgraph_params(options, action)
432                                 success, depgraph, favorites = backtrack_depgraph(
433                                         self.settings, self.trees, options, params, action, atoms, None)
434                                 depgraph.display_problems()
435                                 result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
436                 finally:
437                         portage.util.noiselimit = global_noiselimit
438                         _emerge.emergelog._disable = global_emergelog_disable
439
440                 return result
441
442         def run_TestCase(self, test_case):
443                 if not isinstance(test_case, ResolverPlaygroundTestCase):
444                         raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase")
445                 for atoms in test_case.requests:
446                         result = self.run(atoms, test_case.options, test_case.action)
447                         if not test_case.compare_with_result(result):
448                                 return
449
450         def cleanup(self):
451                 portdb = self.trees[self.root]["porttree"].dbapi
452                 portdb.close_caches()
453                 portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
454                 if self.debug:
455                         print("\nEROOT=%s" % self.eroot)
456                 else:
457                         shutil.rmtree(self.eroot)
458
459 class ResolverPlaygroundTestCase(object):
460
461         def __init__(self, request, **kwargs):
462                 self.all_permutations = kwargs.pop("all_permutations", False)
463                 self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False)
464                 self.check_repo_names = kwargs.pop("check_repo_names", False)
465
466                 if self.all_permutations:
467                         self.requests = list(permutations(request))
468                 else:
469                         self.requests = [request]
470
471                 self.options = kwargs.pop("options", {})
472                 self.action = kwargs.pop("action", None)
473                 self.test_success = True
474                 self.fail_msg = None
475                 self._checks = kwargs.copy()
476
477         def compare_with_result(self, result):
478                 checks = dict.fromkeys(result.checks)
479                 for key, value in self._checks.items():
480                         if not key in checks:
481                                 raise KeyError("Not an avaiable check: '%s'" % key)
482                         checks[key] = value
483
484                 fail_msgs = []
485                 for key, value in checks.items():
486                         got = getattr(result, key)
487                         expected = value
488
489                         if key in result.optional_checks and expected is None:
490                                 continue
491
492                         if key == "mergelist":
493                                 if not self.check_repo_names:
494                                         #Strip repo names if we don't check them
495                                         if got:
496                                                 new_got = []
497                                                 for cpv in got:
498                                                         a = Atom("="+cpv, allow_repo=True)
499                                                         new_got.append(a.cpv)
500                                                 got = new_got
501                                         if expected:
502                                                 new_expected = []
503                                                 for cpv in expected:
504                                                         a = Atom("="+cpv, allow_repo=True)
505                                                         new_expected.append(a.cpv)
506                                                 expected = new_expected
507                                 if self.ignore_mergelist_order and got is not None:
508                                         got = set(got)
509                                         expected = set(expected)
510                         elif key == "unstable_keywords" and expected is not None:
511                                 expected = set(expected)
512
513                         if got != expected:
514                                 fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \
515                                         key + ", expected: " + str(expected) + ", got: " + str(got))
516                 if fail_msgs:
517                         self.test_success = False
518                         self.fail_msg = "\n".join(fail_msgs)
519                         return False
520                 return True
521
522 class ResolverPlaygroundResult(object):
523
524         checks = (
525                 "success", "mergelist", "use_changes", "unstable_keywords", "slot_collision_solutions",
526                 "circular_dependency_solutions",
527                 )
528         optional_checks = (
529                 )
530
531         def __init__(self, atoms, success, mydepgraph, favorites):
532                 self.atoms = atoms
533                 self.success = success
534                 self.depgraph = mydepgraph
535                 self.favorites = favorites
536                 self.mergelist = None
537                 self.use_changes = None
538                 self.unstable_keywords = None
539                 self.slot_collision_solutions = None
540                 self.circular_dependency_solutions = None
541
542                 if self.depgraph._dynamic_config._serialized_tasks_cache is not None:
543                         self.mergelist = []
544                         for x in self.depgraph._dynamic_config._serialized_tasks_cache:
545                                 if isinstance(x, Blocker):
546                                         self.mergelist.append(x.atom)
547                                 else:
548                                         repo_str = ""
549                                         if x.metadata["repository"] != "test_repo":
550                                                 repo_str = _repo_separator + x.metadata["repository"]
551                                         self.mergelist.append(x.cpv + repo_str)
552
553                 if self.depgraph._dynamic_config._needed_use_config_changes:
554                         self.use_changes = {}
555                         for pkg, needed_use_config_changes in \
556                                 self.depgraph._dynamic_config._needed_use_config_changes.items():
557                                 new_use, changes = needed_use_config_changes
558                                 self.use_changes[pkg.cpv] = changes
559
560                 if self.depgraph._dynamic_config._needed_unstable_keywords:
561                         self.unstable_keywords = set()
562                         for pkg in self.depgraph._dynamic_config._needed_unstable_keywords:
563                                 self.unstable_keywords.add(pkg.cpv)
564
565                 if self.depgraph._dynamic_config._slot_conflict_handler is not None:
566                         self.slot_collision_solutions  = []
567                         handler = self.depgraph._dynamic_config._slot_conflict_handler
568
569                         for solution in handler.solutions:
570                                 s = {}
571                                 for pkg in solution:
572                                         changes = {}
573                                         for flag, state in solution[pkg].items():
574                                                 if state == "enabled":
575                                                         changes[flag] = True
576                                                 else:
577                                                         changes[flag] = False
578                                         s[pkg.cpv] = changes
579                                 self.slot_collision_solutions.append(s)
580
581                 if self.depgraph._dynamic_config._circular_dependency_handler is not None:
582                         handler = self.depgraph._dynamic_config._circular_dependency_handler
583                         sol = handler.solutions
584                         self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) )
585
586 class ResolverPlaygroundDepcleanResult(object):
587
588         checks = (
589                 "success", "cleanlist", "ordered", "req_pkg_count",
590                 )
591         optional_checks = (
592                 "ordered", "req_pkg_count",
593                 )
594
595         def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count):
596                 self.atoms = atoms
597                 self.success = rval == 0
598                 self.cleanlist = cleanlist
599                 self.ordered = ordered
600                 self.req_pkg_count = req_pkg_count