pym/portage/tests/resolver/ResolverPlayground.py

   1 # Copyright 2010 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from itertools import chain, permutations
   5 import shutil
   6 import tempfile
   7 import portage
   8 from portage import os
   9 from portage.const import PORTAGE_BASE_PATH
  10 from portage.dbapi.vartree import vartree
  11 from portage.dbapi.porttree import portagetree
  12 from portage.dbapi.bintree import binarytree
  13 from portage.dep import Atom
  14 from portage.package.ebuild.config import config
  15 from portage._sets import load_default_config
  16 from portage.versions import catsplit
  17
  18 import _emerge
  19 from _emerge.actions import calc_depclean
  20 from _emerge.Blocker import Blocker
  21 from _emerge.create_depgraph_params import create_depgraph_params
  22 from _emerge.depgraph import backtrack_depgraph
  23 from _emerge.RootConfig import RootConfig
  24
  25 class ResolverPlayground(object):
  26         """
  27         This class help to create the necessary files on disk and
  28         the needed settings instances, etc. for the resolver to do
  29         it's work.
  30         """
  31
  32         config_files = frozenset(("package.use", "package.mask", "package.keywords", \
  33                 "package.unmask", "package.properties", "package.license"))
  34
  35         def __init__(self, ebuilds={}, installed={}, profile={}, user_config={}, sets={}, world=[], debug=False):
  36                 """
  37                 ebuilds: cpv -> metadata mapping simulating avaiable ebuilds.
  38                 installed: cpv -> metadata mapping simulating installed packages.
  39                         If a metadata key is missing, it gets a default value.
  40                 profile: settings defined by the profile.
  41                 """
  42                 self.debug = debug
  43                 self.root = "/"
  44                 self.eprefix = tempfile.mkdtemp()
  45                 self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep
  46                 self.portdir = os.path.join(self.eroot, "usr/portage")
  47                 self.vdbdir = os.path.join(self.eroot, "var/db/pkg")
  48                 os.makedirs(self.portdir)
  49                 os.makedirs(self.vdbdir)
  50
  51                 if not debug:
  52                         portage.util.noiselimit = -2
  53
  54                 self._create_ebuilds(ebuilds)
  55                 self._create_installed(installed)
  56                 self._create_profile(ebuilds, installed, profile, user_config, sets)
  57                 self._create_world(world)
  58
  59                 self.settings, self.trees = self._load_config()
  60
  61                 self._create_ebuild_manifests(ebuilds)
  62
  63                 portage.util.noiselimit = 0
  64
  65         def _create_ebuilds(self, ebuilds):
  66                 for cpv in ebuilds:
  67                         a = Atom("=" + cpv)
  68                         ebuild_dir = os.path.join(self.portdir, a.cp)
  69                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
  70                         try:
  71                                 os.makedirs(ebuild_dir)
  72                         except os.error:
  73                                 pass
  74
  75                         metadata = ebuilds[cpv].copy()
  76                         eapi = metadata.pop("EAPI", 0)
  77                         lic = metadata.pop("LICENSE", 0)
  78                         properties = metadata.pop("PROPERTIES", "")
  79                         slot = metadata.pop("SLOT", 0)
  80                         keywords = metadata.pop("KEYWORDS", "x86")
  81                         iuse = metadata.pop("IUSE", "")
  82                         depend = metadata.pop("DEPEND", "")
  83                         rdepend = metadata.pop("RDEPEND", None)
  84                         pdepend = metadata.pop("PDEPEND", None)
  85                         required_use = metadata.pop("REQUIRED_USE", None)
  86
  87                         if metadata:
  88                                 raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
  89
  90                         f = open(ebuild_path, "w")
  91                         f.write('EAPI="' + str(eapi) + '"\n')
  92                         f.write('LICENSE="' + str(lic) + '"\n')
  93                         f.write('PROPERTIES="' + str(properties) + '"\n')
  94                         f.write('SLOT="' + str(slot) + '"\n')
  95                         f.write('KEYWORDS="' + str(keywords) + '"\n')
  96                         f.write('IUSE="' + str(iuse) + '"\n')
  97                         f.write('DEPEND="' + str(depend) + '"\n')
  98                         if rdepend is not None:
  99                                 f.write('RDEPEND="' + str(rdepend) + '"\n')
 100                         if pdepend is not None:
 101                                 f.write('PDEPEND="' + str(pdepend) + '"\n')
 102                         if required_use is not None:
 103                                 f.write('REQUIRED_USE="' + str(required_use) + '"\n')
 104                         f.close()
 105
 106         def _create_ebuild_manifests(self, ebuilds):
 107                 for cpv in ebuilds:
 108                         a = Atom("=" + cpv)
 109                         ebuild_dir = os.path.join(self.portdir, a.cp)
 110                         ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild")
 111
 112                         portage.util.noiselimit = -1
 113                         tmpsettings = config(clone=self.settings)
 114                         portdb = self.trees[self.root]["porttree"].dbapi
 115                         portage.doebuild(ebuild_path, "digest", self.root, tmpsettings,
 116                                 tree="porttree", mydbapi=portdb)
 117                         portage.util.noiselimit = 0
 118
 119         def _create_installed(self, installed):
 120                 for cpv in installed:
 121                         a = Atom("=" + cpv)
 122                         vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv)
 123                         try:
 124                                 os.makedirs(vdb_pkg_dir)
 125                         except os.error:
 126                                 pass
 127
 128                         metadata = installed[cpv].copy()
 129                         eapi = metadata.pop("EAPI", 0)
 130                         lic = metadata.pop("LICENSE", "")
 131                         properties = metadata.pop("PROPERTIES", "")
 132                         slot = metadata.pop("SLOT", 0)
 133                         keywords = metadata.pop("KEYWORDS", "~x86")
 134                         iuse = metadata.pop("IUSE", "")
 135                         use = metadata.pop("USE", "")
 136                         depend = metadata.pop("DEPEND", "")
 137                         rdepend = metadata.pop("RDEPEND", None)
 138                         pdepend = metadata.pop("PDEPEND", None)
 139                         required_use = metadata.pop("REQUIRED_USE", None)
 140
 141                         if metadata:
 142                                 raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys()))
 143
 144                         def write_key(key, value):
 145                                 f = open(os.path.join(vdb_pkg_dir, key), "w")
 146                                 f.write(str(value) + "\n")
 147                                 f.close()
 148
 149                         write_key("EAPI", eapi)
 150                         write_key("LICENSE", lic)
 151                         write_key("PROPERTIES", properties)
 152                         write_key("SLOT", slot)
 153                         write_key("KEYWORDS", keywords)
 154                         write_key("IUSE", iuse)
 155                         write_key("USE", use)
 156                         write_key("DEPEND", depend)
 157                         if rdepend is not None:
 158                                 write_key("RDEPEND", rdepend)
 159                         if pdepend is not None:
 160                                 write_key("PDEPEND", pdepend)
 161                         if required_use is not None:
 162                                 write_key("REQUIRED_USE", required_use)
 163
 164         def _create_profile(self, ebuilds, installed, profile, user_config, sets):
 165                 #Create $PORTDIR/profiles/categories
 166                 categories = set()
 167                 for cpv in chain(ebuilds.keys(), installed.keys()):
 168                         categories.add(catsplit(cpv)[0])
 169
 170                 profile_dir = os.path.join(self.portdir, "profiles")
 171                 try:
 172                         os.makedirs(profile_dir)
 173                 except os.error:
 174                         pass
 175
 176                 categories_file = os.path.join(profile_dir, "categories")
 177
 178                 f = open(categories_file, "w")
 179                 for cat in categories:
 180                         f.write(cat + "\n")
 181                 f.close()
 182
 183
 184                 #Create $REPO/profiles/license_groups
 185                 license_file = os.path.join(profile_dir, "license_groups")
 186                 f = open(license_file, "w")
 187                 f.write("EULA TEST\n")
 188                 f.close()
 189
 190                 #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there)
 191                 os.makedirs(os.path.join(self.portdir, "eclass"))
 192
 193                 sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile")
 194                 os.makedirs(sub_profile_dir)
 195
 196                 eapi_file = os.path.join(sub_profile_dir, "eapi")
 197                 f = open(eapi_file, "w")
 198                 f.write("0\n")
 199                 f.close()
 200
 201                 make_defaults_file = os.path.join(sub_profile_dir, "make.defaults")
 202                 f = open(make_defaults_file, "w")
 203                 f.write("ARCH=\"x86\"\n")
 204                 f.write("ACCEPT_KEYWORDS=\"x86\"\n")
 205                 f.close()
 206
 207                 use_force_file = os.path.join(sub_profile_dir, "use.force")
 208                 f = open(use_force_file, "w")
 209                 f.write("x86\n")
 210                 f.close()
 211
 212                 if profile:
 213                         #This is meant to allow the consumer to set up his own profile,
 214                         #with package.mask and what not.
 215                         raise NotImplementedError()
 216
 217                 #Create profile symlink
 218                 os.makedirs(os.path.join(self.eroot, "etc"))
 219                 os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile"))
 220
 221                 user_config_dir = os.path.join(self.eroot, "etc", "portage")
 222
 223                 try:
 224                         os.makedirs(user_config_dir)
 225                 except os.error:
 226                         pass
 227
 228                 for config_file, lines in user_config.items():
 229                         if config_file not in self.config_files:
 230                                 raise ValueError("Unknown config file: '%s'" % config_file)
 231
 232                         file_name = os.path.join(user_config_dir, config_file)
 233                         f = open(file_name, "w")
 234                         for line in lines:
 235                                 f.write("%s\n" % line)
 236                         f.close()
 237
 238                 #Create /usr/share/portage/config/sets/portage.conf
 239                 default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets")
 240
 241                 try:
 242                         os.makedirs(default_sets_conf_dir)
 243                 except os.error:
 244                         pass
 245
 246                 provided_sets_portage_conf = \
 247                         os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf")
 248                 os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf"))
 249
 250                 set_config_dir = os.path.join(user_config_dir, "sets")
 251
 252                 try:
 253                         os.makedirs(set_config_dir)
 254                 except os.error:
 255                         pass
 256
 257                 for sets_file, lines in sets.items():
 258                         file_name = os.path.join(set_config_dir, sets_file)
 259                         f = open(file_name, "w")
 260                         for line in lines:
 261                                 f.write("%s\n" % line)
 262                         f.close()
 263
 264         def _create_world(self, world):
 265                 #Create /var/lib/portage/world
 266                 var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage")
 267                 os.makedirs(var_lib_portage)
 268
 269                 world_file = os.path.join(var_lib_portage, "world")
 270
 271                 f = open(world_file, "w")
 272                 for atom in world:
 273                         f.write("%s\n" % atom)
 274                 f.close()
 275
 276         def _load_config(self):
 277                 env = {
 278                         "ACCEPT_KEYWORDS": "x86",
 279                         "PORTDIR": self.portdir,
 280                         'PORTAGE_TMPDIR'       : os.path.join(self.eroot, 'var/tmp'),
 281                 }
 282
 283                 # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
 284                 # need to be inherited by ebuild subprocesses.
 285                 if 'PORTAGE_USERNAME' in os.environ:
 286                         env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
 287                 if 'PORTAGE_GRPNAME' in os.environ:
 288                         env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
 289
 290                 settings = config(_eprefix=self.eprefix, env=env)
 291                 settings.lock()
 292
 293                 trees = {
 294                         self.root: {
 295                                         "vartree": vartree(settings=settings),
 296                                         "porttree": portagetree(self.root, settings=settings),
 297                                         "bintree": binarytree(self.root,
 298                                                 os.path.join(self.eroot, "usr/portage/packages"),
 299                                                 settings=settings)
 300                                 }
 301                         }
 302
 303                 for root, root_trees in trees.items():
 304                         settings = root_trees["vartree"].settings
 305                         settings._init_dirs()
 306                         setconfig = load_default_config(settings, root_trees)
 307                         root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
 308
 309                 return settings, trees
 310
 311         def run(self, atoms, options={}, action=None):
 312                 options = options.copy()
 313                 options["--pretend"] = True
 314                 options["--quiet"] = True
 315                 if self.debug:
 316                         options["--debug"] = True
 317
 318                 if not self.debug:
 319                         portage.util.noiselimit = -2
 320                 _emerge.emergelog._disable = True
 321
 322                 if options.get("--depclean"):
 323                         rval, cleanlist, ordered, req_pkg_count = \
 324                                 calc_depclean(self.settings, self.trees, None,
 325                                 options, "depclean", atoms, None)
 326                         result = ResolverPlaygroundDepcleanResult( \
 327                                 atoms, rval, cleanlist, ordered, req_pkg_count)
 328                 else:
 329                         params = create_depgraph_params(options, action)
 330                         success, depgraph, favorites = backtrack_depgraph(
 331                                 self.settings, self.trees, options, params, action, atoms, None)
 332                         depgraph.display_problems()
 333                         result = ResolverPlaygroundResult(atoms, success, depgraph, favorites)
 334
 335                 portage.util.noiselimit = 0
 336                 _emerge.emergelog._disable = False
 337
 338                 return result
 339
 340         def run_TestCase(self, test_case):
 341                 if not isinstance(test_case, ResolverPlaygroundTestCase):
 342                         raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase")
 343                 for atoms in test_case.requests:
 344                         result = self.run(atoms, test_case.options, test_case.action)
 345                         if not test_case.compare_with_result(result):
 346                                 return
 347
 348         def cleanup(self):
 349                 portdb = self.trees[self.root]["porttree"].dbapi
 350                 portdb.close_caches()
 351                 portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb)
 352                 if self.debug:
 353                         print("\nEROOT=%s" % self.eroot)
 354                 else:
 355                         shutil.rmtree(self.eroot)
 356
 357 class ResolverPlaygroundTestCase(object):
 358
 359         def __init__(self, request, **kwargs):
 360                 self.all_permutations = kwargs.pop("all_permutations", False)
 361                 self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False)
 362
 363                 if self.all_permutations:
 364                         self.requests = list(permutations(request))
 365                 else:
 366                         self.requests = [request]
 367
 368                 self.options = kwargs.pop("options", {})
 369                 self.action = kwargs.pop("action", None)
 370                 self.test_success = True
 371                 self.fail_msg = None
 372                 self._checks = kwargs.copy()
 373
 374         def compare_with_result(self, result):
 375                 checks = dict.fromkeys(result.checks)
 376                 for key, value in self._checks.items():
 377                         if not key in checks:
 378                                 raise KeyError("Not an avaiable check: '%s'" % key)
 379                         checks[key] = value
 380
 381                 fail_msgs = []
 382                 for key, value in checks.items():
 383                         got = getattr(result, key)
 384                         expected = value
 385
 386                         if key in result.optional_checks and expected is None:
 387                                 continue
 388
 389                         if key == "mergelist" and self.ignore_mergelist_order and got is not None :
 390                                 got = set(got)
 391                                 expected = set(expected)
 392                         elif key == "unstable_keywords" and expected is not None:
 393                                 expected = set(expected)
 394
 395                         if got != expected:
 396                                 fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \
 397                                         key + ", expected: " + str(expected) + ", got: " + str(got))
 398                 if fail_msgs:
 399                         self.test_success = False
 400                         self.fail_msg = "\n".join(fail_msgs)
 401                         return False
 402                 return True
 403
 404 class ResolverPlaygroundResult(object):
 405
 406         checks = (
 407                 "success", "mergelist", "use_changes", "unstable_keywords", "slot_collision_solutions",
 408                 "circular_dependency_solutions",
 409                 )
 410         optional_checks = (
 411                 )
 412
 413         def __init__(self, atoms, success, mydepgraph, favorites):
 414                 self.atoms = atoms
 415                 self.success = success
 416                 self.depgraph = mydepgraph
 417                 self.favorites = favorites
 418                 self.mergelist = None
 419                 self.use_changes = None
 420                 self.unstable_keywords = None
 421                 self.slot_collision_solutions = None
 422                 self.circular_dependency_solutions = None
 423
 424                 if self.depgraph._dynamic_config._serialized_tasks_cache is not None:
 425                         self.mergelist = []
 426                         for x in self.depgraph._dynamic_config._serialized_tasks_cache:
 427                                 if isinstance(x, Blocker):
 428                                         self.mergelist.append(x.atom)
 429                                 else:
 430                                         self.mergelist.append(x.cpv)
 431
 432                 if self.depgraph._dynamic_config._needed_use_config_changes:
 433                         self.use_changes = {}
 434                         for pkg, needed_use_config_changes in \
 435                                 self.depgraph._dynamic_config._needed_use_config_changes.items():
 436                                 new_use, changes = needed_use_config_changes
 437                                 self.use_changes[pkg.cpv] = changes
 438
 439                 if self.depgraph._dynamic_config._needed_unstable_keywords:
 440                         self.unstable_keywords = set()
 441                         for pkg in self.depgraph._dynamic_config._needed_unstable_keywords:
 442                                 self.unstable_keywords.add(pkg.cpv)
 443
 444                 if self.depgraph._dynamic_config._slot_conflict_handler is not None:
 445                         self.slot_collision_solutions  = []
 446                         handler = self.depgraph._dynamic_config._slot_conflict_handler
 447
 448                         for solution in handler.solutions:
 449                                 s = {}
 450                                 for pkg in solution:
 451                                         changes = {}
 452                                         for flag, state in solution[pkg].items():
 453                                                 if state == "enabled":
 454                                                         changes[flag] = True
 455                                                 else:
 456                                                         changes[flag] = False
 457                                         s[pkg.cpv] = changes
 458                                 self.slot_collision_solutions.append(s)
 459
 460                 if self.depgraph._dynamic_config._circular_dependency_handler is not None:
 461                         handler = self.depgraph._dynamic_config._circular_dependency_handler
 462                         sol = handler.solutions
 463                         self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) )
 464
 465 class ResolverPlaygroundDepcleanResult(object):
 466
 467         checks = (
 468                 "success", "cleanlist", "ordered", "req_pkg_count",
 469                 )
 470         optional_checks = (
 471                 "ordered", "req_pkg_count",
 472                 )
 473
 474         def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count):
 475                 self.atoms = atoms
 476                 self.success = rval == 0
 477                 self.cleanlist = cleanlist
 478                 self.ordered = ordered
 479                 self.req_pkg_count = req_pkg_count