pym/_emerge/actions.py

   1 # Copyright 1999-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import print_function, unicode_literals
   5
   6 import errno
   7 import logging
   8 import operator
   9 import platform
  10 import pwd
  11 import random
  12 import re
  13 import signal
  14 import socket
  15 import stat
  16 import subprocess
  17 import sys
  18 import tempfile
  19 import textwrap
  20 import time
  21 from itertools import chain
  22
  23 import portage
  24 portage.proxy.lazyimport.lazyimport(globals(),
  25         'portage.dbapi._similar_name_search:similar_name_search',
  26         'portage.debug',
  27         'portage.news:count_unread_news,display_news_notifications',
  28         'portage.util._get_vm_info:get_vm_info',
  29         '_emerge.chk_updated_cfg_files:chk_updated_cfg_files',
  30         '_emerge.help:help@emerge_help',
  31         '_emerge.post_emerge:display_news_notification,post_emerge',
  32         '_emerge.stdout_spinner:stdout_spinner',
  33 )
  34
  35 from portage.localization import _
  36 from portage import os
  37 from portage import shutil
  38 from portage import eapi_is_supported, _encodings, _unicode_decode
  39 from portage.cache.cache_errors import CacheError
  40 from portage.const import GLOBAL_CONFIG_PATH, VCS_DIRS, _DEPCLEAN_LIB_CHECK_DEFAULT
  41 from portage.dbapi.dep_expand import dep_expand
  42 from portage.dbapi._expand_new_virt import expand_new_virt
  43 from portage.dep import Atom
  44 from portage.eclass_cache import hashed_path
  45 from portage.exception import InvalidAtom, InvalidData
  46 from portage.output import blue, bold, colorize, create_color_func, darkgreen, \
  47         red, xtermTitle, xtermTitleReset, yellow
  48 good = create_color_func("GOOD")
  49 bad = create_color_func("BAD")
  50 warn = create_color_func("WARN")
  51 from portage.package.ebuild._ipc.QueryCommand import QueryCommand
  52 from portage.package.ebuild.doebuild import _check_temp_dir
  53 from portage._sets import load_default_config, SETPREFIX
  54 from portage._sets.base import InternalPackageSet
  55 from portage.util import cmp_sort_key, writemsg, varexpand, \
  56         writemsg_level, writemsg_stdout
  57 from portage.util.digraph import digraph
  58 from portage.util._async.run_main_scheduler import run_main_scheduler
  59 from portage.util._async.SchedulerInterface import SchedulerInterface
  60 from portage.util._eventloop.global_event_loop import global_event_loop
  61 from portage._global_updates import _global_updates
  62
  63 from _emerge.clear_caches import clear_caches
  64 from _emerge.countdown import countdown
  65 from _emerge.create_depgraph_params import create_depgraph_params
  66 from _emerge.Dependency import Dependency
  67 from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph
  68 from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange
  69 from _emerge.emergelog import emergelog
  70 from _emerge.is_valid_package_atom import is_valid_package_atom
  71 from _emerge.MetadataRegen import MetadataRegen
  72 from _emerge.Package import Package
  73 from _emerge.ProgressHandler import ProgressHandler
  74 from _emerge.RootConfig import RootConfig
  75 from _emerge.Scheduler import Scheduler
  76 from _emerge.search import search
  77 from _emerge.SetArg import SetArg
  78 from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice
  79 from _emerge.sync.getaddrinfo_validate import getaddrinfo_validate
  80 from _emerge.sync.old_tree_timestamp import old_tree_timestamp_warn
  81 from _emerge.unmerge import unmerge
  82 from _emerge.UnmergeDepPriority import UnmergeDepPriority
  83 from _emerge.UseFlagDisplay import pkg_use_display
  84 from _emerge.userquery import userquery
  85
  86 if sys.hexversion >= 0x3000000:
  87         long = int
  88         _unicode = str
  89 else:
  90         _unicode = unicode
  91
  92 def action_build(settings, trees, mtimedb,
  93         myopts, myaction, myfiles, spinner):
  94
  95         if '--usepkgonly' not in myopts:
  96                 old_tree_timestamp_warn(settings['PORTDIR'], settings)
  97
  98         # It's best for config updates in /etc/portage to be processed
  99         # before we get here, so warn if they're not (bug #267103).
 100         chk_updated_cfg_files(settings['EROOT'], ['/etc/portage'])
 101
 102         # validate the state of the resume data
 103         # so that we can make assumptions later.
 104         for k in ("resume", "resume_backup"):
 105                 if k not in mtimedb:
 106                         continue
 107                 resume_data = mtimedb[k]
 108                 if not isinstance(resume_data, dict):
 109                         del mtimedb[k]
 110                         continue
 111                 mergelist = resume_data.get("mergelist")
 112                 if not isinstance(mergelist, list):
 113                         del mtimedb[k]
 114                         continue
 115                 for x in mergelist:
 116                         if not (isinstance(x, list) and len(x) == 4):
 117                                 continue
 118                         pkg_type, pkg_root, pkg_key, pkg_action = x
 119                         if pkg_root not in trees:
 120                                 # Current $ROOT setting differs,
 121                                 # so the list must be stale.
 122                                 mergelist = None
 123                                 break
 124                 if not mergelist:
 125                         del mtimedb[k]
 126                         continue
 127                 resume_opts = resume_data.get("myopts")
 128                 if not isinstance(resume_opts, (dict, list)):
 129                         del mtimedb[k]
 130                         continue
 131                 favorites = resume_data.get("favorites")
 132                 if not isinstance(favorites, list):
 133                         del mtimedb[k]
 134                         continue
 135
 136         resume = False
 137         if "--resume" in myopts and \
 138                 ("resume" in mtimedb or
 139                 "resume_backup" in mtimedb):
 140                 resume = True
 141                 if "resume" not in mtimedb:
 142                         mtimedb["resume"] = mtimedb["resume_backup"]
 143                         del mtimedb["resume_backup"]
 144                         mtimedb.commit()
 145                 # "myopts" is a list for backward compatibility.
 146                 resume_opts = mtimedb["resume"].get("myopts", [])
 147                 if isinstance(resume_opts, list):
 148                         resume_opts = dict((k,True) for k in resume_opts)
 149                 for opt in ("--ask", "--color", "--skipfirst", "--tree"):
 150                         resume_opts.pop(opt, None)
 151
 152                 # Current options always override resume_opts.
 153                 resume_opts.update(myopts)
 154                 myopts.clear()
 155                 myopts.update(resume_opts)
 156
 157                 if "--debug" in myopts:
 158                         writemsg_level("myopts %s\n" % (myopts,))
 159
 160                 # Adjust config according to options of the command being resumed.
 161                 for myroot in trees:
 162                         mysettings =  trees[myroot]["vartree"].settings
 163                         mysettings.unlock()
 164                         adjust_config(myopts, mysettings)
 165                         mysettings.lock()
 166                         del myroot, mysettings
 167
 168         ldpath_mtimes = mtimedb["ldpath"]
 169         favorites=[]
 170         buildpkgonly = "--buildpkgonly" in myopts
 171         pretend = "--pretend" in myopts
 172         fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts
 173         ask = "--ask" in myopts
 174         enter_invalid = '--ask-enter-invalid' in myopts
 175         nodeps = "--nodeps" in myopts
 176         oneshot = "--oneshot" in myopts or "--onlydeps" in myopts
 177         tree = "--tree" in myopts
 178         if nodeps and tree:
 179                 tree = False
 180                 del myopts["--tree"]
 181                 portage.writemsg(colorize("WARN", " * ") + \
 182                         "--tree is broken with --nodeps. Disabling...\n")
 183         debug = "--debug" in myopts
 184         verbose = "--verbose" in myopts
 185         quiet = "--quiet" in myopts
 186         myparams = create_depgraph_params(myopts, myaction)
 187         mergelist_shown = False
 188
 189         if pretend or fetchonly:
 190                 # make the mtimedb readonly
 191                 mtimedb.filename = None
 192         if '--digest' in myopts or 'digest' in settings.features:
 193                 if '--digest' in myopts:
 194                         msg = "The --digest option"
 195                 else:
 196                         msg = "The FEATURES=digest setting"
 197
 198                 msg += " can prevent corruption from being" + \
 199                         " noticed. The `repoman manifest` command is the preferred" + \
 200                         " way to generate manifests and it is capable of doing an" + \
 201                         " entire repository or category at once."
 202                 prefix = bad(" * ")
 203                 writemsg(prefix + "\n")
 204                 for line in textwrap.wrap(msg, 72):
 205                         writemsg("%s%s\n" % (prefix, line))
 206                 writemsg(prefix + "\n")
 207
 208         if resume:
 209                 favorites = mtimedb["resume"].get("favorites")
 210                 if not isinstance(favorites, list):
 211                         favorites = []
 212
 213                 resume_data = mtimedb["resume"]
 214                 mergelist = resume_data["mergelist"]
 215                 if mergelist and "--skipfirst" in myopts:
 216                         for i, task in enumerate(mergelist):
 217                                 if isinstance(task, list) and \
 218                                         task and task[-1] == "merge":
 219                                         del mergelist[i]
 220                                         break
 221
 222                 success = False
 223                 mydepgraph = None
 224                 try:
 225                         success, mydepgraph, dropped_tasks = resume_depgraph(
 226                                 settings, trees, mtimedb, myopts, myparams, spinner)
 227                 except (portage.exception.PackageNotFound,
 228                         depgraph.UnsatisfiedResumeDep) as e:
 229                         if isinstance(e, depgraph.UnsatisfiedResumeDep):
 230                                 mydepgraph = e.depgraph
 231
 232                         from portage.output import EOutput
 233                         out = EOutput()
 234
 235                         resume_data = mtimedb["resume"]
 236                         mergelist = resume_data.get("mergelist")
 237                         if not isinstance(mergelist, list):
 238                                 mergelist = []
 239                         if mergelist and debug or (verbose and not quiet):
 240                                 out.eerror("Invalid resume list:")
 241                                 out.eerror("")
 242                                 indent = "  "
 243                                 for task in mergelist:
 244                                         if isinstance(task, list):
 245                                                 out.eerror(indent + str(tuple(task)))
 246                                 out.eerror("")
 247
 248                         if isinstance(e, depgraph.UnsatisfiedResumeDep):
 249                                 out.eerror("One or more packages are either masked or " + \
 250                                         "have missing dependencies:")
 251                                 out.eerror("")
 252                                 indent = "  "
 253                                 for dep in e.value:
 254                                         if dep.atom is None:
 255                                                 out.eerror(indent + "Masked package:")
 256                                                 out.eerror(2 * indent + str(dep.parent))
 257                                                 out.eerror("")
 258                                         else:
 259                                                 out.eerror(indent + str(dep.atom) + " pulled in by:")
 260                                                 out.eerror(2 * indent + str(dep.parent))
 261                                                 out.eerror("")
 262                                 msg = "The resume list contains packages " + \
 263                                         "that are either masked or have " + \
 264                                         "unsatisfied dependencies. " + \
 265                                         "Please restart/continue " + \
 266                                         "the operation manually, or use --skipfirst " + \
 267                                         "to skip the first package in the list and " + \
 268                                         "any other packages that may be " + \
 269                                         "masked or have missing dependencies."
 270                                 for line in textwrap.wrap(msg, 72):
 271                                         out.eerror(line)
 272                         elif isinstance(e, portage.exception.PackageNotFound):
 273                                 out.eerror("An expected package is " + \
 274                                         "not available: %s" % str(e))
 275                                 out.eerror("")
 276                                 msg = "The resume list contains one or more " + \
 277                                         "packages that are no longer " + \
 278                                         "available. Please restart/continue " + \
 279                                         "the operation manually."
 280                                 for line in textwrap.wrap(msg, 72):
 281                                         out.eerror(line)
 282
 283                 if success:
 284                         if dropped_tasks:
 285                                 portage.writemsg("!!! One or more packages have been " + \
 286                                         "dropped due to\n" + \
 287                                         "!!! masking or unsatisfied dependencies:\n\n",
 288                                         noiselevel=-1)
 289                                 for task, atoms in dropped_tasks.items():
 290                                         if not atoms:
 291                                                 writemsg("  %s is masked or unavailable\n" %
 292                                                         (task,), noiselevel=-1)
 293                                         else:
 294                                                 writemsg("  %s requires %s\n" %
 295                                                         (task, ", ".join(atoms)), noiselevel=-1)
 296
 297                                 portage.writemsg("\n", noiselevel=-1)
 298                         del dropped_tasks
 299                 else:
 300                         if mydepgraph is not None:
 301                                 mydepgraph.display_problems()
 302                         if not (ask or pretend):
 303                                 # delete the current list and also the backup
 304                                 # since it's probably stale too.
 305                                 for k in ("resume", "resume_backup"):
 306                                         mtimedb.pop(k, None)
 307                                 mtimedb.commit()
 308
 309                         return 1
 310         else:
 311                 if ("--resume" in myopts):
 312                         print(darkgreen("emerge: It seems we have nothing to resume..."))
 313                         return os.EX_OK
 314
 315                 try:
 316                         success, mydepgraph, favorites = backtrack_depgraph(
 317                                 settings, trees, myopts, myparams, myaction, myfiles, spinner)
 318                 except portage.exception.PackageSetNotFound as e:
 319                         root_config = trees[settings['EROOT']]['root_config']
 320                         display_missing_pkg_set(root_config, e.value)
 321                         return 1
 322
 323                 if not success:
 324                         mydepgraph.display_problems()
 325                         return 1
 326
 327         mergecount = None
 328         if "--pretend" not in myopts and \
 329                 ("--ask" in myopts or "--tree" in myopts or \
 330                 "--verbose" in myopts) and \
 331                 not ("--quiet" in myopts and "--ask" not in myopts):
 332                 if "--resume" in myopts:
 333                         mymergelist = mydepgraph.altlist()
 334                         if len(mymergelist) == 0:
 335                                 print(colorize("INFORM", "emerge: It seems we have nothing to resume..."))
 336                                 return os.EX_OK
 337                         favorites = mtimedb["resume"]["favorites"]
 338                         retval = mydepgraph.display(
 339                                 mydepgraph.altlist(reversed=tree),
 340                                 favorites=favorites)
 341                         mydepgraph.display_problems()
 342                         mergelist_shown = True
 343                         if retval != os.EX_OK:
 344                                 return retval
 345                         prompt="Would you like to resume merging these packages?"
 346                 else:
 347                         retval = mydepgraph.display(
 348                                 mydepgraph.altlist(reversed=("--tree" in myopts)),
 349                                 favorites=favorites)
 350                         mydepgraph.display_problems()
 351                         mergelist_shown = True
 352                         if retval != os.EX_OK:
 353                                 return retval
 354                         mergecount=0
 355                         for x in mydepgraph.altlist():
 356                                 if isinstance(x, Package) and x.operation == "merge":
 357                                         mergecount += 1
 358
 359                         prompt = None
 360                         if mergecount==0:
 361                                 sets = trees[settings['EROOT']]['root_config'].sets
 362                                 world_candidates = None
 363                                 if "selective" in myparams and \
 364                                         not oneshot and favorites:
 365                                         # Sets that are not world candidates are filtered
 366                                         # out here since the favorites list needs to be
 367                                         # complete for depgraph.loadResumeCommand() to
 368                                         # operate correctly.
 369                                         world_candidates = [x for x in favorites \
 370                                                 if not (x.startswith(SETPREFIX) and \
 371                                                 not sets[x[1:]].world_candidate)]
 372
 373                                 if "selective" in myparams and \
 374                                         not oneshot and world_candidates:
 375                                         # Prompt later, inside saveNomergeFavorites.
 376                                         prompt = None
 377                                 elif settings["AUTOCLEAN"] and "yes"==settings["AUTOCLEAN"]:
 378                                         prompt="Nothing to merge; would you like to auto-clean packages?"
 379                                 else:
 380                                         print()
 381                                         print("Nothing to merge; quitting.")
 382                                         print()
 383                                         return os.EX_OK
 384                         elif "--fetchonly" in myopts or "--fetch-all-uri" in myopts:
 385                                 prompt="Would you like to fetch the source files for these packages?"
 386                         else:
 387                                 prompt="Would you like to merge these packages?"
 388                 print()
 389                 if prompt is not None and "--ask" in myopts and \
 390                         userquery(prompt, enter_invalid) == "No":
 391                         print()
 392                         print("Quitting.")
 393                         print()
 394                         return 128 + signal.SIGINT
 395                 # Don't ask again (e.g. when auto-cleaning packages after merge)
 396                 if mergecount != 0:
 397                         myopts.pop("--ask", None)
 398
 399         if ("--pretend" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts):
 400                 if ("--resume" in myopts):
 401                         mymergelist = mydepgraph.altlist()
 402                         if len(mymergelist) == 0:
 403                                 print(colorize("INFORM", "emerge: It seems we have nothing to resume..."))
 404                                 return os.EX_OK
 405                         favorites = mtimedb["resume"]["favorites"]
 406                         retval = mydepgraph.display(
 407                                 mydepgraph.altlist(reversed=tree),
 408                                 favorites=favorites)
 409                         mydepgraph.display_problems()
 410                         mergelist_shown = True
 411                         if retval != os.EX_OK:
 412                                 return retval
 413                 else:
 414                         retval = mydepgraph.display(
 415                                 mydepgraph.altlist(reversed=("--tree" in myopts)),
 416                                 favorites=favorites)
 417                         mydepgraph.display_problems()
 418                         mergelist_shown = True
 419                         if retval != os.EX_OK:
 420                                 return retval
 421                         if "--buildpkgonly" in myopts:
 422                                 graph_copy = mydepgraph._dynamic_config.digraph.copy()
 423                                 removed_nodes = set()
 424                                 for node in graph_copy:
 425                                         if not isinstance(node, Package) or \
 426                                                 node.operation == "nomerge":
 427                                                 removed_nodes.add(node)
 428                                 graph_copy.difference_update(removed_nodes)
 429                                 if not graph_copy.hasallzeros(ignore_priority = \
 430                                         DepPrioritySatisfiedRange.ignore_medium):
 431                                         print("\n!!! --buildpkgonly requires all dependencies to be merged.")
 432                                         print("!!! You have to merge the dependencies before you can build this package.\n")
 433                                         return 1
 434         else:
 435                 if "--buildpkgonly" in myopts:
 436                         graph_copy = mydepgraph._dynamic_config.digraph.copy()
 437                         removed_nodes = set()
 438                         for node in graph_copy:
 439                                 if not isinstance(node, Package) or \
 440                                         node.operation == "nomerge":
 441                                         removed_nodes.add(node)
 442                         graph_copy.difference_update(removed_nodes)
 443                         if not graph_copy.hasallzeros(ignore_priority = \
 444                                 DepPrioritySatisfiedRange.ignore_medium):
 445                                 print("\n!!! --buildpkgonly requires all dependencies to be merged.")
 446                                 print("!!! Cannot merge requested packages. Merge deps and try again.\n")
 447                                 return 1
 448
 449                 if not mergelist_shown:
 450                         # If we haven't already shown the merge list above, at
 451                         # least show warnings about missed updates and such.
 452                         mydepgraph.display_problems()
 453
 454                 if ("--resume" in myopts):
 455                         favorites=mtimedb["resume"]["favorites"]
 456
 457                 else:
 458                         if "resume" in mtimedb and \
 459                         "mergelist" in mtimedb["resume"] and \
 460                         len(mtimedb["resume"]["mergelist"]) > 1:
 461                                 mtimedb["resume_backup"] = mtimedb["resume"]
 462                                 del mtimedb["resume"]
 463                                 mtimedb.commit()
 464
 465                         mydepgraph.saveNomergeFavorites()
 466
 467                 if mergecount == 0:
 468                         retval = os.EX_OK
 469                 else:
 470                         mergetask = Scheduler(settings, trees, mtimedb, myopts,
 471                                 spinner, favorites=favorites,
 472                                 graph_config=mydepgraph.schedulerGraph())
 473
 474                         del mydepgraph
 475                         clear_caches(trees)
 476
 477                         retval = mergetask.merge()
 478
 479                         if retval == os.EX_OK and \
 480                                 not (buildpkgonly or fetchonly or pretend):
 481                                 if "yes" == settings.get("AUTOCLEAN"):
 482                                         portage.writemsg_stdout(">>> Auto-cleaning packages...\n")
 483                                         unmerge(trees[settings['EROOT']]['root_config'],
 484                                                 myopts, "clean", [],
 485                                                 ldpath_mtimes, autoclean=1)
 486                                 else:
 487                                         portage.writemsg_stdout(colorize("WARN", "WARNING:")
 488                                                 + " AUTOCLEAN is disabled.  This can cause serious"
 489                                                 + " problems due to overlapping packages.\n")
 490
 491                 return retval
 492
 493 def action_config(settings, trees, myopts, myfiles):
 494         enter_invalid = '--ask-enter-invalid' in myopts
 495         if len(myfiles) != 1:
 496                 print(red("!!! config can only take a single package atom at this time\n"))
 497                 sys.exit(1)
 498         if not is_valid_package_atom(myfiles[0], allow_repo=True):
 499                 portage.writemsg("!!! '%s' is not a valid package atom.\n" % myfiles[0],
 500                         noiselevel=-1)
 501                 portage.writemsg("!!! Please check ebuild(5) for full details.\n")
 502                 portage.writemsg("!!! (Did you specify a version but forget to prefix with '='?)\n")
 503                 sys.exit(1)
 504         print()
 505         try:
 506                 pkgs = trees[settings['EROOT']]['vartree'].dbapi.match(myfiles[0])
 507         except portage.exception.AmbiguousPackageName as e:
 508                 # Multiple matches thrown from cpv_expand
 509                 pkgs = e.args[0]
 510         if len(pkgs) == 0:
 511                 print("No packages found.\n")
 512                 sys.exit(0)
 513         elif len(pkgs) > 1:
 514                 if "--ask" in myopts:
 515                         options = []
 516                         print("Please select a package to configure:")
 517                         idx = 0
 518                         for pkg in pkgs:
 519                                 idx += 1
 520                                 options.append(str(idx))
 521                                 print(options[-1]+") "+pkg)
 522                         print("X) Cancel")
 523                         options.append("X")
 524                         idx = userquery("Selection?", enter_invalid, responses=options)
 525                         if idx == "X":
 526                                 sys.exit(128 + signal.SIGINT)
 527                         pkg = pkgs[int(idx)-1]
 528                 else:
 529                         print("The following packages available:")
 530                         for pkg in pkgs:
 531                                 print("* "+pkg)
 532                         print("\nPlease use a specific atom or the --ask option.")
 533                         sys.exit(1)
 534         else:
 535                 pkg = pkgs[0]
 536
 537         print()
 538         if "--ask" in myopts:
 539                 if userquery("Ready to configure %s?" % pkg, enter_invalid) == "No":
 540                         sys.exit(128 + signal.SIGINT)
 541         else:
 542                 print("Configuring pkg...")
 543         print()
 544         ebuildpath = trees[settings['EROOT']]['vartree'].dbapi.findname(pkg)
 545         mysettings = portage.config(clone=settings)
 546         vardb = trees[mysettings['EROOT']]['vartree'].dbapi
 547         debug = mysettings.get("PORTAGE_DEBUG") == "1"
 548         retval = portage.doebuild(ebuildpath, "config", settings=mysettings,
 549                 debug=(settings.get("PORTAGE_DEBUG", "") == 1), cleanup=True,
 550                 mydbapi = trees[settings['EROOT']]['vartree'].dbapi, tree="vartree")
 551         if retval == os.EX_OK:
 552                 portage.doebuild(ebuildpath, "clean", settings=mysettings,
 553                         debug=debug, mydbapi=vardb, tree="vartree")
 554         print()
 555
 556 def action_depclean(settings, trees, ldpath_mtimes,
 557         myopts, action, myfiles, spinner, scheduler=None):
 558         # Kill packages that aren't explicitly merged or are required as a
 559         # dependency of another package. World file is explicit.
 560
 561         # Global depclean or prune operations are not very safe when there are
 562         # missing dependencies since it's unknown how badly incomplete
 563         # the dependency graph is, and we might accidentally remove packages
 564         # that should have been pulled into the graph. On the other hand, it's
 565         # relatively safe to ignore missing deps when only asked to remove
 566         # specific packages.
 567
 568         msg = []
 569         if "preserve-libs" not in settings.features and \
 570                 not myopts.get("--depclean-lib-check", _DEPCLEAN_LIB_CHECK_DEFAULT) != "n":
 571                 msg.append("Depclean may break link level dependencies. Thus, it is\n")
 572                 msg.append("recommended to use a tool such as " + good("`revdep-rebuild`") + " (from\n")
 573                 msg.append("app-portage/gentoolkit) in order to detect such breakage.\n")
 574                 msg.append("\n")
 575         msg.append("Always study the list of packages to be cleaned for any obvious\n")
 576         msg.append("mistakes. Packages that are part of the world set will always\n")
 577         msg.append("be kept.  They can be manually added to this set with\n")
 578         msg.append(good("`emerge --noreplace <atom>`") + ".  Packages that are listed in\n")
 579         msg.append("package.provided (see portage(5)) will be removed by\n")
 580         msg.append("depclean, even if they are part of the world set.\n")
 581         msg.append("\n")
 582         msg.append("As a safety measure, depclean will not remove any packages\n")
 583         msg.append("unless *all* required dependencies have been resolved.  As a\n")
 584         msg.append("consequence, it is often necessary to run %s\n" % \
 585                 good("`emerge --update"))
 586         msg.append(good("--newuse --deep @world`") + \
 587                 " prior to depclean.\n")
 588
 589         if action == "depclean" and "--quiet" not in myopts and not myfiles:
 590                 portage.writemsg_stdout("\n")
 591                 for x in msg:
 592                         portage.writemsg_stdout(colorize("WARN", " * ") + x)
 593
 594         root_config = trees[settings['EROOT']]['root_config']
 595         vardb = root_config.trees['vartree'].dbapi
 596
 597         args_set = InternalPackageSet(allow_repo=True)
 598         if myfiles:
 599                 args_set.update(myfiles)
 600                 matched_packages = False
 601                 for x in args_set:
 602                         if vardb.match(x):
 603                                 matched_packages = True
 604                         else:
 605                                 writemsg_level("--- Couldn't find '%s' to %s.\n" % \
 606                                         (x.replace("null/", ""), action),
 607                                         level=logging.WARN, noiselevel=-1)
 608                 if not matched_packages:
 609                         writemsg_level(">>> No packages selected for removal by %s\n" % \
 610                                 action)
 611                         return 0
 612
 613         # The calculation is done in a separate function so that depgraph
 614         # references go out of scope and the corresponding memory
 615         # is freed before we call unmerge().
 616         rval, cleanlist, ordered, req_pkg_count = \
 617                 calc_depclean(settings, trees, ldpath_mtimes,
 618                         myopts, action, args_set, spinner)
 619
 620         clear_caches(trees)
 621
 622         if rval != os.EX_OK:
 623                 return rval
 624
 625         if cleanlist:
 626                 rval = unmerge(root_config, myopts, "unmerge",
 627                         cleanlist, ldpath_mtimes, ordered=ordered,
 628                         scheduler=scheduler)
 629
 630         if action == "prune":
 631                 return rval
 632
 633         if not cleanlist and "--quiet" in myopts:
 634                 return rval
 635
 636         set_atoms = {}
 637         for k in ("system", "selected"):
 638                 try:
 639                         set_atoms[k] = root_config.setconfig.getSetAtoms(k)
 640                 except portage.exception.PackageSetNotFound:
 641                         # A nested set could not be resolved, so ignore nested sets.
 642                         set_atoms[k] = root_config.sets[k].getAtoms()
 643
 644         print("Packages installed:   " + str(len(vardb.cpv_all())))
 645         print("Packages in world:    %d" % len(set_atoms["selected"]))
 646         print("Packages in system:   %d" % len(set_atoms["system"]))
 647         print("Required packages:    "+str(req_pkg_count))
 648         if "--pretend" in myopts:
 649                 print("Number to remove:     "+str(len(cleanlist)))
 650         else:
 651                 print("Number removed:       "+str(len(cleanlist)))
 652
 653         return rval
 654
 655 def calc_depclean(settings, trees, ldpath_mtimes,
 656         myopts, action, args_set, spinner):
 657         allow_missing_deps = bool(args_set)
 658
 659         debug = '--debug' in myopts
 660         xterm_titles = "notitles" not in settings.features
 661         root_len = len(settings["ROOT"])
 662         eroot = settings['EROOT']
 663         root_config = trees[eroot]["root_config"]
 664         psets = root_config.setconfig.psets
 665         deselect = myopts.get('--deselect') != 'n'
 666         required_sets = {}
 667         required_sets['world'] = psets['world']
 668
 669         # When removing packages, a temporary version of the world 'selected'
 670         # set may be used which excludes packages that are intended to be
 671         # eligible for removal.
 672         selected_set = psets['selected']
 673         required_sets['selected'] = selected_set
 674         protected_set = InternalPackageSet()
 675         protected_set_name = '____depclean_protected_set____'
 676         required_sets[protected_set_name] = protected_set
 677         system_set = psets["system"]
 678
 679         set_atoms = {}
 680         for k in ("system", "selected"):
 681                 try:
 682                         set_atoms[k] = root_config.setconfig.getSetAtoms(k)
 683                 except portage.exception.PackageSetNotFound:
 684                         # A nested set could not be resolved, so ignore nested sets.
 685                         set_atoms[k] = root_config.sets[k].getAtoms()
 686
 687         if not set_atoms["system"] or not set_atoms["selected"]:
 688
 689                 if not set_atoms["system"]:
 690                         writemsg_level("!!! You have no system list.\n",
 691                                 level=logging.ERROR, noiselevel=-1)
 692
 693                 if not set_atoms["selected"]:
 694                         writemsg_level("!!! You have no world file.\n",
 695                                         level=logging.WARNING, noiselevel=-1)
 696
 697                 writemsg_level("!!! Proceeding is likely to " + \
 698                         "break your installation.\n",
 699                         level=logging.WARNING, noiselevel=-1)
 700                 if "--pretend" not in myopts:
 701                         countdown(int(settings["EMERGE_WARNING_DELAY"]), ">>> Depclean")
 702
 703         if action == "depclean":
 704                 emergelog(xterm_titles, " >>> depclean")
 705
 706         writemsg_level("\nCalculating dependencies  ")
 707         resolver_params = create_depgraph_params(myopts, "remove")
 708         resolver = depgraph(settings, trees, myopts, resolver_params, spinner)
 709         resolver._load_vdb()
 710         vardb = resolver._frozen_config.trees[eroot]["vartree"].dbapi
 711         real_vardb = trees[eroot]["vartree"].dbapi
 712
 713         if action == "depclean":
 714
 715                 if args_set:
 716
 717                         if deselect:
 718                                 # Start with an empty set.
 719                                 selected_set = InternalPackageSet()
 720                                 required_sets['selected'] = selected_set
 721                                 # Pull in any sets nested within the selected set.
 722                                 selected_set.update(psets['selected'].getNonAtoms())
 723
 724                         # Pull in everything that's installed but not matched
 725                         # by an argument atom since we don't want to clean any
 726                         # package if something depends on it.
 727                         for pkg in vardb:
 728                                 if spinner:
 729                                         spinner.update()
 730
 731                                 try:
 732                                         if args_set.findAtomForPackage(pkg) is None:
 733                                                 protected_set.add("=" + pkg.cpv)
 734                                                 continue
 735                                 except portage.exception.InvalidDependString as e:
 736                                         show_invalid_depstring_notice(pkg,
 737                                                 pkg._metadata["PROVIDE"], _unicode(e))
 738                                         del e
 739                                         protected_set.add("=" + pkg.cpv)
 740                                         continue
 741
 742         elif action == "prune":
 743
 744                 if deselect:
 745                         # Start with an empty set.
 746                         selected_set = InternalPackageSet()
 747                         required_sets['selected'] = selected_set
 748                         # Pull in any sets nested within the selected set.
 749                         selected_set.update(psets['selected'].getNonAtoms())
 750
 751                 # Pull in everything that's installed since we don't
 752                 # to prune a package if something depends on it.
 753                 protected_set.update(vardb.cp_all())
 754
 755                 if not args_set:
 756
 757                         # Try to prune everything that's slotted.
 758                         for cp in vardb.cp_all():
 759                                 if len(vardb.cp_list(cp)) > 1:
 760                                         args_set.add(cp)
 761
 762                 # Remove atoms from world that match installed packages
 763                 # that are also matched by argument atoms, but do not remove
 764                 # them if they match the highest installed version.
 765                 for pkg in vardb:
 766                         if spinner is not None:
 767                                 spinner.update()
 768                         pkgs_for_cp = vardb.match_pkgs(pkg.cp)
 769                         if not pkgs_for_cp or pkg not in pkgs_for_cp:
 770                                 raise AssertionError("package expected in matches: " + \
 771                                         "cp = %s, cpv = %s matches = %s" % \
 772                                         (pkg.cp, pkg.cpv, [str(x) for x in pkgs_for_cp]))
 773
 774                         highest_version = pkgs_for_cp[-1]
 775                         if pkg == highest_version:
 776                                 # pkg is the highest version
 777                                 protected_set.add("=" + pkg.cpv)
 778                                 continue
 779
 780                         if len(pkgs_for_cp) <= 1:
 781                                 raise AssertionError("more packages expected: " + \
 782                                         "cp = %s, cpv = %s matches = %s" % \
 783                                         (pkg.cp, pkg.cpv, [str(x) for x in pkgs_for_cp]))
 784
 785                         try:
 786                                 if args_set.findAtomForPackage(pkg) is None:
 787                                         protected_set.add("=" + pkg.cpv)
 788                                         continue
 789                         except portage.exception.InvalidDependString as e:
 790                                 show_invalid_depstring_notice(pkg,
 791                                         pkg._metadata["PROVIDE"], _unicode(e))
 792                                 del e
 793                                 protected_set.add("=" + pkg.cpv)
 794                                 continue
 795
 796         if resolver._frozen_config.excluded_pkgs:
 797                 excluded_set = resolver._frozen_config.excluded_pkgs
 798                 required_sets['__excluded__'] = InternalPackageSet()
 799
 800                 for pkg in vardb:
 801                         if spinner:
 802                                 spinner.update()
 803
 804                         try:
 805                                 if excluded_set.findAtomForPackage(pkg):
 806                                         required_sets['__excluded__'].add("=" + pkg.cpv)
 807                         except portage.exception.InvalidDependString as e:
 808                                 show_invalid_depstring_notice(pkg,
 809                                         pkg._metadata["PROVIDE"], _unicode(e))
 810                                 del e
 811                                 required_sets['__excluded__'].add("=" + pkg.cpv)
 812
 813         success = resolver._complete_graph(required_sets={eroot:required_sets})
 814         writemsg_level("\b\b... done!\n")
 815
 816         resolver.display_problems()
 817
 818         if not success:
 819                 return 1, [], False, 0
 820
 821         def unresolved_deps():
 822
 823                 unresolvable = set()
 824                 for dep in resolver._dynamic_config._initially_unsatisfied_deps:
 825                         if isinstance(dep.parent, Package) and \
 826                                 (dep.priority > UnmergeDepPriority.SOFT):
 827                                 unresolvable.add((dep.atom, dep.parent.cpv))
 828
 829                 if not unresolvable:
 830                         return False
 831
 832                 if unresolvable and not allow_missing_deps:
 833
 834                         if "--debug" in myopts:
 835                                 writemsg("\ndigraph:\n\n", noiselevel=-1)
 836                                 resolver._dynamic_config.digraph.debug_print()
 837                                 writemsg("\n", noiselevel=-1)
 838
 839                         prefix = bad(" * ")
 840                         msg = []
 841                         msg.append("Dependencies could not be completely resolved due to")
 842                         msg.append("the following required packages not being installed:")
 843                         msg.append("")
 844                         for atom, parent in unresolvable:
 845                                 if atom != atom.unevaluated_atom and \
 846                                         vardb.match(_unicode(atom)):
 847                                         msg.append("  %s (%s) pulled in by:" %
 848                                                 (atom.unevaluated_atom, atom))
 849                                 else:
 850                                         msg.append("  %s pulled in by:" % (atom,))
 851                                 msg.append("    %s" % (parent,))
 852                                 msg.append("")
 853                         msg.extend(textwrap.wrap(
 854                                 "Have you forgotten to do a complete update prior " + \
 855                                 "to depclean? The most comprehensive command for this " + \
 856                                 "purpose is as follows:", 65
 857                         ))
 858                         msg.append("")
 859                         msg.append("  " + \
 860                                 good("emerge --update --newuse --deep --with-bdeps=y @world"))
 861                         msg.append("")
 862                         msg.extend(textwrap.wrap(
 863                                 "Note that the --with-bdeps=y option is not required in " + \
 864                                 "many situations. Refer to the emerge manual page " + \
 865                                 "(run `man emerge`) for more information about " + \
 866                                 "--with-bdeps.", 65
 867                         ))
 868                         msg.append("")
 869                         msg.extend(textwrap.wrap(
 870                                 "Also, note that it may be necessary to manually uninstall " + \
 871                                 "packages that no longer exist in the portage tree, since " + \
 872                                 "it may not be possible to satisfy their dependencies.", 65
 873                         ))
 874                         if action == "prune":
 875                                 msg.append("")
 876                                 msg.append("If you would like to ignore " + \
 877                                         "dependencies then use %s." % good("--nodeps"))
 878                         writemsg_level("".join("%s%s\n" % (prefix, line) for line in msg),
 879                                 level=logging.ERROR, noiselevel=-1)
 880                         return True
 881                 return False
 882
 883         if unresolved_deps():
 884                 return 1, [], False, 0
 885
 886         graph = resolver._dynamic_config.digraph.copy()
 887         required_pkgs_total = 0
 888         for node in graph:
 889                 if isinstance(node, Package):
 890                         required_pkgs_total += 1
 891
 892         def show_parents(child_node):
 893                 parent_atoms = \
 894                         resolver._dynamic_config._parent_atoms.get(child_node, [])
 895
 896                 # Never display the special internal protected_set.
 897                 parent_atoms = [parent_atom for parent_atom in parent_atoms
 898                         if not (isinstance(parent_atom[0], SetArg) and
 899                         parent_atom[0].name == protected_set_name)]
 900
 901                 if not parent_atoms:
 902                         # With --prune, the highest version can be pulled in without any
 903                         # real parent since all installed packages are pulled in.  In that
 904                         # case there's nothing to show here.
 905                         return
 906                 parent_atom_dict = {}
 907                 for parent, atom in parent_atoms:
 908                         parent_atom_dict.setdefault(parent, []).append(atom)
 909
 910                 parent_strs = []
 911                 for parent, atoms in parent_atom_dict.items():
 912                         parent_strs.append("%s requires %s" %
 913                                 (getattr(parent, "cpv", parent), ", ".join(atoms)))
 914                 parent_strs.sort()
 915                 msg = []
 916                 msg.append("  %s pulled in by:\n" % (child_node.cpv,))
 917                 for parent_str in parent_strs:
 918                         msg.append("    %s\n" % (parent_str,))
 919                 msg.append("\n")
 920                 portage.writemsg_stdout("".join(msg), noiselevel=-1)
 921
 922         def cmp_pkg_cpv(pkg1, pkg2):
 923                 """Sort Package instances by cpv."""
 924                 if pkg1.cpv > pkg2.cpv:
 925                         return 1
 926                 elif pkg1.cpv == pkg2.cpv:
 927                         return 0
 928                 else:
 929                         return -1
 930
 931         def create_cleanlist():
 932
 933                 if "--debug" in myopts:
 934                         writemsg("\ndigraph:\n\n", noiselevel=-1)
 935                         graph.debug_print()
 936                         writemsg("\n", noiselevel=-1)
 937
 938                 pkgs_to_remove = []
 939
 940                 if action == "depclean":
 941                         if args_set:
 942
 943                                 for pkg in sorted(vardb, key=cmp_sort_key(cmp_pkg_cpv)):
 944                                         arg_atom = None
 945                                         try:
 946                                                 arg_atom = args_set.findAtomForPackage(pkg)
 947                                         except portage.exception.InvalidDependString:
 948                                                 # this error has already been displayed by now
 949                                                 continue
 950
 951                                         if arg_atom:
 952                                                 if pkg not in graph:
 953                                                         pkgs_to_remove.append(pkg)
 954                                                 elif "--verbose" in myopts:
 955                                                         show_parents(pkg)
 956
 957                         else:
 958                                 for pkg in sorted(vardb, key=cmp_sort_key(cmp_pkg_cpv)):
 959                                         if pkg not in graph:
 960                                                 pkgs_to_remove.append(pkg)
 961                                         elif "--verbose" in myopts:
 962                                                 show_parents(pkg)
 963
 964                 elif action == "prune":
 965
 966                         for atom in args_set:
 967                                 for pkg in vardb.match_pkgs(atom):
 968                                         if pkg not in graph:
 969                                                 pkgs_to_remove.append(pkg)
 970                                         elif "--verbose" in myopts:
 971                                                 show_parents(pkg)
 972
 973                 if not pkgs_to_remove:
 974                         writemsg_level(
 975                                 ">>> No packages selected for removal by %s\n" % action)
 976                         if "--verbose" not in myopts:
 977                                 writemsg_level(
 978                                         ">>> To see reverse dependencies, use %s\n" % \
 979                                                 good("--verbose"))
 980                         if action == "prune":
 981                                 writemsg_level(
 982                                         ">>> To ignore dependencies, use %s\n" % \
 983                                                 good("--nodeps"))
 984
 985                 return pkgs_to_remove
 986
 987         cleanlist = create_cleanlist()
 988         clean_set = set(cleanlist)
 989
 990         if cleanlist and \
 991                 real_vardb._linkmap is not None and \
 992                 myopts.get("--depclean-lib-check", _DEPCLEAN_LIB_CHECK_DEFAULT) != "n" and \
 993                 "preserve-libs" not in settings.features:
 994
 995                 # Check if any of these packages are the sole providers of libraries
 996                 # with consumers that have not been selected for removal. If so, these
 997                 # packages and any dependencies need to be added to the graph.
 998                 linkmap = real_vardb._linkmap
 999                 consumer_cache = {}
1000                 provider_cache = {}
1001                 consumer_map = {}
1002
1003                 writemsg_level(">>> Checking for lib consumers...\n")
1004
1005                 for pkg in cleanlist:
1006                         pkg_dblink = real_vardb._dblink(pkg.cpv)
1007                         consumers = {}
1008
1009                         for lib in pkg_dblink.getcontents():
1010                                 lib = lib[root_len:]
1011                                 lib_key = linkmap._obj_key(lib)
1012                                 lib_consumers = consumer_cache.get(lib_key)
1013                                 if lib_consumers is None:
1014                                         try:
1015                                                 lib_consumers = linkmap.findConsumers(lib_key)
1016                                         except KeyError:
1017                                                 continue
1018                                         consumer_cache[lib_key] = lib_consumers
1019                                 if lib_consumers:
1020                                         consumers[lib_key] = lib_consumers
1021
1022                         if not consumers:
1023                                 continue
1024
1025                         for lib, lib_consumers in list(consumers.items()):
1026                                 for consumer_file in list(lib_consumers):
1027                                         if pkg_dblink.isowner(consumer_file):
1028                                                 lib_consumers.remove(consumer_file)
1029                                 if not lib_consumers:
1030                                         del consumers[lib]
1031
1032                         if not consumers:
1033                                 continue
1034
1035                         for lib, lib_consumers in consumers.items():
1036
1037                                 soname = linkmap.getSoname(lib)
1038
1039                                 consumer_providers = []
1040                                 for lib_consumer in lib_consumers:
1041                                         providers = provider_cache.get(lib)
1042                                         if providers is None:
1043                                                 providers = linkmap.findProviders(lib_consumer)
1044                                                 provider_cache[lib_consumer] = providers
1045                                         if soname not in providers:
1046                                                 # Why does this happen?
1047                                                 continue
1048                                         consumer_providers.append(
1049                                                 (lib_consumer, providers[soname]))
1050
1051                                 consumers[lib] = consumer_providers
1052
1053                         consumer_map[pkg] = consumers
1054
1055                 if consumer_map:
1056
1057                         search_files = set()
1058                         for consumers in consumer_map.values():
1059                                 for lib, consumer_providers in consumers.items():
1060                                         for lib_consumer, providers in consumer_providers:
1061                                                 search_files.add(lib_consumer)
1062                                                 search_files.update(providers)
1063
1064                         writemsg_level(">>> Assigning files to packages...\n")
1065                         file_owners = {}
1066                         for f in search_files:
1067                                 owner_set = set()
1068                                 for owner in linkmap.getOwners(f):
1069                                         owner_dblink = real_vardb._dblink(owner)
1070                                         if owner_dblink.exists():
1071                                                 owner_set.add(owner_dblink)
1072                                 if owner_set:
1073                                         file_owners[f] = owner_set
1074
1075                         for pkg, consumers in list(consumer_map.items()):
1076                                 for lib, consumer_providers in list(consumers.items()):
1077                                         lib_consumers = set()
1078
1079                                         for lib_consumer, providers in consumer_providers:
1080                                                 owner_set = file_owners.get(lib_consumer)
1081                                                 provider_dblinks = set()
1082                                                 provider_pkgs = set()
1083
1084                                                 if len(providers) > 1:
1085                                                         for provider in providers:
1086                                                                 provider_set = file_owners.get(provider)
1087                                                                 if provider_set is not None:
1088                                                                         provider_dblinks.update(provider_set)
1089
1090                                                 if len(provider_dblinks) > 1:
1091                                                         for provider_dblink in provider_dblinks:
1092                                                                 provider_pkg = resolver._pkg(
1093                                                                         provider_dblink.mycpv, "installed",
1094                                                                         root_config, installed=True)
1095                                                                 if provider_pkg not in clean_set:
1096                                                                         provider_pkgs.add(provider_pkg)
1097
1098                                                 if provider_pkgs:
1099                                                         continue
1100
1101                                                 if owner_set is not None:
1102                                                         lib_consumers.update(owner_set)
1103
1104                                         for consumer_dblink in list(lib_consumers):
1105                                                 if resolver._pkg(consumer_dblink.mycpv, "installed",
1106                                                         root_config, installed=True) in clean_set:
1107                                                         lib_consumers.remove(consumer_dblink)
1108                                                         continue
1109
1110                                         if lib_consumers:
1111                                                 consumers[lib] = lib_consumers
1112                                         else:
1113                                                 del consumers[lib]
1114                                 if not consumers:
1115                                         del consumer_map[pkg]
1116
1117                 if consumer_map:
1118                         # TODO: Implement a package set for rebuilding consumer packages.
1119
1120                         msg = "In order to avoid breakage of link level " + \
1121                                 "dependencies, one or more packages will not be removed. " + \
1122                                 "This can be solved by rebuilding " + \
1123                                 "the packages that pulled them in."
1124
1125                         prefix = bad(" * ")
1126                         writemsg_level("".join(prefix + "%s\n" % line for \
1127                                 line in textwrap.wrap(msg, 70)), level=logging.WARNING, noiselevel=-1)
1128
1129                         msg = []
1130                         for pkg in sorted(consumer_map, key=cmp_sort_key(cmp_pkg_cpv)):
1131                                 consumers = consumer_map[pkg]
1132                                 consumer_libs = {}
1133                                 for lib, lib_consumers in consumers.items():
1134                                         for consumer in lib_consumers:
1135                                                 consumer_libs.setdefault(
1136                                                         consumer.mycpv, set()).add(linkmap.getSoname(lib))
1137                                 unique_consumers = set(chain(*consumers.values()))
1138                                 unique_consumers = sorted(consumer.mycpv \
1139                                         for consumer in unique_consumers)
1140                                 msg.append("")
1141                                 msg.append("  %s pulled in by:" % (pkg.cpv,))
1142                                 for consumer in unique_consumers:
1143                                         libs = consumer_libs[consumer]
1144                                         msg.append("    %s needs %s" % \
1145                                                 (consumer, ', '.join(sorted(libs))))
1146                         msg.append("")
1147                         writemsg_level("".join(prefix + "%s\n" % line for line in msg),
1148                                 level=logging.WARNING, noiselevel=-1)
1149
1150                         # Add lib providers to the graph as children of lib consumers,
1151                         # and also add any dependencies pulled in by the provider.
1152                         writemsg_level(">>> Adding lib providers to graph...\n")
1153
1154                         for pkg, consumers in consumer_map.items():
1155                                 for consumer_dblink in set(chain(*consumers.values())):
1156                                         consumer_pkg = resolver._pkg(consumer_dblink.mycpv,
1157                                                 "installed", root_config, installed=True)
1158                                         if not resolver._add_pkg(pkg,
1159                                                 Dependency(parent=consumer_pkg,
1160                                                 priority=UnmergeDepPriority(runtime=True),
1161                                                 root=pkg.root)):
1162                                                 resolver.display_problems()
1163                                                 return 1, [], False, 0
1164
1165                         writemsg_level("\nCalculating dependencies  ")
1166                         success = resolver._complete_graph(
1167                                 required_sets={eroot:required_sets})
1168                         writemsg_level("\b\b... done!\n")
1169                         resolver.display_problems()
1170                         if not success:
1171                                 return 1, [], False, 0
1172                         if unresolved_deps():
1173                                 return 1, [], False, 0
1174
1175                         graph = resolver._dynamic_config.digraph.copy()
1176                         required_pkgs_total = 0
1177                         for node in graph:
1178                                 if isinstance(node, Package):
1179                                         required_pkgs_total += 1
1180                         cleanlist = create_cleanlist()
1181                         if not cleanlist:
1182                                 return 0, [], False, required_pkgs_total
1183                         clean_set = set(cleanlist)
1184
1185         if clean_set:
1186                 writemsg_level(">>> Calculating removal order...\n")
1187                 # Use a topological sort to create an unmerge order such that
1188                 # each package is unmerged before it's dependencies. This is
1189                 # necessary to avoid breaking things that may need to run
1190                 # during pkg_prerm or pkg_postrm phases.
1191
1192                 # Create a new graph to account for dependencies between the
1193                 # packages being unmerged.
1194                 graph = digraph()
1195                 del cleanlist[:]
1196
1197                 runtime = UnmergeDepPriority(runtime=True)
1198                 runtime_post = UnmergeDepPriority(runtime_post=True)
1199                 buildtime = UnmergeDepPriority(buildtime=True)
1200                 priority_map = {
1201                         "RDEPEND": runtime,
1202                         "PDEPEND": runtime_post,
1203                         "HDEPEND": buildtime,
1204                         "DEPEND": buildtime,
1205                 }
1206
1207                 for node in clean_set:
1208                         graph.add(node, None)
1209                         for dep_type in Package._dep_keys:
1210                                 depstr = node._metadata[dep_type]
1211                                 if not depstr:
1212                                         continue
1213                                 priority = priority_map[dep_type]
1214
1215                                 if debug:
1216                                         writemsg_level("\nParent:    %s\n"
1217                                                 % (node,), noiselevel=-1, level=logging.DEBUG)
1218                                         writemsg_level(  "Depstring: %s\n"
1219                                                 % (depstr,), noiselevel=-1, level=logging.DEBUG)
1220                                         writemsg_level(  "Priority:  %s\n"
1221                                                 % (priority,), noiselevel=-1, level=logging.DEBUG)
1222
1223                                 try:
1224                                         atoms = resolver._select_atoms(eroot, depstr,
1225                                                 myuse=node.use.enabled, parent=node,
1226                                                 priority=priority)[node]
1227                                 except portage.exception.InvalidDependString:
1228                                         # Ignore invalid deps of packages that will
1229                                         # be uninstalled anyway.
1230                                         continue
1231
1232                                 if debug:
1233                                         writemsg_level("Candidates: [%s]\n" % \
1234                                                 ', '.join("'%s'" % (x,) for x in atoms),
1235                                                 noiselevel=-1, level=logging.DEBUG)
1236
1237                                 for atom in atoms:
1238                                         if not isinstance(atom, portage.dep.Atom):
1239                                                 # Ignore invalid atoms returned from dep_check().
1240                                                 continue
1241                                         if atom.blocker:
1242                                                 continue
1243                                         matches = vardb.match_pkgs(atom)
1244                                         if not matches:
1245                                                 continue
1246                                         for child_node in matches:
1247                                                 if child_node in clean_set:
1248                                                         graph.add(child_node, node, priority=priority)
1249
1250                 if debug:
1251                         writemsg_level("\nunmerge digraph:\n\n",
1252                                 noiselevel=-1, level=logging.DEBUG)
1253                         graph.debug_print()
1254                         writemsg_level("\n", noiselevel=-1, level=logging.DEBUG)
1255
1256                 ordered = True
1257                 if len(graph.order) == len(graph.root_nodes()):
1258                         # If there are no dependencies between packages
1259                         # let unmerge() group them by cat/pn.
1260                         ordered = False
1261                         cleanlist = [pkg.cpv for pkg in graph.order]
1262                 else:
1263                         # Order nodes from lowest to highest overall reference count for
1264                         # optimal root node selection (this can help minimize issues
1265                         # with unaccounted implicit dependencies).
1266                         node_refcounts = {}
1267                         for node in graph.order:
1268                                 node_refcounts[node] = len(graph.parent_nodes(node))
1269                         def cmp_reference_count(node1, node2):
1270                                 return node_refcounts[node1] - node_refcounts[node2]
1271                         graph.order.sort(key=cmp_sort_key(cmp_reference_count))
1272
1273                         ignore_priority_range = [None]
1274                         ignore_priority_range.extend(
1275                                 range(UnmergeDepPriority.MIN, UnmergeDepPriority.MAX + 1))
1276                         while graph:
1277                                 for ignore_priority in ignore_priority_range:
1278                                         nodes = graph.root_nodes(ignore_priority=ignore_priority)
1279                                         if nodes:
1280                                                 break
1281                                 if not nodes:
1282                                         raise AssertionError("no root nodes")
1283                                 if ignore_priority is not None:
1284                                         # Some deps have been dropped due to circular dependencies,
1285                                         # so only pop one node in order to minimize the number that
1286                                         # are dropped.
1287                                         del nodes[1:]
1288                                 for node in nodes:
1289                                         graph.remove(node)
1290                                         cleanlist.append(node.cpv)
1291
1292                 return 0, cleanlist, ordered, required_pkgs_total
1293         return 0, [], False, required_pkgs_total
1294
1295 def action_deselect(settings, trees, opts, atoms):
1296         enter_invalid = '--ask-enter-invalid' in opts
1297         root_config = trees[settings['EROOT']]['root_config']
1298         world_set = root_config.sets['selected']
1299         if not hasattr(world_set, 'update'):
1300                 writemsg_level("World @selected set does not appear to be mutable.\n",
1301                         level=logging.ERROR, noiselevel=-1)
1302                 return 1
1303
1304         pretend = '--pretend' in opts
1305         locked = False
1306         if not pretend and hasattr(world_set, 'lock'):
1307                 world_set.lock()
1308                 locked = True
1309         try:
1310                 world_set.load()
1311                 world_atoms = world_set.getAtoms()
1312                 vardb = root_config.trees["vartree"].dbapi
1313                 expanded_atoms = set(atoms)
1314
1315                 for atom in atoms:
1316                         if not atom.startswith(SETPREFIX):
1317                                 if atom.cp.startswith("null/"):
1318                                         # try to expand category from world set
1319                                         null_cat, pn = portage.catsplit(atom.cp)
1320                                         for world_atom in world_atoms:
1321                                                 cat, world_pn = portage.catsplit(world_atom.cp)
1322                                                 if pn == world_pn:
1323                                                         expanded_atoms.add(
1324                                                                 Atom(atom.replace("null", cat, 1),
1325                                                                 allow_repo=True, allow_wildcard=True))
1326
1327                                 for cpv in vardb.match(atom):
1328                                         pkg = vardb._pkg_str(cpv, None)
1329                                         expanded_atoms.add(Atom("%s:%s" % (pkg.cp, pkg.slot)))
1330
1331                 discard_atoms = set()
1332                 for atom in world_set:
1333                         for arg_atom in expanded_atoms:
1334                                 if arg_atom.startswith(SETPREFIX):
1335                                         if atom.startswith(SETPREFIX) and \
1336                                                 arg_atom == atom:
1337                                                 discard_atoms.add(atom)
1338                                                 break
1339                                 else:
1340                                         if not atom.startswith(SETPREFIX) and \
1341                                                 arg_atom.intersects(atom) and \
1342                                                 not (arg_atom.slot and not atom.slot) and \
1343                                                 not (arg_atom.repo and not atom.repo):
1344                                                 discard_atoms.add(atom)
1345                                                 break
1346                 if discard_atoms:
1347                         for atom in sorted(discard_atoms):
1348
1349                                 if pretend:
1350                                         action_desc = "Would remove"
1351                                 else:
1352                                         action_desc = "Removing"
1353
1354                                 if atom.startswith(SETPREFIX):
1355                                         filename = "world_sets"
1356                                 else:
1357                                         filename = "world"
1358
1359                                 writemsg_stdout(
1360                                         ">>> %s %s from \"%s\" favorites file...\n" %
1361                                         (action_desc, colorize("INFORM", _unicode(atom)),
1362                                         filename), noiselevel=-1)
1363
1364                         if '--ask' in opts:
1365                                 prompt = "Would you like to remove these " + \
1366                                         "packages from your world favorites?"
1367                                 if userquery(prompt, enter_invalid) == 'No':
1368                                         return 128 + signal.SIGINT
1369
1370                         remaining = set(world_set)
1371                         remaining.difference_update(discard_atoms)
1372                         if not pretend:
1373                                 world_set.replace(remaining)
1374                 else:
1375                         print(">>> No matching atoms found in \"world\" favorites file...")
1376         finally:
1377                 if locked:
1378                         world_set.unlock()
1379         return os.EX_OK
1380
1381 class _info_pkgs_ver(object):
1382         def __init__(self, ver, repo_suffix, provide_suffix):
1383                 self.ver = ver
1384                 self.repo_suffix = repo_suffix
1385                 self.provide_suffix = provide_suffix
1386
1387         def __lt__(self, other):
1388                 return portage.versions.vercmp(self.ver, other.ver) < 0
1389
1390         def toString(self):
1391                 """
1392                 This may return unicode if repo_name contains unicode.
1393                 Don't use __str__ and str() since unicode triggers compatibility
1394                 issues between python 2.x and 3.x.
1395                 """
1396                 return self.ver + self.repo_suffix + self.provide_suffix
1397
1398 def action_info(settings, trees, myopts, myfiles):
1399
1400         # See if we can find any packages installed matching the strings
1401         # passed on the command line
1402         mypkgs = []
1403         eroot = settings['EROOT']
1404         vardb = trees[eroot]["vartree"].dbapi
1405         portdb = trees[eroot]['porttree'].dbapi
1406         bindb = trees[eroot]["bintree"].dbapi
1407         for x in myfiles:
1408                 any_match = False
1409                 cp_exists = bool(vardb.match(x.cp))
1410                 installed_match = vardb.match(x)
1411                 for installed in installed_match:
1412                         mypkgs.append((installed, "installed"))
1413                         any_match = True
1414
1415                 if any_match:
1416                         continue
1417
1418                 for db, pkg_type in ((portdb, "ebuild"), (bindb, "binary")):
1419                         if pkg_type == "binary" and "--usepkg" not in myopts:
1420                                 continue
1421
1422                         # Use match instead of cp_list, to account for old-style virtuals.
1423                         if not cp_exists and db.match(x.cp):
1424                                 cp_exists = True
1425                         # Search for masked packages too.
1426                         if not cp_exists and hasattr(db, "xmatch") and \
1427                                 db.xmatch("match-all", x.cp):
1428                                 cp_exists = True
1429
1430                         matches = db.match(x)
1431                         matches.reverse()
1432                         for match in matches:
1433                                 if pkg_type == "binary":
1434                                         if db.bintree.isremote(match):
1435                                                 continue
1436                                 auxkeys = ["EAPI", "DEFINED_PHASES"]
1437                                 metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys)))
1438                                 if metadata["EAPI"] not in ("0", "1", "2", "3") and \
1439                                         "info" in metadata["DEFINED_PHASES"].split():
1440                                         mypkgs.append((match, pkg_type))
1441                                         break
1442
1443                 if not cp_exists:
1444                         xinfo = '"%s"' % x.unevaluated_atom
1445                         # Discard null/ from failed cpv_expand category expansion.
1446                         xinfo = xinfo.replace("null/", "")
1447                         if settings["ROOT"] != "/":
1448                                 xinfo = "%s for %s" % (xinfo, eroot)
1449                         writemsg("\nemerge: there are no ebuilds to satisfy %s.\n" %
1450                                 colorize("INFORM", xinfo), noiselevel=-1)
1451
1452                         if myopts.get("--misspell-suggestions", "y") != "n":
1453
1454                                 writemsg("\nemerge: searching for similar names..."
1455                                         , noiselevel=-1)
1456
1457                                 dbs = [vardb]
1458                                 #if "--usepkgonly" not in myopts:
1459                                 dbs.append(portdb)
1460                                 if "--usepkg" in myopts:
1461                                         dbs.append(bindb)
1462
1463                                 matches = similar_name_search(dbs, x)
1464
1465                                 if len(matches) == 1:
1466                                         writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n"
1467                                                 , noiselevel=-1)
1468                                 elif len(matches) > 1:
1469                                         writemsg(
1470                                                 "\nemerge: Maybe you meant any of these: %s?\n" % \
1471                                                 (", ".join(matches),), noiselevel=-1)
1472                                 else:
1473                                         # Generally, this would only happen if
1474                                         # all dbapis are empty.
1475                                         writemsg(" nothing similar found.\n"
1476                                                 , noiselevel=-1)
1477
1478                         return 1
1479
1480         output_buffer = []
1481         append = output_buffer.append
1482         root_config = trees[settings['EROOT']]['root_config']
1483         running_eroot = trees._running_eroot
1484         chost = settings.get("CHOST")
1485
1486         append(getportageversion(settings["PORTDIR"], None,
1487                 settings.profile_path, settings["CHOST"],
1488                 trees[settings['EROOT']]["vartree"].dbapi))
1489
1490         header_width = 65
1491         header_title = "System Settings"
1492         if myfiles:
1493                 append(header_width * "=")
1494                 append(header_title.rjust(int(header_width/2 + len(header_title)/2)))
1495         append(header_width * "=")
1496         append("System uname: %s" % (platform.platform(aliased=1),))
1497
1498         vm_info = get_vm_info()
1499         if "ram.total" in vm_info:
1500                 line = "%-9s %10d total" % ("KiB Mem:", vm_info["ram.total"] / 1024)
1501                 if "ram.free" in vm_info:
1502                         line += ",%10d free" % (vm_info["ram.free"] / 1024,)
1503                 append(line)
1504         if "swap.total" in vm_info:
1505                 line = "%-9s %10d total" % ("KiB Swap:", vm_info["swap.total"] / 1024)
1506                 if "swap.free" in vm_info:
1507                         line += ",%10d free" % (vm_info["swap.free"] / 1024,)
1508                 append(line)
1509
1510         lastSync = portage.grabfile(os.path.join(
1511                 settings["PORTDIR"], "metadata", "timestamp.chk"))
1512         if lastSync:
1513                 lastSync = lastSync[0]
1514         else:
1515                 lastSync = "Unknown"
1516         append("Timestamp of tree: %s" % (lastSync,))
1517
1518         ld_names = []
1519         if chost:
1520                 ld_names.append(chost + "-ld")
1521         ld_names.append("ld")
1522         for name in ld_names:
1523                 try:
1524                         proc = subprocess.Popen([name, "--version"],
1525                                 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1526                 except OSError:
1527                         pass
1528                 else:
1529                         output = _unicode_decode(proc.communicate()[0]).splitlines()
1530                         proc.wait()
1531                         if proc.wait() == os.EX_OK and output:
1532                                 append("ld %s" % (output[0]))
1533                                 break
1534
1535         try:
1536                 proc = subprocess.Popen(["distcc", "--version"],
1537                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1538         except OSError:
1539                 output = (1, None)
1540         else:
1541                 output = _unicode_decode(proc.communicate()[0]).rstrip("\n")
1542                 output = (proc.wait(), output)
1543         if output[0] == os.EX_OK:
1544                 distcc_str = output[1].split("\n", 1)[0]
1545                 if "distcc" in settings.features:
1546                         distcc_str += " [enabled]"
1547                 else:
1548                         distcc_str += " [disabled]"
1549                 append(distcc_str)
1550
1551         try:
1552                 proc = subprocess.Popen(["ccache", "-V"],
1553                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1554         except OSError:
1555                 output = (1, None)
1556         else:
1557                 output = _unicode_decode(proc.communicate()[0]).rstrip("\n")
1558                 output = (proc.wait(), output)
1559         if output[0] == os.EX_OK:
1560                 ccache_str = output[1].split("\n", 1)[0]
1561                 if "ccache" in settings.features:
1562                         ccache_str += " [enabled]"
1563                 else:
1564                         ccache_str += " [disabled]"
1565                 append(ccache_str)
1566
1567         myvars  = ["sys-devel/autoconf", "sys-devel/automake", "virtual/os-headers",
1568                    "sys-devel/binutils", "sys-devel/libtool",  "dev-lang/python"]
1569         myvars += portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_pkgs")
1570         atoms = []
1571         vardb = trees[running_eroot]['vartree'].dbapi
1572         for x in myvars:
1573                 try:
1574                         x = Atom(x)
1575                 except InvalidAtom:
1576                         append("%-20s %s" % (x+":", "[NOT VALID]"))
1577                 else:
1578                         for atom in expand_new_virt(vardb, x):
1579                                 if not atom.blocker:
1580                                         atoms.append((x, atom))
1581
1582         myvars = sorted(set(atoms))
1583
1584         portdb = trees[running_eroot]['porttree'].dbapi
1585         main_repo = portdb.getRepositoryName(portdb.porttree_root)
1586         cp_map = {}
1587         cp_max_len = 0
1588
1589         for orig_atom, x in myvars:
1590                         pkg_matches = vardb.match(x)
1591
1592                         versions = []
1593                         for cpv in pkg_matches:
1594                                 matched_cp = portage.versions.cpv_getkey(cpv)
1595                                 ver = portage.versions.cpv_getversion(cpv)
1596                                 ver_map = cp_map.setdefault(matched_cp, {})
1597                                 prev_match = ver_map.get(ver)
1598                                 if prev_match is not None:
1599                                         if prev_match.provide_suffix:
1600                                                 # prefer duplicate matches that include
1601                                                 # additional virtual provider info
1602                                                 continue
1603
1604                                 if len(matched_cp) > cp_max_len:
1605                                         cp_max_len = len(matched_cp)
1606                                 repo = vardb.aux_get(cpv, ["repository"])[0]
1607                                 if repo == main_repo:
1608                                         repo_suffix = ""
1609                                 elif not repo:
1610                                         repo_suffix = "::<unknown repository>"
1611                                 else:
1612                                         repo_suffix = "::" + repo
1613
1614                                 if matched_cp == orig_atom.cp:
1615                                         provide_suffix = ""
1616                                 else:
1617                                         provide_suffix = " (%s)" % (orig_atom,)
1618
1619                                 ver_map[ver] = _info_pkgs_ver(ver, repo_suffix, provide_suffix)
1620
1621         for cp in sorted(cp_map):
1622                 versions = sorted(cp_map[cp].values())
1623                 versions = ", ".join(ver.toString() for ver in versions)
1624                 append("%s %s" % \
1625                         ((cp + ":").ljust(cp_max_len + 1), versions))
1626
1627         repos = portdb.settings.repositories
1628         if "--verbose" in myopts:
1629                 append("Repositories:\n")
1630                 for repo in repos:
1631                         append(repo.info_string())
1632         else:
1633                 append("Repositories: %s" % \
1634                         " ".join(repo.name for repo in repos))
1635
1636         installed_sets = sorted(s for s in
1637                 root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX))
1638         if installed_sets:
1639                 sets_line = "Installed sets: "
1640                 sets_line += ", ".join(installed_sets)
1641                 append(sets_line)
1642
1643         if "--verbose" in myopts:
1644                 myvars = list(settings)
1645         else:
1646                 myvars = ['GENTOO_MIRRORS', 'CONFIG_PROTECT', 'CONFIG_PROTECT_MASK',
1647                           'PORTDIR', 'DISTDIR', 'PKGDIR', 'PORTAGE_TMPDIR',
1648                           'PORTDIR_OVERLAY', 'PORTAGE_BUNZIP2_COMMAND',
1649                           'PORTAGE_BZIP2_COMMAND',
1650                           'USE', 'CHOST', 'CFLAGS', 'CXXFLAGS',
1651                           'ACCEPT_KEYWORDS', 'ACCEPT_LICENSE', 'SYNC', 'FEATURES',
1652                           'EMERGE_DEFAULT_OPTS']
1653
1654                 myvars.extend(portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_vars"))
1655
1656         myvars_ignore_defaults = {
1657                 'PORTAGE_BZIP2_COMMAND' : 'bzip2',
1658         }
1659
1660         myvars = portage.util.unique_array(myvars)
1661         use_expand = settings.get('USE_EXPAND', '').split()
1662         use_expand.sort()
1663         unset_vars = []
1664         myvars.sort()
1665         for k in myvars:
1666                 v = settings.get(k)
1667                 if v is not None:
1668                         if k != "USE":
1669                                 default = myvars_ignore_defaults.get(k)
1670                                 if default is not None and \
1671                                         default == v:
1672                                         continue
1673                                 append('%s="%s"' % (k, v))
1674                         else:
1675                                 use = set(v.split())
1676                                 for varname in use_expand:
1677                                         flag_prefix = varname.lower() + "_"
1678                                         for f in list(use):
1679                                                 if f.startswith(flag_prefix):
1680                                                         use.remove(f)
1681                                 use = list(use)
1682                                 use.sort()
1683                                 use = ['USE="%s"' % " ".join(use)]
1684                                 for varname in use_expand:
1685                                         myval = settings.get(varname)
1686                                         if myval:
1687                                                 use.append('%s="%s"' % (varname, myval))
1688                                 append(" ".join(use))
1689                 else:
1690                         unset_vars.append(k)
1691         if unset_vars:
1692                 append("Unset:  "+", ".join(unset_vars))
1693         append("")
1694         append("")
1695         writemsg_stdout("\n".join(output_buffer),
1696                 noiselevel=-1)
1697
1698         # If some packages were found...
1699         if mypkgs:
1700                 # Get our global settings (we only print stuff if it varies from
1701                 # the current config)
1702                 mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS' ]
1703                 auxkeys = mydesiredvars + list(vardb._aux_cache_keys)
1704                 auxkeys.append('DEFINED_PHASES')
1705                 pkgsettings = portage.config(clone=settings)
1706
1707                 # Loop through each package
1708                 # Only print settings if they differ from global settings
1709                 header_title = "Package Settings"
1710                 print(header_width * "=")
1711                 print(header_title.rjust(int(header_width/2 + len(header_title)/2)))
1712                 print(header_width * "=")
1713                 from portage.output import EOutput
1714                 out = EOutput()
1715                 for mypkg in mypkgs:
1716                         cpv = mypkg[0]
1717                         pkg_type = mypkg[1]
1718                         # Get all package specific variables
1719                         if pkg_type == "installed":
1720                                 metadata = dict(zip(auxkeys, vardb.aux_get(cpv, auxkeys)))
1721                         elif pkg_type == "ebuild":
1722                                 metadata = dict(zip(auxkeys, portdb.aux_get(cpv, auxkeys)))
1723                         elif pkg_type == "binary":
1724                                 metadata = dict(zip(auxkeys, bindb.aux_get(cpv, auxkeys)))
1725
1726                         pkg = Package(built=(pkg_type!="ebuild"), cpv=cpv,
1727                                 installed=(pkg_type=="installed"), metadata=zip(Package.metadata_keys,
1728                                 (metadata.get(x, '') for x in Package.metadata_keys)),
1729                                 root_config=root_config, type_name=pkg_type)
1730
1731                         if pkg_type == "installed":
1732                                 print("\n%s was built with the following:" % \
1733                                         colorize("INFORM", str(pkg.cpv)))
1734                         elif pkg_type == "ebuild":
1735                                 print("\n%s would be build with the following:" % \
1736                                         colorize("INFORM", str(pkg.cpv)))
1737                         elif pkg_type == "binary":
1738                                 print("\n%s (non-installed binary) was built with the following:" % \
1739                                         colorize("INFORM", str(pkg.cpv)))
1740
1741                         writemsg_stdout('%s\n' % pkg_use_display(pkg, myopts),
1742                                 noiselevel=-1)
1743                         if pkg_type == "installed":
1744                                 for myvar in mydesiredvars:
1745                                         if metadata[myvar].split() != settings.get(myvar, '').split():
1746                                                 print("%s=\"%s\"" % (myvar, metadata[myvar]))
1747                         print()
1748
1749                         if metadata['DEFINED_PHASES']:
1750                                 if 'info' not in metadata['DEFINED_PHASES'].split():
1751                                         continue
1752
1753                         print(">>> Attempting to run pkg_info() for '%s'" % pkg.cpv)
1754
1755                         if pkg_type == "installed":
1756                                 ebuildpath = vardb.findname(pkg.cpv)
1757                         elif pkg_type == "ebuild":
1758                                 ebuildpath = portdb.findname(pkg.cpv, myrepo=pkg.repo)
1759                         elif pkg_type == "binary":
1760                                 tbz2_file = bindb.bintree.getname(pkg.cpv)
1761                                 ebuild_file_name = pkg.cpv.split("/")[1] + ".ebuild"
1762                                 ebuild_file_contents = portage.xpak.tbz2(tbz2_file).getfile(ebuild_file_name)
1763                                 tmpdir = tempfile.mkdtemp()
1764                                 ebuildpath = os.path.join(tmpdir, ebuild_file_name)
1765                                 file = open(ebuildpath, 'w')
1766                                 file.write(ebuild_file_contents)
1767                                 file.close()
1768
1769                         if not ebuildpath or not os.path.exists(ebuildpath):
1770                                 out.ewarn("No ebuild found for '%s'" % pkg.cpv)
1771                                 continue
1772
1773                         if pkg_type == "installed":
1774                                 portage.doebuild(ebuildpath, "info", settings=pkgsettings,
1775                                         debug=(settings.get("PORTAGE_DEBUG", "") == 1),
1776                                         mydbapi=trees[settings['EROOT']]["vartree"].dbapi,
1777                                         tree="vartree")
1778                         elif pkg_type == "ebuild":
1779                                 portage.doebuild(ebuildpath, "info", settings=pkgsettings,
1780                                         debug=(settings.get("PORTAGE_DEBUG", "") == 1),
1781                                         mydbapi=trees[settings['EROOT']]['porttree'].dbapi,
1782                                         tree="porttree")
1783                         elif pkg_type == "binary":
1784                                 portage.doebuild(ebuildpath, "info", settings=pkgsettings,
1785                                         debug=(settings.get("PORTAGE_DEBUG", "") == 1),
1786                                         mydbapi=trees[settings['EROOT']]["bintree"].dbapi,
1787                                         tree="bintree")
1788                                 shutil.rmtree(tmpdir)
1789
1790 def action_metadata(settings, portdb, myopts, porttrees=None):
1791         if porttrees is None:
1792                 porttrees = portdb.porttrees
1793         portage.writemsg_stdout("\n>>> Updating Portage cache\n")
1794         old_umask = os.umask(0o002)
1795         cachedir = os.path.normpath(settings.depcachedir)
1796         if cachedir in ["/",    "/bin", "/dev",  "/etc",  "/home",
1797                                         "/lib", "/opt", "/proc", "/root", "/sbin",
1798                                         "/sys", "/tmp", "/usr",  "/var"]:
1799                 print("!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + \
1800                         "ROOT DIRECTORY ON YOUR SYSTEM.", file=sys.stderr)
1801                 print("!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir, file=sys.stderr)
1802                 sys.exit(73)
1803         if not os.path.exists(cachedir):
1804                 os.makedirs(cachedir)
1805
1806         auxdbkeys = portdb._known_keys
1807
1808         class TreeData(object):
1809                 __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes')
1810                 def __init__(self, dest_db, eclass_db, path, src_db):
1811                         self.dest_db = dest_db
1812                         self.eclass_db = eclass_db
1813                         self.path = path
1814                         self.src_db = src_db
1815                         self.valid_nodes = set()
1816
1817         porttrees_data = []
1818         for path in porttrees:
1819                 src_db = portdb._pregen_auxdb.get(path)
1820                 if src_db is None:
1821                         # portdbapi does not populate _pregen_auxdb
1822                         # when FEATURES=metadata-transfer is enabled
1823                         src_db = portdb._create_pregen_cache(path)
1824
1825                 if src_db is not None:
1826                         porttrees_data.append(TreeData(portdb.auxdb[path],
1827                                 portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db))
1828
1829         porttrees = [tree_data.path for tree_data in porttrees_data]
1830
1831         quiet = settings.get('TERM') == 'dumb' or \
1832                 '--quiet' in myopts or \
1833                 not sys.stdout.isatty()
1834
1835         onProgress = None
1836         if not quiet:
1837                 progressBar = portage.output.TermProgressBar()
1838                 progressHandler = ProgressHandler()
1839                 onProgress = progressHandler.onProgress
1840                 def display():
1841                         progressBar.set(progressHandler.curval, progressHandler.maxval)
1842                 progressHandler.display = display
1843                 def sigwinch_handler(signum, frame):
1844                         lines, progressBar.term_columns = \
1845                                 portage.output.get_term_size()
1846                 signal.signal(signal.SIGWINCH, sigwinch_handler)
1847
1848         # Temporarily override portdb.porttrees so portdb.cp_all()
1849         # will only return the relevant subset.
1850         portdb_porttrees = portdb.porttrees
1851         portdb.porttrees = porttrees
1852         try:
1853                 cp_all = portdb.cp_all()
1854         finally:
1855                 portdb.porttrees = portdb_porttrees
1856
1857         curval = 0
1858         maxval = len(cp_all)
1859         if onProgress is not None:
1860                 onProgress(maxval, curval)
1861
1862         # TODO: Display error messages, but do not interfere with the progress bar.
1863         # Here's how:
1864         #  1) erase the progress bar
1865         #  2) show the error message
1866         #  3) redraw the progress bar on a new line
1867
1868         for cp in cp_all:
1869                 for tree_data in porttrees_data:
1870
1871                         src_chf = tree_data.src_db.validation_chf
1872                         dest_chf = tree_data.dest_db.validation_chf
1873                         dest_chf_key = '_%s_' % dest_chf
1874                         dest_chf_getter = operator.attrgetter(dest_chf)
1875
1876                         for cpv in portdb.cp_list(cp, mytree=tree_data.path):
1877                                 tree_data.valid_nodes.add(cpv)
1878                                 try:
1879                                         src = tree_data.src_db[cpv]
1880                                 except (CacheError, KeyError):
1881                                         continue
1882
1883                                 ebuild_location = portdb.findname(cpv, mytree=tree_data.path)
1884                                 if ebuild_location is None:
1885                                         continue
1886                                 ebuild_hash = hashed_path(ebuild_location)
1887
1888                                 try:
1889                                         if not tree_data.src_db.validate_entry(src,
1890                                                 ebuild_hash, tree_data.eclass_db):
1891                                                 continue
1892                                 except CacheError:
1893                                         continue
1894
1895                                 eapi = src.get('EAPI')
1896                                 if not eapi:
1897                                         eapi = '0'
1898                                 eapi_supported = eapi_is_supported(eapi)
1899                                 if not eapi_supported:
1900                                         continue
1901
1902                                 dest = None
1903                                 try:
1904                                         dest = tree_data.dest_db[cpv]
1905                                 except (KeyError, CacheError):
1906                                         pass
1907
1908                                 for d in (src, dest):
1909                                         if d is not None and d.get('EAPI') in ('', '0'):
1910                                                 del d['EAPI']
1911
1912                                 if src_chf != 'mtime':
1913                                         # src may contain an irrelevant _mtime_ which corresponds
1914                                         # to the time that the cache entry was written
1915                                         src.pop('_mtime_', None)
1916
1917                                 if src_chf != dest_chf:
1918                                         # populate src entry with dest_chf_key
1919                                         # (the validity of the dest_chf that we generate from the
1920                                         # ebuild here relies on the fact that we already used
1921                                         # validate_entry to validate the ebuild with src_chf)
1922                                         src[dest_chf_key] = dest_chf_getter(ebuild_hash)
1923
1924                                 if dest is not None:
1925                                         if not (dest[dest_chf_key] == src[dest_chf_key] and \
1926                                                 tree_data.eclass_db.validate_and_rewrite_cache(
1927                                                         dest['_eclasses_'], tree_data.dest_db.validation_chf,
1928                                                         tree_data.dest_db.store_eclass_paths) is not None and \
1929                                                 set(dest['_eclasses_']) == set(src['_eclasses_'])):
1930                                                 dest = None
1931                                         else:
1932                                                 # We don't want to skip the write unless we're really
1933                                                 # sure that the existing cache is identical, so don't
1934                                                 # trust _mtime_ and _eclasses_ alone.
1935                                                 for k in auxdbkeys:
1936                                                         if dest.get(k, '') != src.get(k, ''):
1937                                                                 dest = None
1938                                                                 break
1939
1940                                 if dest is not None:
1941                                         # The existing data is valid and identical,
1942                                         # so there's no need to overwrite it.
1943                                         continue
1944
1945                                 try:
1946                                         tree_data.dest_db[cpv] = src
1947                                 except CacheError:
1948                                         # ignore it; can't do anything about it.
1949                                         pass
1950
1951                 curval += 1
1952                 if onProgress is not None:
1953                         onProgress(maxval, curval)
1954
1955         if onProgress is not None:
1956                 onProgress(maxval, curval)
1957
1958         for tree_data in porttrees_data:
1959                 try:
1960                         dead_nodes = set(tree_data.dest_db)
1961                 except CacheError as e:
1962                         writemsg_level("Error listing cache entries for " + \
1963                                 "'%s': %s, continuing...\n" % (tree_data.path, e),
1964                                 level=logging.ERROR, noiselevel=-1)
1965                         del e
1966                 else:
1967                         dead_nodes.difference_update(tree_data.valid_nodes)
1968                         for cpv in dead_nodes:
1969                                 try:
1970                                         del tree_data.dest_db[cpv]
1971                                 except (KeyError, CacheError):
1972                                         pass
1973
1974         if not quiet:
1975                 # make sure the final progress is displayed
1976                 progressHandler.display()
1977                 print()
1978                 signal.signal(signal.SIGWINCH, signal.SIG_DFL)
1979
1980         sys.stdout.flush()
1981         os.umask(old_umask)
1982
1983 def action_regen(settings, portdb, max_jobs, max_load):
1984         xterm_titles = "notitles" not in settings.features
1985         emergelog(xterm_titles, " === regen")
1986         #regenerate cache entries
1987         sys.stdout.flush()
1988
1989         regen = MetadataRegen(portdb, max_jobs=max_jobs,
1990                 max_load=max_load, main=True)
1991
1992         signum = run_main_scheduler(regen)
1993         if signum is not None:
1994                 sys.exit(128 + signum)
1995
1996         portage.writemsg_stdout("done!\n")
1997         return regen.returncode
1998
1999 def action_search(root_config, myopts, myfiles, spinner):
2000         if not myfiles:
2001                 print("emerge: no search terms provided.")
2002         else:
2003                 searchinstance = search(root_config,
2004                         spinner, "--searchdesc" in myopts,
2005                         "--quiet" not in myopts, "--usepkg" in myopts,
2006                         "--usepkgonly" in myopts)
2007                 for mysearch in myfiles:
2008                         try:
2009                                 searchinstance.execute(mysearch)
2010                         except re.error as comment:
2011                                 print("\n!!! Regular expression error in \"%s\": %s" % ( mysearch, comment ))
2012                                 sys.exit(1)
2013                         searchinstance.output()
2014
2015 def action_sync(settings, trees, mtimedb, myopts, myaction):
2016         enter_invalid = '--ask-enter-invalid' in myopts
2017         xterm_titles = "notitles" not in settings.features
2018         emergelog(xterm_titles, " === sync")
2019         portdb = trees[settings['EROOT']]['porttree'].dbapi
2020         myportdir = portdb.porttree_root
2021         if not myportdir:
2022                 myportdir = settings.get('PORTDIR', '')
2023                 if myportdir and myportdir.strip():
2024                         myportdir = os.path.realpath(myportdir)
2025                 else:
2026                         myportdir = None
2027         out = portage.output.EOutput()
2028         global_config_path = GLOBAL_CONFIG_PATH
2029         if settings['EPREFIX']:
2030                 global_config_path = os.path.join(settings['EPREFIX'],
2031                                 GLOBAL_CONFIG_PATH.lstrip(os.sep))
2032         if not myportdir:
2033                 sys.stderr.write("!!! PORTDIR is undefined.  " + \
2034                         "Is %s/make.globals missing?\n" % global_config_path)
2035                 sys.exit(1)
2036         if myportdir[-1]=="/":
2037                 myportdir=myportdir[:-1]
2038         try:
2039                 st = os.stat(myportdir)
2040         except OSError:
2041                 st = None
2042         if st is None:
2043                 print(">>>",myportdir,"not found, creating it.")
2044                 portage.util.ensure_dirs(myportdir, mode=0o755)
2045                 st = os.stat(myportdir)
2046
2047         usersync_uid = None
2048         spawn_kwargs = {}
2049         spawn_kwargs["env"] = settings.environ()
2050         if 'usersync' in settings.features and \
2051                 portage.data.secpass >= 2 and \
2052                 (st.st_uid != os.getuid() and st.st_mode & 0o700 or \
2053                 st.st_gid != os.getgid() and st.st_mode & 0o070):
2054                 try:
2055                         homedir = pwd.getpwuid(st.st_uid).pw_dir
2056                 except KeyError:
2057                         pass
2058                 else:
2059                         # Drop privileges when syncing, in order to match
2060                         # existing uid/gid settings.
2061                         usersync_uid = st.st_uid
2062                         spawn_kwargs["uid"]    = st.st_uid
2063                         spawn_kwargs["gid"]    = st.st_gid
2064                         spawn_kwargs["groups"] = [st.st_gid]
2065                         spawn_kwargs["env"]["HOME"] = homedir
2066                         umask = 0o002
2067                         if not st.st_mode & 0o020:
2068                                 umask = umask | 0o020
2069                         spawn_kwargs["umask"] = umask
2070
2071         if usersync_uid is not None:
2072                 # PORTAGE_TMPDIR is used below, so validate it and
2073                 # bail out if necessary.
2074                 rval = _check_temp_dir(settings)
2075                 if rval != os.EX_OK:
2076                         return rval
2077
2078         syncuri = settings.get("SYNC", "").strip()
2079         if not syncuri:
2080                 writemsg_level("!!! SYNC is undefined. " + \
2081                         "Is %s/make.globals missing?\n" % global_config_path,
2082                         noiselevel=-1, level=logging.ERROR)
2083                 return 1
2084
2085         vcs_dirs = frozenset(VCS_DIRS)
2086         vcs_dirs = vcs_dirs.intersection(os.listdir(myportdir))
2087
2088         os.umask(0o022)
2089         dosyncuri = syncuri
2090         updatecache_flg = False
2091         git = False
2092         if myaction == "metadata":
2093                 print("skipping sync")
2094                 updatecache_flg = True
2095         elif ".git" in vcs_dirs:
2096                 # Update existing git repository, and ignore the syncuri. We are
2097                 # going to trust the user and assume that the user is in the branch
2098                 # that he/she wants updated. We'll let the user manage branches with
2099                 # git directly.
2100                 if portage.process.find_binary("git") is None:
2101                         msg = ["Command not found: git",
2102                         "Type \"emerge dev-util/git\" to enable git support."]
2103                         for l in msg:
2104                                 writemsg_level("!!! %s\n" % l,
2105                                         level=logging.ERROR, noiselevel=-1)
2106                         return 1
2107                 msg = ">>> Starting git pull in %s..." % myportdir
2108                 emergelog(xterm_titles, msg )
2109                 writemsg_level(msg + "\n")
2110                 exitcode = portage.process.spawn_bash("cd %s ; git pull" % \
2111                         (portage._shell_quote(myportdir),),
2112                         **portage._native_kwargs(spawn_kwargs))
2113                 if exitcode != os.EX_OK:
2114                         msg = "!!! git pull error in %s." % myportdir
2115                         emergelog(xterm_titles, msg)
2116                         writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
2117                         return exitcode
2118                 msg = ">>> Git pull in %s successful" % myportdir
2119                 emergelog(xterm_titles, msg)
2120                 writemsg_level(msg + "\n")
2121                 git = True
2122         elif syncuri[:8]=="rsync://" or syncuri[:6]=="ssh://":
2123                 for vcs_dir in vcs_dirs:
2124                         writemsg_level(("!!! %s appears to be under revision " + \
2125                                 "control (contains %s).\n!!! Aborting rsync sync.\n") % \
2126                                 (myportdir, vcs_dir), level=logging.ERROR, noiselevel=-1)
2127                         return 1
2128                 rsync_binary = portage.process.find_binary("rsync")
2129                 if rsync_binary is None:
2130                         print("!!! /usr/bin/rsync does not exist, so rsync support is disabled.")
2131                         print("!!! Type \"emerge net-misc/rsync\" to enable rsync support.")
2132                         sys.exit(1)
2133                 mytimeout=180
2134
2135                 rsync_opts = []
2136                 if settings["PORTAGE_RSYNC_OPTS"] == "":
2137                         portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
2138                         rsync_opts.extend([
2139                                 "--recursive",    # Recurse directories
2140                                 "--links",        # Consider symlinks
2141                                 "--safe-links",   # Ignore links outside of tree
2142                                 "--perms",        # Preserve permissions
2143                                 "--times",        # Preserive mod times
2144                                 "--compress",     # Compress the data transmitted
2145                                 "--force",        # Force deletion on non-empty dirs
2146                                 "--whole-file",   # Don't do block transfers, only entire files
2147                                 "--delete",       # Delete files that aren't in the master tree
2148                                 "--stats",        # Show final statistics about what was transfered
2149                                 "--human-readable",
2150                                 "--timeout="+str(mytimeout), # IO timeout if not done in X seconds
2151                                 "--exclude=/distfiles",   # Exclude distfiles from consideration
2152                                 "--exclude=/local",       # Exclude local     from consideration
2153                                 "--exclude=/packages",    # Exclude packages  from consideration
2154                         ])
2155
2156                 else:
2157                         # The below validation is not needed when using the above hardcoded
2158                         # defaults.
2159
2160                         portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
2161                         rsync_opts.extend(portage.util.shlex_split(
2162                                 settings.get("PORTAGE_RSYNC_OPTS", "")))
2163                         for opt in ("--recursive", "--times"):
2164                                 if opt not in rsync_opts:
2165                                         portage.writemsg(yellow("WARNING:") + " adding required option " + \
2166                                         "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
2167                                         rsync_opts.append(opt)
2168
2169                         for exclude in ("distfiles", "local", "packages"):
2170                                 opt = "--exclude=/%s" % exclude
2171                                 if opt not in rsync_opts:
2172                                         portage.writemsg(yellow("WARNING:") + \
2173                                         " adding required option %s not included in "  % opt + \
2174                                         "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
2175                                         rsync_opts.append(opt)
2176
2177                         if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
2178                                 def rsync_opt_startswith(opt_prefix):
2179                                         for x in rsync_opts:
2180                                                 if x.startswith(opt_prefix):
2181                                                         return True
2182                                         return False
2183
2184                                 if not rsync_opt_startswith("--timeout="):
2185                                         rsync_opts.append("--timeout=%d" % mytimeout)
2186
2187                                 for opt in ("--compress", "--whole-file"):
2188                                         if opt not in rsync_opts:
2189                                                 portage.writemsg(yellow("WARNING:") + " adding required option " + \
2190                                                 "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
2191                                                 rsync_opts.append(opt)
2192
2193                 if "--quiet" in myopts:
2194                         rsync_opts.append("--quiet")    # Shut up a lot
2195                 else:
2196                         rsync_opts.append("--verbose")  # Print filelist
2197
2198                 if "--verbose" in myopts:
2199                         rsync_opts.append("--progress")  # Progress meter for each file
2200
2201                 if "--debug" in myopts:
2202                         rsync_opts.append("--checksum") # Force checksum on all files
2203
2204                 # Real local timestamp file.
2205                 servertimestampfile = os.path.join(
2206                         myportdir, "metadata", "timestamp.chk")
2207
2208                 content = portage.util.grabfile(servertimestampfile)
2209                 mytimestamp = 0
2210                 if content:
2211                         try:
2212                                 mytimestamp = time.mktime(time.strptime(content[0],
2213                                         "%a, %d %b %Y %H:%M:%S +0000"))
2214                         except (OverflowError, ValueError):
2215                                 pass
2216                 del content
2217
2218                 try:
2219                         rsync_initial_timeout = \
2220                                 int(settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
2221                 except ValueError:
2222                         rsync_initial_timeout = 15
2223
2224                 try:
2225                         maxretries=int(settings["PORTAGE_RSYNC_RETRIES"])
2226                 except SystemExit as e:
2227                         raise # Needed else can't exit
2228                 except:
2229                         maxretries = -1 #default number of retries
2230
2231                 retries=0
2232                 try:
2233                         proto, user_name, hostname, port = re.split(
2234                                 r"(rsync|ssh)://([^:/]+@)?(\[[:\da-fA-F]*\]|[^:/]*)(:[0-9]+)?",
2235                                 syncuri, maxsplit=4)[1:5]
2236                 except ValueError:
2237                         writemsg_level("!!! SYNC is invalid: %s\n" % syncuri,
2238                                 noiselevel=-1, level=logging.ERROR)
2239                         return 1
2240                 if port is None:
2241                         port=""
2242                 if user_name is None:
2243                         user_name=""
2244                 if re.match(r"^\[[:\da-fA-F]*\]$", hostname) is None:
2245                         getaddrinfo_host = hostname
2246                 else:
2247                         # getaddrinfo needs the brackets stripped
2248                         getaddrinfo_host = hostname[1:-1]
2249                 updatecache_flg=True
2250                 all_rsync_opts = set(rsync_opts)
2251                 extra_rsync_opts = portage.util.shlex_split(
2252                         settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
2253                 all_rsync_opts.update(extra_rsync_opts)
2254
2255                 family = socket.AF_UNSPEC
2256                 if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
2257                         family = socket.AF_INET
2258                 elif socket.has_ipv6 and \
2259                         ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
2260                         family = socket.AF_INET6
2261
2262                 addrinfos = None
2263                 uris = []
2264
2265                 try:
2266                         addrinfos = getaddrinfo_validate(
2267                                 socket.getaddrinfo(getaddrinfo_host, None,
2268                                 family, socket.SOCK_STREAM))
2269                 except socket.error as e:
2270                         writemsg_level(
2271                                 "!!! getaddrinfo failed for '%s': %s\n" % (hostname, e),
2272                                 noiselevel=-1, level=logging.ERROR)
2273
2274                 if addrinfos:
2275
2276                         AF_INET = socket.AF_INET
2277                         AF_INET6 = None
2278                         if socket.has_ipv6:
2279                                 AF_INET6 = socket.AF_INET6
2280
2281                         ips_v4 = []
2282                         ips_v6 = []
2283
2284                         for addrinfo in addrinfos:
2285                                 if addrinfo[0] == AF_INET:
2286                                         ips_v4.append("%s" % addrinfo[4][0])
2287                                 elif AF_INET6 is not None and addrinfo[0] == AF_INET6:
2288                                         # IPv6 addresses need to be enclosed in square brackets
2289                                         ips_v6.append("[%s]" % addrinfo[4][0])
2290
2291                         random.shuffle(ips_v4)
2292                         random.shuffle(ips_v6)
2293
2294                         # Give priority to the address family that
2295                         # getaddrinfo() returned first.
2296                         if AF_INET6 is not None and addrinfos and \
2297                                 addrinfos[0][0] == AF_INET6:
2298                                 ips = ips_v6 + ips_v4
2299                         else:
2300                                 ips = ips_v4 + ips_v6
2301
2302                         for ip in ips:
2303                                 uris.append(syncuri.replace(
2304                                         "//" + user_name + hostname + port + "/",
2305                                         "//" + user_name + ip + port + "/", 1))
2306
2307                 if not uris:
2308                         # With some configurations we need to use the plain hostname
2309                         # rather than try to resolve the ip addresses (bug #340817).
2310                         uris.append(syncuri)
2311
2312                 # reverse, for use with pop()
2313                 uris.reverse()
2314
2315                 effective_maxretries = maxretries
2316                 if effective_maxretries < 0:
2317                         effective_maxretries = len(uris) - 1
2318
2319                 SERVER_OUT_OF_DATE = -1
2320                 EXCEEDED_MAX_RETRIES = -2
2321                 while (1):
2322                         if uris:
2323                                 dosyncuri = uris.pop()
2324                         else:
2325                                 writemsg("!!! Exhausted addresses for %s\n" % \
2326                                         hostname, noiselevel=-1)
2327                                 return 1
2328
2329                         if (retries==0):
2330                                 if "--ask" in myopts:
2331                                         if userquery("Do you want to sync your Portage tree " + \
2332                                                 "with the mirror at\n" + blue(dosyncuri) + bold("?"),
2333                                                 enter_invalid) == "No":
2334                                                 print()
2335                                                 print("Quitting.")
2336                                                 print()
2337                                                 sys.exit(128 + signal.SIGINT)
2338                                 emergelog(xterm_titles, ">>> Starting rsync with " + dosyncuri)
2339                                 if "--quiet" not in myopts:
2340                                         print(">>> Starting rsync with "+dosyncuri+"...")
2341                         else:
2342                                 emergelog(xterm_titles,
2343                                         ">>> Starting retry %d of %d with %s" % \
2344                                                 (retries, effective_maxretries, dosyncuri))
2345                                 writemsg_stdout(
2346                                         "\n\n>>> Starting retry %d of %d with %s\n" % \
2347                                         (retries, effective_maxretries, dosyncuri), noiselevel=-1)
2348
2349                         if dosyncuri.startswith('ssh://'):
2350                                 dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
2351
2352                         if mytimestamp != 0 and "--quiet" not in myopts:
2353                                 print(">>> Checking server timestamp ...")
2354
2355                         rsynccommand = [rsync_binary] + rsync_opts + extra_rsync_opts
2356
2357                         if "--debug" in myopts:
2358                                 print(rsynccommand)
2359
2360                         exitcode = os.EX_OK
2361                         servertimestamp = 0
2362                         # Even if there's no timestamp available locally, fetch the
2363                         # timestamp anyway as an initial probe to verify that the server is
2364                         # responsive.  This protects us from hanging indefinitely on a
2365                         # connection attempt to an unresponsive server which rsync's
2366                         # --timeout option does not prevent.
2367                         if True:
2368                                 # Temporary file for remote server timestamp comparison.
2369                                 # NOTE: If FEATURES=usersync is enabled then the tempfile
2370                                 # needs to be in a directory that's readable by the usersync
2371                                 # user. We assume that PORTAGE_TMPDIR will satisfy this
2372                                 # requirement, since that's not necessarily true for the
2373                                 # default directory used by the tempfile module.
2374                                 if usersync_uid is not None:
2375                                         tmpdir = settings['PORTAGE_TMPDIR']
2376                                 else:
2377                                         # use default dir from tempfile module
2378                                         tmpdir = None
2379                                 fd, tmpservertimestampfile = \
2380                                         tempfile.mkstemp(dir=tmpdir)
2381                                 os.close(fd)
2382                                 if usersync_uid is not None:
2383                                         portage.util.apply_permissions(tmpservertimestampfile,
2384                                                 uid=usersync_uid)
2385                                 mycommand = rsynccommand[:]
2386                                 mycommand.append(dosyncuri.rstrip("/") + \
2387                                         "/metadata/timestamp.chk")
2388                                 mycommand.append(tmpservertimestampfile)
2389                                 content = None
2390                                 mypids = []
2391                                 try:
2392                                         # Timeout here in case the server is unresponsive.  The
2393                                         # --timeout rsync option doesn't apply to the initial
2394                                         # connection attempt.
2395                                         try:
2396                                                 if rsync_initial_timeout:
2397                                                         portage.exception.AlarmSignal.register(
2398                                                                 rsync_initial_timeout)
2399
2400                                                 mypids.extend(portage.process.spawn(
2401                                                         mycommand, returnpid=True,
2402                                                         **portage._native_kwargs(spawn_kwargs)))
2403                                                 exitcode = os.waitpid(mypids[0], 0)[1]
2404                                                 if usersync_uid is not None:
2405                                                         portage.util.apply_permissions(tmpservertimestampfile,
2406                                                                 uid=os.getuid())
2407                                                 content = portage.grabfile(tmpservertimestampfile)
2408                                         finally:
2409                                                 if rsync_initial_timeout:
2410                                                         portage.exception.AlarmSignal.unregister()
2411                                                 try:
2412                                                         os.unlink(tmpservertimestampfile)
2413                                                 except OSError:
2414                                                         pass
2415                                 except portage.exception.AlarmSignal:
2416                                         # timed out
2417                                         print('timed out')
2418                                         # With waitpid and WNOHANG, only check the
2419                                         # first element of the tuple since the second
2420                                         # element may vary (bug #337465).
2421                                         if mypids and os.waitpid(mypids[0], os.WNOHANG)[0] == 0:
2422                                                 os.kill(mypids[0], signal.SIGTERM)
2423                                                 os.waitpid(mypids[0], 0)
2424                                         # This is the same code rsync uses for timeout.
2425                                         exitcode = 30
2426                                 else:
2427                                         if exitcode != os.EX_OK:
2428                                                 if exitcode & 0xff:
2429                                                         exitcode = (exitcode & 0xff) << 8
2430                                                 else:
2431                                                         exitcode = exitcode >> 8
2432                                 if mypids:
2433                                         portage.process.spawned_pids.remove(mypids[0])
2434                                 if content:
2435                                         try:
2436                                                 servertimestamp = time.mktime(time.strptime(
2437                                                         content[0], "%a, %d %b %Y %H:%M:%S +0000"))
2438                                         except (OverflowError, ValueError):
2439                                                 pass
2440                                 del mycommand, mypids, content
2441                         if exitcode == os.EX_OK:
2442                                 if (servertimestamp != 0) and (servertimestamp == mytimestamp):
2443                                         emergelog(xterm_titles,
2444                                                 ">>> Cancelling sync -- Already current.")
2445                                         print()
2446                                         print(">>>")
2447                                         print(">>> Timestamps on the server and in the local repository are the same.")
2448                                         print(">>> Cancelling all further sync action. You are already up to date.")
2449                                         print(">>>")
2450                                         print(">>> In order to force sync, remove '%s'." % servertimestampfile)
2451                                         print(">>>")
2452                                         print()
2453                                         sys.exit(0)
2454                                 elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
2455                                         emergelog(xterm_titles,
2456                                                 ">>> Server out of date: %s" % dosyncuri)
2457                                         print()
2458                                         print(">>>")
2459                                         print(">>> SERVER OUT OF DATE: %s" % dosyncuri)
2460                                         print(">>>")
2461                                         print(">>> In order to force sync, remove '%s'." % servertimestampfile)
2462                                         print(">>>")
2463                                         print()
2464                                         exitcode = SERVER_OUT_OF_DATE
2465                                 elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
2466                                         # actual sync
2467                                         mycommand = rsynccommand + [dosyncuri+"/", myportdir]
2468                                         exitcode = portage.process.spawn(mycommand,
2469                                                 **portage._native_kwargs(spawn_kwargs))
2470                                         if exitcode in [0,1,3,4,11,14,20,21]:
2471                                                 break
2472                         elif exitcode in [1,3,4,11,14,20,21]:
2473                                 break
2474                         else:
2475                                 # Code 2 indicates protocol incompatibility, which is expected
2476                                 # for servers with protocol < 29 that don't support
2477                                 # --prune-empty-directories.  Retry for a server that supports
2478                                 # at least rsync protocol version 29 (>=rsync-2.6.4).
2479                                 pass
2480
2481                         retries=retries+1
2482
2483                         if maxretries < 0 or retries <= maxretries:
2484                                 print(">>> Retrying...")
2485                         else:
2486                                 # over retries
2487                                 # exit loop
2488                                 updatecache_flg=False
2489                                 exitcode = EXCEEDED_MAX_RETRIES
2490                                 break
2491
2492                 if (exitcode==0):
2493                         emergelog(xterm_titles, "=== Sync completed with %s" % dosyncuri)
2494                 elif exitcode == SERVER_OUT_OF_DATE:
2495                         sys.exit(1)
2496                 elif exitcode == EXCEEDED_MAX_RETRIES:
2497                         sys.stderr.write(
2498                                 ">>> Exceeded PORTAGE_RSYNC_RETRIES: %s\n" % maxretries)
2499                         sys.exit(1)
2500                 elif (exitcode>0):
2501                         msg = []
2502                         if exitcode==1:
2503                                 msg.append("Rsync has reported that there is a syntax error. Please ensure")
2504                                 msg.append("that your SYNC statement is proper.")
2505                                 msg.append("SYNC=" + settings["SYNC"])
2506                         elif exitcode==11:
2507                                 msg.append("Rsync has reported that there is a File IO error. Normally")
2508                                 msg.append("this means your disk is full, but can be caused by corruption")
2509                                 msg.append("on the filesystem that contains PORTDIR. Please investigate")
2510                                 msg.append("and try again after the problem has been fixed.")
2511                                 msg.append("PORTDIR=" + settings["PORTDIR"])
2512                         elif exitcode==20:
2513                                 msg.append("Rsync was killed before it finished.")
2514                         else:
2515                                 msg.append("Rsync has not successfully finished. It is recommended that you keep")
2516                                 msg.append("trying or that you use the 'emerge-webrsync' option if you are unable")
2517                                 msg.append("to use rsync due to firewall or other restrictions. This should be a")
2518                                 msg.append("temporary problem unless complications exist with your network")
2519                                 msg.append("(and possibly your system's filesystem) configuration.")
2520                         for line in msg:
2521                                 out.eerror(line)
2522                         sys.exit(exitcode)
2523         elif syncuri[:6]=="cvs://":
2524                 if not os.path.exists("/usr/bin/cvs"):
2525                         print("!!! /usr/bin/cvs does not exist, so CVS support is disabled.")
2526                         print("!!! Type \"emerge dev-vcs/cvs\" to enable CVS support.")
2527                         sys.exit(1)
2528                 cvsroot=syncuri[6:]
2529                 cvsdir=os.path.dirname(myportdir)
2530                 if not os.path.exists(myportdir+"/CVS"):
2531                         #initial checkout
2532                         print(">>> Starting initial cvs checkout with "+syncuri+"...")
2533                         if os.path.exists(cvsdir+"/gentoo-x86"):
2534                                 print("!!! existing",cvsdir+"/gentoo-x86 directory; exiting.")
2535                                 sys.exit(1)
2536                         try:
2537                                 os.rmdir(myportdir)
2538                         except OSError as e:
2539                                 if e.errno != errno.ENOENT:
2540                                         sys.stderr.write(
2541                                                 "!!! existing '%s' directory; exiting.\n" % myportdir)
2542                                         sys.exit(1)
2543                                 del e
2544                         if portage.process.spawn_bash(
2545                                         "cd %s; exec cvs -z0 -d %s co -P gentoo-x86" % \
2546                                         (portage._shell_quote(cvsdir), portage._shell_quote(cvsroot)),
2547                                         **portage._native_kwargs(spawn_kwargs)) != os.EX_OK:
2548                                 print("!!! cvs checkout error; exiting.")
2549                                 sys.exit(1)
2550                         os.rename(os.path.join(cvsdir, "gentoo-x86"), myportdir)
2551                 else:
2552                         #cvs update
2553                         print(">>> Starting cvs update with "+syncuri+"...")
2554                         retval = portage.process.spawn_bash(
2555                                 "cd %s; exec cvs -z0 -q update -dP" % \
2556                                 (portage._shell_quote(myportdir),),
2557                                 **portage._native_kwargs(spawn_kwargs))
2558                         if retval != os.EX_OK:
2559                                 writemsg_level("!!! cvs update error; exiting.\n",
2560                                         noiselevel=-1, level=logging.ERROR)
2561                                 sys.exit(retval)
2562                 dosyncuri = syncuri
2563         else:
2564                 writemsg_level("!!! Unrecognized protocol: SYNC='%s'\n" % (syncuri,),
2565                         noiselevel=-1, level=logging.ERROR)
2566                 return 1
2567
2568         # Reload the whole config from scratch.
2569         portage._sync_disabled_warnings = False
2570         settings, trees, mtimedb = load_emerge_config(trees=trees)
2571         adjust_configs(myopts, trees)
2572         root_config = trees[settings['EROOT']]['root_config']
2573         portdb = trees[settings['EROOT']]['porttree'].dbapi
2574
2575         if git:
2576                 # NOTE: Do this after reloading the config, in case
2577                 # it did not exist prior to sync, so that the config
2578                 # and portdb properly account for its existence.
2579                 exitcode = git_sync_timestamps(portdb, myportdir)
2580                 if exitcode == os.EX_OK:
2581                         updatecache_flg = True
2582
2583         if updatecache_flg and  \
2584                 myaction != "metadata" and \
2585                 "metadata-transfer" not in settings.features:
2586                 updatecache_flg = False
2587
2588         if updatecache_flg and \
2589                 os.path.exists(os.path.join(myportdir, 'metadata', 'cache')):
2590
2591                 # Only update cache for myportdir since that's
2592                 # the only one that's been synced here.
2593                 action_metadata(settings, portdb, myopts, porttrees=[myportdir])
2594
2595         if myopts.get('--package-moves') != 'n' and \
2596                 _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)):
2597                 mtimedb.commit()
2598                 # Reload the whole config from scratch.
2599                 settings, trees, mtimedb = load_emerge_config(trees=trees)
2600                 adjust_configs(myopts, trees)
2601                 portdb = trees[settings['EROOT']]['porttree'].dbapi
2602                 root_config = trees[settings['EROOT']]['root_config']
2603
2604         mybestpv = portdb.xmatch("bestmatch-visible",
2605                 portage.const.PORTAGE_PACKAGE_ATOM)
2606         mypvs = portage.best(
2607                 trees[settings['EROOT']]['vartree'].dbapi.match(
2608                 portage.const.PORTAGE_PACKAGE_ATOM))
2609
2610         chk_updated_cfg_files(settings["EROOT"],
2611                 portage.util.shlex_split(settings.get("CONFIG_PROTECT", "")))
2612
2613         if myaction != "metadata":
2614                 postsync = os.path.join(settings["PORTAGE_CONFIGROOT"],
2615                         portage.USER_CONFIG_PATH, "bin", "post_sync")
2616                 if os.access(postsync, os.X_OK):
2617                         retval = portage.process.spawn(
2618                                 [postsync, dosyncuri], env=settings.environ())
2619                         if retval != os.EX_OK:
2620                                 writemsg_level(
2621                                         " %s spawn failed of %s\n" % (bad("*"), postsync,),
2622                                         level=logging.ERROR, noiselevel=-1)
2623
2624         if(mybestpv != mypvs) and not "--quiet" in myopts:
2625                 print()
2626                 print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended")
2627                 print(warn(" * ")+"that you update portage now, before any other packages are updated.")
2628                 print()
2629                 print(warn(" * ")+"To update portage, run 'emerge --oneshot portage' now.")
2630                 print()
2631
2632         display_news_notification(root_config, myopts)
2633         return os.EX_OK
2634
2635 def action_uninstall(settings, trees, ldpath_mtimes,
2636         opts, action, files, spinner):
2637         # For backward compat, some actions do not require leading '='.
2638         ignore_missing_eq = action in ('clean', 'unmerge')
2639         root = settings['ROOT']
2640         eroot = settings['EROOT']
2641         vardb = trees[settings['EROOT']]['vartree'].dbapi
2642         valid_atoms = []
2643         lookup_owners = []
2644
2645         # Ensure atoms are valid before calling unmerge().
2646         # For backward compat, leading '=' is not required.
2647         for x in files:
2648                 if is_valid_package_atom(x, allow_repo=True) or \
2649                         (ignore_missing_eq and is_valid_package_atom('=' + x)):
2650
2651                         try:
2652                                 atom = dep_expand(x, mydb=vardb, settings=settings)
2653                         except portage.exception.AmbiguousPackageName as e:
2654                                 msg = "The short ebuild name \"" + x + \
2655                                         "\" is ambiguous.  Please specify " + \
2656                                         "one of the following " + \
2657                                         "fully-qualified ebuild names instead:"
2658                                 for line in textwrap.wrap(msg, 70):
2659                                         writemsg_level("!!! %s\n" % (line,),
2660                                                 level=logging.ERROR, noiselevel=-1)
2661                                 for i in e.args[0]:
2662                                         writemsg_level("    %s\n" % colorize("INFORM", i),
2663                                                 level=logging.ERROR, noiselevel=-1)
2664                                 writemsg_level("\n", level=logging.ERROR, noiselevel=-1)
2665                                 return 1
2666                         else:
2667                                 if atom.use and atom.use.conditional:
2668                                         writemsg_level(
2669                                                 ("\n\n!!! '%s' contains a conditional " + \
2670                                                 "which is not allowed.\n") % (x,),
2671                                                 level=logging.ERROR, noiselevel=-1)
2672                                         writemsg_level(
2673                                                 "!!! Please check ebuild(5) for full details.\n",
2674                                                 level=logging.ERROR)
2675                                         return 1
2676                                 valid_atoms.append(atom)
2677
2678                 elif x.startswith(os.sep):
2679                         if not x.startswith(eroot):
2680                                 writemsg_level(("!!! '%s' does not start with" + \
2681                                         " $EROOT.\n") % x, level=logging.ERROR, noiselevel=-1)
2682                                 return 1
2683                         # Queue these up since it's most efficient to handle
2684                         # multiple files in a single iter_owners() call.
2685                         lookup_owners.append(x)
2686
2687                 elif x.startswith(SETPREFIX) and action == "deselect":
2688                         valid_atoms.append(x)
2689
2690                 elif "*" in x:
2691                         try:
2692                                 ext_atom = Atom(x, allow_repo=True, allow_wildcard=True)
2693                         except InvalidAtom:
2694                                 msg = []
2695                                 msg.append("'%s' is not a valid package atom." % (x,))
2696                                 msg.append("Please check ebuild(5) for full details.")
2697                                 writemsg_level("".join("!!! %s\n" % line for line in msg),
2698                                         level=logging.ERROR, noiselevel=-1)
2699                                 return 1
2700
2701                         for cpv in vardb.cpv_all():
2702                                 if portage.match_from_list(ext_atom, [cpv]):
2703                                         require_metadata = False
2704                                         atom = portage.cpv_getkey(cpv)
2705                                         if ext_atom.operator == '=*':
2706                                                 atom = "=" + atom + "-" + \
2707                                                         portage.versions.cpv_getversion(cpv)
2708                                         if ext_atom.slot:
2709                                                 atom += ":" + ext_atom.slot
2710                                                 require_metadata = True
2711                                         if ext_atom.repo:
2712                                                 atom += "::" + ext_atom.repo
2713                                                 require_metadata = True
2714
2715                                         atom = Atom(atom, allow_repo=True)
2716                                         if require_metadata:
2717                                                 try:
2718                                                         cpv = vardb._pkg_str(cpv, ext_atom.repo)
2719                                                 except (KeyError, InvalidData):
2720                                                         continue
2721                                                 if not portage.match_from_list(atom, [cpv]):
2722                                                         continue
2723
2724                                         valid_atoms.append(atom)
2725
2726                 else:
2727                         msg = []
2728                         msg.append("'%s' is not a valid package atom." % (x,))
2729                         msg.append("Please check ebuild(5) for full details.")
2730                         writemsg_level("".join("!!! %s\n" % line for line in msg),
2731                                 level=logging.ERROR, noiselevel=-1)
2732                         return 1
2733
2734         if lookup_owners:
2735                 relative_paths = []
2736                 search_for_multiple = False
2737                 if len(lookup_owners) > 1:
2738                         search_for_multiple = True
2739
2740                 for x in lookup_owners:
2741                         if not search_for_multiple and os.path.isdir(x):
2742                                 search_for_multiple = True
2743                         relative_paths.append(x[len(root)-1:])
2744
2745                 owners = set()
2746                 for pkg, relative_path in \
2747                         vardb._owners.iter_owners(relative_paths):
2748                         owners.add(pkg.mycpv)
2749                         if not search_for_multiple:
2750                                 break
2751
2752                 if owners:
2753                         for cpv in owners:
2754                                 pkg = vardb._pkg_str(cpv, None)
2755                                 atom = '%s:%s' % (pkg.cp, pkg.slot)
2756                                 valid_atoms.append(portage.dep.Atom(atom))
2757                 else:
2758                         writemsg_level(("!!! '%s' is not claimed " + \
2759                                 "by any package.\n") % lookup_owners[0],
2760                                 level=logging.WARNING, noiselevel=-1)
2761
2762         if files and not valid_atoms:
2763                 return 1
2764
2765         if action == 'unmerge' and \
2766                 '--quiet' not in opts and \
2767                 '--quiet-unmerge-warn' not in opts:
2768                 msg = "This action can remove important packages! " + \
2769                         "In order to be safer, use " + \
2770                         "`emerge -pv --depclean <atom>` to check for " + \
2771                         "reverse dependencies before removing packages."
2772                 out = portage.output.EOutput()
2773                 for line in textwrap.wrap(msg, 72):
2774                         out.ewarn(line)
2775
2776         if action == 'deselect':
2777                 return action_deselect(settings, trees, opts, valid_atoms)
2778
2779         # Use the same logic as the Scheduler class to trigger redirection
2780         # of ebuild pkg_prerm/postrm phase output to logs as appropriate
2781         # for options such as --jobs, --quiet and --quiet-build.
2782         max_jobs = opts.get("--jobs", 1)
2783         background = (max_jobs is True or max_jobs > 1 or
2784                 "--quiet" in opts or opts.get("--quiet-build") == "y")
2785         sched_iface = SchedulerInterface(global_event_loop(),
2786                 is_background=lambda: background)
2787
2788         if background:
2789                 settings.unlock()
2790                 settings["PORTAGE_BACKGROUND"] = "1"
2791                 settings.backup_changes("PORTAGE_BACKGROUND")
2792                 settings.lock()
2793
2794         if action in ('clean', 'unmerge') or \
2795                 (action == 'prune' and "--nodeps" in opts):
2796                 # When given a list of atoms, unmerge them in the order given.
2797                 ordered = action == 'unmerge'
2798                 rval = unmerge(trees[settings['EROOT']]['root_config'], opts, action,
2799                         valid_atoms, ldpath_mtimes, ordered=ordered,
2800                         scheduler=sched_iface)
2801         else:
2802                 rval = action_depclean(settings, trees, ldpath_mtimes,
2803                         opts, action, valid_atoms, spinner,
2804                         scheduler=sched_iface)
2805
2806         return rval
2807
2808 def adjust_configs(myopts, trees):
2809         for myroot in trees:
2810                 mysettings =  trees[myroot]["vartree"].settings
2811                 mysettings.unlock()
2812                 adjust_config(myopts, mysettings)
2813                 mysettings.lock()
2814
2815 def adjust_config(myopts, settings):
2816         """Make emerge specific adjustments to the config."""
2817
2818         # Kill noauto as it will break merges otherwise.
2819         if "noauto" in settings.features:
2820                 settings.features.remove('noauto')
2821
2822         fail_clean = myopts.get('--fail-clean')
2823         if fail_clean is not None:
2824                 if fail_clean is True and \
2825                         'fail-clean' not in settings.features:
2826                         settings.features.add('fail-clean')
2827                 elif fail_clean == 'n' and \
2828                         'fail-clean' in settings.features:
2829                         settings.features.remove('fail-clean')
2830
2831         CLEAN_DELAY = 5
2832         try:
2833                 CLEAN_DELAY = int(settings.get("CLEAN_DELAY", str(CLEAN_DELAY)))
2834         except ValueError as e:
2835                 portage.writemsg("!!! %s\n" % str(e), noiselevel=-1)
2836                 portage.writemsg("!!! Unable to parse integer: CLEAN_DELAY='%s'\n" % \
2837                         settings["CLEAN_DELAY"], noiselevel=-1)
2838         settings["CLEAN_DELAY"] = str(CLEAN_DELAY)
2839         settings.backup_changes("CLEAN_DELAY")
2840
2841         EMERGE_WARNING_DELAY = 10
2842         try:
2843                 EMERGE_WARNING_DELAY = int(settings.get(
2844                         "EMERGE_WARNING_DELAY", str(EMERGE_WARNING_DELAY)))
2845         except ValueError as e:
2846                 portage.writemsg("!!! %s\n" % str(e), noiselevel=-1)
2847                 portage.writemsg("!!! Unable to parse integer: EMERGE_WARNING_DELAY='%s'\n" % \
2848                         settings["EMERGE_WARNING_DELAY"], noiselevel=-1)
2849         settings["EMERGE_WARNING_DELAY"] = str(EMERGE_WARNING_DELAY)
2850         settings.backup_changes("EMERGE_WARNING_DELAY")
2851
2852         buildpkg = myopts.get("--buildpkg")
2853         if buildpkg is True:
2854                 settings.features.add("buildpkg")
2855         elif buildpkg == 'n':
2856                 settings.features.discard("buildpkg")
2857
2858         if "--quiet" in myopts:
2859                 settings["PORTAGE_QUIET"]="1"
2860                 settings.backup_changes("PORTAGE_QUIET")
2861
2862         if "--verbose" in myopts:
2863                 settings["PORTAGE_VERBOSE"] = "1"
2864                 settings.backup_changes("PORTAGE_VERBOSE")
2865
2866         # Set so that configs will be merged regardless of remembered status
2867         if ("--noconfmem" in myopts):
2868                 settings["NOCONFMEM"]="1"
2869                 settings.backup_changes("NOCONFMEM")
2870
2871         # Set various debug markers... They should be merged somehow.
2872         PORTAGE_DEBUG = 0
2873         try:
2874                 PORTAGE_DEBUG = int(settings.get("PORTAGE_DEBUG", str(PORTAGE_DEBUG)))
2875                 if PORTAGE_DEBUG not in (0, 1):
2876                         portage.writemsg("!!! Invalid value: PORTAGE_DEBUG='%i'\n" % \
2877                                 PORTAGE_DEBUG, noiselevel=-1)
2878                         portage.writemsg("!!! PORTAGE_DEBUG must be either 0 or 1\n",
2879                                 noiselevel=-1)
2880                         PORTAGE_DEBUG = 0
2881         except ValueError as e:
2882                 portage.writemsg("!!! %s\n" % str(e), noiselevel=-1)
2883                 portage.writemsg("!!! Unable to parse integer: PORTAGE_DEBUG='%s'\n" %\
2884                         settings["PORTAGE_DEBUG"], noiselevel=-1)
2885                 del e
2886         if "--debug" in myopts:
2887                 PORTAGE_DEBUG = 1
2888         settings["PORTAGE_DEBUG"] = str(PORTAGE_DEBUG)
2889         settings.backup_changes("PORTAGE_DEBUG")
2890
2891         if settings.get("NOCOLOR") not in ("yes","true"):
2892                 portage.output.havecolor = 1
2893
2894         # The explicit --color < y | n > option overrides the NOCOLOR environment
2895         # variable and stdout auto-detection.
2896         if "--color" in myopts:
2897                 if "y" == myopts["--color"]:
2898                         portage.output.havecolor = 1
2899                         settings["NOCOLOR"] = "false"
2900                 else:
2901                         portage.output.havecolor = 0
2902                         settings["NOCOLOR"] = "true"
2903                 settings.backup_changes("NOCOLOR")
2904         elif settings.get('TERM') == 'dumb' or \
2905                 not sys.stdout.isatty():
2906                 portage.output.havecolor = 0
2907                 settings["NOCOLOR"] = "true"
2908                 settings.backup_changes("NOCOLOR")
2909
2910 def display_missing_pkg_set(root_config, set_name):
2911
2912         msg = []
2913         msg.append(("emerge: There are no sets to satisfy '%s'. " + \
2914                 "The following sets exist:") % \
2915                 colorize("INFORM", set_name))
2916         msg.append("")
2917
2918         for s in sorted(root_config.sets):
2919                 msg.append("    %s" % s)
2920         msg.append("")
2921
2922         writemsg_level("".join("%s\n" % l for l in msg),
2923                 level=logging.ERROR, noiselevel=-1)
2924
2925 def relative_profile_path(portdir, abs_profile):
2926         realpath = os.path.realpath(abs_profile)
2927         basepath   = os.path.realpath(os.path.join(portdir, "profiles"))
2928         if realpath.startswith(basepath):
2929                 profilever = realpath[1 + len(basepath):]
2930         else:
2931                 profilever = None
2932         return profilever
2933
2934 def getportageversion(portdir, _unused, profile, chost, vardb):
2935         profilever = None
2936         repositories = vardb.settings.repositories
2937         if profile:
2938                 profilever = relative_profile_path(portdir, profile)
2939                 if profilever is None:
2940                         try:
2941                                 for parent in portage.grabfile(
2942                                         os.path.join(profile, 'parent')):
2943                                         profilever = relative_profile_path(portdir,
2944                                                 os.path.join(profile, parent))
2945                                         if profilever is not None:
2946                                                 break
2947                                         colon = parent.find(":")
2948                                         if colon != -1:
2949                                                 p_repo_name = parent[:colon]
2950                                                 try:
2951                                                         p_repo_loc = \
2952                                                                 repositories.get_location_for_name(p_repo_name)
2953                                                 except KeyError:
2954                                                         pass
2955                                                 else:
2956                                                         profilever = relative_profile_path(p_repo_loc,
2957                                                                 os.path.join(p_repo_loc, 'profiles',
2958                                                                         parent[colon+1:]))
2959                                                         if profilever is not None:
2960                                                                 break
2961                         except portage.exception.PortageException:
2962                                 pass
2963
2964                         if profilever is None:
2965                                 try:
2966                                         profilever = "!" + os.readlink(profile)
2967                                 except (OSError):
2968                                         pass
2969
2970         if profilever is None:
2971                 profilever = "unavailable"
2972
2973         libcver = []
2974         libclist = set()
2975         for atom in expand_new_virt(vardb, portage.const.LIBC_PACKAGE_ATOM):
2976                 if not atom.blocker:
2977                         libclist.update(vardb.match(atom))
2978         if libclist:
2979                 for cpv in sorted(libclist):
2980                         libc_split = portage.catpkgsplit(cpv)[1:]
2981                         if libc_split[-1] == "r0":
2982                                 libc_split = libc_split[:-1]
2983                         libcver.append("-".join(libc_split))
2984         else:
2985                 libcver = ["unavailable"]
2986
2987         gccver = getgccversion(chost)
2988         unameout=platform.release()+" "+platform.machine()
2989
2990         return "Portage %s (%s, %s, %s, %s)" % \
2991                 (portage.VERSION, profilever, gccver, ",".join(libcver), unameout)
2992
2993 def git_sync_timestamps(portdb, portdir):
2994         """
2995         Since git doesn't preserve timestamps, synchronize timestamps between
2996         entries and ebuilds/eclasses. Assume the cache has the correct timestamp
2997         for a given file as long as the file in the working tree is not modified
2998         (relative to HEAD).
2999         """
3000
3001         cache_db = portdb._pregen_auxdb.get(portdir)
3002
3003         try:
3004                 if cache_db is None:
3005                         # portdbapi does not populate _pregen_auxdb
3006                         # when FEATURES=metadata-transfer is enabled
3007                         cache_db = portdb._create_pregen_cache(portdir)
3008         except CacheError as e:
3009                 writemsg_level("!!! Unable to instantiate cache: %s\n" % (e,),
3010                         level=logging.ERROR, noiselevel=-1)
3011                 return 1
3012
3013         if cache_db is None:
3014                 return os.EX_OK
3015
3016         if cache_db.validation_chf != 'mtime':
3017                 # newer formats like md5-dict do not require mtime sync
3018                 return os.EX_OK
3019
3020         writemsg_level(">>> Synchronizing timestamps...\n")
3021
3022         ec_dir = os.path.join(portdir, "eclass")
3023         try:
3024                 ec_names = set(f[:-7] for f in os.listdir(ec_dir) \
3025                         if f.endswith(".eclass"))
3026         except OSError as e:
3027                 writemsg_level("!!! Unable to list eclasses: %s\n" % (e,),
3028                         level=logging.ERROR, noiselevel=-1)
3029                 return 1
3030
3031         args = [portage.const.BASH_BINARY, "-c",
3032                 "cd %s && git diff-index --name-only --diff-filter=M HEAD" % \
3033                 portage._shell_quote(portdir)]
3034         proc = subprocess.Popen(args, stdout=subprocess.PIPE)
3035         modified_files = set(_unicode_decode(l).rstrip("\n") for l in proc.stdout)
3036         rval = proc.wait()
3037         proc.stdout.close()
3038         if rval != os.EX_OK:
3039                 return rval
3040
3041         modified_eclasses = set(ec for ec in ec_names \
3042                 if os.path.join("eclass", ec + ".eclass") in modified_files)
3043
3044         updated_ec_mtimes = {}
3045
3046         for cpv in cache_db:
3047                 cpv_split = portage.catpkgsplit(cpv)
3048                 if cpv_split is None:
3049                         writemsg_level("!!! Invalid cache entry: %s\n" % (cpv,),
3050                                 level=logging.ERROR, noiselevel=-1)
3051                         continue
3052
3053                 cat, pn, ver, rev = cpv_split
3054                 cat, pf = portage.catsplit(cpv)
3055                 relative_eb_path = os.path.join(cat, pn, pf + ".ebuild")
3056                 if relative_eb_path in modified_files:
3057                         continue
3058
3059                 try:
3060                         cache_entry = cache_db[cpv]
3061                         eb_mtime = cache_entry.get("_mtime_")
3062                         ec_mtimes = cache_entry.get("_eclasses_")
3063                 except KeyError:
3064                         writemsg_level("!!! Missing cache entry: %s\n" % (cpv,),
3065                                 level=logging.ERROR, noiselevel=-1)
3066                         continue
3067                 except CacheError as e:
3068                         writemsg_level("!!! Unable to access cache entry: %s %s\n" % \
3069                                 (cpv, e), level=logging.ERROR, noiselevel=-1)
3070                         continue
3071
3072                 if eb_mtime is None:
3073                         writemsg_level("!!! Missing ebuild mtime: %s\n" % (cpv,),
3074                                 level=logging.ERROR, noiselevel=-1)
3075                         continue
3076
3077                 try:
3078                         eb_mtime = long(eb_mtime)
3079                 except ValueError:
3080                         writemsg_level("!!! Invalid ebuild mtime: %s %s\n" % \
3081                                 (cpv, eb_mtime), level=logging.ERROR, noiselevel=-1)
3082                         continue
3083
3084                 if ec_mtimes is None:
3085                         writemsg_level("!!! Missing eclass mtimes: %s\n" % (cpv,),
3086                                 level=logging.ERROR, noiselevel=-1)
3087                         continue
3088
3089                 if modified_eclasses.intersection(ec_mtimes):
3090                         continue
3091
3092                 missing_eclasses = set(ec_mtimes).difference(ec_names)
3093                 if missing_eclasses:
3094                         writemsg_level("!!! Non-existent eclass(es): %s %s\n" % \
3095                                 (cpv, sorted(missing_eclasses)), level=logging.ERROR,
3096                                 noiselevel=-1)
3097                         continue
3098
3099                 eb_path = os.path.join(portdir, relative_eb_path)
3100                 try:
3101                         current_eb_mtime = os.stat(eb_path)
3102                 except OSError:
3103                         writemsg_level("!!! Missing ebuild: %s\n" % \
3104                                 (cpv,), level=logging.ERROR, noiselevel=-1)
3105                         continue
3106
3107                 inconsistent = False
3108                 for ec, (ec_path, ec_mtime) in ec_mtimes.items():
3109                         updated_mtime = updated_ec_mtimes.get(ec)
3110                         if updated_mtime is not None and updated_mtime != ec_mtime:
3111                                 writemsg_level("!!! Inconsistent eclass mtime: %s %s\n" % \
3112                                         (cpv, ec), level=logging.ERROR, noiselevel=-1)
3113                                 inconsistent = True
3114                                 break
3115
3116                 if inconsistent:
3117                         continue
3118
3119                 if current_eb_mtime != eb_mtime:
3120                         os.utime(eb_path, (eb_mtime, eb_mtime))
3121
3122                 for ec, (ec_path, ec_mtime) in ec_mtimes.items():
3123                         if ec in updated_ec_mtimes:
3124                                 continue
3125                         ec_path = os.path.join(ec_dir, ec + ".eclass")
3126                         current_mtime = os.stat(ec_path)[stat.ST_MTIME]
3127                         if current_mtime != ec_mtime:
3128                                 os.utime(ec_path, (ec_mtime, ec_mtime))
3129                         updated_ec_mtimes[ec] = ec_mtime
3130
3131         return os.EX_OK
3132
3133 def load_emerge_config(trees=None):
3134         kwargs = {}
3135         for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")):
3136                 v = os.environ.get(envvar, None)
3137                 if v and v.strip():
3138                         kwargs[k] = v
3139         trees = portage.create_trees(trees=trees, **portage._native_kwargs(kwargs))
3140
3141         for root_trees in trees.values():
3142                 settings = root_trees["vartree"].settings
3143                 settings._init_dirs()
3144                 setconfig = load_default_config(settings, root_trees)
3145                 root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
3146
3147         settings = trees[trees._target_eroot]['vartree'].settings
3148         mtimedbfile = os.path.join(settings['EROOT'], portage.CACHE_PATH, "mtimedb")
3149         mtimedb = portage.MtimeDB(mtimedbfile)
3150         QueryCommand._db = trees
3151         return settings, trees, mtimedb
3152
3153 def getgccversion(chost):
3154         """
3155         rtype: C{str}
3156         return:  the current in-use gcc version
3157         """
3158
3159         gcc_ver_command = ['gcc', '-dumpversion']
3160         gcc_ver_prefix = 'gcc-'
3161
3162         gcc_not_found_error = red(
3163         "!!! No gcc found. You probably need to 'source /etc/profile'\n" +
3164         "!!! to update the environment of this terminal and possibly\n" +
3165         "!!! other terminals also.\n"
3166         )
3167
3168         try:
3169                 proc = subprocess.Popen(["gcc-config", "-c"],
3170                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
3171         except OSError:
3172                 myoutput = None
3173                 mystatus = 1
3174         else:
3175                 myoutput = _unicode_decode(proc.communicate()[0]).rstrip("\n")
3176                 mystatus = proc.wait()
3177         if mystatus == os.EX_OK and myoutput.startswith(chost + "-"):
3178                 return myoutput.replace(chost + "-", gcc_ver_prefix, 1)
3179
3180         try:
3181                 proc = subprocess.Popen(
3182                         [chost + "-" + gcc_ver_command[0]] + gcc_ver_command[1:],
3183                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
3184         except OSError:
3185                 myoutput = None
3186                 mystatus = 1
3187         else:
3188                 myoutput = _unicode_decode(proc.communicate()[0]).rstrip("\n")
3189                 mystatus = proc.wait()
3190         if mystatus == os.EX_OK:
3191                 return gcc_ver_prefix + myoutput
3192
3193         try:
3194                 proc = subprocess.Popen(gcc_ver_command,
3195                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
3196         except OSError:
3197                 myoutput = None
3198                 mystatus = 1
3199         else:
3200                 myoutput = _unicode_decode(proc.communicate()[0]).rstrip("\n")
3201                 mystatus = proc.wait()
3202         if mystatus == os.EX_OK:
3203                 return gcc_ver_prefix + myoutput
3204
3205         portage.writemsg(gcc_not_found_error, noiselevel=-1)
3206         return "[unavailable]"
3207
3208 # Warn about features that may confuse users and
3209 # lead them to report invalid bugs.
3210 _emerge_features_warn = frozenset(['keeptemp', 'keepwork'])
3211
3212 def validate_ebuild_environment(trees):
3213         features_warn = set()
3214         for myroot in trees:
3215                 settings = trees[myroot]["vartree"].settings
3216                 settings.validate()
3217                 features_warn.update(
3218                         _emerge_features_warn.intersection(settings.features))
3219
3220         if features_warn:
3221                 msg = "WARNING: The FEATURES variable contains one " + \
3222                         "or more values that should be disabled under " + \
3223                         "normal circumstances: %s" % " ".join(features_warn)
3224                 out = portage.output.EOutput()
3225                 for line in textwrap.wrap(msg, 65):
3226                         out.ewarn(line)
3227
3228 def check_procfs():
3229         procfs_path = '/proc'
3230         if platform.system() not in ("Linux",) or \
3231                 os.path.ismount(procfs_path):
3232                 return os.EX_OK
3233         msg = "It seems that %s is not mounted. You have been warned." % procfs_path
3234         writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)),
3235                 level=logging.ERROR, noiselevel=-1)
3236         return 1
3237
3238 def config_protect_check(trees):
3239         for root, root_trees in trees.items():
3240                 settings = root_trees["root_config"].settings
3241                 if not settings.get("CONFIG_PROTECT"):
3242                         msg = "!!! CONFIG_PROTECT is empty"
3243                         if settings["ROOT"] != "/":
3244                                 msg += " for '%s'" % root
3245                         msg += "\n"
3246                         writemsg_level(msg, level=logging.WARN, noiselevel=-1)
3247
3248 def apply_priorities(settings):
3249         ionice(settings)
3250         nice(settings)
3251
3252 def nice(settings):
3253         try:
3254                 os.nice(int(settings.get("PORTAGE_NICENESS", "0")))
3255         except (OSError, ValueError) as e:
3256                 out = portage.output.EOutput()
3257                 out.eerror("Failed to change nice value to '%s'" % \
3258                         settings["PORTAGE_NICENESS"])
3259                 out.eerror("%s\n" % str(e))
3260
3261 def ionice(settings):
3262
3263         ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND")
3264         if ionice_cmd:
3265                 ionice_cmd = portage.util.shlex_split(ionice_cmd)
3266         if not ionice_cmd:
3267                 return
3268
3269         variables = {"PID" : str(os.getpid())}
3270         cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
3271
3272         try:
3273                 rval = portage.process.spawn(cmd, env=os.environ)
3274         except portage.exception.CommandNotFound:
3275                 # The OS kernel probably doesn't support ionice,
3276                 # so return silently.
3277                 return
3278
3279         if rval != os.EX_OK:
3280                 out = portage.output.EOutput()
3281                 out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,))
3282                 out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.")
3283
3284 def setconfig_fallback(root_config):
3285         setconfig = root_config.setconfig
3286         setconfig._create_default_config()
3287         setconfig._parse(update=True)
3288         root_config.sets = setconfig.getSets()
3289
3290 def get_missing_sets(root_config):
3291         # emerge requires existence of "world", "selected", and "system"
3292         missing_sets = []
3293
3294         for s in ("selected", "system", "world",):
3295                 if s not in root_config.sets:
3296                         missing_sets.append(s)
3297
3298         return missing_sets
3299
3300 def missing_sets_warning(root_config, missing_sets):
3301         if len(missing_sets) > 2:
3302                 missing_sets_str = ", ".join('"%s"' % s for s in missing_sets[:-1])
3303                 missing_sets_str += ', and "%s"' % missing_sets[-1]
3304         elif len(missing_sets) == 2:
3305                 missing_sets_str = '"%s" and "%s"' % tuple(missing_sets)
3306         else:
3307                 missing_sets_str = '"%s"' % missing_sets[-1]
3308         msg = ["emerge: incomplete set configuration, " + \
3309                 "missing set(s): %s" % missing_sets_str]
3310         if root_config.sets:
3311                 msg.append("        sets defined: %s" % ", ".join(root_config.sets))
3312         global_config_path = portage.const.GLOBAL_CONFIG_PATH
3313         if root_config.settings['EPREFIX']:
3314                 global_config_path = os.path.join(root_config.settings['EPREFIX'],
3315                                 portage.const.GLOBAL_CONFIG_PATH.lstrip(os.sep))
3316         msg.append("        This usually means that '%s'" % \
3317                 (os.path.join(global_config_path, "sets/portage.conf"),))
3318         msg.append("        is missing or corrupt.")
3319         msg.append("        Falling back to default world and system set configuration!!!")
3320         for line in msg:
3321                 writemsg_level(line + "\n", level=logging.ERROR, noiselevel=-1)
3322
3323 def ensure_required_sets(trees):
3324         warning_shown = False
3325         for root_trees in trees.values():
3326                 missing_sets = get_missing_sets(root_trees["root_config"])
3327                 if missing_sets and not warning_shown:
3328                         warning_shown = True
3329                         missing_sets_warning(root_trees["root_config"], missing_sets)
3330                 if missing_sets:
3331                         setconfig_fallback(root_trees["root_config"])
3332
3333 def expand_set_arguments(myfiles, myaction, root_config):
3334         retval = os.EX_OK
3335         setconfig = root_config.setconfig
3336
3337         sets = setconfig.getSets()
3338
3339         # In order to know exactly which atoms/sets should be added to the
3340         # world file, the depgraph performs set expansion later. It will get
3341         # confused about where the atoms came from if it's not allowed to
3342         # expand them itself.
3343         do_not_expand = myaction is None
3344         newargs = []
3345         for a in myfiles:
3346                 if a in ("system", "world"):
3347                         newargs.append(SETPREFIX+a)
3348                 else:
3349                         newargs.append(a)
3350         myfiles = newargs
3351         del newargs
3352         newargs = []
3353
3354         # separators for set arguments
3355         ARG_START = "{"
3356         ARG_END = "}"
3357
3358         for i in range(0, len(myfiles)):
3359                 if myfiles[i].startswith(SETPREFIX):
3360                         start = 0
3361                         end = 0
3362                         x = myfiles[i][len(SETPREFIX):]
3363                         newset = ""
3364                         while x:
3365                                 start = x.find(ARG_START)
3366                                 end = x.find(ARG_END)
3367                                 if start > 0 and start < end:
3368                                         namepart = x[:start]
3369                                         argpart = x[start+1:end]
3370
3371                                         # TODO: implement proper quoting
3372                                         args = argpart.split(",")
3373                                         options = {}
3374                                         for a in args:
3375                                                 if "=" in a:
3376                                                         k, v  = a.split("=", 1)
3377                                                         options[k] = v
3378                                                 else:
3379                                                         options[a] = "True"
3380                                         setconfig.update(namepart, options)
3381                                         newset += (x[:start-len(namepart)]+namepart)
3382                                         x = x[end+len(ARG_END):]
3383                                 else:
3384                                         newset += x
3385                                         x = ""
3386                         myfiles[i] = SETPREFIX+newset
3387
3388         sets = setconfig.getSets()
3389
3390         # display errors that occurred while loading the SetConfig instance
3391         for e in setconfig.errors:
3392                 print(colorize("BAD", "Error during set creation: %s" % e))
3393
3394         unmerge_actions = ("unmerge", "prune", "clean", "depclean")
3395
3396         for a in myfiles:
3397                 if a.startswith(SETPREFIX):
3398                                 s = a[len(SETPREFIX):]
3399                                 if s not in sets:
3400                                         display_missing_pkg_set(root_config, s)
3401                                         return (None, 1)
3402                                 if s == "installed":
3403                                         msg = ("The @installed set is deprecated and will soon be "
3404                                         "removed. Please refer to bug #387059 for details.")
3405                                         out = portage.output.EOutput()
3406                                         for line in textwrap.wrap(msg, 50):
3407                                                 out.ewarn(line)
3408                                 setconfig.active.append(s)
3409
3410                                 if do_not_expand:
3411                                         # Loading sets can be slow, so skip it here, in order
3412                                         # to allow the depgraph to indicate progress with the
3413                                         # spinner while sets are loading (bug #461412).
3414                                         newargs.append(a)
3415                                         continue
3416
3417                                 try:
3418                                         set_atoms = setconfig.getSetAtoms(s)
3419                                 except portage.exception.PackageSetNotFound as e:
3420                                         writemsg_level(("emerge: the given set '%s' " + \
3421                                                 "contains a non-existent set named '%s'.\n") % \
3422                                                 (s, e), level=logging.ERROR, noiselevel=-1)
3423                                         if s in ('world', 'selected') and \
3424                                                 SETPREFIX + e.value in sets['selected']:
3425                                                 writemsg_level(("Use `emerge --deselect %s%s` to "
3426                                                         "remove this set from world_sets.\n") %
3427                                                         (SETPREFIX, e,), level=logging.ERROR,
3428                                                         noiselevel=-1)
3429                                         return (None, 1)
3430                                 if myaction in unmerge_actions and \
3431                                                 not sets[s].supportsOperation("unmerge"):
3432                                         sys.stderr.write("emerge: the given set '%s' does " % s + \
3433                                                 "not support unmerge operations\n")
3434                                         retval = 1
3435                                 elif not set_atoms:
3436                                         print("emerge: '%s' is an empty set" % s)
3437                                 else:
3438                                         newargs.extend(set_atoms)
3439                                 for e in sets[s].errors:
3440                                         print(e)
3441                 else:
3442                         newargs.append(a)
3443         return (newargs, retval)
3444
3445 def repo_name_check(trees):
3446         missing_repo_names = set()
3447         for root_trees in trees.values():
3448                 porttree = root_trees.get("porttree")
3449                 if porttree:
3450                         portdb = porttree.dbapi
3451                         missing_repo_names.update(portdb.getMissingRepoNames())
3452                         if portdb.porttree_root in missing_repo_names and \
3453                                 not os.path.exists(os.path.join(
3454                                 portdb.porttree_root, "profiles")):
3455                                 # This is normal if $PORTDIR happens to be empty,
3456                                 # so don't warn about it.
3457                                 missing_repo_names.remove(portdb.porttree_root)
3458
3459         if missing_repo_names:
3460                 msg = []
3461                 msg.append("WARNING: One or more repositories " + \
3462                         "have missing repo_name entries:")
3463                 msg.append("")
3464                 for p in missing_repo_names:
3465                         msg.append("\t%s/profiles/repo_name" % (p,))
3466                 msg.append("")
3467                 msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \
3468                         "should be a plain text file containing a unique " + \
3469                         "name for the repository on the first line.", 70))
3470                 msg.append("\n")
3471                 writemsg_level("".join("%s\n" % l for l in msg),
3472                         level=logging.WARNING, noiselevel=-1)
3473
3474         return bool(missing_repo_names)
3475
3476 def repo_name_duplicate_check(trees):
3477         ignored_repos = {}
3478         for root, root_trees in trees.items():
3479                 if 'porttree' in root_trees:
3480                         portdb = root_trees['porttree'].dbapi
3481                         if portdb.settings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0':
3482                                 for repo_name, paths in portdb.getIgnoredRepos():
3483                                         k = (root, repo_name, portdb.getRepositoryPath(repo_name))
3484                                         ignored_repos.setdefault(k, []).extend(paths)
3485
3486         if ignored_repos:
3487                 msg = []
3488                 msg.append('WARNING: One or more repositories ' + \
3489                         'have been ignored due to duplicate')
3490                 msg.append('  profiles/repo_name entries:')
3491                 msg.append('')
3492                 for k in sorted(ignored_repos):
3493                         msg.append('  %s overrides' % ", ".join(k))
3494                         for path in ignored_repos[k]:
3495                                 msg.append('    %s' % (path,))
3496                         msg.append('')
3497                 msg.extend('  ' + x for x in textwrap.wrap(
3498                         "All profiles/repo_name entries must be unique in order " + \
3499                         "to avoid having duplicates ignored. " + \
3500                         "Set PORTAGE_REPO_DUPLICATE_WARN=\"0\" in " + \
3501                         "/etc/make.conf if you would like to disable this warning."))
3502                 msg.append("\n")
3503                 writemsg_level(''.join('%s\n' % l for l in msg),
3504                         level=logging.WARNING, noiselevel=-1)
3505
3506         return bool(ignored_repos)
3507
3508 def run_action(settings, trees, mtimedb, myaction, myopts, myfiles,
3509         gc_locals=None):
3510
3511         # The caller may have its local variables garbage collected, so
3512         # they don't consume any memory during this long-running function.
3513         if gc_locals is not None:
3514                 gc_locals()
3515                 gc_locals = None
3516
3517         # skip global updates prior to sync, since it's called after sync
3518         if myaction not in ('help', 'info', 'sync', 'version') and \
3519                 myopts.get('--package-moves') != 'n' and \
3520                 _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)):
3521                 mtimedb.commit()
3522                 # Reload the whole config from scratch.
3523                 settings, trees, mtimedb = load_emerge_config(trees=trees)
3524
3525         xterm_titles = "notitles" not in settings.features
3526         if xterm_titles:
3527                 xtermTitle("emerge")
3528
3529         if "--digest" in myopts:
3530                 os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest"
3531                 # Reload the whole config from scratch so that the portdbapi internal
3532                 # config is updated with new FEATURES.
3533                 settings, trees, mtimedb = load_emerge_config(trees=trees)
3534
3535         # NOTE: adjust_configs() can map options to FEATURES, so any relevant
3536         # options adjustments should be made prior to calling adjust_configs().
3537         if "--buildpkgonly" in myopts:
3538                 myopts["--buildpkg"] = True
3539
3540         if "getbinpkg" in settings.features:
3541                 myopts["--getbinpkg"] = True
3542
3543         if "--getbinpkgonly" in myopts:
3544                 myopts["--getbinpkg"] = True
3545
3546         if "--getbinpkgonly" in myopts:
3547                 myopts["--usepkgonly"] = True
3548
3549         if "--getbinpkg" in myopts:
3550                 myopts["--usepkg"] = True
3551
3552         if "--usepkgonly" in myopts:
3553                 myopts["--usepkg"] = True
3554
3555         if "--buildpkgonly" in myopts:
3556                 # --buildpkgonly will not merge anything, so
3557                 # it cancels all binary package options.
3558                 for opt in ("--getbinpkg", "--getbinpkgonly",
3559                         "--usepkg", "--usepkgonly"):
3560                         myopts.pop(opt, None)
3561
3562         adjust_configs(myopts, trees)
3563         apply_priorities(settings)
3564
3565         if myaction == 'version':
3566                 writemsg_stdout(getportageversion(
3567                         settings["PORTDIR"], None,
3568                         settings.profile_path, settings["CHOST"],
3569                         trees[settings['EROOT']]['vartree'].dbapi) + '\n', noiselevel=-1)
3570                 return 0
3571         elif myaction == 'help':
3572                 emerge_help()
3573                 return 0
3574
3575         spinner = stdout_spinner()
3576         if "candy" in settings.features:
3577                 spinner.update = spinner.update_scroll
3578
3579         if "--quiet" not in myopts:
3580                 portage.deprecated_profile_check(settings=settings)
3581                 if portage.const._ENABLE_REPO_NAME_WARN:
3582                         # Bug #248603 - Disable warnings about missing
3583                         # repo_name entries for stable branch.
3584                         repo_name_check(trees)
3585                 repo_name_duplicate_check(trees)
3586                 config_protect_check(trees)
3587         check_procfs()
3588
3589         for mytrees in trees.values():
3590                 mydb = mytrees["porttree"].dbapi
3591                 # Freeze the portdbapi for performance (memoize all xmatch results).
3592                 mydb.freeze()
3593
3594                 if myaction in ('search', None) and \
3595                         "--usepkg" in myopts:
3596                         # Populate the bintree with current --getbinpkg setting.
3597                         # This needs to happen before expand_set_arguments(), in case
3598                         # any sets use the bintree.
3599                         mytrees["bintree"].populate(
3600                                 getbinpkgs="--getbinpkg" in myopts)
3601
3602         del mytrees, mydb
3603
3604         for x in myfiles:
3605                 if x.endswith((".ebuild", ".tbz2")) and \
3606                         os.path.exists(os.path.abspath(x)):
3607                         print(colorize("BAD", "\n*** emerging by path is broken "
3608                                 "and may not always work!!!\n"))
3609                         break
3610
3611         root_config = trees[settings['EROOT']]['root_config']
3612
3613         if myaction == "list-sets":
3614                 writemsg_stdout("".join("%s\n" % s for s in sorted(root_config.sets)))
3615                 return os.EX_OK
3616         elif myaction == "check-news":
3617                 news_counts = count_unread_news(
3618                         root_config.trees["porttree"].dbapi,
3619                         root_config.trees["vartree"].dbapi)
3620                 if any(news_counts.values()):
3621                         display_news_notifications(news_counts)
3622                 elif "--quiet" not in myopts:
3623                         print("", colorize("GOOD", "*"), "No news items were found.")
3624                 return os.EX_OK
3625
3626         ensure_required_sets(trees)
3627
3628         # only expand sets for actions taking package arguments
3629         oldargs = myfiles[:]
3630         if myaction in ("clean", "config", "depclean",
3631                 "info", "prune", "unmerge", None):
3632                 myfiles, retval = expand_set_arguments(myfiles, myaction, root_config)
3633                 if retval != os.EX_OK:
3634                         return retval
3635
3636                 # Need to handle empty sets specially, otherwise emerge will react
3637                 # with the help message for empty argument lists
3638                 if oldargs and not myfiles:
3639                         print("emerge: no targets left after set expansion")
3640                         return 0
3641
3642         if ("--tree" in myopts) and ("--columns" in myopts):
3643                 print("emerge: can't specify both of \"--tree\" and \"--columns\".")
3644                 return 1
3645
3646         if '--emptytree' in myopts and '--noreplace' in myopts:
3647                 writemsg_level("emerge: can't specify both of " + \
3648                         "\"--emptytree\" and \"--noreplace\".\n",
3649                         level=logging.ERROR, noiselevel=-1)
3650                 return 1
3651
3652         if ("--quiet" in myopts):
3653                 spinner.update = spinner.update_quiet
3654                 portage.util.noiselimit = -1
3655
3656         if "--fetch-all-uri" in myopts:
3657                 myopts["--fetchonly"] = True
3658
3659         if "--skipfirst" in myopts and "--resume" not in myopts:
3660                 myopts["--resume"] = True
3661
3662         # Allow -p to remove --ask
3663         if "--pretend" in myopts:
3664                 myopts.pop("--ask", None)
3665
3666         # forbid --ask when not in a terminal
3667         # note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway.
3668         if ("--ask" in myopts) and (not sys.stdin.isatty()):
3669                 portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n",
3670                         noiselevel=-1)
3671                 return 1
3672
3673         if settings.get("PORTAGE_DEBUG", "") == "1":
3674                 spinner.update = spinner.update_quiet
3675                 portage.util.noiselimit = 0
3676                 if "python-trace" in settings.features:
3677                         portage.debug.set_trace(True)
3678
3679         if not ("--quiet" in myopts):
3680                 if '--nospinner' in myopts or \
3681                         settings.get('TERM') == 'dumb' or \
3682                         not sys.stdout.isatty():
3683                         spinner.update = spinner.update_basic
3684
3685         if "--debug" in myopts:
3686                 print("myaction", myaction)
3687                 print("myopts", myopts)
3688
3689         if not myaction and not myfiles and "--resume" not in myopts:
3690                 emerge_help()
3691                 return 1
3692
3693         pretend = "--pretend" in myopts
3694         fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts
3695         buildpkgonly = "--buildpkgonly" in myopts
3696
3697         # check if root user is the current user for the actions where emerge needs this
3698         if portage.data.secpass < 2:
3699                 # We've already allowed "--version" and "--help" above.
3700                 if "--pretend" not in myopts and myaction not in ("search","info"):
3701                         need_superuser = myaction in ('clean', 'depclean', 'deselect',
3702                                 'prune', 'unmerge') or not \
3703                                 (fetchonly or \
3704                                 (buildpkgonly and portage.data.secpass >= 1) or \
3705                                 myaction in ("metadata", "regen", "sync"))
3706                         if portage.data.secpass < 1 or \
3707                                 need_superuser:
3708                                 if need_superuser:
3709                                         access_desc = "superuser"
3710                                 else:
3711                                         access_desc = "portage group"
3712                                 # Always show portage_group_warning() when only portage group
3713                                 # access is required but the user is not in the portage group.
3714                                 if "--ask" in myopts:
3715                                         writemsg_stdout("This action requires %s access...\n" % \
3716                                                 (access_desc,), noiselevel=-1)
3717                                         if portage.data.secpass < 1 and not need_superuser:
3718                                                 portage.data.portage_group_warning()
3719                                         if userquery("Would you like to add --pretend to options?",
3720                                                 "--ask-enter-invalid" in myopts) == "No":
3721                                                 return 128 + signal.SIGINT
3722                                         myopts["--pretend"] = True
3723                                         myopts.pop("--ask")
3724                                 else:
3725                                         sys.stderr.write(("emerge: %s access is required\n") \
3726                                                 % access_desc)
3727                                         if portage.data.secpass < 1 and not need_superuser:
3728                                                 portage.data.portage_group_warning()
3729                                         return 1
3730
3731         # Disable emergelog for everything except build or unmerge operations.
3732         # This helps minimize parallel emerge.log entries that can confuse log
3733         # parsers like genlop.
3734         disable_emergelog = False
3735         for x in ("--pretend", "--fetchonly", "--fetch-all-uri"):
3736                 if x in myopts:
3737                         disable_emergelog = True
3738                         break
3739         if disable_emergelog:
3740                 pass
3741         elif myaction in ("search", "info"):
3742                 disable_emergelog = True
3743         elif portage.data.secpass < 1:
3744                 disable_emergelog = True
3745
3746         import _emerge.emergelog
3747         _emerge.emergelog._disable = disable_emergelog
3748
3749         if not disable_emergelog:
3750                 if 'EMERGE_LOG_DIR' in settings:
3751                         try:
3752                                 # At least the parent needs to exist for the lock file.
3753                                 portage.util.ensure_dirs(settings['EMERGE_LOG_DIR'])
3754                         except portage.exception.PortageException as e:
3755                                 writemsg_level("!!! Error creating directory for " + \
3756                                         "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \
3757                                         (settings['EMERGE_LOG_DIR'], e),
3758                                         noiselevel=-1, level=logging.ERROR)
3759                                 portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir)
3760                         else:
3761                                 _emerge.emergelog._emerge_log_dir = settings["EMERGE_LOG_DIR"]
3762                 else:
3763                         _emerge.emergelog._emerge_log_dir = os.path.join(os.sep,
3764                                 settings["EPREFIX"].lstrip(os.sep), "var", "log")
3765                         portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir)
3766
3767         if not "--pretend" in myopts:
3768                 time_fmt = "%b %d, %Y %H:%M:%S"
3769                 if sys.hexversion < 0x3000000:
3770                         time_fmt = portage._unicode_encode(time_fmt)
3771                 time_str = time.strftime(time_fmt, time.localtime(time.time()))
3772                 # Avoid potential UnicodeDecodeError in Python 2, since strftime
3773                 # returns bytes in Python 2, and %b may contain non-ascii chars.
3774                 time_str = _unicode_decode(time_str,
3775                         encoding=_encodings['content'], errors='replace')
3776                 emergelog(xterm_titles, "Started emerge on: %s" % time_str)
3777                 myelogstr=""
3778                 if myopts:
3779                         opt_list = []
3780                         for opt, arg in myopts.items():
3781                                 if arg is True:
3782                                         opt_list.append(opt)
3783                                 elif isinstance(arg, list):
3784                                         # arguments like --exclude that use 'append' action
3785                                         for x in arg:
3786                                                 opt_list.append("%s=%s" % (opt, x))
3787                                 else:
3788                                         opt_list.append("%s=%s" % (opt, arg))
3789                         myelogstr=" ".join(opt_list)
3790                 if myaction:
3791                         myelogstr += " --" + myaction
3792                 if myfiles:
3793                         myelogstr += " " + " ".join(oldargs)
3794                 emergelog(xterm_titles, " *** emerge " + myelogstr)
3795
3796         oldargs = None
3797
3798         def emergeexitsig(signum, frame):
3799                 signal.signal(signal.SIGTERM, signal.SIG_IGN)
3800                 portage.util.writemsg(
3801                         "\n\nExiting on signal %(signal)s\n" % {"signal":signum})
3802                 sys.exit(128 + signum)
3803
3804         signal.signal(signal.SIGTERM, emergeexitsig)
3805
3806         def emergeexit():
3807                 """This gets out final log message in before we quit."""
3808                 if "--pretend" not in myopts:
3809                         emergelog(xterm_titles, " *** terminating.")
3810                 if xterm_titles:
3811                         xtermTitleReset()
3812         portage.atexit_register(emergeexit)
3813
3814         if myaction in ("config", "metadata", "regen", "sync"):
3815                 if "--pretend" in myopts:
3816                         sys.stderr.write(("emerge: The '%s' action does " + \
3817                                 "not support '--pretend'.\n") % myaction)
3818                         return 1
3819
3820         if "sync" == myaction:
3821                 return action_sync(settings, trees, mtimedb, myopts, myaction)
3822         elif "metadata" == myaction:
3823                 action_metadata(settings,
3824                         trees[settings['EROOT']]['porttree'].dbapi, myopts)
3825         elif myaction=="regen":
3826                 validate_ebuild_environment(trees)
3827                 return action_regen(settings,
3828                         trees[settings['EROOT']]['porttree'].dbapi, myopts.get("--jobs"),
3829                         myopts.get("--load-average"))
3830         # HELP action
3831         elif "config"==myaction:
3832                 validate_ebuild_environment(trees)
3833                 action_config(settings, trees, myopts, myfiles)
3834
3835         # SEARCH action
3836         elif "search"==myaction:
3837                 validate_ebuild_environment(trees)
3838                 action_search(trees[settings['EROOT']]['root_config'],
3839                         myopts, myfiles, spinner)
3840
3841         elif myaction in ('clean', 'depclean', 'deselect', 'prune', 'unmerge'):
3842                 validate_ebuild_environment(trees)
3843                 rval = action_uninstall(settings, trees, mtimedb["ldpath"],
3844                         myopts, myaction, myfiles, spinner)
3845                 if not (myaction == 'deselect' or
3846                         buildpkgonly or fetchonly or pretend):
3847                         post_emerge(myaction, myopts, myfiles, settings['EROOT'],
3848                                 trees, mtimedb, rval)
3849                 return rval
3850
3851         elif myaction == 'info':
3852
3853                 # Ensure atoms are valid before calling unmerge().
3854                 vardb = trees[settings['EROOT']]['vartree'].dbapi
3855                 portdb = trees[settings['EROOT']]['porttree'].dbapi
3856                 bindb = trees[settings['EROOT']]["bintree"].dbapi
3857                 valid_atoms = []
3858                 for x in myfiles:
3859                         if is_valid_package_atom(x, allow_repo=True):
3860                                 try:
3861                                         #look at the installed files first, if there is no match
3862                                         #look at the ebuilds, since EAPI 4 allows running pkg_info
3863                                         #on non-installed packages
3864                                         valid_atom = dep_expand(x, mydb=vardb, settings=settings)
3865                                         if valid_atom.cp.split("/")[0] == "null":
3866                                                 valid_atom = dep_expand(x,
3867                                                         mydb=portdb, settings=settings)
3868
3869                                         if valid_atom.cp.split("/")[0] == "null" and \
3870                                                 "--usepkg" in myopts:
3871                                                 valid_atom = dep_expand(x,
3872                                                         mydb=bindb, settings=settings)
3873
3874                                         valid_atoms.append(valid_atom)
3875
3876                                 except portage.exception.AmbiguousPackageName as e:
3877                                         msg = "The short ebuild name \"" + x + \
3878                                                 "\" is ambiguous.  Please specify " + \
3879                                                 "one of the following " + \
3880                                                 "fully-qualified ebuild names instead:"
3881                                         for line in textwrap.wrap(msg, 70):
3882                                                 writemsg_level("!!! %s\n" % (line,),
3883                                                         level=logging.ERROR, noiselevel=-1)
3884                                         for i in e.args[0]:
3885                                                 writemsg_level("    %s\n" % colorize("INFORM", i),
3886                                                         level=logging.ERROR, noiselevel=-1)
3887                                         writemsg_level("\n", level=logging.ERROR, noiselevel=-1)
3888                                         return 1
3889                                 continue
3890                         msg = []
3891                         msg.append("'%s' is not a valid package atom." % (x,))
3892                         msg.append("Please check ebuild(5) for full details.")
3893                         writemsg_level("".join("!!! %s\n" % line for line in msg),
3894                                 level=logging.ERROR, noiselevel=-1)
3895                         return 1
3896
3897                 return action_info(settings, trees, myopts, valid_atoms)
3898
3899         # "update", "system", or just process files:
3900         else:
3901                 validate_ebuild_environment(trees)
3902
3903                 for x in myfiles:
3904                         if x.startswith(SETPREFIX) or \
3905                                 is_valid_package_atom(x, allow_repo=True):
3906                                 continue
3907                         if x[:1] == os.sep:
3908                                 continue
3909                         try:
3910                                 os.lstat(x)
3911                                 continue
3912                         except OSError:
3913                                 pass
3914                         msg = []
3915                         msg.append("'%s' is not a valid package atom." % (x,))
3916                         msg.append("Please check ebuild(5) for full details.")
3917                         writemsg_level("".join("!!! %s\n" % line for line in msg),
3918                                 level=logging.ERROR, noiselevel=-1)
3919                         return 1
3920
3921                 # GLEP 42 says to display news *after* an emerge --pretend
3922                 if "--pretend" not in myopts:
3923                         display_news_notification(root_config, myopts)
3924                 retval = action_build(settings, trees, mtimedb,
3925                         myopts, myaction, myfiles, spinner)
3926                 post_emerge(myaction, myopts, myfiles, settings['EROOT'],
3927                         trees, mtimedb, retval)
3928
3929                 return retval