pym/_emerge/__init__.py

   1 #!/usr/bin/python -O
   2 # Copyright 1999-2006 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # $Id$
   5
   6 import logging
   7 import pwd
   8 import shlex
   9 import signal
  10 import sys
  11 import textwrap
  12 import os, stat
  13 import platform
  14
  15 try:
  16         import portage
  17 except ImportError:
  18         from os import path as osp
  19         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  20         import portage
  21
  22 from portage import digraph
  23 from portage.const import NEWS_LIB_PATH
  24
  25 import _emerge.help
  26 import portage.xpak, commands, errno, re, socket, time
  27 from portage.output import blue, bold, colorize, darkgreen, \
  28         red, xtermTitleReset, yellow
  29 from portage.output import create_color_func
  30 good = create_color_func("GOOD")
  31 bad = create_color_func("BAD")
  32
  33 import portage.elog
  34 import portage.dep
  35 portage.dep._dep_check_strict = True
  36 import portage.util
  37 import portage.locks
  38 import portage.exception
  39 from portage.cache.cache_errors import CacheError
  40 from portage.data import secpass
  41 from portage.util import normalize_path as normpath
  42 from portage.util import cmp_sort_key, writemsg, writemsg_level
  43 from portage.sets import load_default_config, SETPREFIX
  44 from portage.sets.base import InternalPackageSet
  45
  46 from itertools import chain, izip
  47
  48 from _emerge.clear_caches import clear_caches
  49 from _emerge.countdown import countdown
  50 from _emerge.create_depgraph_params import create_depgraph_params
  51 from _emerge.Dependency import Dependency
  52 from _emerge.depgraph import depgraph, resume_depgraph
  53 from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange
  54 from _emerge.emergelog import emergelog
  55 from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo
  56 from _emerge.is_valid_package_atom import is_valid_package_atom
  57 from _emerge.MetadataRegen import MetadataRegen
  58 from _emerge.Package import Package
  59 from _emerge.ProgressHandler import ProgressHandler
  60 from _emerge.RootConfig import RootConfig
  61 from _emerge.Scheduler import Scheduler
  62 from _emerge.search import search
  63 from _emerge.SetArg import SetArg
  64 from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice
  65 from _emerge.stdout_spinner import stdout_spinner
  66 from _emerge.unmerge import unmerge
  67 from _emerge.UnmergeDepPriority import UnmergeDepPriority
  68 from _emerge.UseFlagDisplay import UseFlagDisplay
  69 from _emerge.userquery import userquery
  70
  71
  72 actions = frozenset([
  73 "clean", "config", "depclean",
  74 "info", "list-sets", "metadata",
  75 "prune", "regen",  "search",
  76 "sync",  "unmerge", "version",
  77 ])
  78 options=[
  79 "--ask",          "--alphabetical",
  80 "--buildpkg",     "--buildpkgonly",
  81 "--changelog",    "--columns",
  82 "--complete-graph",
  83 "--debug",        "--deep",
  84 "--digest",
  85 "--emptytree",
  86 "--fetchonly",    "--fetch-all-uri",
  87 "--getbinpkg",    "--getbinpkgonly",
  88 "--help",         "--ignore-default-opts",
  89 "--keep-going",
  90 "--noconfmem",
  91 "--newuse",
  92 "--nodeps",       "--noreplace",
  93 "--nospinner",    "--oneshot",
  94 "--onlydeps",     "--pretend",
  95 "--quiet",        "--resume",
  96 "--searchdesc",   "--selective",
  97 "--skipfirst",
  98 "--tree",
  99 "--update",
 100 "--usepkg",       "--usepkgonly",
 101 "--verbose",
 102 ]
 103
 104 shortmapping={
 105 "1":"--oneshot",
 106 "a":"--ask",
 107 "b":"--buildpkg",  "B":"--buildpkgonly",
 108 "c":"--clean",     "C":"--unmerge",
 109 "d":"--debug",     "D":"--deep",
 110 "e":"--emptytree",
 111 "f":"--fetchonly", "F":"--fetch-all-uri",
 112 "g":"--getbinpkg", "G":"--getbinpkgonly",
 113 "h":"--help",
 114 "k":"--usepkg",    "K":"--usepkgonly",
 115 "l":"--changelog",
 116 "n":"--noreplace", "N":"--newuse",
 117 "o":"--onlydeps",  "O":"--nodeps",
 118 "p":"--pretend",   "P":"--prune",
 119 "q":"--quiet",
 120 "s":"--search",    "S":"--searchdesc",
 121 "t":"--tree",
 122 "u":"--update",
 123 "v":"--verbose",   "V":"--version"
 124 }
 125
 126 def getgccversion(chost):
 127         """
 128         rtype: C{str}
 129         return:  the current in-use gcc version
 130         """
 131
 132         gcc_ver_command = 'gcc -dumpversion'
 133         gcc_ver_prefix = 'gcc-'
 134
 135         gcc_not_found_error = red(
 136         "!!! No gcc found. You probably need to 'source /etc/profile'\n" +
 137         "!!! to update the environment of this terminal and possibly\n" +
 138         "!!! other terminals also.\n"
 139         )
 140
 141         mystatus, myoutput = commands.getstatusoutput("gcc-config -c")
 142         if mystatus == os.EX_OK and myoutput.startswith(chost + "-"):
 143                 return myoutput.replace(chost + "-", gcc_ver_prefix, 1)
 144
 145         mystatus, myoutput = commands.getstatusoutput(
 146                 chost + "-" + gcc_ver_command)
 147         if mystatus == os.EX_OK:
 148                 return gcc_ver_prefix + myoutput
 149
 150         mystatus, myoutput = commands.getstatusoutput(gcc_ver_command)
 151         if mystatus == os.EX_OK:
 152                 return gcc_ver_prefix + myoutput
 153
 154         portage.writemsg(gcc_not_found_error, noiselevel=-1)
 155         return "[unavailable]"
 156
 157 def getportageversion(portdir, target_root, profile, chost, vardb):
 158         profilever = "unavailable"
 159         if profile:
 160                 realpath = os.path.realpath(profile)
 161                 basepath   = os.path.realpath(os.path.join(portdir, "profiles"))
 162                 if realpath.startswith(basepath):
 163                         profilever = realpath[1 + len(basepath):]
 164                 else:
 165                         try:
 166                                 profilever = "!" + os.readlink(profile)
 167                         except (OSError):
 168                                 pass
 169                 del realpath, basepath
 170
 171         libcver=[]
 172         libclist  = vardb.match("virtual/libc")
 173         libclist += vardb.match("virtual/glibc")
 174         libclist  = portage.util.unique_array(libclist)
 175         for x in libclist:
 176                 xs=portage.catpkgsplit(x)
 177                 if libcver:
 178                         libcver+=","+"-".join(xs[1:])
 179                 else:
 180                         libcver="-".join(xs[1:])
 181         if libcver==[]:
 182                 libcver="unavailable"
 183
 184         gccver = getgccversion(chost)
 185         unameout=platform.release()+" "+platform.machine()
 186
 187         return "Portage " + portage.VERSION +" ("+profilever+", "+gccver+", "+libcver+", "+unameout+")"
 188
 189 def chk_updated_info_files(root, infodirs, prev_mtimes, retval):
 190
 191         if os.path.exists("/usr/bin/install-info"):
 192                 out = portage.output.EOutput()
 193                 regen_infodirs=[]
 194                 for z in infodirs:
 195                         if z=='':
 196                                 continue
 197                         inforoot=normpath(root+z)
 198                         if os.path.isdir(inforoot):
 199                                 infomtime = long(os.stat(inforoot).st_mtime)
 200                                 if inforoot not in prev_mtimes or \
 201                                         prev_mtimes[inforoot] != infomtime:
 202                                                 regen_infodirs.append(inforoot)
 203
 204                 if not regen_infodirs:
 205                         portage.writemsg_stdout("\n")
 206                         out.einfo("GNU info directory index is up-to-date.")
 207                 else:
 208                         portage.writemsg_stdout("\n")
 209                         out.einfo("Regenerating GNU info directory index...")
 210
 211                         dir_extensions = ("", ".gz", ".bz2")
 212                         icount=0
 213                         badcount=0
 214                         errmsg = ""
 215                         for inforoot in regen_infodirs:
 216                                 if inforoot=='':
 217                                         continue
 218
 219                                 if not os.path.isdir(inforoot) or \
 220                                         not os.access(inforoot, os.W_OK):
 221                                         continue
 222
 223                                 file_list = os.listdir(inforoot)
 224                                 file_list.sort()
 225                                 dir_file = os.path.join(inforoot, "dir")
 226                                 moved_old_dir = False
 227                                 processed_count = 0
 228                                 for x in file_list:
 229                                         if x.startswith(".") or \
 230                                                 os.path.isdir(os.path.join(inforoot, x)):
 231                                                 continue
 232                                         if x.startswith("dir"):
 233                                                 skip = False
 234                                                 for ext in dir_extensions:
 235                                                         if x == "dir" + ext or \
 236                                                                 x == "dir" + ext + ".old":
 237                                                                 skip = True
 238                                                                 break
 239                                                 if skip:
 240                                                         continue
 241                                         if processed_count == 0:
 242                                                 for ext in dir_extensions:
 243                                                         try:
 244                                                                 os.rename(dir_file + ext, dir_file + ext + ".old")
 245                                                                 moved_old_dir = True
 246                                                         except EnvironmentError, e:
 247                                                                 if e.errno != errno.ENOENT:
 248                                                                         raise
 249                                                                 del e
 250                                         processed_count += 1
 251                                         myso=commands.getstatusoutput("LANG=C LANGUAGE=C /usr/bin/install-info --dir-file="+inforoot+"/dir "+inforoot+"/"+x)[1]
 252                                         existsstr="already exists, for file `"
 253                                         if myso!="":
 254                                                 if re.search(existsstr,myso):
 255                                                         # Already exists... Don't increment the count for this.
 256                                                         pass
 257                                                 elif myso[:44]=="install-info: warning: no info dir entry in ":
 258                                                         # This info file doesn't contain a DIR-header: install-info produces this
 259                                                         # (harmless) warning (the --quiet switch doesn't seem to work).
 260                                                         # Don't increment the count for this.
 261                                                         pass
 262                                                 else:
 263                                                         badcount=badcount+1
 264                                                         errmsg += myso + "\n"
 265                                         icount=icount+1
 266
 267                                 if moved_old_dir and not os.path.exists(dir_file):
 268                                         # We didn't generate a new dir file, so put the old file
 269                                         # back where it was originally found.
 270                                         for ext in dir_extensions:
 271                                                 try:
 272                                                         os.rename(dir_file + ext + ".old", dir_file + ext)
 273                                                 except EnvironmentError, e:
 274                                                         if e.errno != errno.ENOENT:
 275                                                                 raise
 276                                                         del e
 277
 278                                 # Clean dir.old cruft so that they don't prevent
 279                                 # unmerge of otherwise empty directories.
 280                                 for ext in dir_extensions:
 281                                         try:
 282                                                 os.unlink(dir_file + ext + ".old")
 283                                         except EnvironmentError, e:
 284                                                 if e.errno != errno.ENOENT:
 285                                                         raise
 286                                                 del e
 287
 288                                 #update mtime so we can potentially avoid regenerating.
 289                                 prev_mtimes[inforoot] = long(os.stat(inforoot).st_mtime)
 290
 291                         if badcount:
 292                                 out.eerror("Processed %d info files; %d errors." % \
 293                                         (icount, badcount))
 294                                 writemsg_level(errmsg, level=logging.ERROR, noiselevel=-1)
 295                         else:
 296                                 if icount > 0:
 297                                         out.einfo("Processed %d info files." % (icount,))
 298
 299
 300 def display_news_notification(root_config, myopts):
 301         target_root = root_config.root
 302         trees = root_config.trees
 303         settings = trees["vartree"].settings
 304         portdb = trees["porttree"].dbapi
 305         vardb = trees["vartree"].dbapi
 306         NEWS_PATH = os.path.join("metadata", "news")
 307         UNREAD_PATH = os.path.join(target_root, NEWS_LIB_PATH, "news")
 308         newsReaderDisplay = False
 309         update = "--pretend" not in myopts
 310
 311         for repo in portdb.getRepositories():
 312                 unreadItems = checkUpdatedNewsItems(
 313                         portdb, vardb, NEWS_PATH, UNREAD_PATH, repo, update=update)
 314                 if unreadItems:
 315                         if not newsReaderDisplay:
 316                                 newsReaderDisplay = True
 317                                 print
 318                         print colorize("WARN", " * IMPORTANT:"),
 319                         print "%s news items need reading for repository '%s'." % (unreadItems, repo)
 320
 321
 322         if newsReaderDisplay:
 323                 print colorize("WARN", " *"),
 324                 print "Use " + colorize("GOOD", "eselect news") + " to read news items."
 325                 print
 326
 327 def display_preserved_libs(vardbapi):
 328         MAX_DISPLAY = 3
 329
 330         # Ensure the registry is consistent with existing files.
 331         vardbapi.plib_registry.pruneNonExisting()
 332
 333         if vardbapi.plib_registry.hasEntries():
 334                 print
 335                 print colorize("WARN", "!!!") + " existing preserved libs:"
 336                 plibdata = vardbapi.plib_registry.getPreservedLibs()
 337                 linkmap = vardbapi.linkmap
 338                 consumer_map = {}
 339                 owners = {}
 340                 linkmap_broken = False
 341
 342                 try:
 343                         linkmap.rebuild()
 344                 except portage.exception.CommandNotFound, e:
 345                         writemsg_level("!!! Command Not Found: %s\n" % (e,),
 346                                 level=logging.ERROR, noiselevel=-1)
 347                         del e
 348                         linkmap_broken = True
 349                 else:
 350                         search_for_owners = set()
 351                         for cpv in plibdata:
 352                                 internal_plib_keys = set(linkmap._obj_key(f) \
 353                                         for f in plibdata[cpv])
 354                                 for f in plibdata[cpv]:
 355                                         if f in consumer_map:
 356                                                 continue
 357                                         consumers = []
 358                                         for c in linkmap.findConsumers(f):
 359                                                 # Filter out any consumers that are also preserved libs
 360                                                 # belonging to the same package as the provider.
 361                                                 if linkmap._obj_key(c) not in internal_plib_keys:
 362                                                         consumers.append(c)
 363                                         consumers.sort()
 364                                         consumer_map[f] = consumers
 365                                         search_for_owners.update(consumers[:MAX_DISPLAY+1])
 366
 367                         owners = vardbapi._owners.getFileOwnerMap(search_for_owners)
 368
 369                 for cpv in plibdata:
 370                         print colorize("WARN", ">>>") + " package: %s" % cpv
 371                         samefile_map = {}
 372                         for f in plibdata[cpv]:
 373                                 obj_key = linkmap._obj_key(f)
 374                                 alt_paths = samefile_map.get(obj_key)
 375                                 if alt_paths is None:
 376                                         alt_paths = set()
 377                                         samefile_map[obj_key] = alt_paths
 378                                 alt_paths.add(f)
 379
 380                         for alt_paths in samefile_map.itervalues():
 381                                 alt_paths = sorted(alt_paths)
 382                                 for p in alt_paths:
 383                                         print colorize("WARN", " * ") + " - %s" % (p,)
 384                                 f = alt_paths[0]
 385                                 consumers = consumer_map.get(f, [])
 386                                 for c in consumers[:MAX_DISPLAY]:
 387                                         print colorize("WARN", " * ") + "     used by %s (%s)" % \
 388                                                 (c, ", ".join(x.mycpv for x in owners.get(c, [])))
 389                                 if len(consumers) == MAX_DISPLAY + 1:
 390                                         print colorize("WARN", " * ") + "     used by %s (%s)" % \
 391                                                 (consumers[MAX_DISPLAY], ", ".join(x.mycpv \
 392                                                 for x in owners.get(consumers[MAX_DISPLAY], [])))
 393                                 elif len(consumers) > MAX_DISPLAY:
 394                                         print colorize("WARN", " * ") + "     used by %d other files" % (len(consumers) - MAX_DISPLAY)
 395                 print "Use " + colorize("GOOD", "emerge @preserved-rebuild") + " to rebuild packages using these libraries"
 396
 397
 398 def post_emerge(root_config, myopts, mtimedb, retval):
 399         """
 400         Misc. things to run at the end of a merge session.
 401
 402         Update Info Files
 403         Update Config Files
 404         Update News Items
 405         Commit mtimeDB
 406         Display preserved libs warnings
 407         Exit Emerge
 408
 409         @param trees: A dictionary mapping each ROOT to it's package databases
 410         @type trees: dict
 411         @param mtimedb: The mtimeDB to store data needed across merge invocations
 412         @type mtimedb: MtimeDB class instance
 413         @param retval: Emerge's return value
 414         @type retval: Int
 415         @rype: None
 416         @returns:
 417         1.  Calls sys.exit(retval)
 418         """
 419
 420         target_root = root_config.root
 421         trees = { target_root : root_config.trees }
 422         vardbapi = trees[target_root]["vartree"].dbapi
 423         settings = vardbapi.settings
 424         info_mtimes = mtimedb["info"]
 425
 426         # Load the most current variables from ${ROOT}/etc/profile.env
 427         settings.unlock()
 428         settings.reload()
 429         settings.regenerate()
 430         settings.lock()
 431
 432         config_protect = settings.get("CONFIG_PROTECT","").split()
 433         infodirs = settings.get("INFOPATH","").split(":") + \
 434                 settings.get("INFODIR","").split(":")
 435
 436         os.chdir("/")
 437
 438         if retval == os.EX_OK:
 439                 exit_msg = " *** exiting successfully."
 440         else:
 441                 exit_msg = " *** exiting unsuccessfully with status '%s'." % retval
 442         emergelog("notitles" not in settings.features, exit_msg)
 443
 444         _flush_elog_mod_echo()
 445
 446         counter_hash = settings.get("PORTAGE_COUNTER_HASH")
 447         if "--pretend" in myopts or (counter_hash is not None and \
 448                 counter_hash == vardbapi._counter_hash()):
 449                 display_news_notification(root_config, myopts)
 450                 # If vdb state has not changed then there's nothing else to do.
 451                 sys.exit(retval)
 452
 453         vdb_path = os.path.join(target_root, portage.VDB_PATH)
 454         portage.util.ensure_dirs(vdb_path)
 455         vdb_lock = None
 456         if os.access(vdb_path, os.W_OK) and not "--pretend" in myopts:
 457                 vdb_lock = portage.locks.lockdir(vdb_path)
 458
 459         if vdb_lock:
 460                 try:
 461                         if "noinfo" not in settings.features:
 462                                 chk_updated_info_files(target_root,
 463                                         infodirs, info_mtimes, retval)
 464                         mtimedb.commit()
 465                 finally:
 466                         if vdb_lock:
 467                                 portage.locks.unlockdir(vdb_lock)
 468
 469         chk_updated_cfg_files(target_root, config_protect)
 470
 471         display_news_notification(root_config, myopts)
 472         if retval in (None, os.EX_OK) or (not "--pretend" in myopts):
 473                 display_preserved_libs(vardbapi)
 474
 475         sys.exit(retval)
 476
 477
 478 def chk_updated_cfg_files(target_root, config_protect):
 479         if config_protect:
 480                 #number of directories with some protect files in them
 481                 procount=0
 482                 for x in config_protect:
 483                         x = os.path.join(target_root, x.lstrip(os.path.sep))
 484                         if not os.access(x, os.W_OK):
 485                                 # Avoid Permission denied errors generated
 486                                 # later by `find`.
 487                                 continue
 488                         try:
 489                                 mymode = os.lstat(x).st_mode
 490                         except OSError:
 491                                 continue
 492                         if stat.S_ISLNK(mymode):
 493                                 # We want to treat it like a directory if it
 494                                 # is a symlink to an existing directory.
 495                                 try:
 496                                         real_mode = os.stat(x).st_mode
 497                                         if stat.S_ISDIR(real_mode):
 498                                                 mymode = real_mode
 499                                 except OSError:
 500                                         pass
 501                         if stat.S_ISDIR(mymode):
 502                                 mycommand = "find '%s' -name '.*' -type d -prune -o -name '._cfg????_*'" % x
 503                         else:
 504                                 mycommand = "find '%s' -maxdepth 1 -name '._cfg????_%s'" % \
 505                                         os.path.split(x.rstrip(os.path.sep))
 506                         mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0"
 507                         a = commands.getstatusoutput(mycommand)
 508                         if a[0] != 0:
 509                                 sys.stderr.write(" %s error scanning '%s': " % (bad("*"), x))
 510                                 sys.stderr.flush()
 511                                 # Show the error message alone, sending stdout to /dev/null.
 512                                 os.system(mycommand + " 1>/dev/null")
 513                         else:
 514                                 files = a[1].split('\0')
 515                                 # split always produces an empty string as the last element
 516                                 if files and not files[-1]:
 517                                         del files[-1]
 518                                 if files:
 519                                         procount += 1
 520                                         print "\n"+colorize("WARN", " * IMPORTANT:"),
 521                                         if stat.S_ISDIR(mymode):
 522                                                  print "%d config files in '%s' need updating." % \
 523                                                         (len(files), x)
 524                                         else:
 525                                                  print "config file '%s' needs updating." % x
 526
 527                 if procount:
 528                         print " "+yellow("*")+" See the "+colorize("INFORM","CONFIGURATION FILES")+ \
 529                                 " section of the " + bold("emerge")
 530                         print " "+yellow("*")+" man page to learn how to update config files."
 531
 532 def checkUpdatedNewsItems(portdb, vardb, NEWS_PATH, UNREAD_PATH, repo_id,
 533         update=False):
 534         """
 535         Examines news items in repodir + '/' + NEWS_PATH and attempts to find unread items
 536         Returns the number of unread (yet relevent) items.
 537
 538         @param portdb: a portage tree database
 539         @type portdb: pordbapi
 540         @param vardb: an installed package database
 541         @type vardb: vardbapi
 542         @param NEWS_PATH:
 543         @type NEWS_PATH:
 544         @param UNREAD_PATH:
 545         @type UNREAD_PATH:
 546         @param repo_id:
 547         @type repo_id:
 548         @rtype: Integer
 549         @returns:
 550         1.  The number of unread but relevant news items.
 551
 552         """
 553         from portage.news import NewsManager
 554         manager = NewsManager(portdb, vardb, NEWS_PATH, UNREAD_PATH)
 555         return manager.getUnreadItems( repo_id, update=update )
 556
 557 def action_sync(settings, trees, mtimedb, myopts, myaction):
 558         xterm_titles = "notitles" not in settings.features
 559         emergelog(xterm_titles, " === sync")
 560         portdb = trees[settings["ROOT"]]["porttree"].dbapi
 561         myportdir = portdb.porttree_root
 562         out = portage.output.EOutput()
 563         if not myportdir:
 564                 sys.stderr.write("!!! PORTDIR is undefined.  Is /etc/make.globals missing?\n")
 565                 sys.exit(1)
 566         if myportdir[-1]=="/":
 567                 myportdir=myportdir[:-1]
 568         try:
 569                 st = os.stat(myportdir)
 570         except OSError:
 571                 st = None
 572         if st is None:
 573                 print ">>>",myportdir,"not found, creating it."
 574                 os.makedirs(myportdir,0755)
 575                 st = os.stat(myportdir)
 576
 577         spawn_kwargs = {}
 578         spawn_kwargs["env"] = settings.environ()
 579         if 'usersync' in settings.features and \
 580                 portage.data.secpass >= 2 and \
 581                 (st.st_uid != os.getuid() and st.st_mode & 0700 or \
 582                 st.st_gid != os.getgid() and st.st_mode & 0070):
 583                 try:
 584                         homedir = pwd.getpwuid(st.st_uid).pw_dir
 585                 except KeyError:
 586                         pass
 587                 else:
 588                         # Drop privileges when syncing, in order to match
 589                         # existing uid/gid settings.
 590                         spawn_kwargs["uid"]    = st.st_uid
 591                         spawn_kwargs["gid"]    = st.st_gid
 592                         spawn_kwargs["groups"] = [st.st_gid]
 593                         spawn_kwargs["env"]["HOME"] = homedir
 594                         umask = 0002
 595                         if not st.st_mode & 0020:
 596                                 umask = umask | 0020
 597                         spawn_kwargs["umask"] = umask
 598
 599         syncuri = settings.get("SYNC", "").strip()
 600         if not syncuri:
 601                 writemsg_level("!!! SYNC is undefined. Is /etc/make.globals missing?\n",
 602                         noiselevel=-1, level=logging.ERROR)
 603                 return 1
 604
 605         vcs_dirs = frozenset([".git", ".svn", "CVS", ".hg"])
 606         vcs_dirs = vcs_dirs.intersection(os.listdir(myportdir))
 607
 608         os.umask(0022)
 609         dosyncuri = syncuri
 610         updatecache_flg = False
 611         if myaction == "metadata":
 612                 print "skipping sync"
 613                 updatecache_flg = True
 614         elif ".git" in vcs_dirs:
 615                 # Update existing git repository, and ignore the syncuri. We are
 616                 # going to trust the user and assume that the user is in the branch
 617                 # that he/she wants updated. We'll let the user manage branches with
 618                 # git directly.
 619                 if portage.process.find_binary("git") is None:
 620                         msg = ["Command not found: git",
 621                         "Type \"emerge dev-util/git\" to enable git support."]
 622                         for l in msg:
 623                                 writemsg_level("!!! %s\n" % l,
 624                                         level=logging.ERROR, noiselevel=-1)
 625                         return 1
 626                 msg = ">>> Starting git pull in %s..." % myportdir
 627                 emergelog(xterm_titles, msg )
 628                 writemsg_level(msg + "\n")
 629                 exitcode = portage.process.spawn_bash("cd %s ; git pull" % \
 630                         (portage._shell_quote(myportdir),), **spawn_kwargs)
 631                 if exitcode != os.EX_OK:
 632                         msg = "!!! git pull error in %s." % myportdir
 633                         emergelog(xterm_titles, msg)
 634                         writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
 635                         return exitcode
 636                 msg = ">>> Git pull in %s successful" % myportdir
 637                 emergelog(xterm_titles, msg)
 638                 writemsg_level(msg + "\n")
 639                 exitcode = git_sync_timestamps(settings, myportdir)
 640                 if exitcode == os.EX_OK:
 641                         updatecache_flg = True
 642         elif syncuri[:8]=="rsync://":
 643                 for vcs_dir in vcs_dirs:
 644                         writemsg_level(("!!! %s appears to be under revision " + \
 645                                 "control (contains %s).\n!!! Aborting rsync sync.\n") % \
 646                                 (myportdir, vcs_dir), level=logging.ERROR, noiselevel=-1)
 647                         return 1
 648                 if not os.path.exists("/usr/bin/rsync"):
 649                         print "!!! /usr/bin/rsync does not exist, so rsync support is disabled."
 650                         print "!!! Type \"emerge net-misc/rsync\" to enable rsync support."
 651                         sys.exit(1)
 652                 mytimeout=180
 653
 654                 rsync_opts = []
 655                 if settings["PORTAGE_RSYNC_OPTS"] == "":
 656                         portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n")
 657                         rsync_opts.extend([
 658                                 "--recursive",    # Recurse directories
 659                                 "--links",        # Consider symlinks
 660                                 "--safe-links",   # Ignore links outside of tree
 661                                 "--perms",        # Preserve permissions
 662                                 "--times",        # Preserive mod times
 663                                 "--compress",     # Compress the data transmitted
 664                                 "--force",        # Force deletion on non-empty dirs
 665                                 "--whole-file",   # Don't do block transfers, only entire files
 666                                 "--delete",       # Delete files that aren't in the master tree
 667                                 "--stats",        # Show final statistics about what was transfered
 668                                 "--timeout="+str(mytimeout), # IO timeout if not done in X seconds
 669                                 "--exclude=/distfiles",   # Exclude distfiles from consideration
 670                                 "--exclude=/local",       # Exclude local     from consideration
 671                                 "--exclude=/packages",    # Exclude packages  from consideration
 672                         ])
 673
 674                 else:
 675                         # The below validation is not needed when using the above hardcoded
 676                         # defaults.
 677
 678                         portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1)
 679                         rsync_opts.extend(
 680                                 shlex.split(settings.get("PORTAGE_RSYNC_OPTS","")))
 681                         for opt in ("--recursive", "--times"):
 682                                 if opt not in rsync_opts:
 683                                         portage.writemsg(yellow("WARNING:") + " adding required option " + \
 684                                         "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
 685                                         rsync_opts.append(opt)
 686
 687                         for exclude in ("distfiles", "local", "packages"):
 688                                 opt = "--exclude=/%s" % exclude
 689                                 if opt not in rsync_opts:
 690                                         portage.writemsg(yellow("WARNING:") + \
 691                                         " adding required option %s not included in "  % opt + \
 692                                         "PORTAGE_RSYNC_OPTS (can be overridden with --exclude='!')\n")
 693                                         rsync_opts.append(opt)
 694
 695                         if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"):
 696                                 def rsync_opt_startswith(opt_prefix):
 697                                         for x in rsync_opts:
 698                                                 if x.startswith(opt_prefix):
 699                                                         return True
 700                                         return False
 701
 702                                 if not rsync_opt_startswith("--timeout="):
 703                                         rsync_opts.append("--timeout=%d" % mytimeout)
 704
 705                                 for opt in ("--compress", "--whole-file"):
 706                                         if opt not in rsync_opts:
 707                                                 portage.writemsg(yellow("WARNING:") + " adding required option " + \
 708                                                 "%s not included in PORTAGE_RSYNC_OPTS\n" % opt)
 709                                                 rsync_opts.append(opt)
 710
 711                 if "--quiet" in myopts:
 712                         rsync_opts.append("--quiet")    # Shut up a lot
 713                 else:
 714                         rsync_opts.append("--verbose")  # Print filelist
 715
 716                 if "--verbose" in myopts:
 717                         rsync_opts.append("--progress")  # Progress meter for each file
 718
 719                 if "--debug" in myopts:
 720                         rsync_opts.append("--checksum") # Force checksum on all files
 721
 722                 # Real local timestamp file.
 723                 servertimestampfile = os.path.join(
 724                         myportdir, "metadata", "timestamp.chk")
 725
 726                 content = portage.util.grabfile(servertimestampfile)
 727                 mytimestamp = 0
 728                 if content:
 729                         try:
 730                                 mytimestamp = time.mktime(time.strptime(content[0],
 731                                         "%a, %d %b %Y %H:%M:%S +0000"))
 732                         except (OverflowError, ValueError):
 733                                 pass
 734                 del content
 735
 736                 try:
 737                         rsync_initial_timeout = \
 738                                 int(settings.get("PORTAGE_RSYNC_INITIAL_TIMEOUT", "15"))
 739                 except ValueError:
 740                         rsync_initial_timeout = 15
 741
 742                 try:
 743                         maxretries=int(settings["PORTAGE_RSYNC_RETRIES"])
 744                 except SystemExit, e:
 745                         raise # Needed else can't exit
 746                 except:
 747                         maxretries=3 #default number of retries
 748
 749                 retries=0
 750                 user_name, hostname, port = re.split(
 751                         "rsync://([^:/]+@)?([^:/]*)(:[0-9]+)?", syncuri, maxsplit=3)[1:4]
 752                 if port is None:
 753                         port=""
 754                 if user_name is None:
 755                         user_name=""
 756                 updatecache_flg=True
 757                 all_rsync_opts = set(rsync_opts)
 758                 extra_rsync_opts = shlex.split(
 759                         settings.get("PORTAGE_RSYNC_EXTRA_OPTS",""))
 760                 all_rsync_opts.update(extra_rsync_opts)
 761                 family = socket.AF_INET
 762                 if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
 763                         family = socket.AF_INET
 764                 elif socket.has_ipv6 and \
 765                         ("-6" in all_rsync_opts or "--ipv6" in all_rsync_opts):
 766                         family = socket.AF_INET6
 767                 ips=[]
 768                 SERVER_OUT_OF_DATE = -1
 769                 EXCEEDED_MAX_RETRIES = -2
 770                 while (1):
 771                         if ips:
 772                                 del ips[0]
 773                         if ips==[]:
 774                                 try:
 775                                         for addrinfo in socket.getaddrinfo(
 776                                                 hostname, None, family, socket.SOCK_STREAM):
 777                                                 if socket.has_ipv6 and addrinfo[0] == socket.AF_INET6:
 778                                                         # IPv6 addresses need to be enclosed in square brackets
 779                                                         ips.append("[%s]" % addrinfo[4][0])
 780                                                 else:
 781                                                         ips.append(addrinfo[4][0])
 782                                         from random import shuffle
 783                                         shuffle(ips)
 784                                 except SystemExit, e:
 785                                         raise # Needed else can't exit
 786                                 except Exception, e:
 787                                         print "Notice:",str(e)
 788                                         dosyncuri=syncuri
 789
 790                         if ips:
 791                                 try:
 792                                         dosyncuri = syncuri.replace(
 793                                                 "//" + user_name + hostname + port + "/",
 794                                                 "//" + user_name + ips[0] + port + "/", 1)
 795                                 except SystemExit, e:
 796                                         raise # Needed else can't exit
 797                                 except Exception, e:
 798                                         print "Notice:",str(e)
 799                                         dosyncuri=syncuri
 800
 801                         if (retries==0):
 802                                 if "--ask" in myopts:
 803                                         if userquery("Do you want to sync your Portage tree with the mirror at\n" + blue(dosyncuri) + bold("?"))=="No":
 804                                                 print
 805                                                 print "Quitting."
 806                                                 print
 807                                                 sys.exit(0)
 808                                 emergelog(xterm_titles, ">>> Starting rsync with " + dosyncuri)
 809                                 if "--quiet" not in myopts:
 810                                         print ">>> Starting rsync with "+dosyncuri+"..."
 811                         else:
 812                                 emergelog(xterm_titles,
 813                                         ">>> Starting retry %d of %d with %s" % \
 814                                                 (retries,maxretries,dosyncuri))
 815                                 print "\n\n>>> Starting retry %d of %d with %s" % (retries,maxretries,dosyncuri)
 816
 817                         if mytimestamp != 0 and "--quiet" not in myopts:
 818                                 print ">>> Checking server timestamp ..."
 819
 820                         rsynccommand = ["/usr/bin/rsync"] + rsync_opts + extra_rsync_opts
 821
 822                         if "--debug" in myopts:
 823                                 print rsynccommand
 824
 825                         exitcode = os.EX_OK
 826                         servertimestamp = 0
 827                         # Even if there's no timestamp available locally, fetch the
 828                         # timestamp anyway as an initial probe to verify that the server is
 829                         # responsive.  This protects us from hanging indefinitely on a
 830                         # connection attempt to an unresponsive server which rsync's
 831                         # --timeout option does not prevent.
 832                         if True:
 833                                 # Temporary file for remote server timestamp comparison.
 834                                 from tempfile import mkstemp
 835                                 fd, tmpservertimestampfile = mkstemp()
 836                                 os.close(fd)
 837                                 mycommand = rsynccommand[:]
 838                                 mycommand.append(dosyncuri.rstrip("/") + \
 839                                         "/metadata/timestamp.chk")
 840                                 mycommand.append(tmpservertimestampfile)
 841                                 content = None
 842                                 mypids = []
 843                                 try:
 844                                         def timeout_handler(signum, frame):
 845                                                 raise portage.exception.PortageException("timed out")
 846                                         signal.signal(signal.SIGALRM, timeout_handler)
 847                                         # Timeout here in case the server is unresponsive.  The
 848                                         # --timeout rsync option doesn't apply to the initial
 849                                         # connection attempt.
 850                                         if rsync_initial_timeout:
 851                                                 signal.alarm(rsync_initial_timeout)
 852                                         try:
 853                                                 mypids.extend(portage.process.spawn(
 854                                                         mycommand, env=settings.environ(), returnpid=True))
 855                                                 exitcode = os.waitpid(mypids[0], 0)[1]
 856                                                 content = portage.grabfile(tmpservertimestampfile)
 857                                         finally:
 858                                                 if rsync_initial_timeout:
 859                                                         signal.alarm(0)
 860                                                 try:
 861                                                         os.unlink(tmpservertimestampfile)
 862                                                 except OSError:
 863                                                         pass
 864                                 except portage.exception.PortageException, e:
 865                                         # timed out
 866                                         print e
 867                                         del e
 868                                         if mypids and os.waitpid(mypids[0], os.WNOHANG) == (0,0):
 869                                                 os.kill(mypids[0], signal.SIGTERM)
 870                                                 os.waitpid(mypids[0], 0)
 871                                         # This is the same code rsync uses for timeout.
 872                                         exitcode = 30
 873                                 else:
 874                                         if exitcode != os.EX_OK:
 875                                                 if exitcode & 0xff:
 876                                                         exitcode = (exitcode & 0xff) << 8
 877                                                 else:
 878                                                         exitcode = exitcode >> 8
 879                                 if mypids:
 880                                         portage.process.spawned_pids.remove(mypids[0])
 881                                 if content:
 882                                         try:
 883                                                 servertimestamp = time.mktime(time.strptime(
 884                                                         content[0], "%a, %d %b %Y %H:%M:%S +0000"))
 885                                         except (OverflowError, ValueError):
 886                                                 pass
 887                                 del mycommand, mypids, content
 888                         if exitcode == os.EX_OK:
 889                                 if (servertimestamp != 0) and (servertimestamp == mytimestamp):
 890                                         emergelog(xterm_titles,
 891                                                 ">>> Cancelling sync -- Already current.")
 892                                         print
 893                                         print ">>>"
 894                                         print ">>> Timestamps on the server and in the local repository are the same."
 895                                         print ">>> Cancelling all further sync action. You are already up to date."
 896                                         print ">>>"
 897                                         print ">>> In order to force sync, remove '%s'." % servertimestampfile
 898                                         print ">>>"
 899                                         print
 900                                         sys.exit(0)
 901                                 elif (servertimestamp != 0) and (servertimestamp < mytimestamp):
 902                                         emergelog(xterm_titles,
 903                                                 ">>> Server out of date: %s" % dosyncuri)
 904                                         print
 905                                         print ">>>"
 906                                         print ">>> SERVER OUT OF DATE: %s" % dosyncuri
 907                                         print ">>>"
 908                                         print ">>> In order to force sync, remove '%s'." % servertimestampfile
 909                                         print ">>>"
 910                                         print
 911                                         exitcode = SERVER_OUT_OF_DATE
 912                                 elif (servertimestamp == 0) or (servertimestamp > mytimestamp):
 913                                         # actual sync
 914                                         mycommand = rsynccommand + [dosyncuri+"/", myportdir]
 915                                         exitcode = portage.process.spawn(mycommand, **spawn_kwargs)
 916                                         if exitcode in [0,1,3,4,11,14,20,21]:
 917                                                 break
 918                         elif exitcode in [1,3,4,11,14,20,21]:
 919                                 break
 920                         else:
 921                                 # Code 2 indicates protocol incompatibility, which is expected
 922                                 # for servers with protocol < 29 that don't support
 923                                 # --prune-empty-directories.  Retry for a server that supports
 924                                 # at least rsync protocol version 29 (>=rsync-2.6.4).
 925                                 pass
 926
 927                         retries=retries+1
 928
 929                         if retries<=maxretries:
 930                                 print ">>> Retrying..."
 931                                 time.sleep(11)
 932                         else:
 933                                 # over retries
 934                                 # exit loop
 935                                 updatecache_flg=False
 936                                 exitcode = EXCEEDED_MAX_RETRIES
 937                                 break
 938
 939                 if (exitcode==0):
 940                         emergelog(xterm_titles, "=== Sync completed with %s" % dosyncuri)
 941                 elif exitcode == SERVER_OUT_OF_DATE:
 942                         sys.exit(1)
 943                 elif exitcode == EXCEEDED_MAX_RETRIES:
 944                         sys.stderr.write(
 945                                 ">>> Exceeded PORTAGE_RSYNC_RETRIES: %s\n" % maxretries)
 946                         sys.exit(1)
 947                 elif (exitcode>0):
 948                         msg = []
 949                         if exitcode==1:
 950                                 msg.append("Rsync has reported that there is a syntax error. Please ensure")
 951                                 msg.append("that your SYNC statement is proper.")
 952                                 msg.append("SYNC=" + settings["SYNC"])
 953                         elif exitcode==11:
 954                                 msg.append("Rsync has reported that there is a File IO error. Normally")
 955                                 msg.append("this means your disk is full, but can be caused by corruption")
 956                                 msg.append("on the filesystem that contains PORTDIR. Please investigate")
 957                                 msg.append("and try again after the problem has been fixed.")
 958                                 msg.append("PORTDIR=" + settings["PORTDIR"])
 959                         elif exitcode==20:
 960                                 msg.append("Rsync was killed before it finished.")
 961                         else:
 962                                 msg.append("Rsync has not successfully finished. It is recommended that you keep")
 963                                 msg.append("trying or that you use the 'emerge-webrsync' option if you are unable")
 964                                 msg.append("to use rsync due to firewall or other restrictions. This should be a")
 965                                 msg.append("temporary problem unless complications exist with your network")
 966                                 msg.append("(and possibly your system's filesystem) configuration.")
 967                         for line in msg:
 968                                 out.eerror(line)
 969                         sys.exit(exitcode)
 970         elif syncuri[:6]=="cvs://":
 971                 if not os.path.exists("/usr/bin/cvs"):
 972                         print "!!! /usr/bin/cvs does not exist, so CVS support is disabled."
 973                         print "!!! Type \"emerge dev-util/cvs\" to enable CVS support."
 974                         sys.exit(1)
 975                 cvsroot=syncuri[6:]
 976                 cvsdir=os.path.dirname(myportdir)
 977                 if not os.path.exists(myportdir+"/CVS"):
 978                         #initial checkout
 979                         print ">>> Starting initial cvs checkout with "+syncuri+"..."
 980                         if os.path.exists(cvsdir+"/gentoo-x86"):
 981                                 print "!!! existing",cvsdir+"/gentoo-x86 directory; exiting."
 982                                 sys.exit(1)
 983                         try:
 984                                 os.rmdir(myportdir)
 985                         except OSError, e:
 986                                 if e.errno != errno.ENOENT:
 987                                         sys.stderr.write(
 988                                                 "!!! existing '%s' directory; exiting.\n" % myportdir)
 989                                         sys.exit(1)
 990                                 del e
 991                         if portage.spawn("cd "+cvsdir+"; cvs -z0 -d "+cvsroot+" co -P gentoo-x86",settings,free=1):
 992                                 print "!!! cvs checkout error; exiting."
 993                                 sys.exit(1)
 994                         os.rename(os.path.join(cvsdir, "gentoo-x86"), myportdir)
 995                 else:
 996                         #cvs update
 997                         print ">>> Starting cvs update with "+syncuri+"..."
 998                         retval = portage.process.spawn_bash(
 999                                 "cd %s; cvs -z0 -q update -dP" % \
1000                                 (portage._shell_quote(myportdir),), **spawn_kwargs)
1001                         if retval != os.EX_OK:
1002                                 sys.exit(retval)
1003                 dosyncuri = syncuri
1004         else:
1005                 writemsg_level("!!! Unrecognized protocol: SYNC='%s'\n" % (syncuri,),
1006                         noiselevel=-1, level=logging.ERROR)
1007                 return 1
1008
1009         if updatecache_flg and  \
1010                 myaction != "metadata" and \
1011                 "metadata-transfer" not in settings.features:
1012                 updatecache_flg = False
1013
1014         # Reload the whole config from scratch.
1015         settings, trees, mtimedb = load_emerge_config(trees=trees)
1016         root_config = trees[settings["ROOT"]]["root_config"]
1017         portdb = trees[settings["ROOT"]]["porttree"].dbapi
1018
1019         if updatecache_flg and \
1020                 os.path.exists(os.path.join(myportdir, 'metadata', 'cache')):
1021
1022                 # Only update cache for myportdir since that's
1023                 # the only one that's been synced here.
1024                 action_metadata(settings, portdb, myopts, porttrees=[myportdir])
1025
1026         if portage._global_updates(trees, mtimedb["updates"]):
1027                 mtimedb.commit()
1028                 # Reload the whole config from scratch.
1029                 settings, trees, mtimedb = load_emerge_config(trees=trees)
1030                 portdb = trees[settings["ROOT"]]["porttree"].dbapi
1031                 root_config = trees[settings["ROOT"]]["root_config"]
1032
1033         mybestpv = portdb.xmatch("bestmatch-visible",
1034                 portage.const.PORTAGE_PACKAGE_ATOM)
1035         mypvs = portage.best(
1036                 trees[settings["ROOT"]]["vartree"].dbapi.match(
1037                 portage.const.PORTAGE_PACKAGE_ATOM))
1038
1039         chk_updated_cfg_files("/", settings.get("CONFIG_PROTECT","").split())
1040
1041         if myaction != "metadata":
1042                 if os.access(portage.USER_CONFIG_PATH + "/bin/post_sync", os.X_OK):
1043                         retval = portage.process.spawn(
1044                                 [os.path.join(portage.USER_CONFIG_PATH, "bin", "post_sync"),
1045                                 dosyncuri], env=settings.environ())
1046                         if retval != os.EX_OK:
1047                                 print red(" * ")+bold("spawn failed of "+ portage.USER_CONFIG_PATH + "/bin/post_sync")
1048
1049         if(mybestpv != mypvs) and not "--quiet" in myopts:
1050                 print
1051                 print red(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended"
1052                 print red(" * ")+"that you update portage now, before any other packages are updated."
1053                 print
1054                 print red(" * ")+"To update portage, run 'emerge portage' now."
1055                 print
1056
1057         display_news_notification(root_config, myopts)
1058         return os.EX_OK
1059
1060 def git_sync_timestamps(settings, portdir):
1061         """
1062         Since git doesn't preserve timestamps, synchronize timestamps between
1063         entries and ebuilds/eclasses. Assume the cache has the correct timestamp
1064         for a given file as long as the file in the working tree is not modified
1065         (relative to HEAD).
1066         """
1067         cache_dir = os.path.join(portdir, "metadata", "cache")
1068         if not os.path.isdir(cache_dir):
1069                 return os.EX_OK
1070         writemsg_level(">>> Synchronizing timestamps...\n")
1071
1072         from portage.cache.cache_errors import CacheError
1073         try:
1074                 cache_db = settings.load_best_module("portdbapi.metadbmodule")(
1075                         portdir, "metadata/cache", portage.auxdbkeys[:], readonly=True)
1076         except CacheError, e:
1077                 writemsg_level("!!! Unable to instantiate cache: %s\n" % (e,),
1078                         level=logging.ERROR, noiselevel=-1)
1079                 return 1
1080
1081         ec_dir = os.path.join(portdir, "eclass")
1082         try:
1083                 ec_names = set(f[:-7] for f in os.listdir(ec_dir) \
1084                         if f.endswith(".eclass"))
1085         except OSError, e:
1086                 writemsg_level("!!! Unable to list eclasses: %s\n" % (e,),
1087                         level=logging.ERROR, noiselevel=-1)
1088                 return 1
1089
1090         args = [portage.const.BASH_BINARY, "-c",
1091                 "cd %s && git diff-index --name-only --diff-filter=M HEAD" % \
1092                 portage._shell_quote(portdir)]
1093         import subprocess
1094         proc = subprocess.Popen(args, stdout=subprocess.PIPE)
1095         modified_files = set(l.rstrip("\n") for l in proc.stdout)
1096         rval = proc.wait()
1097         if rval != os.EX_OK:
1098                 return rval
1099
1100         modified_eclasses = set(ec for ec in ec_names \
1101                 if os.path.join("eclass", ec + ".eclass") in modified_files)
1102
1103         updated_ec_mtimes = {}
1104
1105         for cpv in cache_db:
1106                 cpv_split = portage.catpkgsplit(cpv)
1107                 if cpv_split is None:
1108                         writemsg_level("!!! Invalid cache entry: %s\n" % (cpv,),
1109                                 level=logging.ERROR, noiselevel=-1)
1110                         continue
1111
1112                 cat, pn, ver, rev = cpv_split
1113                 cat, pf = portage.catsplit(cpv)
1114                 relative_eb_path = os.path.join(cat, pn, pf + ".ebuild")
1115                 if relative_eb_path in modified_files:
1116                         continue
1117
1118                 try:
1119                         cache_entry = cache_db[cpv]
1120                         eb_mtime = cache_entry.get("_mtime_")
1121                         ec_mtimes = cache_entry.get("_eclasses_")
1122                 except KeyError:
1123                         writemsg_level("!!! Missing cache entry: %s\n" % (cpv,),
1124                                 level=logging.ERROR, noiselevel=-1)
1125                         continue
1126                 except CacheError, e:
1127                         writemsg_level("!!! Unable to access cache entry: %s %s\n" % \
1128                                 (cpv, e), level=logging.ERROR, noiselevel=-1)
1129                         continue
1130
1131                 if eb_mtime is None:
1132                         writemsg_level("!!! Missing ebuild mtime: %s\n" % (cpv,),
1133                                 level=logging.ERROR, noiselevel=-1)
1134                         continue
1135
1136                 try:
1137                         eb_mtime = long(eb_mtime)
1138                 except ValueError:
1139                         writemsg_level("!!! Invalid ebuild mtime: %s %s\n" % \
1140                                 (cpv, eb_mtime), level=logging.ERROR, noiselevel=-1)
1141                         continue
1142
1143                 if ec_mtimes is None:
1144                         writemsg_level("!!! Missing eclass mtimes: %s\n" % (cpv,),
1145                                 level=logging.ERROR, noiselevel=-1)
1146                         continue
1147
1148                 if modified_eclasses.intersection(ec_mtimes):
1149                         continue
1150
1151                 missing_eclasses = set(ec_mtimes).difference(ec_names)
1152                 if missing_eclasses:
1153                         writemsg_level("!!! Non-existent eclass(es): %s %s\n" % \
1154                                 (cpv, sorted(missing_eclasses)), level=logging.ERROR,
1155                                 noiselevel=-1)
1156                         continue
1157
1158                 eb_path = os.path.join(portdir, relative_eb_path)
1159                 try:
1160                         current_eb_mtime = os.stat(eb_path)
1161                 except OSError:
1162                         writemsg_level("!!! Missing ebuild: %s\n" % \
1163                                 (cpv,), level=logging.ERROR, noiselevel=-1)
1164                         continue
1165
1166                 inconsistent = False
1167                 for ec, (ec_path, ec_mtime) in ec_mtimes.iteritems():
1168                         updated_mtime = updated_ec_mtimes.get(ec)
1169                         if updated_mtime is not None and updated_mtime != ec_mtime:
1170                                 writemsg_level("!!! Inconsistent eclass mtime: %s %s\n" % \
1171                                         (cpv, ec), level=logging.ERROR, noiselevel=-1)
1172                                 inconsistent = True
1173                                 break
1174
1175                 if inconsistent:
1176                         continue
1177
1178                 if current_eb_mtime != eb_mtime:
1179                         os.utime(eb_path, (eb_mtime, eb_mtime))
1180
1181                 for ec, (ec_path, ec_mtime) in ec_mtimes.iteritems():
1182                         if ec in updated_ec_mtimes:
1183                                 continue
1184                         ec_path = os.path.join(ec_dir, ec + ".eclass")
1185                         current_mtime = long(os.stat(ec_path).st_mtime)
1186                         if current_mtime != ec_mtime:
1187                                 os.utime(ec_path, (ec_mtime, ec_mtime))
1188                         updated_ec_mtimes[ec] = ec_mtime
1189
1190         return os.EX_OK
1191
1192 def action_metadata(settings, portdb, myopts, porttrees=None):
1193         if porttrees is None:
1194                 porttrees = portdb.porttrees
1195         portage.writemsg_stdout("\n>>> Updating Portage cache\n")
1196         old_umask = os.umask(0002)
1197         cachedir = os.path.normpath(settings.depcachedir)
1198         if cachedir in ["/",    "/bin", "/dev",  "/etc",  "/home",
1199                                         "/lib", "/opt", "/proc", "/root", "/sbin",
1200                                         "/sys", "/tmp", "/usr",  "/var"]:
1201                 print >> sys.stderr, "!!! PORTAGE_DEPCACHEDIR IS SET TO A PRIMARY " + \
1202                         "ROOT DIRECTORY ON YOUR SYSTEM."
1203                 print >> sys.stderr, \
1204                         "!!! This is ALMOST CERTAINLY NOT what you want: '%s'" % cachedir
1205                 sys.exit(73)
1206         if not os.path.exists(cachedir):
1207                 os.makedirs(cachedir)
1208
1209         auxdbkeys = [x for x in portage.auxdbkeys if not x.startswith("UNUSED_0")]
1210         auxdbkeys = tuple(auxdbkeys)
1211
1212         class TreeData(object):
1213                 __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes')
1214                 def __init__(self, dest_db, eclass_db, path, src_db):
1215                         self.dest_db = dest_db
1216                         self.eclass_db = eclass_db
1217                         self.path = path
1218                         self.src_db = src_db
1219                         self.valid_nodes = set()
1220
1221         porttrees_data = []
1222         for path in porttrees:
1223                 src_db = portdb._pregen_auxdb.get(path)
1224                 if src_db is None and \
1225                         os.path.isdir(os.path.join(path, 'metadata', 'cache')):
1226                         src_db = portdb.metadbmodule(
1227                                 path, 'metadata/cache', auxdbkeys, readonly=True)
1228                         try:
1229                                 src_db.ec = portdb._repo_info[path].eclass_db
1230                         except AttributeError:
1231                                 pass
1232
1233                 if src_db is not None:
1234                         porttrees_data.append(TreeData(portdb.auxdb[path],
1235                                 portdb._repo_info[path].eclass_db, path, src_db))
1236
1237         porttrees = [tree_data.path for tree_data in porttrees_data]
1238
1239         isatty = sys.stdout.isatty()
1240         quiet = not isatty or '--quiet' in myopts
1241         onProgress = None
1242         if not quiet:
1243                 progressBar = portage.output.TermProgressBar()
1244                 progressHandler = ProgressHandler()
1245                 onProgress = progressHandler.onProgress
1246                 def display():
1247                         progressBar.set(progressHandler.curval, progressHandler.maxval)
1248                 progressHandler.display = display
1249                 def sigwinch_handler(signum, frame):
1250                         lines, progressBar.term_columns = \
1251                                 portage.output.get_term_size()
1252                 signal.signal(signal.SIGWINCH, sigwinch_handler)
1253
1254         # Temporarily override portdb.porttrees so portdb.cp_all()
1255         # will only return the relevant subset.
1256         portdb_porttrees = portdb.porttrees
1257         portdb.porttrees = porttrees
1258         try:
1259                 cp_all = portdb.cp_all()
1260         finally:
1261                 portdb.porttrees = portdb_porttrees
1262
1263         curval = 0
1264         maxval = len(cp_all)
1265         if onProgress is not None:
1266                 onProgress(maxval, curval)
1267
1268         from portage.cache.util import quiet_mirroring
1269         from portage import eapi_is_supported, \
1270                 _validate_cache_for_unsupported_eapis
1271
1272         # TODO: Display error messages, but do not interfere with the progress bar.
1273         # Here's how:
1274         #  1) erase the progress bar
1275         #  2) show the error message
1276         #  3) redraw the progress bar on a new line
1277         noise = quiet_mirroring()
1278
1279         for cp in cp_all:
1280                 for tree_data in porttrees_data:
1281                         for cpv in portdb.cp_list(cp, mytree=tree_data.path):
1282                                 tree_data.valid_nodes.add(cpv)
1283                                 try:
1284                                         src = tree_data.src_db[cpv]
1285                                 except KeyError, e:
1286                                         noise.missing_entry(cpv)
1287                                         del e
1288                                         continue
1289                                 except CacheError, ce:
1290                                         noise.exception(cpv, ce)
1291                                         del ce
1292                                         continue
1293
1294                                 eapi = src.get('EAPI')
1295                                 if not eapi:
1296                                         eapi = '0'
1297                                 eapi = eapi.lstrip('-')
1298                                 eapi_supported = eapi_is_supported(eapi)
1299                                 if not eapi_supported:
1300                                         if not _validate_cache_for_unsupported_eapis:
1301                                                 noise.misc(cpv, "unable to validate " + \
1302                                                         "cache for EAPI='%s'" % eapi)
1303                                                 continue
1304
1305                                 dest = None
1306                                 try:
1307                                         dest = tree_data.dest_db[cpv]
1308                                 except (KeyError, CacheError):
1309                                         pass
1310
1311                                 for d in (src, dest):
1312                                         if d is not None and d.get('EAPI') in ('', '0'):
1313                                                 del d['EAPI']
1314
1315                                 if dest is not None:
1316                                         if not (dest['_mtime_'] == src['_mtime_'] and \
1317                                                 tree_data.eclass_db.is_eclass_data_valid(
1318                                                         dest['_eclasses_']) and \
1319                                                 set(dest['_eclasses_']) == set(src['_eclasses_'])):
1320                                                 dest = None
1321                                         else:
1322                                                 # We don't want to skip the write unless we're really
1323                                                 # sure that the existing cache is identical, so don't
1324                                                 # trust _mtime_ and _eclasses_ alone.
1325                                                 for k in set(chain(src, dest)).difference(
1326                                                         ('_mtime_', '_eclasses_')):
1327                                                         if dest.get(k, '') != src.get(k, ''):
1328                                                                 dest = None
1329                                                                 break
1330
1331                                 if dest is not None:
1332                                         # The existing data is valid and identical,
1333                                         # so there's no need to overwrite it.
1334                                         continue
1335
1336                                 try:
1337                                         inherited = src.get('INHERITED', '')
1338                                         eclasses = src.get('_eclasses_')
1339                                 except CacheError, ce:
1340                                         noise.exception(cpv, ce)
1341                                         del ce
1342                                         continue
1343
1344                                 if eclasses is not None:
1345                                         if not tree_data.eclass_db.is_eclass_data_valid(
1346                                                 src['_eclasses_']):
1347                                                 noise.eclass_stale(cpv)
1348                                                 continue
1349                                         inherited = eclasses
1350                                 else:
1351                                         inherited = inherited.split()
1352
1353                                 if tree_data.src_db.complete_eclass_entries and \
1354                                         eclasses is None:
1355                                         noise.corruption(cpv, "missing _eclasses_ field")
1356                                         continue
1357
1358                                 if inherited:
1359                                         # Even if _eclasses_ already exists, replace it with data from
1360                                         # eclass_cache, in order to insert local eclass paths.
1361                                         try:
1362                                                 eclasses = tree_data.eclass_db.get_eclass_data(inherited)
1363                                         except KeyError:
1364                                                 # INHERITED contains a non-existent eclass.
1365                                                 noise.eclass_stale(cpv)
1366                                                 continue
1367
1368                                         if eclasses is None:
1369                                                 noise.eclass_stale(cpv)
1370                                                 continue
1371                                         src['_eclasses_'] = eclasses
1372                                 else:
1373                                         src['_eclasses_'] = {}
1374
1375                                 if not eapi_supported:
1376                                         src = {
1377                                                 'EAPI'       : '-' + eapi,
1378                                                 '_mtime_'    : src['_mtime_'],
1379                                                 '_eclasses_' : src['_eclasses_'],
1380                                         }
1381
1382                                 try:
1383                                         tree_data.dest_db[cpv] = src
1384                                 except CacheError, ce:
1385                                         noise.exception(cpv, ce)
1386                                         del ce
1387
1388                 curval += 1
1389                 if onProgress is not None:
1390                         onProgress(maxval, curval)
1391
1392         if onProgress is not None:
1393                 onProgress(maxval, curval)
1394
1395         for tree_data in porttrees_data:
1396                 try:
1397                         dead_nodes = set(tree_data.dest_db.iterkeys())
1398                 except CacheError, e:
1399                         writemsg_level("Error listing cache entries for " + \
1400                                 "'%s': %s, continuing...\n" % (tree_data.path, e),
1401                                 level=logging.ERROR, noiselevel=-1)
1402                         del e
1403                 else:
1404                         dead_nodes.difference_update(tree_data.valid_nodes)
1405                         for cpv in dead_nodes:
1406                                 try:
1407                                         del tree_data.dest_db[cpv]
1408                                 except (KeyError, CacheError):
1409                                         pass
1410
1411         if not quiet:
1412                 # make sure the final progress is displayed
1413                 progressHandler.display()
1414                 print
1415                 signal.signal(signal.SIGWINCH, signal.SIG_DFL)
1416
1417         sys.stdout.flush()
1418         os.umask(old_umask)
1419
1420 def action_regen(settings, portdb, max_jobs, max_load):
1421         xterm_titles = "notitles" not in settings.features
1422         emergelog(xterm_titles, " === regen")
1423         #regenerate cache entries
1424         portage.writemsg_stdout("Regenerating cache entries...\n")
1425         try:
1426                 os.close(sys.stdin.fileno())
1427         except SystemExit, e:
1428                 raise # Needed else can't exit
1429         except:
1430                 pass
1431         sys.stdout.flush()
1432
1433         regen = MetadataRegen(portdb, max_jobs=max_jobs, max_load=max_load)
1434         regen.run()
1435
1436         portage.writemsg_stdout("done!\n")
1437         return regen.returncode
1438
1439 def action_config(settings, trees, myopts, myfiles):
1440         if len(myfiles) != 1:
1441                 print red("!!! config can only take a single package atom at this time\n")
1442                 sys.exit(1)
1443         if not is_valid_package_atom(myfiles[0]):
1444                 portage.writemsg("!!! '%s' is not a valid package atom.\n" % myfiles[0],
1445                         noiselevel=-1)
1446                 portage.writemsg("!!! Please check ebuild(5) for full details.\n")
1447                 portage.writemsg("!!! (Did you specify a version but forget to prefix with '='?)\n")
1448                 sys.exit(1)
1449         print
1450         try:
1451                 pkgs = trees[settings["ROOT"]]["vartree"].dbapi.match(myfiles[0])
1452         except portage.exception.AmbiguousPackageName, e:
1453                 # Multiple matches thrown from cpv_expand
1454                 pkgs = e.args[0]
1455         if len(pkgs) == 0:
1456                 print "No packages found.\n"
1457                 sys.exit(0)
1458         elif len(pkgs) > 1:
1459                 if "--ask" in myopts:
1460                         options = []
1461                         print "Please select a package to configure:"
1462                         idx = 0
1463                         for pkg in pkgs:
1464                                 idx += 1
1465                                 options.append(str(idx))
1466                                 print options[-1]+") "+pkg
1467                         print "X) Cancel"
1468                         options.append("X")
1469                         idx = userquery("Selection?", options)
1470                         if idx == "X":
1471                                 sys.exit(0)
1472                         pkg = pkgs[int(idx)-1]
1473                 else:
1474                         print "The following packages available:"
1475                         for pkg in pkgs:
1476                                 print "* "+pkg
1477                         print "\nPlease use a specific atom or the --ask option."
1478                         sys.exit(1)
1479         else:
1480                 pkg = pkgs[0]
1481
1482         print
1483         if "--ask" in myopts:
1484                 if userquery("Ready to configure "+pkg+"?") == "No":
1485                         sys.exit(0)
1486         else:
1487                 print "Configuring pkg..."
1488         print
1489         ebuildpath = trees[settings["ROOT"]]["vartree"].dbapi.findname(pkg)
1490         mysettings = portage.config(clone=settings)
1491         vardb = trees[mysettings["ROOT"]]["vartree"].dbapi
1492         debug = mysettings.get("PORTAGE_DEBUG") == "1"
1493         retval = portage.doebuild(ebuildpath, "config", mysettings["ROOT"],
1494                 mysettings,
1495                 debug=(settings.get("PORTAGE_DEBUG", "") == 1), cleanup=True,
1496                 mydbapi=trees[settings["ROOT"]]["vartree"].dbapi, tree="vartree")
1497         if retval == os.EX_OK:
1498                 portage.doebuild(ebuildpath, "clean", mysettings["ROOT"],
1499                         mysettings, debug=debug, mydbapi=vardb, tree="vartree")
1500         print
1501
1502 def action_info(settings, trees, myopts, myfiles):
1503         print getportageversion(settings["PORTDIR"], settings["ROOT"],
1504                 settings.profile_path, settings["CHOST"],
1505                 trees[settings["ROOT"]]["vartree"].dbapi)
1506         header_width = 65
1507         header_title = "System Settings"
1508         if myfiles:
1509                 print header_width * "="
1510                 print header_title.rjust(int(header_width/2 + len(header_title)/2))
1511         print header_width * "="
1512         print "System uname: "+platform.platform(aliased=1)
1513
1514         lastSync = portage.grabfile(os.path.join(
1515                 settings["PORTDIR"], "metadata", "timestamp.chk"))
1516         print "Timestamp of tree:",
1517         if lastSync:
1518                 print lastSync[0]
1519         else:
1520                 print "Unknown"
1521
1522         output=commands.getstatusoutput("distcc --version")
1523         if not output[0]:
1524                 print str(output[1].split("\n",1)[0]),
1525                 if "distcc" in settings.features:
1526                         print "[enabled]"
1527                 else:
1528                         print "[disabled]"
1529
1530         output=commands.getstatusoutput("ccache -V")
1531         if not output[0]:
1532                 print str(output[1].split("\n",1)[0]),
1533                 if "ccache" in settings.features:
1534                         print "[enabled]"
1535                 else:
1536                         print "[disabled]"
1537
1538         myvars  = ["sys-devel/autoconf", "sys-devel/automake", "virtual/os-headers",
1539                    "sys-devel/binutils", "sys-devel/libtool",  "dev-lang/python"]
1540         myvars += portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_pkgs")
1541         myvars  = portage.util.unique_array(myvars)
1542         myvars.sort()
1543
1544         for x in myvars:
1545                 if portage.isvalidatom(x):
1546                         pkg_matches = trees["/"]["vartree"].dbapi.match(x)
1547                         pkg_matches = [portage.catpkgsplit(cpv)[1:] for cpv in pkg_matches]
1548                         pkg_matches.sort(key=cmp_sort_key(portage.pkgcmp))
1549                         pkgs = []
1550                         for pn, ver, rev in pkg_matches:
1551                                 if rev != "r0":
1552                                         pkgs.append(ver + "-" + rev)
1553                                 else:
1554                                         pkgs.append(ver)
1555                         if pkgs:
1556                                 pkgs = ", ".join(pkgs)
1557                                 print "%-20s %s" % (x+":", pkgs)
1558                 else:
1559                         print "%-20s %s" % (x+":", "[NOT VALID]")
1560
1561         libtool_vers = ",".join(trees["/"]["vartree"].dbapi.match("sys-devel/libtool"))
1562
1563         if "--verbose" in myopts:
1564                 myvars=settings.keys()
1565         else:
1566                 myvars = ['GENTOO_MIRRORS', 'CONFIG_PROTECT', 'CONFIG_PROTECT_MASK',
1567                           'PORTDIR', 'DISTDIR', 'PKGDIR', 'PORTAGE_TMPDIR',
1568                           'PORTDIR_OVERLAY', 'USE', 'CHOST', 'CFLAGS', 'CXXFLAGS',
1569                           'ACCEPT_KEYWORDS', 'SYNC', 'FEATURES', 'EMERGE_DEFAULT_OPTS']
1570
1571                 myvars.extend(portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_vars"))
1572
1573         myvars = portage.util.unique_array(myvars)
1574         use_expand = settings.get('USE_EXPAND', '').split()
1575         use_expand.sort()
1576         use_expand_hidden = set(
1577                 settings.get('USE_EXPAND_HIDDEN', '').upper().split())
1578         alphabetical_use = '--alphabetical' in myopts
1579         root_config = trees[settings["ROOT"]]['root_config']
1580         unset_vars = []
1581         myvars.sort()
1582         for x in myvars:
1583                 if x in settings:
1584                         if x != "USE":
1585                                 print '%s="%s"' % (x, settings[x])
1586                         else:
1587                                 use = set(settings["USE"].split())
1588                                 for varname in use_expand:
1589                                         flag_prefix = varname.lower() + "_"
1590                                         for f in list(use):
1591                                                 if f.startswith(flag_prefix):
1592                                                         use.remove(f)
1593                                 use = list(use)
1594                                 use.sort()
1595                                 print 'USE="%s"' % " ".join(use),
1596                                 for varname in use_expand:
1597                                         myval = settings.get(varname)
1598                                         if myval:
1599                                                 print '%s="%s"' % (varname, myval),
1600                                 print
1601                 else:
1602                         unset_vars.append(x)
1603         if unset_vars:
1604                 print "Unset:  "+", ".join(unset_vars)
1605         print
1606
1607         if "--debug" in myopts:
1608                 for x in dir(portage):
1609                         module = getattr(portage, x)
1610                         if "cvs_id_string" in dir(module):
1611                                 print "%s: %s" % (str(x), str(module.cvs_id_string))
1612
1613         # See if we can find any packages installed matching the strings
1614         # passed on the command line
1615         mypkgs = []
1616         vardb = trees[settings["ROOT"]]["vartree"].dbapi
1617         portdb = trees[settings["ROOT"]]["porttree"].dbapi
1618         for x in myfiles:
1619                 mypkgs.extend(vardb.match(x))
1620
1621         # If some packages were found...
1622         if mypkgs:
1623                 # Get our global settings (we only print stuff if it varies from
1624                 # the current config)
1625                 mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS' ]
1626                 auxkeys = mydesiredvars + list(vardb._aux_cache_keys)
1627                 auxkeys.append('DEFINED_PHASES')
1628                 global_vals = {}
1629                 pkgsettings = portage.config(clone=settings)
1630
1631                 # Loop through each package
1632                 # Only print settings if they differ from global settings
1633                 header_title = "Package Settings"
1634                 print header_width * "="
1635                 print header_title.rjust(int(header_width/2 + len(header_title)/2))
1636                 print header_width * "="
1637                 from portage.output import EOutput
1638                 out = EOutput()
1639                 for cpv in mypkgs:
1640                         # Get all package specific variables
1641                         metadata = dict(izip(auxkeys, vardb.aux_get(cpv, auxkeys)))
1642                         pkg = Package(built=True, cpv=cpv,
1643                                 installed=True, metadata=izip(Package.metadata_keys,
1644                                 (metadata.get(x, '') for x in Package.metadata_keys)),
1645                                 root_config=root_config, type_name='installed')
1646
1647                         print "\n%s was built with the following:" % \
1648                                 colorize("INFORM", str(pkg.cpv))
1649
1650                         pkgsettings.setcpv(pkg)
1651                         forced_flags = set(chain(pkgsettings.useforce,
1652                                 pkgsettings.usemask))
1653                         use = set(pkg.use.enabled)
1654                         use.discard(pkgsettings.get('ARCH'))
1655                         use_expand_flags = set()
1656                         use_enabled = {}
1657                         use_disabled = {}
1658                         for varname in use_expand:
1659                                 flag_prefix = varname.lower() + "_"
1660                                 for f in use:
1661                                         if f.startswith(flag_prefix):
1662                                                 use_expand_flags.add(f)
1663                                                 use_enabled.setdefault(
1664                                                         varname.upper(), []).append(f[len(flag_prefix):])
1665
1666                                 for f in pkg.iuse.all:
1667                                         if f.startswith(flag_prefix):
1668                                                 use_expand_flags.add(f)
1669                                                 if f not in use:
1670                                                         use_disabled.setdefault(
1671                                                                 varname.upper(), []).append(f[len(flag_prefix):])
1672
1673                         var_order = set(use_enabled)
1674                         var_order.update(use_disabled)
1675                         var_order = sorted(var_order)
1676                         var_order.insert(0, 'USE')
1677                         use.difference_update(use_expand_flags)
1678                         use_enabled['USE'] = list(use)
1679                         use_disabled['USE'] = []
1680
1681                         for f in pkg.iuse.all:
1682                                 if f not in use and \
1683                                         f not in use_expand_flags:
1684                                         use_disabled['USE'].append(f)
1685
1686                         for varname in var_order:
1687                                 if varname in use_expand_hidden:
1688                                         continue
1689                                 flags = []
1690                                 for f in use_enabled.get(varname, []):
1691                                         flags.append(UseFlagDisplay(f, True, f in forced_flags))
1692                                 for f in use_disabled.get(varname, []):
1693                                         flags.append(UseFlagDisplay(f, False, f in forced_flags))
1694                                 if alphabetical_use:
1695                                         flags.sort(key=UseFlagDisplay.sort_combined)
1696                                 else:
1697                                         flags.sort(key=UseFlagDisplay.sort_separated)
1698                                 print '%s="%s"' % (varname, ' '.join(str(f) for f in flags)),
1699                         print
1700
1701                         for myvar in mydesiredvars:
1702                                 if metadata[myvar].split() != settings.get(myvar, '').split():
1703                                         print "%s=\"%s\"" % (myvar, metadata[myvar])
1704                         print
1705
1706                         if metadata['DEFINED_PHASES']:
1707                                 if 'info' not in metadata['DEFINED_PHASES'].split():
1708                                         continue
1709
1710                         print ">>> Attempting to run pkg_info() for '%s'" % pkg.cpv
1711                         ebuildpath = vardb.findname(pkg.cpv)
1712                         if not ebuildpath or not os.path.exists(ebuildpath):
1713                                 out.ewarn("No ebuild found for '%s'" % pkg.cpv)
1714                                 continue
1715                         portage.doebuild(ebuildpath, "info", pkgsettings["ROOT"],
1716                                 pkgsettings, debug=(settings.get("PORTAGE_DEBUG", "") == 1),
1717                                 mydbapi=trees[settings["ROOT"]]["vartree"].dbapi,
1718                                 tree="vartree")
1719
1720 def action_search(root_config, myopts, myfiles, spinner):
1721         if not myfiles:
1722                 print "emerge: no search terms provided."
1723         else:
1724                 searchinstance = search(root_config,
1725                         spinner, "--searchdesc" in myopts,
1726                         "--quiet" not in myopts, "--usepkg" in myopts,
1727                         "--usepkgonly" in myopts)
1728                 for mysearch in myfiles:
1729                         try:
1730                                 searchinstance.execute(mysearch)
1731                         except re.error, comment:
1732                                 print "\n!!! Regular expression error in \"%s\": %s" % ( mysearch, comment )
1733                                 sys.exit(1)
1734                         searchinstance.output()
1735
1736 def action_uninstall(settings, trees, ldpath_mtimes,
1737         opts, action, files, spinner):
1738
1739         # For backward compat, some actions do not require leading '='.
1740         ignore_missing_eq = action in ('clean', 'unmerge')
1741         root = settings['ROOT']
1742         vardb = trees[root]['vartree'].dbapi
1743         valid_atoms = []
1744         lookup_owners = []
1745
1746         # Ensure atoms are valid before calling unmerge().
1747         # For backward compat, leading '=' is not required.
1748         for x in files:
1749                 if is_valid_package_atom(x) or \
1750                         (ignore_missing_eq and is_valid_package_atom('=' + x)):
1751
1752                         try:
1753                                 valid_atoms.append(
1754                                         portage.dep_expand(x, mydb=vardb, settings=settings))
1755                         except portage.exception.AmbiguousPackageName, e:
1756                                 msg = "The short ebuild name \"" + x + \
1757                                         "\" is ambiguous.  Please specify " + \
1758                                         "one of the following " + \
1759                                         "fully-qualified ebuild names instead:"
1760                                 for line in textwrap.wrap(msg, 70):
1761                                         writemsg_level("!!! %s\n" % (line,),
1762                                                 level=logging.ERROR, noiselevel=-1)
1763                                 for i in e[0]:
1764                                         writemsg_level("    %s\n" % colorize("INFORM", i),
1765                                                 level=logging.ERROR, noiselevel=-1)
1766                                 writemsg_level("\n", level=logging.ERROR, noiselevel=-1)
1767                                 return 1
1768
1769                 elif x.startswith(os.sep):
1770                         if not x.startswith(root):
1771                                 writemsg_level(("!!! '%s' does not start with" + \
1772                                         " $ROOT.\n") % x, level=logging.ERROR, noiselevel=-1)
1773                                 return 1
1774                         # Queue these up since it's most efficient to handle
1775                         # multiple files in a single iter_owners() call.
1776                         lookup_owners.append(x)
1777
1778                 else:
1779                         msg = []
1780                         msg.append("'%s' is not a valid package atom." % (x,))
1781                         msg.append("Please check ebuild(5) for full details.")
1782                         writemsg_level("".join("!!! %s\n" % line for line in msg),
1783                                 level=logging.ERROR, noiselevel=-1)
1784                         return 1
1785
1786         if lookup_owners:
1787                 relative_paths = []
1788                 search_for_multiple = False
1789                 if len(lookup_owners) > 1:
1790                         search_for_multiple = True
1791
1792                 for x in lookup_owners:
1793                         if not search_for_multiple and os.path.isdir(x):
1794                                 search_for_multiple = True
1795                         relative_paths.append(x[len(root):])
1796
1797                 owners = set()
1798                 for pkg, relative_path in \
1799                         vardb._owners.iter_owners(relative_paths):
1800                         owners.add(pkg.mycpv)
1801                         if not search_for_multiple:
1802                                 break
1803
1804                 if owners:
1805                         for cpv in owners:
1806                                 slot = vardb.aux_get(cpv, ['SLOT'])[0]
1807                                 if not slot:
1808                                         # portage now masks packages with missing slot, but it's
1809                                         # possible that one was installed by an older version
1810                                         atom = portage.cpv_getkey(cpv)
1811                                 else:
1812                                         atom = '%s:%s' % (portage.cpv_getkey(cpv), slot)
1813                                 valid_atoms.append(portage.dep.Atom(atom))
1814                 else:
1815                         writemsg_level(("!!! '%s' is not claimed " + \
1816                                 "by any package.\n") % lookup_owners[0],
1817                                 level=logging.WARNING, noiselevel=-1)
1818
1819         if files and not valid_atoms:
1820                 return 1
1821
1822         if action in ('clean', 'unmerge') or \
1823                 (action == 'prune' and "--nodeps" in opts):
1824                 # When given a list of atoms, unmerge them in the order given.
1825                 ordered = action == 'unmerge'
1826                 unmerge(trees[settings["ROOT"]]['root_config'], opts, action,
1827                         valid_atoms, ldpath_mtimes, ordered=ordered)
1828                 rval = os.EX_OK
1829         elif action == 'deselect':
1830                 rval = action_deselect(settings, trees, opts, valid_atoms)
1831         else:
1832                 rval = action_depclean(settings, trees, ldpath_mtimes,
1833                         opts, action, valid_atoms, spinner)
1834
1835         return rval
1836
1837 def action_deselect(settings, trees, opts, atoms):
1838         root_config = trees[settings['ROOT']]['root_config']
1839         world_set = root_config.sets['world']
1840         if not hasattr(world_set, 'update'):
1841                 writemsg_level("World set does not appear to be mutable.\n",
1842                         level=logging.ERROR, noiselevel=-1)
1843                 return 1
1844
1845         vardb = root_config.trees['vartree'].dbapi
1846         expanded_atoms = set(atoms)
1847         from portage.dep import Atom
1848         for atom in atoms:
1849                 for cpv in vardb.match(atom):
1850                         slot, = vardb.aux_get(cpv, ['SLOT'])
1851                         if not slot:
1852                                 slot = '0'
1853                         expanded_atoms.add(Atom('%s:%s' % (portage.cpv_getkey(cpv), slot)))
1854
1855         pretend = '--pretend' in opts
1856         locked = False
1857         if not pretend and hasattr(world_set, 'lock'):
1858                 world_set.lock()
1859                 locked = True
1860         try:
1861                 discard_atoms = set()
1862                 world_set.load()
1863                 for atom in world_set:
1864                         if not isinstance(atom, Atom):
1865                                 # nested set
1866                                 continue
1867                         for arg_atom in expanded_atoms:
1868                                 if arg_atom.intersects(atom) and \
1869                                         not (arg_atom.slot and not atom.slot):
1870                                         discard_atoms.add(atom)
1871                                         break
1872                 if discard_atoms:
1873                         for atom in sorted(discard_atoms):
1874                                 print ">>> Removing %s from \"world\" favorites file..." % \
1875                                         colorize("INFORM", str(atom))
1876
1877                         if '--ask' in opts:
1878                                 prompt = "Would you like to remove these " + \
1879                                         "packages from your world favorites?"
1880                                 if userquery(prompt) == 'No':
1881                                         return os.EX_OK
1882
1883                         remaining = set(world_set)
1884                         remaining.difference_update(discard_atoms)
1885                         if not pretend:
1886                                 world_set.replace(remaining)
1887                 else:
1888                         print ">>> No matching atoms found in \"world\" favorites file..."
1889         finally:
1890                 if locked:
1891                         world_set.unlock()
1892         return os.EX_OK
1893
1894 def action_depclean(settings, trees, ldpath_mtimes,
1895         myopts, action, myfiles, spinner):
1896         # Kill packages that aren't explicitly merged or are required as a
1897         # dependency of another package. World file is explicit.
1898
1899         # Global depclean or prune operations are not very safe when there are
1900         # missing dependencies since it's unknown how badly incomplete
1901         # the dependency graph is, and we might accidentally remove packages
1902         # that should have been pulled into the graph. On the other hand, it's
1903         # relatively safe to ignore missing deps when only asked to remove
1904         # specific packages.
1905         allow_missing_deps = len(myfiles) > 0
1906
1907         msg = []
1908         msg.append("Always study the list of packages to be cleaned for any obvious\n")
1909         msg.append("mistakes. Packages that are part of the world set will always\n")
1910         msg.append("be kept.  They can be manually added to this set with\n")
1911         msg.append(good("`emerge --noreplace <atom>`") + ".  Packages that are listed in\n")
1912         msg.append("package.provided (see portage(5)) will be removed by\n")
1913         msg.append("depclean, even if they are part of the world set.\n")
1914         msg.append("\n")
1915         msg.append("As a safety measure, depclean will not remove any packages\n")
1916         msg.append("unless *all* required dependencies have been resolved.  As a\n")
1917         msg.append("consequence, it is often necessary to run %s\n" % \
1918                 good("`emerge --update"))
1919         msg.append(good("--newuse --deep @system @world`") + \
1920                 " prior to depclean.\n")
1921
1922         if action == "depclean" and "--quiet" not in myopts and not myfiles:
1923                 portage.writemsg_stdout("\n")
1924                 for x in msg:
1925                         portage.writemsg_stdout(colorize("WARN", " * ") + x)
1926
1927         xterm_titles = "notitles" not in settings.features
1928         myroot = settings["ROOT"]
1929         root_config = trees[myroot]["root_config"]
1930         getSetAtoms = root_config.setconfig.getSetAtoms
1931         vardb = trees[myroot]["vartree"].dbapi
1932         deselect = myopts.get('--deselect') != 'n'
1933
1934         required_set_names = ("system", "world")
1935         required_sets = {}
1936         set_args = []
1937
1938         for s in required_set_names:
1939                 required_sets[s] = InternalPackageSet(
1940                         initial_atoms=getSetAtoms(s))
1941
1942
1943         # When removing packages, use a temporary version of world
1944         # which excludes packages that are intended to be eligible for
1945         # removal.
1946         world_temp_set = required_sets["world"]
1947         system_set = required_sets["system"]
1948
1949         if not system_set or not world_temp_set:
1950
1951                 if not system_set:
1952                         writemsg_level("!!! You have no system list.\n",
1953                                 level=logging.ERROR, noiselevel=-1)
1954
1955                 if not world_temp_set:
1956                         writemsg_level("!!! You have no world file.\n",
1957                                         level=logging.WARNING, noiselevel=-1)
1958
1959                 writemsg_level("!!! Proceeding is likely to " + \
1960                         "break your installation.\n",
1961                         level=logging.WARNING, noiselevel=-1)
1962                 if "--pretend" not in myopts:
1963                         countdown(int(settings["EMERGE_WARNING_DELAY"]), ">>> Depclean")
1964
1965         if action == "depclean":
1966                 emergelog(xterm_titles, " >>> depclean")
1967
1968         import textwrap
1969         args_set = InternalPackageSet()
1970         if myfiles:
1971                 args_set.update(myfiles)
1972                 matched_packages = False
1973                 for x in args_set:
1974                         if vardb.match(x):
1975                                 matched_packages = True
1976                                 break
1977                 if not matched_packages:
1978                         writemsg_level(">>> No packages selected for removal by %s\n" % \
1979                                 action)
1980                         return
1981
1982         writemsg_level("\nCalculating dependencies  ")
1983         resolver_params = create_depgraph_params(myopts, "remove")
1984         resolver = depgraph(settings, trees, myopts, resolver_params, spinner)
1985         vardb = resolver.trees[myroot]["vartree"].dbapi
1986
1987         if action == "depclean":
1988
1989                 if args_set:
1990
1991                         if deselect:
1992                                 world_temp_set.clear()
1993
1994                         # Pull in everything that's installed but not matched
1995                         # by an argument atom since we don't want to clean any
1996                         # package if something depends on it.
1997                         for pkg in vardb:
1998                                 spinner.update()
1999
2000                                 try:
2001                                         if args_set.findAtomForPackage(pkg) is None:
2002                                                 world_temp_set.add("=" + pkg.cpv)
2003                                                 continue
2004                                 except portage.exception.InvalidDependString, e:
2005                                         show_invalid_depstring_notice(pkg,
2006                                                 pkg.metadata["PROVIDE"], str(e))
2007                                         del e
2008                                         world_temp_set.add("=" + pkg.cpv)
2009                                         continue
2010
2011         elif action == "prune":
2012
2013                 if deselect:
2014                         world_temp_set.clear()
2015
2016                 # Pull in everything that's installed since we don't
2017                 # to prune a package if something depends on it.
2018                 world_temp_set.update(vardb.cp_all())
2019
2020                 if not args_set:
2021
2022                         # Try to prune everything that's slotted.
2023                         for cp in vardb.cp_all():
2024                                 if len(vardb.cp_list(cp)) > 1:
2025                                         args_set.add(cp)
2026
2027                 # Remove atoms from world that match installed packages
2028                 # that are also matched by argument atoms, but do not remove
2029                 # them if they match the highest installed version.
2030                 for pkg in vardb:
2031                         spinner.update()
2032                         pkgs_for_cp = vardb.match_pkgs(pkg.cp)
2033                         if not pkgs_for_cp or pkg not in pkgs_for_cp:
2034                                 raise AssertionError("package expected in matches: " + \
2035                                         "cp = %s, cpv = %s matches = %s" % \
2036                                         (pkg.cp, pkg.cpv, [str(x) for x in pkgs_for_cp]))
2037
2038                         highest_version = pkgs_for_cp[-1]
2039                         if pkg == highest_version:
2040                                 # pkg is the highest version
2041                                 world_temp_set.add("=" + pkg.cpv)
2042                                 continue
2043
2044                         if len(pkgs_for_cp) <= 1:
2045                                 raise AssertionError("more packages expected: " + \
2046                                         "cp = %s, cpv = %s matches = %s" % \
2047                                         (pkg.cp, pkg.cpv, [str(x) for x in pkgs_for_cp]))
2048
2049                         try:
2050                                 if args_set.findAtomForPackage(pkg) is None:
2051                                         world_temp_set.add("=" + pkg.cpv)
2052                                         continue
2053                         except portage.exception.InvalidDependString, e:
2054                                 show_invalid_depstring_notice(pkg,
2055                                         pkg.metadata["PROVIDE"], str(e))
2056                                 del e
2057                                 world_temp_set.add("=" + pkg.cpv)
2058                                 continue
2059
2060         set_args = {}
2061         for s, package_set in required_sets.iteritems():
2062                 set_atom = SETPREFIX + s
2063                 set_arg = SetArg(arg=set_atom, set=package_set,
2064                         root_config=resolver.roots[myroot])
2065                 set_args[s] = set_arg
2066                 for atom in set_arg.set:
2067                         resolver._dep_stack.append(
2068                                 Dependency(atom=atom, root=myroot, parent=set_arg))
2069                         resolver.digraph.add(set_arg, None)
2070
2071         success = resolver._complete_graph()
2072         writemsg_level("\b\b... done!\n")
2073
2074         resolver.display_problems()
2075
2076         if not success:
2077                 return 1
2078
2079         def unresolved_deps():
2080
2081                 unresolvable = set()
2082                 for dep in resolver._initially_unsatisfied_deps:
2083                         if isinstance(dep.parent, Package) and \
2084                                 (dep.priority > UnmergeDepPriority.SOFT):
2085                                 unresolvable.add((dep.atom, dep.parent.cpv))
2086
2087                 if not unresolvable:
2088                         return False
2089
2090                 if unresolvable and not allow_missing_deps:
2091                         prefix = bad(" * ")
2092                         msg = []
2093                         msg.append("Dependencies could not be completely resolved due to")
2094                         msg.append("the following required packages not being installed:")
2095                         msg.append("")
2096                         for atom, parent in unresolvable:
2097                                 msg.append("  %s pulled in by:" % (atom,))
2098                                 msg.append("    %s" % (parent,))
2099                                 msg.append("")
2100                         msg.append("Have you forgotten to run " + \
2101                                 good("`emerge --update --newuse --deep @system @world`") + " prior")
2102                         msg.append(("to %s? It may be necessary to manually " + \
2103                                 "uninstall packages that no longer") % action)
2104                         msg.append("exist in the portage tree since " + \
2105                                 "it may not be possible to satisfy their")
2106                         msg.append("dependencies.  Also, be aware of " + \
2107                                 "the --with-bdeps option that is documented")
2108                         msg.append("in " + good("`man emerge`") + ".")
2109                         if action == "prune":
2110                                 msg.append("")
2111                                 msg.append("If you would like to ignore " + \
2112                                         "dependencies then use %s." % good("--nodeps"))
2113                         writemsg_level("".join("%s%s\n" % (prefix, line) for line in msg),
2114                                 level=logging.ERROR, noiselevel=-1)
2115                         return True
2116                 return False
2117
2118         if unresolved_deps():
2119                 return 1
2120
2121         graph = resolver.digraph.copy()
2122         required_pkgs_total = 0
2123         for node in graph:
2124                 if isinstance(node, Package):
2125                         required_pkgs_total += 1
2126
2127         def show_parents(child_node):
2128                 parent_nodes = graph.parent_nodes(child_node)
2129                 if not parent_nodes:
2130                         # With --prune, the highest version can be pulled in without any
2131                         # real parent since all installed packages are pulled in.  In that
2132                         # case there's nothing to show here.
2133                         return
2134                 parent_strs = []
2135                 for node in parent_nodes:
2136                         parent_strs.append(str(getattr(node, "cpv", node)))
2137                 parent_strs.sort()
2138                 msg = []
2139                 msg.append("  %s pulled in by:\n" % (child_node.cpv,))
2140                 for parent_str in parent_strs:
2141                         msg.append("    %s\n" % (parent_str,))
2142                 msg.append("\n")
2143                 portage.writemsg_stdout("".join(msg), noiselevel=-1)
2144
2145         def cmp_pkg_cpv(pkg1, pkg2):
2146                 """Sort Package instances by cpv."""
2147                 if pkg1.cpv > pkg2.cpv:
2148                         return 1
2149                 elif pkg1.cpv == pkg2.cpv:
2150                         return 0
2151                 else:
2152                         return -1
2153
2154         def create_cleanlist():
2155                 pkgs_to_remove = []
2156
2157                 if action == "depclean":
2158                         if args_set:
2159
2160                                 for pkg in sorted(vardb, key=cmp_sort_key(cmp_pkg_cpv)):
2161                                         arg_atom = None
2162                                         try:
2163                                                 arg_atom = args_set.findAtomForPackage(pkg)
2164                                         except portage.exception.InvalidDependString:
2165                                                 # this error has already been displayed by now
2166                                                 continue
2167
2168                                         if arg_atom:
2169                                                 if pkg not in graph:
2170                                                         pkgs_to_remove.append(pkg)
2171                                                 elif "--verbose" in myopts:
2172                                                         show_parents(pkg)
2173
2174                         else:
2175                                 for pkg in sorted(vardb, key=cmp_sort_key(cmp_pkg_cpv)):
2176                                         if pkg not in graph:
2177                                                 pkgs_to_remove.append(pkg)
2178                                         elif "--verbose" in myopts:
2179                                                 show_parents(pkg)
2180
2181                 elif action == "prune":
2182                         # Prune really uses all installed instead of world. It's not
2183                         # a real reverse dependency so don't display it as such.
2184                         graph.remove(set_args["world"])
2185
2186                         for atom in args_set:
2187                                 for pkg in vardb.match_pkgs(atom):
2188                                         if pkg not in graph:
2189                                                 pkgs_to_remove.append(pkg)
2190                                         elif "--verbose" in myopts:
2191                                                 show_parents(pkg)
2192
2193                 if not pkgs_to_remove:
2194                         writemsg_level(
2195                                 ">>> No packages selected for removal by %s\n" % action)
2196                         if "--verbose" not in myopts:
2197                                 writemsg_level(
2198                                         ">>> To see reverse dependencies, use %s\n" % \
2199                                                 good("--verbose"))
2200                         if action == "prune":
2201                                 writemsg_level(
2202                                         ">>> To ignore dependencies, use %s\n" % \
2203                                                 good("--nodeps"))
2204
2205                 return pkgs_to_remove
2206
2207         cleanlist = create_cleanlist()
2208
2209         if len(cleanlist):
2210                 clean_set = set(cleanlist)
2211
2212                 # Check if any of these package are the sole providers of libraries
2213                 # with consumers that have not been selected for removal. If so, these
2214                 # packages and any dependencies need to be added to the graph.
2215                 real_vardb = trees[myroot]["vartree"].dbapi
2216                 linkmap = real_vardb.linkmap
2217                 liblist = linkmap.listLibraryObjects()
2218                 consumer_cache = {}
2219                 provider_cache = {}
2220                 soname_cache = {}
2221                 consumer_map = {}
2222
2223                 writemsg_level(">>> Checking for lib consumers...\n")
2224
2225                 for pkg in cleanlist:
2226                         pkg_dblink = real_vardb._dblink(pkg.cpv)
2227                         provided_libs = set()
2228
2229                         for lib in liblist:
2230                                 if pkg_dblink.isowner(lib, myroot):
2231                                         provided_libs.add(lib)
2232
2233                         if not provided_libs:
2234                                 continue
2235
2236                         consumers = {}
2237                         for lib in provided_libs:
2238                                 lib_consumers = consumer_cache.get(lib)
2239                                 if lib_consumers is None:
2240                                         lib_consumers = linkmap.findConsumers(lib)
2241                                         consumer_cache[lib] = lib_consumers
2242                                 if lib_consumers:
2243                                         consumers[lib] = lib_consumers
2244
2245                         if not consumers:
2246                                 continue
2247
2248                         for lib, lib_consumers in consumers.items():
2249                                 for consumer_file in list(lib_consumers):
2250                                         if pkg_dblink.isowner(consumer_file, myroot):
2251                                                 lib_consumers.remove(consumer_file)
2252                                 if not lib_consumers:
2253                                         del consumers[lib]
2254
2255                         if not consumers:
2256                                 continue
2257
2258                         for lib, lib_consumers in consumers.iteritems():
2259
2260                                 soname = soname_cache.get(lib)
2261                                 if soname is None:
2262                                         soname = linkmap.getSoname(lib)
2263                                         soname_cache[lib] = soname
2264
2265                                 consumer_providers = []
2266                                 for lib_consumer in lib_consumers:
2267                                         providers = provider_cache.get(lib)
2268                                         if providers is None:
2269                                                 providers = linkmap.findProviders(lib_consumer)
2270                                                 provider_cache[lib_consumer] = providers
2271                                         if soname not in providers:
2272                                                 # Why does this happen?
2273                                                 continue
2274                                         consumer_providers.append(
2275                                                 (lib_consumer, providers[soname]))
2276
2277                                 consumers[lib] = consumer_providers
2278
2279                         consumer_map[pkg] = consumers
2280
2281                 if consumer_map:
2282
2283                         search_files = set()
2284                         for consumers in consumer_map.itervalues():
2285                                 for lib, consumer_providers in consumers.iteritems():
2286                                         for lib_consumer, providers in consumer_providers:
2287                                                 search_files.add(lib_consumer)
2288                                                 search_files.update(providers)
2289
2290                         writemsg_level(">>> Assigning files to packages...\n")
2291                         file_owners = real_vardb._owners.getFileOwnerMap(search_files)
2292
2293                         for pkg, consumers in consumer_map.items():
2294                                 for lib, consumer_providers in consumers.items():
2295                                         lib_consumers = set()
2296
2297                                         for lib_consumer, providers in consumer_providers:
2298                                                 owner_set = file_owners.get(lib_consumer)
2299                                                 provider_dblinks = set()
2300                                                 provider_pkgs = set()
2301
2302                                                 if len(providers) > 1:
2303                                                         for provider in providers:
2304                                                                 provider_set = file_owners.get(provider)
2305                                                                 if provider_set is not None:
2306                                                                         provider_dblinks.update(provider_set)
2307
2308                                                 if len(provider_dblinks) > 1:
2309                                                         for provider_dblink in provider_dblinks:
2310                                                                 pkg_key = ("installed", myroot,
2311                                                                         provider_dblink.mycpv, "nomerge")
2312                                                                 if pkg_key not in clean_set:
2313                                                                         provider_pkgs.add(vardb.get(pkg_key))
2314
2315                                                 if provider_pkgs:
2316                                                         continue
2317
2318                                                 if owner_set is not None:
2319                                                         lib_consumers.update(owner_set)
2320
2321                                         for consumer_dblink in list(lib_consumers):
2322                                                 if ("installed", myroot, consumer_dblink.mycpv,
2323                                                         "nomerge") in clean_set:
2324                                                         lib_consumers.remove(consumer_dblink)
2325                                                         continue
2326
2327                                         if lib_consumers:
2328                                                 consumers[lib] = lib_consumers
2329                                         else:
2330                                                 del consumers[lib]
2331                                 if not consumers:
2332                                         del consumer_map[pkg]
2333
2334                 if consumer_map:
2335                         # TODO: Implement a package set for rebuilding consumer packages.
2336
2337                         msg = "In order to avoid breakage of link level " + \
2338                                 "dependencies, one or more packages will not be removed. " + \
2339                                 "This can be solved by rebuilding " + \
2340                                 "the packages that pulled them in."
2341
2342                         prefix = bad(" * ")
2343                         from textwrap import wrap
2344                         writemsg_level("".join(prefix + "%s\n" % line for \
2345                                 line in wrap(msg, 70)), level=logging.WARNING, noiselevel=-1)
2346
2347                         msg = []
2348                         for pkg, consumers in consumer_map.iteritems():
2349                                 unique_consumers = set(chain(*consumers.values()))
2350                                 unique_consumers = sorted(consumer.mycpv \
2351                                         for consumer in unique_consumers)
2352                                 msg.append("")
2353                                 msg.append("  %s pulled in by:" % (pkg.cpv,))
2354                                 for consumer in unique_consumers:
2355                                         msg.append("    %s" % (consumer,))
2356                         msg.append("")
2357                         writemsg_level("".join(prefix + "%s\n" % line for line in msg),
2358                                 level=logging.WARNING, noiselevel=-1)
2359
2360                         # Add lib providers to the graph as children of lib consumers,
2361                         # and also add any dependencies pulled in by the provider.
2362                         writemsg_level(">>> Adding lib providers to graph...\n")
2363
2364                         for pkg, consumers in consumer_map.iteritems():
2365                                 for consumer_dblink in set(chain(*consumers.values())):
2366                                         consumer_pkg = vardb.get(("installed", myroot,
2367                                                 consumer_dblink.mycpv, "nomerge"))
2368                                         if not resolver._add_pkg(pkg,
2369                                                 Dependency(parent=consumer_pkg,
2370                                                 priority=UnmergeDepPriority(runtime=True),
2371                                                 root=pkg.root)):
2372                                                 resolver.display_problems()
2373                                                 return 1
2374
2375                         writemsg_level("\nCalculating dependencies  ")
2376                         success = resolver._complete_graph()
2377                         writemsg_level("\b\b... done!\n")
2378                         resolver.display_problems()
2379                         if not success:
2380                                 return 1
2381                         if unresolved_deps():
2382                                 return 1
2383
2384                         graph = resolver.digraph.copy()
2385                         required_pkgs_total = 0
2386                         for node in graph:
2387                                 if isinstance(node, Package):
2388                                         required_pkgs_total += 1
2389                         cleanlist = create_cleanlist()
2390                         if not cleanlist:
2391                                 return 0
2392                         clean_set = set(cleanlist)
2393
2394                 # Use a topological sort to create an unmerge order such that
2395                 # each package is unmerged before it's dependencies. This is
2396                 # necessary to avoid breaking things that may need to run
2397                 # during pkg_prerm or pkg_postrm phases.
2398
2399                 # Create a new graph to account for dependencies between the
2400                 # packages being unmerged.
2401                 graph = digraph()
2402                 del cleanlist[:]
2403
2404                 dep_keys = ["DEPEND", "RDEPEND", "PDEPEND"]
2405                 runtime = UnmergeDepPriority(runtime=True)
2406                 runtime_post = UnmergeDepPriority(runtime_post=True)
2407                 buildtime = UnmergeDepPriority(buildtime=True)
2408                 priority_map = {
2409                         "RDEPEND": runtime,
2410                         "PDEPEND": runtime_post,
2411                         "DEPEND": buildtime,
2412                 }
2413
2414                 for node in clean_set:
2415                         graph.add(node, None)
2416                         mydeps = []
2417                         node_use = node.metadata["USE"].split()
2418                         for dep_type in dep_keys:
2419                                 depstr = node.metadata[dep_type]
2420                                 if not depstr:
2421                                         continue
2422                                 try:
2423                                         portage.dep._dep_check_strict = False
2424                                         success, atoms = portage.dep_check(depstr, None, settings,
2425                                                 myuse=node_use, trees=resolver._graph_trees,
2426                                                 myroot=myroot)
2427                                 finally:
2428                                         portage.dep._dep_check_strict = True
2429                                 if not success:
2430                                         # Ignore invalid deps of packages that will
2431                                         # be uninstalled anyway.
2432                                         continue
2433
2434                                 priority = priority_map[dep_type]
2435                                 for atom in atoms:
2436                                         if not isinstance(atom, portage.dep.Atom):
2437                                                 # Ignore invalid atoms returned from dep_check().
2438                                                 continue
2439                                         if atom.blocker:
2440                                                 continue
2441                                         matches = vardb.match_pkgs(atom)
2442                                         if not matches:
2443                                                 continue
2444                                         for child_node in matches:
2445                                                 if child_node in clean_set:
2446                                                         graph.add(child_node, node, priority=priority)
2447
2448                 ordered = True
2449                 if len(graph.order) == len(graph.root_nodes()):
2450                         # If there are no dependencies between packages
2451                         # let unmerge() group them by cat/pn.
2452                         ordered = False
2453                         cleanlist = [pkg.cpv for pkg in graph.order]
2454                 else:
2455                         # Order nodes from lowest to highest overall reference count for
2456                         # optimal root node selection.
2457                         node_refcounts = {}
2458                         for node in graph.order:
2459                                 node_refcounts[node] = len(graph.parent_nodes(node))
2460                         def cmp_reference_count(node1, node2):
2461                                 return node_refcounts[node1] - node_refcounts[node2]
2462                         graph.order.sort(key=cmp_sort_key(cmp_reference_count))
2463
2464                         ignore_priority_range = [None]
2465                         ignore_priority_range.extend(
2466                                 xrange(UnmergeDepPriority.MIN, UnmergeDepPriority.MAX + 1))
2467                         while not graph.empty():
2468                                 for ignore_priority in ignore_priority_range:
2469                                         nodes = graph.root_nodes(ignore_priority=ignore_priority)
2470                                         if nodes:
2471                                                 break
2472                                 if not nodes:
2473                                         raise AssertionError("no root nodes")
2474                                 if ignore_priority is not None:
2475                                         # Some deps have been dropped due to circular dependencies,
2476                                         # so only pop one node in order do minimize the number that
2477                                         # are dropped.
2478                                         del nodes[1:]
2479                                 for node in nodes:
2480                                         graph.remove(node)
2481                                         cleanlist.append(node.cpv)
2482
2483                 unmerge(root_config, myopts, "unmerge", cleanlist,
2484                         ldpath_mtimes, ordered=ordered)
2485
2486         if action == "prune":
2487                 return
2488
2489         if not cleanlist and "--quiet" in myopts:
2490                 return
2491
2492         print "Packages installed:   "+str(len(vardb.cpv_all()))
2493         print "Packages in world:    " + \
2494                 str(len(root_config.sets["world"].getAtoms()))
2495         print "Packages in system:   " + \
2496                 str(len(root_config.sets["system"].getAtoms()))
2497         print "Required packages:    "+str(required_pkgs_total)
2498         if "--pretend" in myopts:
2499                 print "Number to remove:     "+str(len(cleanlist))
2500         else:
2501                 print "Number removed:       "+str(len(cleanlist))
2502
2503 def action_build(settings, trees, mtimedb,
2504         myopts, myaction, myfiles, spinner):
2505
2506         # validate the state of the resume data
2507         # so that we can make assumptions later.
2508         for k in ("resume", "resume_backup"):
2509                 if k not in mtimedb:
2510                         continue
2511                 resume_data = mtimedb[k]
2512                 if not isinstance(resume_data, dict):
2513                         del mtimedb[k]
2514                         continue
2515                 mergelist = resume_data.get("mergelist")
2516                 if not isinstance(mergelist, list):
2517                         del mtimedb[k]
2518                         continue
2519                 for x in mergelist:
2520                         if not (isinstance(x, list) and len(x) == 4):
2521                                 continue
2522                         pkg_type, pkg_root, pkg_key, pkg_action = x
2523                         if pkg_root not in trees:
2524                                 # Current $ROOT setting differs,
2525                                 # so the list must be stale.
2526                                 mergelist = None
2527                                 break
2528                 if not mergelist:
2529                         del mtimedb[k]
2530                         continue
2531                 resume_opts = resume_data.get("myopts")
2532                 if not isinstance(resume_opts, (dict, list)):
2533                         del mtimedb[k]
2534                         continue
2535                 favorites = resume_data.get("favorites")
2536                 if not isinstance(favorites, list):
2537                         del mtimedb[k]
2538                         continue
2539
2540         resume = False
2541         if "--resume" in myopts and \
2542                 ("resume" in mtimedb or
2543                 "resume_backup" in mtimedb):
2544                 resume = True
2545                 if "resume" not in mtimedb:
2546                         mtimedb["resume"] = mtimedb["resume_backup"]
2547                         del mtimedb["resume_backup"]
2548                         mtimedb.commit()
2549                 # "myopts" is a list for backward compatibility.
2550                 resume_opts = mtimedb["resume"].get("myopts", [])
2551                 if isinstance(resume_opts, list):
2552                         resume_opts = dict((k,True) for k in resume_opts)
2553                 for opt in ("--ask", "--color", "--skipfirst", "--tree"):
2554                         resume_opts.pop(opt, None)
2555
2556                 # Current options always override resume_opts.
2557                 resume_opts.update(myopts)
2558                 myopts.clear()
2559                 myopts.update(resume_opts)
2560
2561                 if "--debug" in myopts:
2562                         writemsg_level("myopts %s\n" % (myopts,))
2563
2564                 # Adjust config according to options of the command being resumed.
2565                 for myroot in trees:
2566                         mysettings =  trees[myroot]["vartree"].settings
2567                         mysettings.unlock()
2568                         adjust_config(myopts, mysettings)
2569                         mysettings.lock()
2570                         del myroot, mysettings
2571
2572         ldpath_mtimes = mtimedb["ldpath"]
2573         favorites=[]
2574         merge_count = 0
2575         buildpkgonly = "--buildpkgonly" in myopts
2576         pretend = "--pretend" in myopts
2577         fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts
2578         ask = "--ask" in myopts
2579         nodeps = "--nodeps" in myopts
2580         oneshot = "--oneshot" in myopts or "--onlydeps" in myopts
2581         tree = "--tree" in myopts
2582         if nodeps and tree:
2583                 tree = False
2584                 del myopts["--tree"]
2585                 portage.writemsg(colorize("WARN", " * ") + \
2586                         "--tree is broken with --nodeps. Disabling...\n")
2587         debug = "--debug" in myopts
2588         verbose = "--verbose" in myopts
2589         quiet = "--quiet" in myopts
2590         if pretend or fetchonly:
2591                 # make the mtimedb readonly
2592                 mtimedb.filename = None
2593         if '--digest' in myopts or 'digest' in settings.features:
2594                 if '--digest' in myopts:
2595                         msg = "The --digest option"
2596                 else:
2597                         msg = "The FEATURES=digest setting"
2598
2599                 msg += " can prevent corruption from being" + \
2600                         " noticed. The `repoman manifest` command is the preferred" + \
2601                         " way to generate manifests and it is capable of doing an" + \
2602                         " entire repository or category at once."
2603                 prefix = bad(" * ")
2604                 writemsg(prefix + "\n")
2605                 from textwrap import wrap
2606                 for line in wrap(msg, 72):
2607                         writemsg("%s%s\n" % (prefix, line))
2608                 writemsg(prefix + "\n")
2609
2610         if "--quiet" not in myopts and \
2611                 ("--pretend" in myopts or "--ask" in myopts or \
2612                 "--tree" in myopts or "--verbose" in myopts):
2613                 action = ""
2614                 if "--fetchonly" in myopts or "--fetch-all-uri" in myopts:
2615                         action = "fetched"
2616                 elif "--buildpkgonly" in myopts:
2617                         action = "built"
2618                 else:
2619                         action = "merged"
2620                 if "--tree" in myopts and action != "fetched": # Tree doesn't work with fetching
2621                         print
2622                         print darkgreen("These are the packages that would be %s, in reverse order:") % action
2623                         print
2624                 else:
2625                         print
2626                         print darkgreen("These are the packages that would be %s, in order:") % action
2627                         print
2628
2629         show_spinner = "--quiet" not in myopts and "--nodeps" not in myopts
2630         if not show_spinner:
2631                 spinner.update = spinner.update_quiet
2632
2633         if resume:
2634                 favorites = mtimedb["resume"].get("favorites")
2635                 if not isinstance(favorites, list):
2636                         favorites = []
2637
2638                 if show_spinner:
2639                         print "Calculating dependencies  ",
2640                 myparams = create_depgraph_params(myopts, myaction)
2641
2642                 resume_data = mtimedb["resume"]
2643                 mergelist = resume_data["mergelist"]
2644                 if mergelist and "--skipfirst" in myopts:
2645                         for i, task in enumerate(mergelist):
2646                                 if isinstance(task, list) and \
2647                                         task and task[-1] == "merge":
2648                                         del mergelist[i]
2649                                         break
2650
2651                 success = False
2652                 mydepgraph = None
2653                 try:
2654                         success, mydepgraph, dropped_tasks = resume_depgraph(
2655                                 settings, trees, mtimedb, myopts, myparams, spinner)
2656                 except (portage.exception.PackageNotFound,
2657                         depgraph.UnsatisfiedResumeDep), e:
2658                         if isinstance(e, depgraph.UnsatisfiedResumeDep):
2659                                 mydepgraph = e.depgraph
2660                         if show_spinner:
2661                                 print
2662                         from textwrap import wrap
2663                         from portage.output import EOutput
2664                         out = EOutput()
2665
2666                         resume_data = mtimedb["resume"]
2667                         mergelist = resume_data.get("mergelist")
2668                         if not isinstance(mergelist, list):
2669                                 mergelist = []
2670                         if mergelist and debug or (verbose and not quiet):
2671                                 out.eerror("Invalid resume list:")
2672                                 out.eerror("")
2673                                 indent = "  "
2674                                 for task in mergelist:
2675                                         if isinstance(task, list):
2676                                                 out.eerror(indent + str(tuple(task)))
2677                                 out.eerror("")
2678
2679                         if isinstance(e, depgraph.UnsatisfiedResumeDep):
2680                                 out.eerror("One or more packages are either masked or " + \
2681                                         "have missing dependencies:")
2682                                 out.eerror("")
2683                                 indent = "  "
2684                                 for dep in e.value:
2685                                         if dep.atom is None:
2686                                                 out.eerror(indent + "Masked package:")
2687                                                 out.eerror(2 * indent + str(dep.parent))
2688                                                 out.eerror("")
2689                                         else:
2690                                                 out.eerror(indent + str(dep.atom) + " pulled in by:")
2691                                                 out.eerror(2 * indent + str(dep.parent))
2692                                                 out.eerror("")
2693                                 msg = "The resume list contains packages " + \
2694                                         "that are either masked or have " + \
2695                                         "unsatisfied dependencies. " + \
2696                                         "Please restart/continue " + \
2697                                         "the operation manually, or use --skipfirst " + \
2698                                         "to skip the first package in the list and " + \
2699                                         "any other packages that may be " + \
2700                                         "masked or have missing dependencies."
2701                                 for line in wrap(msg, 72):
2702                                         out.eerror(line)
2703                         elif isinstance(e, portage.exception.PackageNotFound):
2704                                 out.eerror("An expected package is " + \
2705                                         "not available: %s" % str(e))
2706                                 out.eerror("")
2707                                 msg = "The resume list contains one or more " + \
2708                                         "packages that are no longer " + \
2709                                         "available. Please restart/continue " + \
2710                                         "the operation manually."
2711                                 for line in wrap(msg, 72):
2712                                         out.eerror(line)
2713                 else:
2714                         if show_spinner:
2715                                 print "\b\b... done!"
2716
2717                 if success:
2718                         if dropped_tasks:
2719                                 portage.writemsg("!!! One or more packages have been " + \
2720                                         "dropped due to\n" + \
2721                                         "!!! masking or unsatisfied dependencies:\n\n",
2722                                         noiselevel=-1)
2723                                 for task in dropped_tasks:
2724                                         portage.writemsg("  " + str(task) + "\n", noiselevel=-1)
2725                                 portage.writemsg("\n", noiselevel=-1)
2726                         del dropped_tasks
2727                 else:
2728                         if mydepgraph is not None:
2729                                 mydepgraph.display_problems()
2730                         if not (ask or pretend):
2731                                 # delete the current list and also the backup
2732                                 # since it's probably stale too.
2733                                 for k in ("resume", "resume_backup"):
2734                                         mtimedb.pop(k, None)
2735                                 mtimedb.commit()
2736
2737                         return 1
2738         else:
2739                 if ("--resume" in myopts):
2740                         print darkgreen("emerge: It seems we have nothing to resume...")
2741                         return os.EX_OK
2742
2743                 myparams = create_depgraph_params(myopts, myaction)
2744                 if "--quiet" not in myopts and "--nodeps" not in myopts:
2745                         print "Calculating dependencies  ",
2746                         sys.stdout.flush()
2747                 mydepgraph = depgraph(settings, trees, myopts, myparams, spinner)
2748                 try:
2749                         retval, favorites = mydepgraph.select_files(myfiles)
2750                 except portage.exception.PackageNotFound, e:
2751                         portage.writemsg("\n!!! %s\n" % str(e), noiselevel=-1)
2752                         return 1
2753                 except portage.exception.PackageSetNotFound, e:
2754                         root_config = trees[settings["ROOT"]]["root_config"]
2755                         display_missing_pkg_set(root_config, e.value)
2756                         return 1
2757                 if show_spinner:
2758                         print "\b\b... done!"
2759                 if not retval:
2760                         mydepgraph.display_problems()
2761                         return 1
2762
2763         if "--pretend" not in myopts and \
2764                 ("--ask" in myopts or "--tree" in myopts or \
2765                 "--verbose" in myopts) and \
2766                 not ("--quiet" in myopts and "--ask" not in myopts):
2767                 if "--resume" in myopts:
2768                         mymergelist = mydepgraph.altlist()
2769                         if len(mymergelist) == 0:
2770                                 print colorize("INFORM", "emerge: It seems we have nothing to resume...")
2771                                 return os.EX_OK
2772                         favorites = mtimedb["resume"]["favorites"]
2773                         retval = mydepgraph.display(
2774                                 mydepgraph.altlist(reversed=tree),
2775                                 favorites=favorites)
2776                         mydepgraph.display_problems()
2777                         if retval != os.EX_OK:
2778                                 return retval
2779                         prompt="Would you like to resume merging these packages?"
2780                 else:
2781                         retval = mydepgraph.display(
2782                                 mydepgraph.altlist(reversed=("--tree" in myopts)),
2783                                 favorites=favorites)
2784                         mydepgraph.display_problems()
2785                         if retval != os.EX_OK:
2786                                 return retval
2787                         mergecount=0
2788                         for x in mydepgraph.altlist():
2789                                 if isinstance(x, Package) and x.operation == "merge":
2790                                         mergecount += 1
2791
2792                         if mergecount==0:
2793                                 sets = trees[settings["ROOT"]]["root_config"].sets
2794                                 world_candidates = None
2795                                 if "--noreplace" in myopts and \
2796                                         not oneshot and favorites:
2797                                         # Sets that are not world candidates are filtered
2798                                         # out here since the favorites list needs to be
2799                                         # complete for depgraph.loadResumeCommand() to
2800                                         # operate correctly.
2801                                         world_candidates = [x for x in favorites \
2802                                                 if not (x.startswith(SETPREFIX) and \
2803                                                 not sets[x[1:]].world_candidate)]
2804                                 if "--noreplace" in myopts and \
2805                                         not oneshot and world_candidates:
2806                                         print
2807                                         for x in world_candidates:
2808                                                 print " %s %s" % (good("*"), x)
2809                                         prompt="Would you like to add these packages to your world favorites?"
2810                                 elif settings["AUTOCLEAN"] and "yes"==settings["AUTOCLEAN"]:
2811                                         prompt="Nothing to merge; would you like to auto-clean packages?"
2812                                 else:
2813                                         print
2814                                         print "Nothing to merge; quitting."
2815                                         print
2816                                         return os.EX_OK
2817                         elif "--fetchonly" in myopts or "--fetch-all-uri" in myopts:
2818                                 prompt="Would you like to fetch the source files for these packages?"
2819                         else:
2820                                 prompt="Would you like to merge these packages?"
2821                 print
2822                 if "--ask" in myopts and userquery(prompt) == "No":
2823                         print
2824                         print "Quitting."
2825                         print
2826                         return os.EX_OK
2827                 # Don't ask again (e.g. when auto-cleaning packages after merge)
2828                 myopts.pop("--ask", None)
2829
2830         if ("--pretend" in myopts) and not ("--fetchonly" in myopts or "--fetch-all-uri" in myopts):
2831                 if ("--resume" in myopts):
2832                         mymergelist = mydepgraph.altlist()
2833                         if len(mymergelist) == 0:
2834                                 print colorize("INFORM", "emerge: It seems we have nothing to resume...")
2835                                 return os.EX_OK
2836                         favorites = mtimedb["resume"]["favorites"]
2837                         retval = mydepgraph.display(
2838                                 mydepgraph.altlist(reversed=tree),
2839                                 favorites=favorites)
2840                         mydepgraph.display_problems()
2841                         if retval != os.EX_OK:
2842                                 return retval
2843                 else:
2844                         retval = mydepgraph.display(
2845                                 mydepgraph.altlist(reversed=("--tree" in myopts)),
2846                                 favorites=favorites)
2847                         mydepgraph.display_problems()
2848                         if retval != os.EX_OK:
2849                                 return retval
2850                         if "--buildpkgonly" in myopts:
2851                                 graph_copy = mydepgraph.digraph.clone()
2852                                 removed_nodes = set()
2853                                 for node in graph_copy:
2854                                         if not isinstance(node, Package) or \
2855                                                 node.operation == "nomerge":
2856                                                 removed_nodes.add(node)
2857                                 graph_copy.difference_update(removed_nodes)
2858                                 if not graph_copy.hasallzeros(ignore_priority = \
2859                                         DepPrioritySatisfiedRange.ignore_medium):
2860                                         print "\n!!! --buildpkgonly requires all dependencies to be merged."
2861                                         print "!!! You have to merge the dependencies before you can build this package.\n"
2862                                         return 1
2863         else:
2864                 if "--buildpkgonly" in myopts:
2865                         graph_copy = mydepgraph.digraph.clone()
2866                         removed_nodes = set()
2867                         for node in graph_copy:
2868                                 if not isinstance(node, Package) or \
2869                                         node.operation == "nomerge":
2870                                         removed_nodes.add(node)
2871                         graph_copy.difference_update(removed_nodes)
2872                         if not graph_copy.hasallzeros(ignore_priority = \
2873                                 DepPrioritySatisfiedRange.ignore_medium):
2874                                 print "\n!!! --buildpkgonly requires all dependencies to be merged."
2875                                 print "!!! Cannot merge requested packages. Merge deps and try again.\n"
2876                                 return 1
2877
2878                 if ("--resume" in myopts):
2879                         favorites=mtimedb["resume"]["favorites"]
2880                         mymergelist = mydepgraph.altlist()
2881                         mydepgraph.break_refs(mymergelist)
2882                         mergetask = Scheduler(settings, trees, mtimedb, myopts,
2883                                 spinner, mymergelist, favorites, mydepgraph.schedulerGraph())
2884                         del mydepgraph, mymergelist
2885                         clear_caches(trees)
2886
2887                         retval = mergetask.merge()
2888                         merge_count = mergetask.curval
2889                 else:
2890                         if "resume" in mtimedb and \
2891                         "mergelist" in mtimedb["resume"] and \
2892                         len(mtimedb["resume"]["mergelist"]) > 1:
2893                                 mtimedb["resume_backup"] = mtimedb["resume"]
2894                                 del mtimedb["resume"]
2895                                 mtimedb.commit()
2896                         mtimedb["resume"]={}
2897                         # Stored as a dict starting with portage-2.1.6_rc1, and supported
2898                         # by >=portage-2.1.3_rc8. Versions <portage-2.1.3_rc8 only support
2899                         # a list type for options.
2900                         mtimedb["resume"]["myopts"] = myopts.copy()
2901
2902                         # Convert Atom instances to plain str.
2903                         mtimedb["resume"]["favorites"] = [str(x) for x in favorites]
2904
2905                         pkglist = mydepgraph.altlist()
2906                         mydepgraph.saveNomergeFavorites()
2907                         mydepgraph.break_refs(pkglist)
2908                         mergetask = Scheduler(settings, trees, mtimedb, myopts,
2909                                 spinner, pkglist, favorites, mydepgraph.schedulerGraph())
2910                         del mydepgraph, pkglist
2911                         clear_caches(trees)
2912
2913                         retval = mergetask.merge()
2914                         merge_count = mergetask.curval
2915
2916                 if retval == os.EX_OK and not (buildpkgonly or fetchonly or pretend):
2917                         if "yes" == settings.get("AUTOCLEAN"):
2918                                 portage.writemsg_stdout(">>> Auto-cleaning packages...\n")
2919                                 unmerge(trees[settings["ROOT"]]["root_config"],
2920                                         myopts, "clean", [],
2921                                         ldpath_mtimes, autoclean=1)
2922                         else:
2923                                 portage.writemsg_stdout(colorize("WARN", "WARNING:")
2924                                         + " AUTOCLEAN is disabled.  This can cause serious"
2925                                         + " problems due to overlapping packages.\n")
2926                         trees[settings["ROOT"]]["vartree"].dbapi.plib_registry.pruneNonExisting()
2927
2928                 return retval
2929
2930 def multiple_actions(action1, action2):
2931         sys.stderr.write("\n!!! Multiple actions requested... Please choose one only.\n")
2932         sys.stderr.write("!!! '%s' or '%s'\n\n" % (action1, action2))
2933         sys.exit(1)
2934
2935 def insert_optional_args(args):
2936         """
2937         Parse optional arguments and insert a value if one has
2938         not been provided. This is done before feeding the args
2939         to the optparse parser since that parser does not support
2940         this feature natively.
2941         """
2942
2943         new_args = []
2944         jobs_opts = ("-j", "--jobs")
2945         default_arg_opts = {
2946                 '--deselect'   : ('n',),
2947                 '--root-deps'  : ('rdeps',),
2948         }
2949         arg_stack = args[:]
2950         arg_stack.reverse()
2951         while arg_stack:
2952                 arg = arg_stack.pop()
2953
2954                 default_arg_choices = default_arg_opts.get(arg)
2955                 if default_arg_choices is not None:
2956                         new_args.append(arg)
2957                         if arg_stack and arg_stack[-1] in default_arg_choices:
2958                                 new_args.append(arg_stack.pop())
2959                         else:
2960                                 # insert default argument
2961                                 new_args.append('True')
2962                         continue
2963
2964                 short_job_opt = bool("j" in arg and arg[:1] == "-" and arg[:2] != "--")
2965                 if not (short_job_opt or arg in jobs_opts):
2966                         new_args.append(arg)
2967                         continue
2968
2969                 # Insert an empty placeholder in order to
2970                 # satisfy the requirements of optparse.
2971
2972                 new_args.append("--jobs")
2973                 job_count = None
2974                 saved_opts = None
2975                 if short_job_opt and len(arg) > 2:
2976                         if arg[:2] == "-j":
2977                                 try:
2978                                         job_count = int(arg[2:])
2979                                 except ValueError:
2980                                         saved_opts = arg[2:]
2981                         else:
2982                                 job_count = "True"
2983                                 saved_opts = arg[1:].replace("j", "")
2984
2985                 if job_count is None and arg_stack:
2986                         try:
2987                                 job_count = int(arg_stack[-1])
2988                         except ValueError:
2989                                 pass
2990                         else:
2991                                 # Discard the job count from the stack
2992                                 # since we're consuming it here.
2993                                 arg_stack.pop()
2994
2995                 if job_count is None:
2996                         # unlimited number of jobs
2997                         new_args.append("True")
2998                 else:
2999                         new_args.append(str(job_count))
3000
3001                 if saved_opts is not None:
3002                         new_args.append("-" + saved_opts)
3003
3004         return new_args
3005
3006 def parse_opts(tmpcmdline, silent=False):
3007         myaction=None
3008         myopts = {}
3009         myfiles=[]
3010
3011         global actions, options, shortmapping
3012
3013         longopt_aliases = {"--cols":"--columns", "--skip-first":"--skipfirst"}
3014         argument_options = {
3015                 "--config-root": {
3016                         "help":"specify the location for portage configuration files",
3017                         "action":"store"
3018                 },
3019                 "--color": {
3020                         "help":"enable or disable color output",
3021                         "type":"choice",
3022                         "choices":("y", "n")
3023                 },
3024
3025                 "--deselect": {
3026                         "help"    : "remove atoms from the world file",
3027                         "type"    : "choice",
3028                         "choices" : ("True", "n")
3029                 },
3030
3031                 "--jobs": {
3032
3033                         "help"   : "Specifies the number of packages to build " + \
3034                                 "simultaneously.",
3035
3036                         "action" : "store"
3037                 },
3038
3039                 "--load-average": {
3040
3041                         "help"   :"Specifies that no new builds should be started " + \
3042                                 "if there are other builds running and the load average " + \
3043                                 "is at least LOAD (a floating-point number).",
3044
3045                         "action" : "store"
3046                 },
3047
3048                 "--with-bdeps": {
3049                         "help":"include unnecessary build time dependencies",
3050                         "type":"choice",
3051                         "choices":("y", "n")
3052                 },
3053                 "--reinstall": {
3054                         "help":"specify conditions to trigger package reinstallation",
3055                         "type":"choice",
3056                         "choices":["changed-use"]
3057                 },
3058                 "--root": {
3059                  "help"   : "specify the target root filesystem for merging packages",
3060                  "action" : "store"
3061                 },
3062
3063                 "--root-deps": {
3064                         "help"    : "modify interpretation of depedencies",
3065                         "type"    : "choice",
3066                         "choices" :("True", "rdeps")
3067                 },
3068         }
3069
3070         from optparse import OptionParser
3071         parser = OptionParser()
3072         if parser.has_option("--help"):
3073                 parser.remove_option("--help")
3074
3075         for action_opt in actions:
3076                 parser.add_option("--" + action_opt, action="store_true",
3077                         dest=action_opt.replace("-", "_"), default=False)
3078         for myopt in options:
3079                 parser.add_option(myopt, action="store_true",
3080                         dest=myopt.lstrip("--").replace("-", "_"), default=False)
3081         for shortopt, longopt in shortmapping.iteritems():
3082                 parser.add_option("-" + shortopt, action="store_true",
3083                         dest=longopt.lstrip("--").replace("-", "_"), default=False)
3084         for myalias, myopt in longopt_aliases.iteritems():
3085                 parser.add_option(myalias, action="store_true",
3086                         dest=myopt.lstrip("--").replace("-", "_"), default=False)
3087
3088         for myopt, kwargs in argument_options.iteritems():
3089                 parser.add_option(myopt,
3090                         dest=myopt.lstrip("--").replace("-", "_"), **kwargs)
3091
3092         tmpcmdline = insert_optional_args(tmpcmdline)
3093
3094         myoptions, myargs = parser.parse_args(args=tmpcmdline)
3095
3096         if myoptions.deselect == "True":
3097                 myoptions.deselect = True
3098
3099         if myoptions.root_deps == "True":
3100                 myoptions.root_deps = True
3101
3102         if myoptions.jobs:
3103                 jobs = None
3104                 if myoptions.jobs == "True":
3105                         jobs = True
3106                 else:
3107                         try:
3108                                 jobs = int(myoptions.jobs)
3109                         except ValueError:
3110                                 jobs = -1
3111
3112                 if jobs is not True and \
3113                         jobs < 1:
3114                         jobs = None
3115                         if not silent:
3116                                 writemsg("!!! Invalid --jobs parameter: '%s'\n" % \
3117                                         (myoptions.jobs,), noiselevel=-1)
3118
3119                 myoptions.jobs = jobs
3120
3121         if myoptions.load_average:
3122                 try:
3123                         load_average = float(myoptions.load_average)
3124                 except ValueError:
3125                         load_average = 0.0
3126
3127                 if load_average <= 0.0:
3128                         load_average = None
3129                         if not silent:
3130                                 writemsg("!!! Invalid --load-average parameter: '%s'\n" % \
3131                                         (myoptions.load_average,), noiselevel=-1)
3132
3133                 myoptions.load_average = load_average
3134
3135         for myopt in options:
3136                 v = getattr(myoptions, myopt.lstrip("--").replace("-", "_"))
3137                 if v:
3138                         myopts[myopt] = True
3139
3140         for myopt in argument_options:
3141                 v = getattr(myoptions, myopt.lstrip("--").replace("-", "_"), None)
3142                 if v is not None:
3143                         myopts[myopt] = v
3144
3145         if myoptions.searchdesc:
3146                 myoptions.search = True
3147
3148         for action_opt in actions:
3149                 v = getattr(myoptions, action_opt.replace("-", "_"))
3150                 if v:
3151                         if myaction:
3152                                 multiple_actions(myaction, action_opt)
3153                                 sys.exit(1)
3154                         myaction = action_opt
3155
3156         if myaction is None and myoptions.deselect is True:
3157                 myaction = 'deselect'
3158
3159         myfiles += myargs
3160
3161         return myaction, myopts, myfiles
3162
3163 def validate_ebuild_environment(trees):
3164         for myroot in trees:
3165                 settings = trees[myroot]["vartree"].settings
3166                 settings.validate()
3167
3168 def load_emerge_config(trees=None):
3169         kwargs = {}
3170         for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")):
3171                 v = os.environ.get(envvar, None)
3172                 if v and v.strip():
3173                         kwargs[k] = v
3174         trees = portage.create_trees(trees=trees, **kwargs)
3175
3176         for root, root_trees in trees.iteritems():
3177                 settings = root_trees["vartree"].settings
3178                 setconfig = load_default_config(settings, root_trees)
3179                 root_trees["root_config"] = RootConfig(settings, root_trees, setconfig)
3180
3181         settings = trees["/"]["vartree"].settings
3182
3183         for myroot in trees:
3184                 if myroot != "/":
3185                         settings = trees[myroot]["vartree"].settings
3186                         break
3187
3188         mtimedbfile = os.path.join("/", portage.CACHE_PATH.lstrip(os.path.sep), "mtimedb")
3189         mtimedb = portage.MtimeDB(mtimedbfile)
3190
3191         return settings, trees, mtimedb
3192
3193 def adjust_config(myopts, settings):
3194         """Make emerge specific adjustments to the config."""
3195
3196         # To enhance usability, make some vars case insensitive by forcing them to
3197         # lower case.
3198         for myvar in ("AUTOCLEAN", "NOCOLOR"):
3199                 if myvar in settings:
3200                         settings[myvar] = settings[myvar].lower()
3201                         settings.backup_changes(myvar)
3202         del myvar
3203
3204         # Kill noauto as it will break merges otherwise.
3205         if "noauto" in settings.features:
3206                 settings.features.remove('noauto')
3207                 settings['FEATURES'] = ' '.join(sorted(settings.features))
3208                 settings.backup_changes("FEATURES")
3209
3210         CLEAN_DELAY = 5
3211         try:
3212                 CLEAN_DELAY = int(settings.get("CLEAN_DELAY", str(CLEAN_DELAY)))
3213         except ValueError, e:
3214                 portage.writemsg("!!! %s\n" % str(e), noiselevel=-1)
3215                 portage.writemsg("!!! Unable to parse integer: CLEAN_DELAY='%s'\n" % \
3216                         settings["CLEAN_DELAY"], noiselevel=-1)
3217         settings["CLEAN_DELAY"] = str(CLEAN_DELAY)
3218         settings.backup_changes("CLEAN_DELAY")
3219
3220         EMERGE_WARNING_DELAY = 10
3221         try:
3222                 EMERGE_WARNING_DELAY = int(settings.get(
3223                         "EMERGE_WARNING_DELAY", str(EMERGE_WARNING_DELAY)))
3224         except ValueError, e:
3225                 portage.writemsg("!!! %s\n" % str(e), noiselevel=-1)
3226                 portage.writemsg("!!! Unable to parse integer: EMERGE_WARNING_DELAY='%s'\n" % \
3227                         settings["EMERGE_WARNING_DELAY"], noiselevel=-1)
3228         settings["EMERGE_WARNING_DELAY"] = str(EMERGE_WARNING_DELAY)
3229         settings.backup_changes("EMERGE_WARNING_DELAY")
3230
3231         if "--quiet" in myopts:
3232                 settings["PORTAGE_QUIET"]="1"
3233                 settings.backup_changes("PORTAGE_QUIET")
3234
3235         if "--verbose" in myopts:
3236                 settings["PORTAGE_VERBOSE"] = "1"
3237                 settings.backup_changes("PORTAGE_VERBOSE")
3238
3239         # Set so that configs will be merged regardless of remembered status
3240         if ("--noconfmem" in myopts):
3241                 settings["NOCONFMEM"]="1"
3242                 settings.backup_changes("NOCONFMEM")
3243
3244         # Set various debug markers... They should be merged somehow.
3245         PORTAGE_DEBUG = 0
3246         try:
3247                 PORTAGE_DEBUG = int(settings.get("PORTAGE_DEBUG", str(PORTAGE_DEBUG)))
3248                 if PORTAGE_DEBUG not in (0, 1):
3249                         portage.writemsg("!!! Invalid value: PORTAGE_DEBUG='%i'\n" % \
3250                                 PORTAGE_DEBUG, noiselevel=-1)
3251                         portage.writemsg("!!! PORTAGE_DEBUG must be either 0 or 1\n",
3252                                 noiselevel=-1)
3253                         PORTAGE_DEBUG = 0
3254         except ValueError, e:
3255                 portage.writemsg("!!! %s\n" % str(e), noiselevel=-1)
3256                 portage.writemsg("!!! Unable to parse integer: PORTAGE_DEBUG='%s'\n" %\
3257                         settings["PORTAGE_DEBUG"], noiselevel=-1)
3258                 del e
3259         if "--debug" in myopts:
3260                 PORTAGE_DEBUG = 1
3261         settings["PORTAGE_DEBUG"] = str(PORTAGE_DEBUG)
3262         settings.backup_changes("PORTAGE_DEBUG")
3263
3264         if settings.get("NOCOLOR") not in ("yes","true"):
3265                 portage.output.havecolor = 1
3266
3267         """The explicit --color < y | n > option overrides the NOCOLOR environment
3268         variable and stdout auto-detection."""
3269         if "--color" in myopts:
3270                 if "y" == myopts["--color"]:
3271                         portage.output.havecolor = 1
3272                         settings["NOCOLOR"] = "false"
3273                 else:
3274                         portage.output.havecolor = 0
3275                         settings["NOCOLOR"] = "true"
3276                 settings.backup_changes("NOCOLOR")
3277         elif not sys.stdout.isatty() and settings.get("NOCOLOR") != "no":
3278                 portage.output.havecolor = 0
3279                 settings["NOCOLOR"] = "true"
3280                 settings.backup_changes("NOCOLOR")
3281
3282 def apply_priorities(settings):
3283         ionice(settings)
3284         nice(settings)
3285
3286 def nice(settings):
3287         try:
3288                 os.nice(int(settings.get("PORTAGE_NICENESS", "0")))
3289         except (OSError, ValueError), e:
3290                 out = portage.output.EOutput()
3291                 out.eerror("Failed to change nice value to '%s'" % \
3292                         settings["PORTAGE_NICENESS"])
3293                 out.eerror("%s\n" % str(e))
3294
3295 def ionice(settings):
3296
3297         ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND")
3298         if ionice_cmd:
3299                 ionice_cmd = shlex.split(ionice_cmd)
3300         if not ionice_cmd:
3301                 return
3302
3303         from portage.util import varexpand
3304         variables = {"PID" : str(os.getpid())}
3305         cmd = [varexpand(x, mydict=variables) for x in ionice_cmd]
3306
3307         try:
3308                 rval = portage.process.spawn(cmd, env=os.environ)
3309         except portage.exception.CommandNotFound:
3310                 # The OS kernel probably doesn't support ionice,
3311                 # so return silently.
3312                 return
3313
3314         if rval != os.EX_OK:
3315                 out = portage.output.EOutput()
3316                 out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,))
3317                 out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.")
3318
3319 def display_missing_pkg_set(root_config, set_name):
3320
3321         msg = []
3322         msg.append(("emerge: There are no sets to satisfy '%s'. " + \
3323                 "The following sets exist:") % \
3324                 colorize("INFORM", set_name))
3325         msg.append("")
3326
3327         for s in sorted(root_config.sets):
3328                 msg.append("    %s" % s)
3329         msg.append("")
3330
3331         writemsg_level("".join("%s\n" % l for l in msg),
3332                 level=logging.ERROR, noiselevel=-1)
3333
3334 def expand_set_arguments(myfiles, myaction, root_config):
3335         retval = os.EX_OK
3336         setconfig = root_config.setconfig
3337
3338         sets = setconfig.getSets()
3339
3340         # In order to know exactly which atoms/sets should be added to the
3341         # world file, the depgraph performs set expansion later. It will get
3342         # confused about where the atoms came from if it's not allowed to
3343         # expand them itself.
3344         do_not_expand = (None, )
3345         newargs = []
3346         for a in myfiles:
3347                 if a in ("system", "world"):
3348                         newargs.append(SETPREFIX+a)
3349                 else:
3350                         newargs.append(a)
3351         myfiles = newargs
3352         del newargs
3353         newargs = []
3354
3355         # separators for set arguments
3356         ARG_START = "{"
3357         ARG_END = "}"
3358
3359         # WARNING: all operators must be of equal length
3360         IS_OPERATOR = "/@"
3361         DIFF_OPERATOR = "-@"
3362         UNION_OPERATOR = "+@"
3363
3364         for i in range(0, len(myfiles)):
3365                 if myfiles[i].startswith(SETPREFIX):
3366                         start = 0
3367                         end = 0
3368                         x = myfiles[i][len(SETPREFIX):]
3369                         newset = ""
3370                         while x:
3371                                 start = x.find(ARG_START)
3372                                 end = x.find(ARG_END)
3373                                 if start > 0 and start < end:
3374                                         namepart = x[:start]
3375                                         argpart = x[start+1:end]
3376
3377                                         # TODO: implement proper quoting
3378                                         args = argpart.split(",")
3379                                         options = {}
3380                                         for a in args:
3381                                                 if "=" in a:
3382                                                         k, v  = a.split("=", 1)
3383                                                         options[k] = v
3384                                                 else:
3385                                                         options[a] = "True"
3386                                         setconfig.update(namepart, options)
3387                                         newset += (x[:start-len(namepart)]+namepart)
3388                                         x = x[end+len(ARG_END):]
3389                                 else:
3390                                         newset += x
3391                                         x = ""
3392                         myfiles[i] = SETPREFIX+newset
3393
3394         sets = setconfig.getSets()
3395
3396         # display errors that occured while loading the SetConfig instance
3397         for e in setconfig.errors:
3398                 print colorize("BAD", "Error during set creation: %s" % e)
3399
3400         # emerge relies on the existance of sets with names "world" and "system"
3401         required_sets = ("world", "system")
3402         missing_sets = []
3403
3404         for s in required_sets:
3405                 if s not in sets:
3406                         missing_sets.append(s)
3407         if missing_sets:
3408                 if len(missing_sets) > 2:
3409                         missing_sets_str = ", ".join('"%s"' % s for s in missing_sets[:-1])
3410                         missing_sets_str += ', and "%s"' % missing_sets[-1]
3411                 elif len(missing_sets) == 2:
3412                         missing_sets_str = '"%s" and "%s"' % tuple(missing_sets)
3413                 else:
3414                         missing_sets_str = '"%s"' % missing_sets[-1]
3415                 msg = ["emerge: incomplete set configuration, " + \
3416                         "missing set(s): %s" % missing_sets_str]
3417                 if sets:
3418                         msg.append("        sets defined: %s" % ", ".join(sets))
3419                 msg.append("        This usually means that '%s'" % \
3420                         (os.path.join(portage.const.GLOBAL_CONFIG_PATH, "sets.conf"),))
3421                 msg.append("        is missing or corrupt.")
3422                 for line in msg:
3423                         writemsg_level(line + "\n", level=logging.ERROR, noiselevel=-1)
3424                 return (None, 1)
3425         unmerge_actions = ("unmerge", "prune", "clean", "depclean")
3426
3427         for a in myfiles:
3428                 if a.startswith(SETPREFIX):
3429                         # support simple set operations (intersection, difference and union)
3430                         # on the commandline. Expressions are evaluated strictly left-to-right
3431                         if IS_OPERATOR in a or DIFF_OPERATOR in a or UNION_OPERATOR in a:
3432                                 expression = a[len(SETPREFIX):]
3433                                 expr_sets = []
3434                                 expr_ops = []
3435                                 while IS_OPERATOR in expression or DIFF_OPERATOR in expression or UNION_OPERATOR in expression:
3436                                         is_pos = expression.rfind(IS_OPERATOR)
3437                                         diff_pos = expression.rfind(DIFF_OPERATOR)
3438                                         union_pos = expression.rfind(UNION_OPERATOR)
3439                                         op_pos = max(is_pos, diff_pos, union_pos)
3440                                         s1 = expression[:op_pos]
3441                                         s2 = expression[op_pos+len(IS_OPERATOR):]
3442                                         op = expression[op_pos:op_pos+len(IS_OPERATOR)]
3443                                         if not s2 in sets:
3444                                                 display_missing_pkg_set(root_config, s2)
3445                                                 return (None, 1)
3446                                         expr_sets.insert(0, s2)
3447                                         expr_ops.insert(0, op)
3448                                         expression = s1
3449                                 if not expression in sets:
3450                                         display_missing_pkg_set(root_config, expression)
3451                                         return (None, 1)
3452                                 expr_sets.insert(0, expression)
3453                                 result = set(setconfig.getSetAtoms(expression))
3454                                 for i in range(0, len(expr_ops)):
3455                                         s2 = setconfig.getSetAtoms(expr_sets[i+1])
3456                                         if expr_ops[i] == IS_OPERATOR:
3457                                                 result.intersection_update(s2)
3458                                         elif expr_ops[i] == DIFF_OPERATOR:
3459                                                 result.difference_update(s2)
3460                                         elif expr_ops[i] == UNION_OPERATOR:
3461                                                 result.update(s2)
3462                                         else:
3463                                                 raise NotImplementedError("unknown set operator %s" % expr_ops[i])
3464                                 newargs.extend(result)
3465                         else:
3466                                 s = a[len(SETPREFIX):]
3467                                 if s not in sets:
3468                                         display_missing_pkg_set(root_config, s)
3469                                         return (None, 1)
3470                                 setconfig.active.append(s)
3471                                 try:
3472                                         set_atoms = setconfig.getSetAtoms(s)
3473                                 except portage.exception.PackageSetNotFound, e:
3474                                         writemsg_level(("emerge: the given set '%s' " + \
3475                                                 "contains a non-existent set named '%s'.\n") % \
3476                                                 (s, e), level=logging.ERROR, noiselevel=-1)
3477                                         return (None, 1)
3478                                 if myaction in unmerge_actions and \
3479                                                 not sets[s].supportsOperation("unmerge"):
3480                                         sys.stderr.write("emerge: the given set '%s' does " % s + \
3481                                                 "not support unmerge operations\n")
3482                                         retval = 1
3483                                 elif not set_atoms:
3484                                         print "emerge: '%s' is an empty set" % s
3485                                 elif myaction not in do_not_expand:
3486                                         newargs.extend(set_atoms)
3487                                 else:
3488                                         newargs.append(SETPREFIX+s)
3489                                 for e in sets[s].errors:
3490                                         print e
3491                 else:
3492                         newargs.append(a)
3493         return (newargs, retval)
3494
3495 def repo_name_check(trees):
3496         missing_repo_names = set()
3497         for root, root_trees in trees.iteritems():
3498                 if "porttree" in root_trees:
3499                         portdb = root_trees["porttree"].dbapi
3500                         missing_repo_names.update(portdb.porttrees)
3501                         repos = portdb.getRepositories()
3502                         for r in repos:
3503                                 missing_repo_names.discard(portdb.getRepositoryPath(r))
3504                         if portdb.porttree_root in missing_repo_names and \
3505                                 not os.path.exists(os.path.join(
3506                                 portdb.porttree_root, "profiles")):
3507                                 # This is normal if $PORTDIR happens to be empty,
3508                                 # so don't warn about it.
3509                                 missing_repo_names.remove(portdb.porttree_root)
3510
3511         if missing_repo_names:
3512                 msg = []
3513                 msg.append("WARNING: One or more repositories " + \
3514                         "have missing repo_name entries:")
3515                 msg.append("")
3516                 for p in missing_repo_names:
3517                         msg.append("\t%s/profiles/repo_name" % (p,))
3518                 msg.append("")
3519                 msg.extend(textwrap.wrap("NOTE: Each repo_name entry " + \
3520                         "should be a plain text file containing a unique " + \
3521                         "name for the repository on the first line.", 70))
3522                 writemsg_level("".join("%s\n" % l for l in msg),
3523                         level=logging.WARNING, noiselevel=-1)
3524
3525         return bool(missing_repo_names)
3526
3527 def repo_name_duplicate_check(trees):
3528         ignored_repos = {}
3529         for root, root_trees in trees.iteritems():
3530                 if 'porttree' in root_trees:
3531                         portdb = root_trees['porttree'].dbapi
3532                         if portdb.mysettings.get('PORTAGE_REPO_DUPLICATE_WARN') != '0':
3533                                 for repo_name, paths in portdb._ignored_repos:
3534                                         k = (root, repo_name, portdb.getRepositoryPath(repo_name))
3535                                         ignored_repos.setdefault(k, []).extend(paths)
3536
3537         if ignored_repos:
3538                 msg = []
3539                 msg.append('WARNING: One or more repositories ' + \
3540                         'have been ignored due to duplicate')
3541                 msg.append('  profiles/repo_name entries:')
3542                 msg.append('')
3543                 for k in sorted(ignored_repos):
3544                         msg.append('  %s overrides' % (k,))
3545                         for path in ignored_repos[k]:
3546                                 msg.append('    %s' % (path,))
3547                         msg.append('')
3548                 msg.extend('  ' + x for x in textwrap.wrap(
3549                         "All profiles/repo_name entries must be unique in order " + \
3550                         "to avoid having duplicates ignored. " + \
3551                         "Set PORTAGE_REPO_DUPLICATE_WARN=\"0\" in " + \
3552                         "/etc/make.conf if you would like to disable this warning."))
3553                 writemsg_level(''.join('%s\n' % l for l in msg),
3554                         level=logging.WARNING, noiselevel=-1)
3555
3556         return bool(ignored_repos)
3557
3558 def config_protect_check(trees):
3559         for root, root_trees in trees.iteritems():
3560                 if not root_trees["root_config"].settings.get("CONFIG_PROTECT"):
3561                         msg = "!!! CONFIG_PROTECT is empty"
3562                         if root != "/":
3563                                 msg += " for '%s'" % root
3564                         writemsg_level(msg, level=logging.WARN, noiselevel=-1)
3565
3566 def profile_check(trees, myaction, myopts):
3567         if myaction in ("info", "sync"):
3568                 return os.EX_OK
3569         elif "--version" in myopts or "--help" in myopts:
3570                 return os.EX_OK
3571         for root, root_trees in trees.iteritems():
3572                 if root_trees["root_config"].settings.profiles:
3573                         continue
3574                 # generate some profile related warning messages
3575                 validate_ebuild_environment(trees)
3576                 msg = "If you have just changed your profile configuration, you " + \
3577                         "should revert back to the previous configuration. Due to " + \
3578                         "your current profile being invalid, allowed actions are " + \
3579                         "limited to --help, --info, --sync, and --version."
3580                 writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)),
3581                         level=logging.ERROR, noiselevel=-1)
3582                 return 1
3583         return os.EX_OK
3584
3585 def emerge_main():
3586         global portage  # NFC why this is necessary now - genone
3587         portage._disable_legacy_globals()
3588         # Disable color until we're sure that it should be enabled (after
3589         # EMERGE_DEFAULT_OPTS has been parsed).
3590         portage.output.havecolor = 0
3591         # This first pass is just for options that need to be known as early as
3592         # possible, such as --config-root.  They will be parsed again later,
3593         # together with EMERGE_DEFAULT_OPTS (which may vary depending on the
3594         # the value of --config-root).
3595         myaction, myopts, myfiles = parse_opts(sys.argv[1:], silent=True)
3596         if "--debug" in myopts:
3597                 os.environ["PORTAGE_DEBUG"] = "1"
3598         if "--config-root" in myopts:
3599                 os.environ["PORTAGE_CONFIGROOT"] = myopts["--config-root"]
3600         if "--root" in myopts:
3601                 os.environ["ROOT"] = myopts["--root"]
3602
3603         # Portage needs to ensure a sane umask for the files it creates.
3604         os.umask(022)
3605         settings, trees, mtimedb = load_emerge_config()
3606         portdb = trees[settings["ROOT"]]["porttree"].dbapi
3607         rval = profile_check(trees, myaction, myopts)
3608         if rval != os.EX_OK:
3609                 return rval
3610
3611         if portage._global_updates(trees, mtimedb["updates"]):
3612                 mtimedb.commit()
3613                 # Reload the whole config from scratch.
3614                 settings, trees, mtimedb = load_emerge_config(trees=trees)
3615                 portdb = trees[settings["ROOT"]]["porttree"].dbapi
3616
3617         xterm_titles = "notitles" not in settings.features
3618
3619         tmpcmdline = []
3620         if "--ignore-default-opts" not in myopts:
3621                 tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split())
3622         tmpcmdline.extend(sys.argv[1:])
3623         myaction, myopts, myfiles = parse_opts(tmpcmdline)
3624
3625         if "--digest" in myopts:
3626                 os.environ["FEATURES"] = os.environ.get("FEATURES","") + " digest"
3627                 # Reload the whole config from scratch so that the portdbapi internal
3628                 # config is updated with new FEATURES.
3629                 settings, trees, mtimedb = load_emerge_config(trees=trees)
3630                 portdb = trees[settings["ROOT"]]["porttree"].dbapi
3631
3632         for myroot in trees:
3633                 mysettings =  trees[myroot]["vartree"].settings
3634                 mysettings.unlock()
3635                 adjust_config(myopts, mysettings)
3636                 if '--pretend' not in myopts and myaction in \
3637                         (None, 'clean', 'depclean', 'prune', 'unmerge'):
3638                         mysettings["PORTAGE_COUNTER_HASH"] = \
3639                                 trees[myroot]["vartree"].dbapi._counter_hash()
3640                         mysettings.backup_changes("PORTAGE_COUNTER_HASH")
3641                 mysettings.lock()
3642                 del myroot, mysettings
3643
3644         apply_priorities(settings)
3645
3646         spinner = stdout_spinner()
3647         if "candy" in settings.features:
3648                 spinner.update = spinner.update_scroll
3649
3650         if "--quiet" not in myopts:
3651                 portage.deprecated_profile_check(settings=settings)
3652                 repo_name_check(trees)
3653                 repo_name_duplicate_check(trees)
3654                 config_protect_check(trees)
3655
3656         for mytrees in trees.itervalues():
3657                 mydb = mytrees["porttree"].dbapi
3658                 # Freeze the portdbapi for performance (memoize all xmatch results).
3659                 mydb.freeze()
3660         del mytrees, mydb
3661
3662         if "moo" in myfiles:
3663                 print """
3664
3665   Larry loves Gentoo (""" + platform.system() + """)
3666
3667  _______________________
3668 < Have you mooed today? >
3669  -----------------------
3670         \   ^__^
3671          \  (oo)\_______
3672             (__)\       )\/\
3673                 ||----w |
3674                 ||     ||
3675
3676 """
3677
3678         for x in myfiles:
3679                 ext = os.path.splitext(x)[1]
3680                 if (ext == ".ebuild" or ext == ".tbz2") and os.path.exists(os.path.abspath(x)):
3681                         print colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n")
3682                         break
3683
3684         root_config = trees[settings["ROOT"]]["root_config"]
3685         if myaction == "list-sets":
3686                 sys.stdout.write("".join("%s\n" % s for s in sorted(root_config.sets)))
3687                 sys.stdout.flush()
3688                 return os.EX_OK
3689
3690         # only expand sets for actions taking package arguments
3691         oldargs = myfiles[:]
3692         if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge", None):
3693                 myfiles, retval = expand_set_arguments(myfiles, myaction, root_config)
3694                 if retval != os.EX_OK:
3695                         return retval
3696
3697                 # Need to handle empty sets specially, otherwise emerge will react
3698                 # with the help message for empty argument lists
3699                 if oldargs and not myfiles:
3700                         print "emerge: no targets left after set expansion"
3701                         return 0
3702
3703         if ("--tree" in myopts) and ("--columns" in myopts):
3704                 print "emerge: can't specify both of \"--tree\" and \"--columns\"."
3705                 return 1
3706
3707         if ("--quiet" in myopts):
3708                 spinner.update = spinner.update_quiet
3709                 portage.util.noiselimit = -1
3710
3711         # Always create packages if FEATURES=buildpkg
3712         # Imply --buildpkg if --buildpkgonly
3713         if ("buildpkg" in settings.features) or ("--buildpkgonly" in myopts):
3714                 if "--buildpkg" not in myopts:
3715                         myopts["--buildpkg"] = True
3716
3717         # Always try and fetch binary packages if FEATURES=getbinpkg
3718         if ("getbinpkg" in settings.features):
3719                 myopts["--getbinpkg"] = True
3720
3721         if "--buildpkgonly" in myopts:
3722                 # --buildpkgonly will not merge anything, so
3723                 # it cancels all binary package options.
3724                 for opt in ("--getbinpkg", "--getbinpkgonly",
3725                         "--usepkg", "--usepkgonly"):
3726                         myopts.pop(opt, None)
3727
3728         if "--fetch-all-uri" in myopts:
3729                 myopts["--fetchonly"] = True
3730
3731         if "--skipfirst" in myopts and "--resume" not in myopts:
3732                 myopts["--resume"] = True
3733
3734         if ("--getbinpkgonly" in myopts) and not ("--usepkgonly" in myopts):
3735                 myopts["--usepkgonly"] = True
3736
3737         if ("--getbinpkgonly" in myopts) and not ("--getbinpkg" in myopts):
3738                 myopts["--getbinpkg"] = True
3739
3740         if ("--getbinpkg" in myopts) and not ("--usepkg" in myopts):
3741                 myopts["--usepkg"] = True
3742
3743         # Also allow -K to apply --usepkg/-k
3744         if ("--usepkgonly" in myopts) and not ("--usepkg" in myopts):
3745                 myopts["--usepkg"] = True
3746
3747         # Allow -p to remove --ask
3748         if "--pretend" in myopts:
3749                 myopts.pop("--ask", None)
3750
3751         # forbid --ask when not in a terminal
3752         # note: this breaks `emerge --ask | tee logfile`, but that doesn't work anyway.
3753         if ("--ask" in myopts) and (not sys.stdin.isatty()):
3754                 portage.writemsg("!!! \"--ask\" should only be used in a terminal. Exiting.\n",
3755                         noiselevel=-1)
3756                 return 1
3757
3758         if settings.get("PORTAGE_DEBUG", "") == "1":
3759                 spinner.update = spinner.update_quiet
3760                 portage.debug=1
3761                 if "python-trace" in settings.features:
3762                         import portage.debug
3763                         portage.debug.set_trace(True)
3764
3765         if not ("--quiet" in myopts):
3766                 if not sys.stdout.isatty() or ("--nospinner" in myopts):
3767                         spinner.update = spinner.update_basic
3768
3769         if myaction == 'version':
3770                 print getportageversion(settings["PORTDIR"], settings["ROOT"],
3771                         settings.profile_path, settings["CHOST"],
3772                         trees[settings["ROOT"]]["vartree"].dbapi)
3773                 return 0
3774         elif "--help" in myopts:
3775                 _emerge.help.help(myaction, myopts, portage.output.havecolor)
3776                 return 0
3777
3778         if "--debug" in myopts:
3779                 print "myaction", myaction
3780                 print "myopts", myopts
3781
3782         if not myaction and not myfiles and "--resume" not in myopts:
3783                 _emerge.help.help(myaction, myopts, portage.output.havecolor)
3784                 return 1
3785
3786         pretend = "--pretend" in myopts
3787         fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts
3788         buildpkgonly = "--buildpkgonly" in myopts
3789
3790         # check if root user is the current user for the actions where emerge needs this
3791         if portage.secpass < 2:
3792                 # We've already allowed "--version" and "--help" above.
3793                 if "--pretend" not in myopts and myaction not in ("search","info"):
3794                         need_superuser = myaction in ('clean', 'depclean', 'deselect',
3795                                 'prune', 'unmerge') or not \
3796                                 (fetchonly or \
3797                                 (buildpkgonly and secpass >= 1) or \
3798                                 myaction in ("metadata", "regen") or \
3799                                 (myaction == "sync" and os.access(settings["PORTDIR"], os.W_OK)))
3800                         if portage.secpass < 1 or \
3801                                 need_superuser:
3802                                 if need_superuser:
3803                                         access_desc = "superuser"
3804                                 else:
3805                                         access_desc = "portage group"
3806                                 # Always show portage_group_warning() when only portage group
3807                                 # access is required but the user is not in the portage group.
3808                                 from portage.data import portage_group_warning
3809                                 if "--ask" in myopts:
3810                                         myopts["--pretend"] = True
3811                                         del myopts["--ask"]
3812                                         print ("%s access is required... " + \
3813                                                 "adding --pretend to options\n") % access_desc
3814                                         if portage.secpass < 1 and not need_superuser:
3815                                                 portage_group_warning()
3816                                 else:
3817                                         sys.stderr.write(("emerge: %s access is required\n") \
3818                                                 % access_desc)
3819                                         if portage.secpass < 1 and not need_superuser:
3820                                                 portage_group_warning()
3821                                         return 1
3822
3823         disable_emergelog = False
3824         for x in ("--pretend", "--fetchonly", "--fetch-all-uri"):
3825                 if x in myopts:
3826                         disable_emergelog = True
3827                         break
3828         if myaction in ("search", "info"):
3829                 disable_emergelog = True
3830         if disable_emergelog:
3831                 """ Disable emergelog for everything except build or unmerge
3832                 operations.  This helps minimize parallel emerge.log entries that can
3833                 confuse log parsers.  We especially want it disabled during
3834                 parallel-fetch, which uses --resume --fetchonly."""
3835                 global emergelog
3836                 def emergelog(*pargs, **kargs):
3837                         pass
3838
3839         else:
3840                 if 'EMERGE_LOG_DIR' in settings:
3841                         try:
3842                                 # At least the parent needs to exist for the lock file.
3843                                 portage.util.ensure_dirs(settings['EMERGE_LOG_DIR'])
3844                         except portage.exception.PortageException, e:
3845                                 writemsg_level("!!! Error creating directory for " + \
3846                                         "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \
3847                                         (settings['EMERGE_LOG_DIR'], e),
3848                                         noiselevel=-1, level=logging.ERROR)
3849                         else:
3850                                 global _emerge_log_dir
3851                                 _emerge_log_dir = settings['EMERGE_LOG_DIR']
3852
3853         if not "--pretend" in myopts:
3854                 emergelog(xterm_titles, "Started emerge on: "+\
3855                         time.strftime("%b %d, %Y %H:%M:%S", time.localtime()))
3856                 myelogstr=""
3857                 if myopts:
3858                         myelogstr=" ".join(myopts)
3859                 if myaction:
3860                         myelogstr+=" "+myaction
3861                 if myfiles:
3862                         myelogstr += " " + " ".join(oldargs)
3863                 emergelog(xterm_titles, " *** emerge " + myelogstr)
3864         del oldargs
3865
3866         def emergeexitsig(signum, frame):
3867                 signal.signal(signal.SIGINT, signal.SIG_IGN)
3868                 signal.signal(signal.SIGTERM, signal.SIG_IGN)
3869                 portage.util.writemsg("\n\nExiting on signal %(signal)s\n" % {"signal":signum})
3870                 sys.exit(100+signum)
3871         signal.signal(signal.SIGINT, emergeexitsig)
3872         signal.signal(signal.SIGTERM, emergeexitsig)
3873
3874         def emergeexit():
3875                 """This gets out final log message in before we quit."""
3876                 if "--pretend" not in myopts:
3877                         emergelog(xterm_titles, " *** terminating.")
3878                 if "notitles" not in settings.features:
3879                         xtermTitleReset()
3880         portage.atexit_register(emergeexit)
3881
3882         if myaction in ("config", "metadata", "regen", "sync"):
3883                 if "--pretend" in myopts:
3884                         sys.stderr.write(("emerge: The '%s' action does " + \
3885                                 "not support '--pretend'.\n") % myaction)
3886                         return 1
3887
3888         if "sync" == myaction:
3889                 return action_sync(settings, trees, mtimedb, myopts, myaction)
3890         elif "metadata" == myaction:
3891                 action_metadata(settings, portdb, myopts)
3892         elif myaction=="regen":
3893                 validate_ebuild_environment(trees)
3894                 return action_regen(settings, portdb, myopts.get("--jobs"),
3895                         myopts.get("--load-average"))
3896         # HELP action
3897         elif "config"==myaction:
3898                 validate_ebuild_environment(trees)
3899                 action_config(settings, trees, myopts, myfiles)
3900
3901         # SEARCH action
3902         elif "search"==myaction:
3903                 validate_ebuild_environment(trees)
3904                 action_search(trees[settings["ROOT"]]["root_config"],
3905                         myopts, myfiles, spinner)
3906
3907         elif myaction in ('clean', 'depclean', 'deselect', 'prune', 'unmerge'):
3908                 validate_ebuild_environment(trees)
3909                 rval = action_uninstall(settings, trees, mtimedb["ldpath"],
3910                         myopts, myaction, myfiles, spinner)
3911                 if not (myaction == 'deselect' or buildpkgonly or fetchonly or pretend):
3912                         post_emerge(root_config, myopts, mtimedb, rval)
3913                 return rval
3914
3915         elif myaction == 'info':
3916
3917                 # Ensure atoms are valid before calling unmerge().
3918                 vardb = trees[settings["ROOT"]]["vartree"].dbapi
3919                 valid_atoms = []
3920                 for x in myfiles:
3921                         if is_valid_package_atom(x):
3922                                 try:
3923                                         valid_atoms.append(
3924                                                 portage.dep_expand(x, mydb=vardb, settings=settings))
3925                                 except portage.exception.AmbiguousPackageName, e:
3926                                         msg = "The short ebuild name \"" + x + \
3927                                                 "\" is ambiguous.  Please specify " + \
3928                                                 "one of the following " + \
3929                                                 "fully-qualified ebuild names instead:"
3930                                         for line in textwrap.wrap(msg, 70):
3931                                                 writemsg_level("!!! %s\n" % (line,),
3932                                                         level=logging.ERROR, noiselevel=-1)
3933                                         for i in e[0]:
3934                                                 writemsg_level("    %s\n" % colorize("INFORM", i),
3935                                                         level=logging.ERROR, noiselevel=-1)
3936                                         writemsg_level("\n", level=logging.ERROR, noiselevel=-1)
3937                                         return 1
3938                                 continue
3939                         msg = []
3940                         msg.append("'%s' is not a valid package atom." % (x,))
3941                         msg.append("Please check ebuild(5) for full details.")
3942                         writemsg_level("".join("!!! %s\n" % line for line in msg),
3943                                 level=logging.ERROR, noiselevel=-1)
3944                         return 1
3945
3946                 return action_info(settings, trees, myopts, valid_atoms)
3947
3948         # "update", "system", or just process files:
3949         else:
3950                 validate_ebuild_environment(trees)
3951
3952                 for x in myfiles:
3953                         if x.startswith(SETPREFIX) or \
3954                                 is_valid_package_atom(x):
3955                                 continue
3956                         if x[:1] == os.sep:
3957                                 continue
3958                         try:
3959                                 os.lstat(x)
3960                                 continue
3961                         except OSError:
3962                                 pass
3963                         msg = []
3964                         msg.append("'%s' is not a valid package atom." % (x,))
3965                         msg.append("Please check ebuild(5) for full details.")
3966                         writemsg_level("".join("!!! %s\n" % line for line in msg),
3967                                 level=logging.ERROR, noiselevel=-1)
3968                         return 1
3969
3970                 if "--pretend" not in myopts:
3971                         display_news_notification(root_config, myopts)
3972                 retval = action_build(settings, trees, mtimedb,
3973                         myopts, myaction, myfiles, spinner)
3974                 root_config = trees[settings["ROOT"]]["root_config"]
3975                 post_emerge(root_config, myopts, mtimedb, retval)
3976
3977                 return retval