pym/portage/package/ebuild/doebuild.py

   1 # Copyright 2010-2011 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild']
   5
   6 import gzip
   7 import errno
   8 import io
   9 from itertools import chain
  10 import logging
  11 import os as _os
  12 import re
  13 import shutil
  14 import signal
  15 import stat
  16 import sys
  17 import tempfile
  18 from textwrap import wrap
  19 import time
  20 import warnings
  21 import zlib
  22
  23 import portage
  24 portage.proxy.lazyimport.lazyimport(globals(),
  25         'portage.package.ebuild.config:check_config_instance',
  26         'portage.package.ebuild.digestcheck:digestcheck',
  27         'portage.package.ebuild.digestgen:digestgen',
  28         'portage.package.ebuild.fetch:fetch',
  29         'portage.package.ebuild._spawn_nofetch:spawn_nofetch',
  30         'portage.util.ExtractKernelVersion:ExtractKernelVersion'
  31 )
  32
  33 from portage import auxdbkeys, bsd_chflags, \
  34         eapi_is_supported, merge, os, selinux, \
  35         unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \
  36         _shell_quote, _unicode_decode, _unicode_encode
  37 from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \
  38         EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY
  39 from portage.data import portage_gid, portage_uid, secpass, \
  40         uid, userpriv_groups
  41 from portage.dbapi.porttree import _parse_uri_map
  42 from portage.dep import Atom, check_required_use, \
  43         human_readable_required_use, paren_enclose, use_reduce
  44 from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \
  45         eapi_exports_replace_vars, eapi_has_required_use, \
  46         eapi_has_src_prepare_and_src_configure, eapi_has_pkg_pretend
  47 from portage.elog import elog_process
  48 from portage.elog.messages import eerror, eqawarn
  49 from portage.exception import DigestException, FileNotFound, \
  50         IncorrectParameter, InvalidDependString, PermissionDenied, \
  51         UnsupportedAPIException
  52 from portage.localization import _
  53 from portage.output import style_to_ansi_code
  54 from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs
  55 from portage.util import apply_recursive_permissions, \
  56         apply_secpass_permissions, noiselimit, normalize_path, \
  57         writemsg, writemsg_stdout, write_atomic
  58 from portage.util.lafilefixer import rewrite_lafile
  59 from portage.versions import _pkgsplit
  60 from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
  61 from _emerge.EbuildBuildDir import EbuildBuildDir
  62 from _emerge.EbuildPhase import EbuildPhase
  63 from _emerge.EbuildSpawnProcess import EbuildSpawnProcess
  64 from _emerge.Package import Package
  65 from _emerge.PollScheduler import PollScheduler
  66 from _emerge.RootConfig import RootConfig
  67
  68 _unsandboxed_phases = frozenset([
  69         "clean", "cleanrm", "config",
  70         "help", "info", "postinst",
  71         "preinst", "pretend", "postrm",
  72         "prerm", "setup"
  73 ])
  74
  75 def _doebuild_spawn(phase, settings, actionmap=None, **kwargs):
  76         """
  77         All proper ebuild phases which execute ebuild.sh are spawned
  78         via this function. No exceptions.
  79         """
  80
  81         if phase in _unsandboxed_phases:
  82                 kwargs['free'] = True
  83
  84         if phase == 'depend':
  85                 kwargs['droppriv'] = 'userpriv' in settings.features
  86
  87         if actionmap is not None and phase in actionmap:
  88                 kwargs.update(actionmap[phase]["args"])
  89                 cmd = actionmap[phase]["cmd"] % phase
  90         else:
  91                 if phase == 'cleanrm':
  92                         ebuild_sh_arg = 'clean'
  93                 else:
  94                         ebuild_sh_arg = phase
  95
  96                 cmd = "%s %s" % (_shell_quote(
  97                         os.path.join(settings["PORTAGE_BIN_PATH"],
  98                         os.path.basename(EBUILD_SH_BINARY))),
  99                         ebuild_sh_arg)
 100
 101         settings['EBUILD_PHASE'] = phase
 102         try:
 103                 return spawn(cmd, settings, **kwargs)
 104         finally:
 105                 settings.pop('EBUILD_PHASE', None)
 106
 107 def _spawn_phase(phase, settings, actionmap=None, **kwargs):
 108         if kwargs.get('returnpid'):
 109                 return _doebuild_spawn(phase, settings, actionmap=actionmap, **kwargs)
 110
 111         ebuild_phase = EbuildPhase(actionmap=actionmap, background=False,
 112                 phase=phase, scheduler=PollScheduler().sched_iface,
 113                 settings=settings)
 114         ebuild_phase.start()
 115         ebuild_phase.wait()
 116         return ebuild_phase.returncode
 117
 118 def doebuild_environment(myebuild, mydo, myroot=None, settings=None,
 119         debug=False, use_cache=None, db=None):
 120         """
 121         Create and store environment variable in the config instance
 122         that's passed in as the "settings" parameter. This will raise
 123         UnsupportedAPIException if the given ebuild has an unsupported
 124         EAPI. All EAPI dependent code comes last, so that essential
 125         variables like PORTAGE_BUILDDIR are still initialized even in
 126         cases when UnsupportedAPIException needs to be raised, which
 127         can be useful when uninstalling a package that has corrupt
 128         EAPI metadata.
 129         The myroot and use_cache parameters are unused.
 130         """
 131         myroot = None
 132         use_cache = None
 133
 134         if settings is None:
 135                 raise TypeError("settings argument is required")
 136
 137         if db is None:
 138                 raise TypeError("db argument is required")
 139
 140         mysettings = settings
 141         mydbapi = db
 142         ebuild_path = os.path.abspath(myebuild)
 143         pkg_dir     = os.path.dirname(ebuild_path)
 144         mytree = os.path.dirname(os.path.dirname(pkg_dir))
 145
 146         if "CATEGORY" in mysettings.configdict["pkg"]:
 147                 cat = mysettings.configdict["pkg"]["CATEGORY"]
 148         else:
 149                 cat = os.path.basename(normalize_path(os.path.join(pkg_dir, "..")))
 150
 151         mypv = os.path.basename(ebuild_path)[:-7]
 152
 153         mycpv = cat+"/"+mypv
 154         mysplit = _pkgsplit(mypv)
 155         if mysplit is None:
 156                 raise IncorrectParameter(
 157                         _("Invalid ebuild path: '%s'") % myebuild)
 158
 159         # Make a backup of PORTAGE_TMPDIR prior to calling config.reset()
 160         # so that the caller can override it.
 161         tmpdir = mysettings["PORTAGE_TMPDIR"]
 162
 163         if mydo == 'depend':
 164                 if mycpv != mysettings.mycpv:
 165                         # Don't pass in mydbapi here since the resulting aux_get
 166                         # call would lead to infinite 'depend' phase recursion.
 167                         mysettings.setcpv(mycpv)
 168         else:
 169                 # If EAPI isn't in configdict["pkg"], it means that setcpv()
 170                 # hasn't been called with the mydb argument, so we have to
 171                 # call it here (portage code always calls setcpv properly,
 172                 # but api consumers might not).
 173                 if mycpv != mysettings.mycpv or \
 174                         "EAPI" not in mysettings.configdict["pkg"]:
 175                         # Reload env.d variables and reset any previous settings.
 176                         mysettings.reload()
 177                         mysettings.reset()
 178                         mysettings.setcpv(mycpv, mydb=mydbapi)
 179
 180         # config.reset() might have reverted a change made by the caller,
 181         # so restore it to its original value. Sandbox needs canonical
 182         # paths, so realpath it.
 183         mysettings["PORTAGE_TMPDIR"] = os.path.realpath(tmpdir)
 184
 185         mysettings.pop("EBUILD_PHASE", None) # remove from backupenv
 186         mysettings["EBUILD_PHASE"] = mydo
 187
 188         # Set requested Python interpreter for Portage helpers.
 189         mysettings['PORTAGE_PYTHON'] = portage._python_interpreter
 190
 191         # This is used by assert_sigpipe_ok() that's used by the ebuild
 192         # unpack() helper. SIGPIPE is typically 13, but its better not
 193         # to assume that.
 194         mysettings['PORTAGE_SIGPIPE_STATUS'] = str(128 + signal.SIGPIPE)
 195
 196         # We are disabling user-specific bashrc files.
 197         mysettings["BASH_ENV"] = INVALID_ENV_FILE
 198
 199         if debug: # Otherwise it overrides emerge's settings.
 200                 # We have no other way to set debug... debug can't be passed in
 201                 # due to how it's coded... Don't overwrite this so we can use it.
 202                 mysettings["PORTAGE_DEBUG"] = "1"
 203
 204         mysettings["EBUILD"]   = ebuild_path
 205         mysettings["O"]        = pkg_dir
 206         mysettings.configdict["pkg"]["CATEGORY"] = cat
 207         mysettings["FILESDIR"] = pkg_dir+"/files"
 208         mysettings["PF"]       = mypv
 209
 210         if hasattr(mydbapi, '_repo_info'):
 211                 repo_info = mydbapi._repo_info[mytree]
 212                 mysettings['PORTDIR'] = repo_info.portdir
 213                 mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay
 214                 mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo_info.name
 215
 216         mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
 217         mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
 218         mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
 219
 220         mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
 221         mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_")
 222
 223         mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles)
 224         mysettings["P"]  = mysplit[0]+"-"+mysplit[1]
 225         mysettings["PN"] = mysplit[0]
 226         mysettings["PV"] = mysplit[1]
 227         mysettings["PR"] = mysplit[2]
 228
 229         if noiselimit < 0:
 230                 mysettings["PORTAGE_QUIET"] = "1"
 231
 232         if mysplit[2] == "r0":
 233                 mysettings["PVR"]=mysplit[1]
 234         else:
 235                 mysettings["PVR"]=mysplit[1]+"-"+mysplit[2]
 236
 237         if "PATH" in mysettings:
 238                 mysplit=mysettings["PATH"].split(":")
 239         else:
 240                 mysplit=[]
 241         # Note: PORTAGE_BIN_PATH may differ from the global constant
 242         # when portage is reinstalling itself.
 243         portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
 244         if portage_bin_path not in mysplit:
 245                 mysettings["PATH"] = portage_bin_path + ":" + mysettings["PATH"]
 246
 247         # All temporary directories should be subdirectories of
 248         # $PORTAGE_TMPDIR/portage, since it's common for /tmp and /var/tmp
 249         # to be mounted with the "noexec" option (see bug #346899).
 250         mysettings["BUILD_PREFIX"] = mysettings["PORTAGE_TMPDIR"]+"/portage"
 251         mysettings["PKG_TMPDIR"]   = mysettings["BUILD_PREFIX"]+"/._unmerge_"
 252
 253         # Package {pre,post}inst and {pre,post}rm may overlap, so they must have separate
 254         # locations in order to prevent interference.
 255         if mydo in ("unmerge", "prerm", "postrm", "cleanrm"):
 256                 mysettings["PORTAGE_BUILDDIR"] = os.path.join(
 257                         mysettings["PKG_TMPDIR"],
 258                         mysettings["CATEGORY"], mysettings["PF"])
 259         else:
 260                 mysettings["PORTAGE_BUILDDIR"] = os.path.join(
 261                         mysettings["BUILD_PREFIX"],
 262                         mysettings["CATEGORY"], mysettings["PF"])
 263
 264         mysettings["HOME"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "homedir")
 265         mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "work")
 266         mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "image") + os.sep
 267         mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp")
 268
 269         # Prefix forward compatability
 270         eprefix_lstrip = mysettings["EPREFIX"].lstrip(os.sep)
 271         mysettings["ED"] = os.path.join(
 272                 mysettings["D"], eprefix_lstrip).rstrip(os.sep) + os.sep
 273
 274         mysettings["PORTAGE_BASHRC"] = os.path.join(
 275                 mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_FILE)
 276         mysettings["PM_EBUILD_HOOK_DIR"] = os.path.join(
 277                 mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_DIR)
 278
 279         # Allow color.map to control colors associated with einfo, ewarn, etc...
 280         mycolors = []
 281         for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET"):
 282                 mycolors.append("%s=$'%s'" % \
 283                         (c, style_to_ansi_code(c)))
 284         mysettings["PORTAGE_COLORMAP"] = "\n".join(mycolors)
 285
 286         # All EAPI dependent code comes last, so that essential variables
 287         # like PORTAGE_BUILDDIR are still initialized even in cases when
 288         # UnsupportedAPIException needs to be raised, which can be useful
 289         # when uninstalling a package that has corrupt EAPI metadata.
 290         eapi = None
 291         if mydo == 'depend' and 'EAPI' not in mysettings.configdict['pkg']:
 292                 if eapi is None and 'parse-eapi-ebuild-head' in mysettings.features:
 293                         with io.open(_unicode_encode(ebuild_path,
 294                                 encoding=_encodings['fs'], errors='strict'),
 295                                 mode='r', encoding=_encodings['content'],
 296                                 errors='replace') as f:
 297                                 eapi = _parse_eapi_ebuild_head(f)
 298
 299                 if eapi is not None:
 300                         if not eapi_is_supported(eapi):
 301                                 raise UnsupportedAPIException(mycpv, eapi)
 302                         mysettings.configdict['pkg']['EAPI'] = eapi
 303
 304         if mydo != "depend":
 305                 # Metadata vars such as EAPI and RESTRICT are
 306                 # set by the above config.setcpv() call.
 307                 eapi = mysettings["EAPI"]
 308                 if not eapi_is_supported(eapi):
 309                         # can't do anything with this.
 310                         raise UnsupportedAPIException(mycpv, eapi)
 311
 312                 if hasattr(mydbapi, "getFetchMap") and \
 313                         ("A" not in mysettings.configdict["pkg"] or \
 314                         "AA" not in mysettings.configdict["pkg"]):
 315                         src_uri, = mydbapi.aux_get(mysettings.mycpv,
 316                                 ["SRC_URI"], mytree=mytree)
 317                         metadata = {
 318                                 "EAPI"    : eapi,
 319                                 "SRC_URI" : src_uri,
 320                         }
 321                         use = frozenset(mysettings["PORTAGE_USE"].split())
 322                         try:
 323                                 uri_map = _parse_uri_map(mysettings.mycpv, metadata, use=use)
 324                         except InvalidDependString:
 325                                 mysettings.configdict["pkg"]["A"] = ""
 326                         else:
 327                                 mysettings.configdict["pkg"]["A"] = " ".join(uri_map)
 328
 329                         try:
 330                                 uri_map = _parse_uri_map(mysettings.mycpv, metadata)
 331                         except InvalidDependString:
 332                                 mysettings.configdict["pkg"]["AA"] = ""
 333                         else:
 334                                 mysettings.configdict["pkg"]["AA"] = " ".join(uri_map)
 335
 336         if not eapi_exports_KV(eapi):
 337                 # Discard KV for EAPIs that don't support it. Cache KV is restored
 338                 # from the backupenv whenever config.reset() is called.
 339                 mysettings.pop('KV', None)
 340         elif mydo != 'depend' and 'KV' not in mysettings and \
 341                 mydo in ('compile', 'config', 'configure', 'info',
 342                 'install', 'nofetch', 'postinst', 'postrm', 'preinst',
 343                 'prepare', 'prerm', 'setup', 'test', 'unpack'):
 344                 mykv, err1 = ExtractKernelVersion(
 345                         os.path.join(mysettings['EROOT'], "usr/src/linux"))
 346                 if mykv:
 347                         # Regular source tree
 348                         mysettings["KV"] = mykv
 349                 else:
 350                         mysettings["KV"] = ""
 351                 mysettings.backup_changes("KV")
 352
 353 _doebuild_manifest_cache = None
 354 _doebuild_broken_ebuilds = set()
 355 _doebuild_broken_manifests = set()
 356 _doebuild_commands_without_builddir = (
 357         'clean', 'cleanrm', 'depend', 'digest',
 358         'fetch', 'fetchall', 'help', 'manifest'
 359 )
 360
 361 def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0,
 362         fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None,
 363         mydbapi=None, vartree=None, prev_mtimes=None,
 364         fd_pipes=None, returnpid=False):
 365         """
 366         Wrapper function that invokes specific ebuild phases through the spawning
 367         of ebuild.sh
 368
 369         @param myebuild: name of the ebuild to invoke the phase on (CPV)
 370         @type myebuild: String
 371         @param mydo: Phase to run
 372         @type mydo: String
 373         @param _unused: Deprecated (use settings["ROOT"] instead)
 374         @type _unused: String
 375         @param settings: Portage Configuration
 376         @type settings: instance of portage.config
 377         @param debug: Turns on various debug information (eg, debug for spawn)
 378         @type debug: Boolean
 379         @param listonly: Used to wrap fetch(); passed such that fetch only lists files required.
 380         @type listonly: Boolean
 381         @param fetchonly: Used to wrap fetch(); passed such that files are only fetched (no other actions)
 382         @type fetchonly: Boolean
 383         @param cleanup: Passed to prepare_build_dirs (TODO: what does it do?)
 384         @type cleanup: Boolean
 385         @param dbkey: A file path where metadata generated by the 'depend' phase
 386                 will be written.
 387         @type dbkey: String
 388         @param use_cache: Enables the cache
 389         @type use_cache: Boolean
 390         @param fetchall: Used to wrap fetch(), fetches all URIs (even ones invalid due to USE conditionals)
 391         @type fetchall: Boolean
 392         @param tree: Which tree to use ('vartree','porttree','bintree', etc..), defaults to 'porttree'
 393         @type tree: String
 394         @param mydbapi: a dbapi instance to pass to various functions; this should be a portdbapi instance.
 395         @type mydbapi: portdbapi instance
 396         @param vartree: A instance of vartree; used for aux_get calls, defaults to db[myroot]['vartree']
 397         @type vartree: vartree instance
 398         @param prev_mtimes: A dict of { filename:mtime } keys used by merge() to do config_protection
 399         @type prev_mtimes: dictionary
 400         @param fd_pipes: A dict of mapping for pipes, { '0': stdin, '1': stdout }
 401                 for example.
 402         @type fd_pipes: Dictionary
 403         @param returnpid: Return a list of process IDs for a successful spawn, or
 404                 an integer value if spawn is unsuccessful. NOTE: This requires the
 405                 caller clean up all returned PIDs.
 406         @type returnpid: Boolean
 407         @rtype: Boolean
 408         @returns:
 409         1. 0 for success
 410         2. 1 for error
 411
 412         Most errors have an accompanying error message.
 413
 414         listonly and fetchonly are only really necessary for operations involving 'fetch'
 415         prev_mtimes are only necessary for merge operations.
 416         Other variables may not be strictly required, many have defaults that are set inside of doebuild.
 417
 418         """
 419
 420         if settings is None:
 421                 raise TypeError("settings parameter is required")
 422         mysettings = settings
 423         myroot = settings['EROOT']
 424
 425         if _unused is not None and _unused != mysettings['EROOT']:
 426                 warnings.warn("The third parameter of the "
 427                         "portage.doebuild() is now unused. Use "
 428                         "settings['ROOT'] instead.",
 429                         DeprecationWarning, stacklevel=2)
 430
 431         if not tree:
 432                 writemsg("Warning: tree not specified to doebuild\n")
 433                 tree = "porttree"
 434
 435         # chunked out deps for each phase, so that ebuild binary can use it
 436         # to collapse targets down.
 437         actionmap_deps={
 438         "pretend"  : [],
 439         "setup":  ["pretend"],
 440         "unpack": ["setup"],
 441         "prepare": ["unpack"],
 442         "configure": ["prepare"],
 443         "compile":["configure"],
 444         "test":   ["compile"],
 445         "install":["test"],
 446         "rpm":    ["install"],
 447         "package":["install"],
 448         "merge"  :["install"],
 449         }
 450
 451         if mydbapi is None:
 452                 mydbapi = portage.db[myroot][tree].dbapi
 453
 454         if vartree is None and mydo in ("merge", "qmerge", "unmerge"):
 455                 vartree = portage.db[myroot]["vartree"]
 456
 457         features = mysettings.features
 458
 459         clean_phases = ("clean", "cleanrm")
 460         validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst",
 461                         "config", "info", "setup", "depend", "pretend",
 462                         "fetch", "fetchall", "digest",
 463                         "unpack", "prepare", "configure", "compile", "test",
 464                         "install", "rpm", "qmerge", "merge",
 465                         "package","unmerge", "manifest"]
 466
 467         if mydo not in validcommands:
 468                 validcommands.sort()
 469                 writemsg("!!! doebuild: '%s' is not one of the following valid commands:" % mydo,
 470                         noiselevel=-1)
 471                 for vcount in range(len(validcommands)):
 472                         if vcount%6 == 0:
 473                                 writemsg("\n!!! ", noiselevel=-1)
 474                         writemsg(validcommands[vcount].ljust(11), noiselevel=-1)
 475                 writemsg("\n", noiselevel=-1)
 476                 return 1
 477
 478         if returnpid and mydo != 'depend':
 479                 warnings.warn("portage.doebuild() called " + \
 480                         "with returnpid parameter enabled. This usage will " + \
 481                         "not be supported in the future.",
 482                         DeprecationWarning, stacklevel=2)
 483
 484         if mydo == "fetchall":
 485                 fetchall = 1
 486                 mydo = "fetch"
 487
 488         parallel_fetchonly = mydo in ("fetch", "fetchall") and \
 489                 "PORTAGE_PARALLEL_FETCHONLY" in mysettings
 490
 491         if mydo not in clean_phases and not os.path.exists(myebuild):
 492                 writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo),
 493                         noiselevel=-1)
 494                 return 1
 495
 496         global _doebuild_manifest_cache
 497         pkgdir = os.path.dirname(myebuild)
 498         manifest_path = os.path.join(pkgdir, "Manifest")
 499         if tree == "porttree":
 500                 repo_config = mysettings.repositories.get_repo_for_location(
 501                         os.path.dirname(os.path.dirname(pkgdir)))
 502         else:
 503                 repo_config = None
 504         mf = None
 505         if "strict" in features and \
 506                 "digest" not in features and \
 507                 tree == "porttree" and \
 508                 not repo_config.thin_manifest and \
 509                 mydo not in ("digest", "manifest", "help") and \
 510                 not portage._doebuild_manifest_exempt_depend and \
 511                 not (repo_config.allow_missing_manifest and not os.path.exists(manifest_path)):
 512                 # Always verify the ebuild checksums before executing it.
 513                 global _doebuild_broken_ebuilds
 514
 515                 if myebuild in _doebuild_broken_ebuilds:
 516                         return 1
 517
 518                 # Avoid checking the same Manifest several times in a row during a
 519                 # regen with an empty cache.
 520                 if _doebuild_manifest_cache is None or \
 521                         _doebuild_manifest_cache.getFullname() != manifest_path:
 522                         _doebuild_manifest_cache = None
 523                         if not os.path.exists(manifest_path):
 524                                 out = portage.output.EOutput()
 525                                 out.eerror(_("Manifest not found for '%s'") % (myebuild,))
 526                                 _doebuild_broken_ebuilds.add(myebuild)
 527                                 return 1
 528                         mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])
 529
 530                 else:
 531                         mf = _doebuild_manifest_cache
 532
 533                 try:
 534                         mf.checkFileHashes("EBUILD", os.path.basename(myebuild))
 535                 except KeyError:
 536                         if not (mf.allow_missing and
 537                                 os.path.basename(myebuild) not in mf.fhashdict["EBUILD"]):
 538                                 out = portage.output.EOutput()
 539                                 out.eerror(_("Missing digest for '%s'") % (myebuild,))
 540                                 _doebuild_broken_ebuilds.add(myebuild)
 541                                 return 1
 542                 except FileNotFound:
 543                         out = portage.output.EOutput()
 544                         out.eerror(_("A file listed in the Manifest "
 545                                 "could not be found: '%s'") % (myebuild,))
 546                         _doebuild_broken_ebuilds.add(myebuild)
 547                         return 1
 548                 except DigestException as e:
 549                         out = portage.output.EOutput()
 550                         out.eerror(_("Digest verification failed:"))
 551                         out.eerror("%s" % e.value[0])
 552                         out.eerror(_("Reason: %s") % e.value[1])
 553                         out.eerror(_("Got: %s") % e.value[2])
 554                         out.eerror(_("Expected: %s") % e.value[3])
 555                         _doebuild_broken_ebuilds.add(myebuild)
 556                         return 1
 557
 558                 if mf.getFullname() in _doebuild_broken_manifests:
 559                         return 1
 560
 561                 if mf is not _doebuild_manifest_cache and not mf.allow_missing:
 562
 563                         # Make sure that all of the ebuilds are
 564                         # actually listed in the Manifest.
 565                         for f in os.listdir(pkgdir):
 566                                 pf = None
 567                                 if f[-7:] == '.ebuild':
 568                                         pf = f[:-7]
 569                                 if pf is not None and not mf.hasFile("EBUILD", f):
 570                                         f = os.path.join(pkgdir, f)
 571                                         if f not in _doebuild_broken_ebuilds:
 572                                                 out = portage.output.EOutput()
 573                                                 out.eerror(_("A file is not listed in the "
 574                                                         "Manifest: '%s'") % (f,))
 575                                         _doebuild_broken_manifests.add(manifest_path)
 576                                         return 1
 577
 578                 # We cache it only after all above checks succeed.
 579                 _doebuild_manifest_cache = mf
 580
 581         logfile=None
 582         builddir_lock = None
 583         tmpdir = None
 584         tmpdir_orig = None
 585
 586         try:
 587                 if mydo in ("digest", "manifest", "help"):
 588                         # Temporarily exempt the depend phase from manifest checks, in case
 589                         # aux_get calls trigger cache generation.
 590                         portage._doebuild_manifest_exempt_depend += 1
 591
 592                 # If we don't need much space and we don't need a constant location,
 593                 # we can temporarily override PORTAGE_TMPDIR with a random temp dir
 594                 # so that there's no need for locking and it can be used even if the
 595                 # user isn't in the portage group.
 596                 if mydo in ("info",):
 597                         tmpdir = tempfile.mkdtemp()
 598                         tmpdir_orig = mysettings["PORTAGE_TMPDIR"]
 599                         mysettings["PORTAGE_TMPDIR"] = tmpdir
 600
 601                 doebuild_environment(myebuild, mydo, myroot, mysettings, debug,
 602                         use_cache, mydbapi)
 603
 604                 if mydo in clean_phases:
 605                         builddir_lock = None
 606                         if not returnpid and \
 607                                 'PORTAGE_BUILDIR_LOCKED' not in mysettings:
 608                                 builddir_lock = EbuildBuildDir(
 609                                         scheduler=PollScheduler().sched_iface,
 610                                         settings=mysettings)
 611                                 builddir_lock.lock()
 612                         try:
 613                                 return _spawn_phase(mydo, mysettings,
 614                                         fd_pipes=fd_pipes, returnpid=returnpid)
 615                         finally:
 616                                 if builddir_lock is not None:
 617                                         builddir_lock.unlock()
 618
 619                 # get possible slot information from the deps file
 620                 if mydo == "depend":
 621                         writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2)
 622                         if returnpid:
 623                                 return _spawn_phase(mydo, mysettings,
 624                                         fd_pipes=fd_pipes, returnpid=returnpid)
 625                         elif isinstance(dbkey, dict):
 626                                 warnings.warn("portage.doebuild() called " + \
 627                                         "with dict dbkey argument. This usage will " + \
 628                                         "not be supported in the future.",
 629                                         DeprecationWarning, stacklevel=2)
 630                                 mysettings["dbkey"] = ""
 631                                 pr, pw = os.pipe()
 632                                 fd_pipes = {
 633                                         0:sys.stdin.fileno(),
 634                                         1:sys.stdout.fileno(),
 635                                         2:sys.stderr.fileno(),
 636                                         9:pw}
 637                                 mypids = _spawn_phase(mydo, mysettings, returnpid=True,
 638                                         fd_pipes=fd_pipes)
 639                                 os.close(pw) # belongs exclusively to the child process now
 640                                 f = os.fdopen(pr, 'rb', 0)
 641                                 for k, v in zip(auxdbkeys,
 642                                         (_unicode_decode(line).rstrip('\n') for line in f)):
 643                                         dbkey[k] = v
 644                                 f.close()
 645                                 retval = os.waitpid(mypids[0], 0)[1]
 646                                 portage.process.spawned_pids.remove(mypids[0])
 647                                 # If it got a signal, return the signal that was sent, but
 648                                 # shift in order to distinguish it from a return value. (just
 649                                 # like portage.process.spawn() would do).
 650                                 if retval & 0xff:
 651                                         retval = (retval & 0xff) << 8
 652                                 else:
 653                                         # Otherwise, return its exit code.
 654                                         retval = retval >> 8
 655                                 if retval == os.EX_OK and len(dbkey) != len(auxdbkeys):
 656                                         # Don't trust bash's returncode if the
 657                                         # number of lines is incorrect.
 658                                         retval = 1
 659                                 return retval
 660                         elif dbkey:
 661                                 mysettings["dbkey"] = dbkey
 662                         else:
 663                                 mysettings["dbkey"] = \
 664                                         os.path.join(mysettings.depcachedir, "aux_db_key_temp")
 665
 666                         return _spawn_phase(mydo, mysettings,
 667                                 fd_pipes=fd_pipes, returnpid=returnpid)
 668
 669                 # Validate dependency metadata here to ensure that ebuilds with invalid
 670                 # data are never installed via the ebuild command. Don't bother when
 671                 # returnpid == True since there's no need to do this every time emerge
 672                 # executes a phase.
 673                 if tree == "porttree":
 674                         rval = _validate_deps(mysettings, myroot, mydo, mydbapi)
 675                         if rval != os.EX_OK:
 676                                 return rval
 677
 678                 # The info phase is special because it uses mkdtemp so and
 679                 # user (not necessarily in the portage group) can run it.
 680                 if mydo not in ('info',) and \
 681                         mydo not in _doebuild_commands_without_builddir:
 682                         rval = _check_temp_dir(mysettings)
 683                         if rval != os.EX_OK:
 684                                 return rval
 685
 686                 if mydo == "unmerge":
 687                         return unmerge(mysettings["CATEGORY"],
 688                                 mysettings["PF"], myroot, mysettings, vartree=vartree)
 689
 690                 phases_to_run = set()
 691                 if "noauto" in mysettings.features or \
 692                         mydo not in actionmap_deps:
 693                         phases_to_run.add(mydo)
 694                 else:
 695                         phase_stack = [mydo]
 696                         while phase_stack:
 697                                 x = phase_stack.pop()
 698                                 if x in phases_to_run:
 699                                         continue
 700                                 phases_to_run.add(x)
 701                                 phase_stack.extend(actionmap_deps.get(x, []))
 702                         del phase_stack
 703
 704                 alist = set(mysettings.configdict["pkg"].get("A", "").split())
 705
 706                 unpacked = False
 707                 if tree != "porttree":
 708                         pass
 709                 elif "unpack" not in phases_to_run:
 710                         unpacked = os.path.exists(os.path.join(
 711                                 mysettings["PORTAGE_BUILDDIR"], ".unpacked"))
 712                 else:
 713                         try:
 714                                 workdir_st = os.stat(mysettings["WORKDIR"])
 715                         except OSError:
 716                                 pass
 717                         else:
 718                                 newstuff = False
 719                                 if not os.path.exists(os.path.join(
 720                                         mysettings["PORTAGE_BUILDDIR"], ".unpacked")):
 721                                         writemsg_stdout(_(
 722                                                 ">>> Not marked as unpacked; recreating WORKDIR...\n"))
 723                                         newstuff = True
 724                                 else:
 725                                         for x in alist:
 726                                                 writemsg_stdout(">>> Checking %s's mtime...\n" % x)
 727                                                 try:
 728                                                         x_st = os.stat(os.path.join(
 729                                                                 mysettings["DISTDIR"], x))
 730                                                 except OSError:
 731                                                         # file not fetched yet
 732                                                         x_st = None
 733
 734                                                 if x_st is None or x_st.st_mtime > workdir_st.st_mtime:
 735                                                         writemsg_stdout(_(">>> Timestamp of "
 736                                                                 "%s has changed; recreating WORKDIR...\n") % x)
 737                                                         newstuff = True
 738                                                         break
 739
 740                                 if newstuff:
 741                                         if builddir_lock is None and \
 742                                                 'PORTAGE_BUILDIR_LOCKED' not in mysettings:
 743                                                 builddir_lock = EbuildBuildDir(
 744                                                         scheduler=PollScheduler().sched_iface,
 745                                                         settings=mysettings)
 746                                                 builddir_lock.lock()
 747                                         try:
 748                                                 _spawn_phase("clean", mysettings)
 749                                         finally:
 750                                                 if builddir_lock is not None:
 751                                                         builddir_lock.unlock()
 752                                                         builddir_lock = None
 753                                 else:
 754                                         writemsg_stdout(_(">>> WORKDIR is up-to-date, keeping...\n"))
 755                                         unpacked = True
 756
 757                 # Build directory creation isn't required for any of these.
 758                 # In the fetch phase, the directory is needed only for RESTRICT=fetch
 759                 # in order to satisfy the sane $PWD requirement (from bug #239560)
 760                 # when pkg_nofetch is spawned.
 761                 have_build_dirs = False
 762                 if not parallel_fetchonly and \
 763                         mydo not in ('digest', 'fetch', 'help', 'manifest'):
 764                         if not returnpid and \
 765                                 'PORTAGE_BUILDIR_LOCKED' not in mysettings:
 766                                 builddir_lock = EbuildBuildDir(
 767                                         scheduler=PollScheduler().sched_iface,
 768                                         settings=mysettings)
 769                                 builddir_lock.lock()
 770                         mystatus = prepare_build_dirs(myroot, mysettings, cleanup)
 771                         if mystatus:
 772                                 return mystatus
 773                         have_build_dirs = True
 774
 775                         # emerge handles logging externally
 776                         if not returnpid:
 777                                 # PORTAGE_LOG_FILE is set by the
 778                                 # above prepare_build_dirs() call.
 779                                 logfile = mysettings.get("PORTAGE_LOG_FILE")
 780
 781                 if have_build_dirs:
 782                         rval = _prepare_env_file(mysettings)
 783                         if rval != os.EX_OK:
 784                                 return rval
 785
 786                 if eapi_exports_merge_type(mysettings["EAPI"]) and \
 787                         "MERGE_TYPE" not in mysettings.configdict["pkg"]:
 788                         if tree == "porttree":
 789                                 mysettings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
 790                                 mysettings.configdict["pkg"]["MERGE_TYPE"] = "source"
 791                         elif tree == "bintree":
 792                                 mysettings.configdict["pkg"]["EMERGE_FROM"] = "binary"
 793                                 mysettings.configdict["pkg"]["MERGE_TYPE"] = "binary"
 794
 795                 # NOTE: It's not possible to set REPLACED_BY_VERSION for prerm
 796                 #       and postrm here, since we don't necessarily know what
 797                 #       versions are being installed. This could be a problem
 798                 #       for API consumers if they don't use dblink.treewalk()
 799                 #       to execute prerm and postrm.
 800                 if eapi_exports_replace_vars(mysettings["EAPI"]) and \
 801                         (mydo in ("postinst", "preinst", "pretend", "setup") or \
 802                         ("noauto" not in features and not returnpid and \
 803                         (mydo in actionmap_deps or mydo in ("merge", "package", "qmerge")))):
 804                         if not vartree:
 805                                 writemsg("Warning: vartree not given to doebuild. " + \
 806                                         "Cannot set REPLACING_VERSIONS in pkg_{pretend,setup}\n")
 807                         else:
 808                                 vardb = vartree.dbapi
 809                                 cpv = mysettings.mycpv
 810                                 cp = portage.versions.cpv_getkey(cpv)
 811                                 slot = mysettings["SLOT"]
 812                                 cpv_slot = cp + ":" + slot
 813                                 mysettings["REPLACING_VERSIONS"] = " ".join(
 814                                         set(portage.versions.cpv_getversion(match) \
 815                                                 for match in vardb.match(cpv_slot) + \
 816                                                 vardb.match('='+cpv)))
 817
 818                 # if any of these are being called, handle them -- running them out of
 819                 # the sandbox -- and stop now.
 820                 if mydo in ("config", "help", "info", "postinst",
 821                         "preinst", "pretend", "postrm", "prerm"):
 822                         return _spawn_phase(mydo, mysettings,
 823                                 fd_pipes=fd_pipes, logfile=logfile, returnpid=returnpid)
 824
 825                 mycpv = "/".join((mysettings["CATEGORY"], mysettings["PF"]))
 826
 827                 # Only try and fetch the files if we are going to need them ...
 828                 # otherwise, if user has FEATURES=noauto and they run `ebuild clean
 829                 # unpack compile install`, we will try and fetch 4 times :/
 830                 need_distfiles = tree == "porttree" and not unpacked and \
 831                         (mydo in ("fetch", "unpack") or \
 832                         mydo not in ("digest", "manifest") and "noauto" not in features)
 833                 if need_distfiles:
 834
 835                         src_uri, = mydbapi.aux_get(mysettings.mycpv,
 836                                 ["SRC_URI"], mytree=os.path.dirname(os.path.dirname(
 837                                 os.path.dirname(myebuild))))
 838                         metadata = {
 839                                 "EAPI"    : mysettings["EAPI"],
 840                                 "SRC_URI" : src_uri,
 841                         }
 842                         use = frozenset(mysettings["PORTAGE_USE"].split())
 843                         try:
 844                                 alist = _parse_uri_map(mysettings.mycpv, metadata, use=use)
 845                                 aalist = _parse_uri_map(mysettings.mycpv, metadata)
 846                         except InvalidDependString as e:
 847                                 writemsg("!!! %s\n" % str(e), noiselevel=-1)
 848                                 writemsg(_("!!! Invalid SRC_URI for '%s'.\n") % mycpv,
 849                                         noiselevel=-1)
 850                                 del e
 851                                 return 1
 852
 853                         if "mirror" in features or fetchall:
 854                                 fetchme = aalist
 855                         else:
 856                                 fetchme = alist
 857
 858                         dist_digests = None
 859                         if mf is not None:
 860                                 dist_digests = mf.getTypeDigests("DIST")
 861                         if not fetch(fetchme, mysettings, listonly=listonly,
 862                                 fetchonly=fetchonly, allow_missing_digests=True,
 863                                 digests=dist_digests):
 864                                 spawn_nofetch(mydbapi, myebuild, settings=mysettings)
 865                                 if listonly:
 866                                         # The convention for listonly mode is to report
 867                                         # success in any case, even though fetch() may
 868                                         # return unsuccessfully in order to trigger the
 869                                         # nofetch phase.
 870                                         return 0
 871                                 return 1
 872
 873                 if need_distfiles:
 874                         # Files are already checked inside fetch(),
 875                         # so do not check them again.
 876                         checkme = []
 877                 elif unpacked:
 878                         # The unpack phase is marked as complete, so it
 879                         # would be wasteful to check distfiles again.
 880                         checkme = []
 881                 else:
 882                         checkme = alist
 883
 884                 if mydo == "fetch" and listonly:
 885                         return 0
 886
 887                 try:
 888                         if mydo == "manifest":
 889                                 mf = None
 890                                 _doebuild_manifest_cache = None
 891                                 return not digestgen(mysettings=mysettings, myportdb=mydbapi)
 892                         elif mydo == "digest":
 893                                 mf = None
 894                                 _doebuild_manifest_cache = None
 895                                 return not digestgen(mysettings=mysettings, myportdb=mydbapi)
 896                         elif mydo != 'fetch' and \
 897                                 "digest" in mysettings.features:
 898                                 # Don't do this when called by emerge or when called just
 899                                 # for fetch (especially parallel-fetch) since it's not needed
 900                                 # and it can interfere with parallel tasks.
 901                                 mf = None
 902                                 _doebuild_manifest_cache = None
 903                                 digestgen(mysettings=mysettings, myportdb=mydbapi)
 904                 except PermissionDenied as e:
 905                         writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1)
 906                         if mydo in ("digest", "manifest"):
 907                                 return 1
 908
 909                 # See above comment about fetching only when needed
 910                 if tree == 'porttree' and \
 911                         not digestcheck(checkme, mysettings, "strict" in features, mf=mf):
 912                         return 1
 913
 914                 if mydo == "fetch":
 915                         return 0
 916
 917                 # remove PORTAGE_ACTUAL_DISTDIR once cvs/svn is supported via SRC_URI
 918                 if tree == 'porttree' and \
 919                         ((mydo != "setup" and "noauto" not in features) \
 920                         or mydo in ("install", "unpack")):
 921                         _prepare_fake_distdir(mysettings, alist)
 922
 923                 #initial dep checks complete; time to process main commands
 924                 actionmap = _spawn_actionmap(mysettings)
 925
 926                 # merge the deps in so we have again a 'full' actionmap
 927                 # be glad when this can die.
 928                 for x in actionmap:
 929                         if len(actionmap_deps.get(x, [])):
 930                                 actionmap[x]["dep"] = ' '.join(actionmap_deps[x])
 931
 932                 if mydo in actionmap:
 933                         bintree = None
 934                         if mydo == "package":
 935                                 # Make sure the package directory exists before executing
 936                                 # this phase. This can raise PermissionDenied if
 937                                 # the current user doesn't have write access to $PKGDIR.
 938                                 if hasattr(portage, 'db'):
 939                                         bintree = portage.db[mysettings['EROOT']]['bintree']
 940                                         mysettings["PORTAGE_BINPKG_TMPFILE"] = \
 941                                                 bintree.getname(mysettings.mycpv) + \
 942                                                 ".%s" % (os.getpid(),)
 943                                         bintree._ensure_dir(os.path.dirname(
 944                                                 mysettings["PORTAGE_BINPKG_TMPFILE"]))
 945                                 else:
 946                                         parent_dir = os.path.join(mysettings["PKGDIR"],
 947                                                 mysettings["CATEGORY"])
 948                                         portage.util.ensure_dirs(parent_dir)
 949                                         if not os.access(parent_dir, os.W_OK):
 950                                                 raise PermissionDenied(
 951                                                         "access('%s', os.W_OK)" % parent_dir)
 952                         retval = spawnebuild(mydo,
 953                                 actionmap, mysettings, debug, logfile=logfile,
 954                                 fd_pipes=fd_pipes, returnpid=returnpid)
 955
 956                         if retval == os.EX_OK:
 957                                 if mydo == "package" and bintree is not None:
 958                                         bintree.inject(mysettings.mycpv,
 959                                                 filename=mysettings["PORTAGE_BINPKG_TMPFILE"])
 960                 elif mydo=="qmerge":
 961                         # check to ensure install was run.  this *only* pops up when users
 962                         # forget it and are using ebuild
 963                         if not os.path.exists(
 964                                 os.path.join(mysettings["PORTAGE_BUILDDIR"], ".installed")):
 965                                 writemsg(_("!!! mydo=qmerge, but the install phase has not been run\n"),
 966                                         noiselevel=-1)
 967                                 return 1
 968                         # qmerge is a special phase that implies noclean.
 969                         if "noclean" not in mysettings.features:
 970                                 mysettings.features.add("noclean")
 971                         #qmerge is specifically not supposed to do a runtime dep check
 972                         retval = merge(
 973                                 mysettings["CATEGORY"], mysettings["PF"], mysettings["D"],
 974                                 os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"),
 975                                 myroot, mysettings, myebuild=mysettings["EBUILD"], mytree=tree,
 976                                 mydbapi=mydbapi, vartree=vartree, prev_mtimes=prev_mtimes)
 977                 elif mydo=="merge":
 978                         retval = spawnebuild("install", actionmap, mysettings, debug,
 979                                 alwaysdep=1, logfile=logfile, fd_pipes=fd_pipes,
 980                                 returnpid=returnpid)
 981                         if retval != os.EX_OK:
 982                                 # The merge phase handles this already.  Callers don't know how
 983                                 # far this function got, so we have to call elog_process() here
 984                                 # so that it's only called once.
 985                                 elog_process(mysettings.mycpv, mysettings)
 986                         if retval == os.EX_OK:
 987                                 retval = merge(mysettings["CATEGORY"], mysettings["PF"],
 988                                         mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"],
 989                                         "build-info"), myroot, mysettings,
 990                                         myebuild=mysettings["EBUILD"], mytree=tree, mydbapi=mydbapi,
 991                                         vartree=vartree, prev_mtimes=prev_mtimes)
 992                 else:
 993                         writemsg_stdout(_("!!! Unknown mydo: %s\n") % mydo, noiselevel=-1)
 994                         return 1
 995
 996                 return retval
 997
 998         finally:
 999
1000                 if builddir_lock is not None:
1001                         builddir_lock.unlock()
1002                 if tmpdir:
1003                         mysettings["PORTAGE_TMPDIR"] = tmpdir_orig
1004                         shutil.rmtree(tmpdir)
1005
1006                 mysettings.pop("REPLACING_VERSIONS", None)
1007
1008                 # Make sure that DISTDIR is restored to it's normal value before we return!
1009                 if "PORTAGE_ACTUAL_DISTDIR" in mysettings:
1010                         mysettings["DISTDIR"] = mysettings["PORTAGE_ACTUAL_DISTDIR"]
1011                         del mysettings["PORTAGE_ACTUAL_DISTDIR"]
1012
1013                 if logfile and not returnpid:
1014                         try:
1015                                 if os.stat(logfile).st_size == 0:
1016                                         os.unlink(logfile)
1017                         except OSError:
1018                                 pass
1019
1020                 if mydo in ("digest", "manifest", "help"):
1021                         # If necessary, depend phase has been triggered by aux_get calls
1022                         # and the exemption is no longer needed.
1023                         portage._doebuild_manifest_exempt_depend -= 1
1024
1025 def _check_temp_dir(settings):
1026         if "PORTAGE_TMPDIR" not in settings or \
1027                 not os.path.isdir(settings["PORTAGE_TMPDIR"]):
1028                 writemsg(_("The directory specified in your "
1029                         "PORTAGE_TMPDIR variable, '%s',\n"
1030                         "does not exist.  Please create this directory or "
1031                         "correct your PORTAGE_TMPDIR setting.\n") % \
1032                         settings.get("PORTAGE_TMPDIR", ""), noiselevel=-1)
1033                 return 1
1034
1035         # as some people use a separate PORTAGE_TMPDIR mount
1036         # we prefer that as the checks below would otherwise be pointless
1037         # for those people.
1038         tmpdir = os.path.realpath(settings["PORTAGE_TMPDIR"])
1039         if os.path.exists(os.path.join(tmpdir, "portage")):
1040                 checkdir = os.path.realpath(os.path.join(tmpdir, "portage"))
1041                 if ("sandbox" in settings.features or
1042                         "usersandox" in settings.features) and \
1043                         not checkdir.startswith(tmpdir + os.sep):
1044                         msg = _("The 'portage' subdirectory of the directory "
1045                         "referenced by the PORTAGE_TMPDIR variable appears to be "
1046                         "a symlink. In order to avoid sandbox violations (see bug "
1047                         "#378379), you must adjust PORTAGE_TMPDIR instead of using "
1048                         "the symlink located at '%s'. A suitable PORTAGE_TMPDIR "
1049                         "setting would be '%s'.") % \
1050                         (os.path.join(tmpdir, "portage"), checkdir)
1051                         lines = []
1052                         lines.append("")
1053                         lines.append("")
1054                         lines.extend(wrap(msg, 72))
1055                         lines.append("")
1056                         for line in lines:
1057                                 if line:
1058                                         line = "!!! %s" % (line,)
1059                                 writemsg("%s\n" % (line,), noiselevel=-1)
1060                         return 1
1061         else:
1062                 checkdir = tmpdir
1063
1064         if not os.access(checkdir, os.W_OK):
1065                 writemsg(_("%s is not writable.\n"
1066                         "Likely cause is that you've mounted it as readonly.\n") % checkdir,
1067                         noiselevel=-1)
1068                 return 1
1069
1070         else:
1071                 fd = tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir)
1072                 os.chmod(fd.name, 0o755)
1073                 if not os.access(fd.name, os.X_OK):
1074                         writemsg(_("Can not execute files in %s\n"
1075                                 "Likely cause is that you've mounted it with one of the\n"
1076                                 "following mount options: 'noexec', 'user', 'users'\n\n"
1077                                 "Please make sure that portage can execute files in this directory.\n") % checkdir,
1078                                 noiselevel=-1)
1079                         return 1
1080
1081         return os.EX_OK
1082
1083 def _prepare_env_file(settings):
1084         """
1085         Extract environment.bz2 if it exists, but only if the destination
1086         environment file doesn't already exist. There are lots of possible
1087         states when doebuild() calls this function, and we want to avoid
1088         clobbering an existing environment file.
1089         """
1090
1091         env_extractor = BinpkgEnvExtractor(background=False,
1092                 scheduler=PollScheduler().sched_iface, settings=settings)
1093
1094         if env_extractor.dest_env_exists():
1095                 # There are lots of possible states when doebuild()
1096                 # calls this function, and we want to avoid
1097                 # clobbering an existing environment file.
1098                 return os.EX_OK
1099
1100         if not env_extractor.saved_env_exists():
1101                 # If the environment.bz2 doesn't exist, then ebuild.sh will
1102                 # source the ebuild as a fallback.
1103                 return os.EX_OK
1104
1105         env_extractor.start()
1106         env_extractor.wait()
1107         return env_extractor.returncode
1108
1109 def _prepare_fake_distdir(settings, alist):
1110         orig_distdir = settings["DISTDIR"]
1111         settings["PORTAGE_ACTUAL_DISTDIR"] = orig_distdir
1112         edpath = settings["DISTDIR"] = \
1113                 os.path.join(settings["PORTAGE_BUILDDIR"], "distdir")
1114         portage.util.ensure_dirs(edpath, gid=portage_gid, mode=0o755)
1115
1116         # Remove any unexpected files or directories.
1117         for x in os.listdir(edpath):
1118                 symlink_path = os.path.join(edpath, x)
1119                 st = os.lstat(symlink_path)
1120                 if x in alist and stat.S_ISLNK(st.st_mode):
1121                         continue
1122                 if stat.S_ISDIR(st.st_mode):
1123                         shutil.rmtree(symlink_path)
1124                 else:
1125                         os.unlink(symlink_path)
1126
1127         # Check for existing symlinks and recreate if necessary.
1128         for x in alist:
1129                 symlink_path = os.path.join(edpath, x)
1130                 target = os.path.join(orig_distdir, x)
1131                 try:
1132                         link_target = os.readlink(symlink_path)
1133                 except OSError:
1134                         os.symlink(target, symlink_path)
1135                 else:
1136                         if link_target != target:
1137                                 os.unlink(symlink_path)
1138                                 os.symlink(target, symlink_path)
1139
1140 def _spawn_actionmap(settings):
1141         features = settings.features
1142         restrict = settings["PORTAGE_RESTRICT"].split()
1143         nosandbox = (("userpriv" in features) and \
1144                 ("usersandbox" not in features) and \
1145                 "userpriv" not in restrict and \
1146                 "nouserpriv" not in restrict)
1147         if nosandbox and ("userpriv" not in features or \
1148                 "userpriv" in restrict or \
1149                 "nouserpriv" in restrict):
1150                 nosandbox = ("sandbox" not in features and \
1151                         "usersandbox" not in features)
1152
1153         if not portage.process.sandbox_capable:
1154                 nosandbox = True
1155
1156         sesandbox = settings.selinux_enabled() and \
1157                 "sesandbox" in features
1158
1159         droppriv = "userpriv" in features and \
1160                 "userpriv" not in restrict and \
1161                 secpass >= 2
1162
1163         fakeroot = "fakeroot" in features
1164
1165         portage_bin_path = settings["PORTAGE_BIN_PATH"]
1166         ebuild_sh_binary = os.path.join(portage_bin_path,
1167                 os.path.basename(EBUILD_SH_BINARY))
1168         misc_sh_binary = os.path.join(portage_bin_path,
1169                 os.path.basename(MISC_SH_BINARY))
1170         ebuild_sh = _shell_quote(ebuild_sh_binary) + " %s"
1171         misc_sh = _shell_quote(misc_sh_binary) + " dyn_%s"
1172
1173         # args are for the to spawn function
1174         actionmap = {
1175 "pretend":  {"cmd":ebuild_sh, "args":{"droppriv":0,        "free":1,         "sesandbox":0,         "fakeroot":0}},
1176 "setup":    {"cmd":ebuild_sh, "args":{"droppriv":0,        "free":1,         "sesandbox":0,         "fakeroot":0}},
1177 "unpack":   {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":0,         "sesandbox":sesandbox, "fakeroot":0}},
1178 "prepare":  {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":0,         "sesandbox":sesandbox, "fakeroot":0}},
1179 "configure":{"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}},
1180 "compile":  {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}},
1181 "test":     {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}},
1182 "install":  {"cmd":ebuild_sh, "args":{"droppriv":0,        "free":0,         "sesandbox":sesandbox, "fakeroot":fakeroot}},
1183 "rpm":      {"cmd":misc_sh,   "args":{"droppriv":0,        "free":0,         "sesandbox":0,         "fakeroot":fakeroot}},
1184 "package":  {"cmd":misc_sh,   "args":{"droppriv":0,        "free":0,         "sesandbox":0,         "fakeroot":fakeroot}},
1185                 }
1186
1187         return actionmap
1188
1189 def _validate_deps(mysettings, myroot, mydo, mydbapi):
1190
1191         invalid_dep_exempt_phases = \
1192                 set(["clean", "cleanrm", "help", "prerm", "postrm"])
1193         all_keys = set(Package.metadata_keys)
1194         all_keys.add("SRC_URI")
1195         all_keys = tuple(all_keys)
1196         metadata = dict(zip(all_keys,
1197                 mydbapi.aux_get(mysettings.mycpv, all_keys,
1198                 myrepo=mysettings.get("PORTAGE_REPO_NAME"))))
1199
1200         class FakeTree(object):
1201                 def __init__(self, mydb):
1202                         self.dbapi = mydb
1203
1204         root_config = RootConfig(mysettings, {"porttree":FakeTree(mydbapi)}, None)
1205
1206         pkg = Package(built=False, cpv=mysettings.mycpv,
1207                 metadata=metadata, root_config=root_config,
1208                 type_name="ebuild")
1209
1210         msgs = []
1211         if pkg.invalid:
1212                 for k, v in pkg.invalid.items():
1213                         for msg in v:
1214                                 msgs.append("  %s\n" % (msg,))
1215
1216         if msgs:
1217                 portage.util.writemsg_level(_("Error(s) in metadata for '%s':\n") % \
1218                         (mysettings.mycpv,), level=logging.ERROR, noiselevel=-1)
1219                 for x in msgs:
1220                         portage.util.writemsg_level(x,
1221                                 level=logging.ERROR, noiselevel=-1)
1222                 if mydo not in invalid_dep_exempt_phases:
1223                         return 1
1224
1225         if not pkg.built and \
1226                 mydo not in ("digest", "help", "manifest") and \
1227                 pkg.metadata["REQUIRED_USE"] and \
1228                 eapi_has_required_use(pkg.metadata["EAPI"]):
1229                 result = check_required_use(pkg.metadata["REQUIRED_USE"],
1230                         pkg.use.enabled, pkg.iuse.is_valid_flag)
1231                 if not result:
1232                         reduced_noise = result.tounicode()
1233                         writemsg("\n  %s\n" % _("The following REQUIRED_USE flag" + \
1234                                 " constraints are unsatisfied:"), noiselevel=-1)
1235                         writemsg("    %s\n" % reduced_noise,
1236                                 noiselevel=-1)
1237                         normalized_required_use = \
1238                                 " ".join(pkg.metadata["REQUIRED_USE"].split())
1239                         if reduced_noise != normalized_required_use:
1240                                 writemsg("\n  %s\n" % _("The above constraints " + \
1241                                         "are a subset of the following complete expression:"),
1242                                         noiselevel=-1)
1243                                 writemsg("    %s\n" % \
1244                                         human_readable_required_use(normalized_required_use),
1245                                         noiselevel=-1)
1246                         writemsg("\n", noiselevel=-1)
1247                         return 1
1248
1249         return os.EX_OK
1250
1251 # XXX This would be to replace getstatusoutput completely.
1252 # XXX Issue: cannot block execution. Deadlock condition.
1253 def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakeroot=0, **keywords):
1254         """
1255         Spawn a subprocess with extra portage-specific options.
1256         Optiosn include:
1257
1258         Sandbox: Sandbox means the spawned process will be limited in its ability t
1259         read and write files (normally this means it is restricted to ${D}/)
1260         SElinux Sandbox: Enables sandboxing on SElinux
1261         Reduced Privileges: Drops privilages such that the process runs as portage:portage
1262         instead of as root.
1263
1264         Notes: os.system cannot be used because it messes with signal handling.  Instead we
1265         use the portage.process spawn* family of functions.
1266
1267         This function waits for the process to terminate.
1268
1269         @param mystring: Command to run
1270         @type mystring: String
1271         @param mysettings: Either a Dict of Key,Value pairs or an instance of portage.config
1272         @type mysettings: Dictionary or config instance
1273         @param debug: Ignored
1274         @type debug: Boolean
1275         @param free: Enable sandboxing for this process
1276         @type free: Boolean
1277         @param droppriv: Drop to portage:portage when running this command
1278         @type droppriv: Boolean
1279         @param sesandbox: Enable SELinux Sandboxing (toggles a context switch)
1280         @type sesandbox: Boolean
1281         @param fakeroot: Run this command with faked root privileges
1282         @type fakeroot: Boolean
1283         @param keywords: Extra options encoded as a dict, to be passed to spawn
1284         @type keywords: Dictionary
1285         @rtype: Integer
1286         @returns:
1287         1. The return code of the spawned process.
1288         """
1289
1290         check_config_instance(mysettings)
1291
1292         fd_pipes = keywords.get("fd_pipes")
1293         if fd_pipes is None:
1294                 fd_pipes = {
1295                         0:sys.stdin.fileno(),
1296                         1:sys.stdout.fileno(),
1297                         2:sys.stderr.fileno(),
1298                 }
1299         # In some cases the above print statements don't flush stdout, so
1300         # it needs to be flushed before allowing a child process to use it
1301         # so that output always shows in the correct order.
1302         stdout_filenos = (sys.stdout.fileno(), sys.stderr.fileno())
1303         for fd in fd_pipes.values():
1304                 if fd in stdout_filenos:
1305                         sys.stdout.flush()
1306                         sys.stderr.flush()
1307                         break
1308
1309         features = mysettings.features
1310         # TODO: Enable fakeroot to be used together with droppriv.  The
1311         # fake ownership/permissions will have to be converted to real
1312         # permissions in the merge phase.
1313         fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable
1314         if droppriv and not uid and portage_gid and portage_uid:
1315                 keywords.update({"uid":portage_uid,"gid":portage_gid,
1316                         "groups":userpriv_groups,"umask":0o02})
1317         if not free:
1318                 free=((droppriv and "usersandbox" not in features) or \
1319                         (not droppriv and "sandbox" not in features and \
1320                         "usersandbox" not in features and not fakeroot))
1321
1322         if not free and not (fakeroot or portage.process.sandbox_capable):
1323                 free = True
1324
1325         if mysettings.mycpv is not None:
1326                 keywords["opt_name"] = "[%s]" % mysettings.mycpv
1327         else:
1328                 keywords["opt_name"] = "[%s/%s]" % \
1329                         (mysettings.get("CATEGORY",""), mysettings.get("PF",""))
1330
1331         if free or "SANDBOX_ACTIVE" in os.environ:
1332                 keywords["opt_name"] += " bash"
1333                 spawn_func = portage.process.spawn_bash
1334         elif fakeroot:
1335                 keywords["opt_name"] += " fakeroot"
1336                 keywords["fakeroot_state"] = os.path.join(mysettings["T"], "fakeroot.state")
1337                 spawn_func = portage.process.spawn_fakeroot
1338         else:
1339                 keywords["opt_name"] += " sandbox"
1340                 spawn_func = portage.process.spawn_sandbox
1341
1342         if sesandbox:
1343                 spawn_func = selinux.spawn_wrapper(spawn_func,
1344                         mysettings["PORTAGE_SANDBOX_T"])
1345
1346         if keywords.get("returnpid"):
1347                 return spawn_func(mystring, env=mysettings.environ(), **keywords)
1348
1349         proc = EbuildSpawnProcess(
1350                 background=False, args=mystring,
1351                 scheduler=PollScheduler().sched_iface, spawn_func=spawn_func,
1352                 settings=mysettings, **keywords)
1353
1354         proc.start()
1355         proc.wait()
1356
1357         return proc.returncode
1358
1359 # parse actionmap to spawn ebuild with the appropriate args
1360 def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0,
1361         logfile=None, fd_pipes=None, returnpid=False):
1362
1363         if returnpid:
1364                 warnings.warn("portage.spawnebuild() called " + \
1365                         "with returnpid parameter enabled. This usage will " + \
1366                         "not be supported in the future.",
1367                         DeprecationWarning, stacklevel=2)
1368
1369         if not returnpid and \
1370                 (alwaysdep or "noauto" not in mysettings.features):
1371                 # process dependency first
1372                 if "dep" in actionmap[mydo]:
1373                         retval = spawnebuild(actionmap[mydo]["dep"], actionmap,
1374                                 mysettings, debug, alwaysdep=alwaysdep, logfile=logfile,
1375                                 fd_pipes=fd_pipes, returnpid=returnpid)
1376                         if retval:
1377                                 return retval
1378
1379         eapi = mysettings["EAPI"]
1380
1381         if mydo in ("configure", "prepare") and not eapi_has_src_prepare_and_src_configure(eapi):
1382                 return os.EX_OK
1383
1384         if mydo == "pretend" and not eapi_has_pkg_pretend(eapi):
1385                 return os.EX_OK
1386
1387         if not (mydo == "install" and "noauto" in mysettings.features):
1388                 check_file = os.path.join(
1389                         mysettings["PORTAGE_BUILDDIR"], ".%sed" % mydo.rstrip('e'))
1390                 if os.path.exists(check_file):
1391                         writemsg_stdout(_(">>> It appears that "
1392                                 "'%(action)s' has already executed for '%(pkg)s'; skipping.\n") %
1393                                 {"action":mydo, "pkg":mysettings["PF"]})
1394                         writemsg_stdout(_(">>> Remove '%(file)s' to force %(action)s.\n") %
1395                                 {"file":check_file, "action":mydo})
1396                         return os.EX_OK
1397
1398         return _spawn_phase(mydo, mysettings,
1399                 actionmap=actionmap, logfile=logfile,
1400                 fd_pipes=fd_pipes, returnpid=returnpid)
1401
1402 _post_phase_cmds = {
1403
1404         "install" : [
1405                 "install_qa_check",
1406                 "install_symlink_html_docs",
1407                 "install_hooks"],
1408
1409         "preinst" : [
1410                 "preinst_sfperms",
1411                 "preinst_selinux_labels",
1412                 "preinst_suid_scan",
1413                 "preinst_mask"]
1414 }
1415
1416 def _post_phase_userpriv_perms(mysettings):
1417         if "userpriv" in mysettings.features and secpass >= 2:
1418                 """ Privileged phases may have left files that need to be made
1419                 writable to a less privileged user."""
1420                 apply_recursive_permissions(mysettings["T"],
1421                         uid=portage_uid, gid=portage_gid, dirmode=0o70, dirmask=0,
1422                         filemode=0o60, filemask=0)
1423
1424 def _check_build_log(mysettings, out=None):
1425         """
1426         Search the content of $PORTAGE_LOG_FILE if it exists
1427         and generate the following QA Notices when appropriate:
1428
1429           * Automake "maintainer mode"
1430           * command not found
1431           * Unrecognized configure options
1432         """
1433         logfile = mysettings.get("PORTAGE_LOG_FILE")
1434         if logfile is None:
1435                 return
1436         try:
1437                 f = open(_unicode_encode(logfile, encoding=_encodings['fs'],
1438                         errors='strict'), mode='rb')
1439         except EnvironmentError:
1440                 return
1441
1442         f_real = None
1443         if logfile.endswith('.gz'):
1444                 f_real = f
1445                 f =  gzip.GzipFile(filename='', mode='rb', fileobj=f)
1446
1447         am_maintainer_mode = []
1448         bash_command_not_found = []
1449         bash_command_not_found_re = re.compile(
1450                 r'(.*): line (\d*): (.*): command not found$')
1451         command_not_found_exclude_re = re.compile(r'/configure: line ')
1452         helper_missing_file = []
1453         helper_missing_file_re = re.compile(
1454                 r'^!!! (do|new).*: .* does not exist$')
1455
1456         configure_opts_warn = []
1457         configure_opts_warn_re = re.compile(
1458                 r'^configure: WARNING: [Uu]nrecognized options: ')
1459
1460         # Exclude output from dev-libs/yaz-3.0.47 which looks like this:
1461         #
1462         #Configuration:
1463         #  Automake:                   ${SHELL} /var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run automake-1.10
1464         am_maintainer_mode_re = re.compile(r'/missing --run ')
1465         am_maintainer_mode_exclude_re = \
1466                 re.compile(r'(/missing --run (autoheader|autotest|help2man|makeinfo)|^\s*Automake:\s)')
1467
1468         make_jobserver_re = \
1469                 re.compile(r'g?make\[\d+\]: warning: jobserver unavailable:')
1470         make_jobserver = []
1471
1472         def _eerror(lines):
1473                 for line in lines:
1474                         eerror(line, phase="install", key=mysettings.mycpv, out=out)
1475
1476         try:
1477                 for line in f:
1478                         line = _unicode_decode(line)
1479                         if am_maintainer_mode_re.search(line) is not None and \
1480                                 am_maintainer_mode_exclude_re.search(line) is None:
1481                                 am_maintainer_mode.append(line.rstrip("\n"))
1482
1483                         if bash_command_not_found_re.match(line) is not None and \
1484                                 command_not_found_exclude_re.search(line) is None:
1485                                 bash_command_not_found.append(line.rstrip("\n"))
1486
1487                         if helper_missing_file_re.match(line) is not None:
1488                                 helper_missing_file.append(line.rstrip("\n"))
1489
1490                         if configure_opts_warn_re.match(line) is not None:
1491                                 configure_opts_warn.append(line.rstrip("\n"))
1492
1493                         if make_jobserver_re.match(line) is not None:
1494                                 make_jobserver.append(line.rstrip("\n"))
1495
1496         except zlib.error as e:
1497                 _eerror(["portage encountered a zlib error: '%s'" % (e,),
1498                         "while reading the log file: '%s'" % logfile])
1499         finally:
1500                 f.close()
1501
1502         def _eqawarn(lines):
1503                 for line in lines:
1504                         eqawarn(line, phase="install", key=mysettings.mycpv, out=out)
1505         wrap_width = 70
1506
1507         if am_maintainer_mode:
1508                 msg = [_("QA Notice: Automake \"maintainer mode\" detected:")]
1509                 msg.append("")
1510                 msg.extend("\t" + line for line in am_maintainer_mode)
1511                 msg.append("")
1512                 msg.extend(wrap(_(
1513                         "If you patch Makefile.am, "
1514                         "configure.in,  or configure.ac then you "
1515                         "should use autotools.eclass and "
1516                         "eautomake or eautoreconf. Exceptions "
1517                         "are limited to system packages "
1518                         "for which it is impossible to run "
1519                         "autotools during stage building. "
1520                         "See http://www.gentoo.org/p"
1521                         "roj/en/qa/autofailure.xml for more information."),
1522                         wrap_width))
1523                 _eqawarn(msg)
1524
1525         if bash_command_not_found:
1526                 msg = [_("QA Notice: command not found:")]
1527                 msg.append("")
1528                 msg.extend("\t" + line for line in bash_command_not_found)
1529                 _eqawarn(msg)
1530
1531         if helper_missing_file:
1532                 msg = [_("QA Notice: file does not exist:")]
1533                 msg.append("")
1534                 msg.extend("\t" + line[4:] for line in helper_missing_file)
1535                 _eqawarn(msg)
1536
1537         if configure_opts_warn:
1538                 msg = [_("QA Notice: Unrecognized configure options:")]
1539                 msg.append("")
1540                 msg.extend("\t" + line for line in configure_opts_warn)
1541                 _eqawarn(msg)
1542
1543         if make_jobserver:
1544                 msg = [_("QA Notice: make jobserver unavailable:")]
1545                 msg.append("")
1546                 msg.extend("\t" + line for line in make_jobserver)
1547                 _eqawarn(msg)
1548
1549         f.close()
1550         if f_real is not None:
1551                 f_real.close()
1552
1553 def _post_src_install_chost_fix(settings):
1554         """
1555         It's possible that the ebuild has changed the
1556         CHOST variable, so revert it to the initial
1557         setting. Also, revert IUSE in case it's corrupted
1558         due to local environment settings like in bug #386829.
1559         """
1560
1561         build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info')
1562
1563         for k in ('IUSE',):
1564                 v = settings.get(k)
1565                 if v is not None:
1566                         write_atomic(os.path.join(build_info_dir, k), v + '\n')
1567
1568         # The following variables are irrelevant for virtual packages.
1569         if settings.get('CATEGORY') != 'virtual':
1570
1571                 for k in ('CHOST',):
1572                         v = settings.get(k)
1573                         if v is not None:
1574                                 write_atomic(os.path.join(build_info_dir, k), v + '\n')
1575
1576 _vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND',
1577         'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',)
1578 _vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND'])
1579
1580 def _preinst_bsdflags(mysettings):
1581         if bsd_chflags:
1582                 # Save all the file flags for restoration later.
1583                 os.system("mtree -c -p %s -k flags > %s" % \
1584                         (_shell_quote(mysettings["D"]),
1585                         _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree"))))
1586
1587                 # Remove all the file flags to avoid EPERM errors.
1588                 os.system("chflags -R noschg,nouchg,nosappnd,nouappnd %s" % \
1589                         (_shell_quote(mysettings["D"]),))
1590                 os.system("chflags -R nosunlnk,nouunlnk %s 2>/dev/null" % \
1591                         (_shell_quote(mysettings["D"]),))
1592
1593
1594 def _postinst_bsdflags(mysettings):
1595         if bsd_chflags:
1596                 # Restore all of the flags saved above.
1597                 os.system("mtree -e -p %s -U -k flags < %s > /dev/null" % \
1598                         (_shell_quote(mysettings["ROOT"]),
1599                         _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree"))))
1600
1601 def _post_src_install_uid_fix(mysettings, out):
1602         """
1603         Files in $D with user and group bits that match the "portage"
1604         user or group are automatically mapped to PORTAGE_INST_UID and
1605         PORTAGE_INST_GID if necessary. The chown system call may clear
1606         S_ISUID and S_ISGID bits, so those bits are restored if
1607         necessary.
1608         """
1609
1610         os = _os_merge
1611
1612         inst_uid = int(mysettings["PORTAGE_INST_UID"])
1613         inst_gid = int(mysettings["PORTAGE_INST_GID"])
1614
1615         _preinst_bsdflags(mysettings)
1616
1617         destdir = mysettings["D"]
1618         unicode_errors = []
1619
1620         while True:
1621
1622                 unicode_error = False
1623                 size = 0
1624                 counted_inodes = set()
1625                 fixlafiles_announced = False
1626                 fixlafiles = "fixlafiles" in mysettings.features
1627
1628                 for parent, dirs, files in os.walk(destdir):
1629                         try:
1630                                 parent = _unicode_decode(parent,
1631                                         encoding=_encodings['merge'], errors='strict')
1632                         except UnicodeDecodeError:
1633                                 new_parent = _unicode_decode(parent,
1634                                         encoding=_encodings['merge'], errors='replace')
1635                                 new_parent = _unicode_encode(new_parent,
1636                                         encoding='ascii', errors='backslashreplace')
1637                                 new_parent = _unicode_decode(new_parent,
1638                                         encoding=_encodings['merge'], errors='replace')
1639                                 os.rename(parent, new_parent)
1640                                 unicode_error = True
1641                                 unicode_errors.append(new_parent[len(destdir):])
1642                                 break
1643
1644                         for fname in chain(dirs, files):
1645                                 try:
1646                                         fname = _unicode_decode(fname,
1647                                                 encoding=_encodings['merge'], errors='strict')
1648                                 except UnicodeDecodeError:
1649                                         fpath = _os.path.join(
1650                                                 parent.encode(_encodings['merge']), fname)
1651                                         new_fname = _unicode_decode(fname,
1652                                                 encoding=_encodings['merge'], errors='replace')
1653                                         new_fname = _unicode_encode(new_fname,
1654                                                 encoding='ascii', errors='backslashreplace')
1655                                         new_fname = _unicode_decode(new_fname,
1656                                                 encoding=_encodings['merge'], errors='replace')
1657                                         new_fpath = os.path.join(parent, new_fname)
1658                                         os.rename(fpath, new_fpath)
1659                                         unicode_error = True
1660                                         unicode_errors.append(new_fpath[len(destdir):])
1661                                         fname = new_fname
1662                                         fpath = new_fpath
1663                                 else:
1664                                         fpath = os.path.join(parent, fname)
1665
1666                                 if fixlafiles and \
1667                                         fname.endswith(".la") and os.path.isfile(fpath):
1668                                         f = open(_unicode_encode(fpath,
1669                                                 encoding=_encodings['merge'], errors='strict'),
1670                                                 mode='rb')
1671                                         has_lafile_header = b'.la - a libtool library file' \
1672                                                 in f.readline()
1673                                         f.seek(0)
1674                                         contents = f.read()
1675                                         f.close()
1676                                         try:
1677                                                 needs_update, new_contents = rewrite_lafile(contents)
1678                                         except portage.exception.InvalidData as e:
1679                                                 needs_update = False
1680                                                 if not fixlafiles_announced:
1681                                                         fixlafiles_announced = True
1682                                                         writemsg("Fixing .la files\n", fd=out)
1683
1684                                                 # Suppress warnings if the file does not have the
1685                                                 # expected header (bug #340725). Even if the header is
1686                                                 # missing, we still call rewrite_lafile() since some
1687                                                 # valid libtool archives may not have the header.
1688                                                 msg = "   %s is not a valid libtool archive, skipping\n" % fpath[len(destdir):]
1689                                                 qa_msg = "QA Notice: invalid .la file found: %s, %s" % (fpath[len(destdir):], e)
1690                                                 if has_lafile_header:
1691                                                         writemsg(msg, fd=out)
1692                                                         eqawarn(qa_msg, key=mysettings.mycpv, out=out)
1693
1694                                         if needs_update:
1695                                                 if not fixlafiles_announced:
1696                                                         fixlafiles_announced = True
1697                                                         writemsg("Fixing .la files\n", fd=out)
1698                                                 writemsg("   %s\n" % fpath[len(destdir):], fd=out)
1699                                                 # write_atomic succeeds even in some cases in which
1700                                                 # a normal write might fail due to file permission
1701                                                 # settings on some operating systems such as HP-UX
1702                                                 write_atomic(_unicode_encode(fpath,
1703                                                         encoding=_encodings['merge'], errors='strict'),
1704                                                         new_contents, mode='wb')
1705
1706                                 mystat = os.lstat(fpath)
1707                                 if stat.S_ISREG(mystat.st_mode) and \
1708                                         mystat.st_ino not in counted_inodes:
1709                                         counted_inodes.add(mystat.st_ino)
1710                                         size += mystat.st_size
1711                                 if mystat.st_uid != portage_uid and \
1712                                         mystat.st_gid != portage_gid:
1713                                         continue
1714                                 myuid = -1
1715                                 mygid = -1
1716                                 if mystat.st_uid == portage_uid:
1717                                         myuid = inst_uid
1718                                 if mystat.st_gid == portage_gid:
1719                                         mygid = inst_gid
1720                                 apply_secpass_permissions(
1721                                         _unicode_encode(fpath, encoding=_encodings['merge']),
1722                                         uid=myuid, gid=mygid,
1723                                         mode=mystat.st_mode, stat_cached=mystat,
1724                                         follow_links=False)
1725
1726                         if unicode_error:
1727                                 break
1728
1729                 if not unicode_error:
1730                         break
1731
1732         if unicode_errors:
1733                 for l in _merge_unicode_error(unicode_errors):
1734                         eerror(l, phase='install', key=mysettings.mycpv, out=out)
1735
1736         build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'],
1737                 'build-info')
1738
1739         f = io.open(_unicode_encode(os.path.join(build_info_dir,
1740                 'SIZE'), encoding=_encodings['fs'], errors='strict'),
1741                 mode='w', encoding=_encodings['repo.content'],
1742                 errors='strict')
1743         f.write(_unicode_decode(str(size) + '\n'))
1744         f.close()
1745
1746         f = io.open(_unicode_encode(os.path.join(build_info_dir,
1747                 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'),
1748                 mode='w', encoding=_encodings['repo.content'],
1749                 errors='strict')
1750         f.write(_unicode_decode("%.0f\n" % (time.time(),)))
1751         f.close()
1752
1753         use = frozenset(mysettings['PORTAGE_USE'].split())
1754         for k in _vdb_use_conditional_keys:
1755                 v = mysettings.configdict['pkg'].get(k)
1756                 filename = os.path.join(build_info_dir, k)
1757                 if v is None:
1758                         try:
1759                                 os.unlink(filename)
1760                         except OSError:
1761                                 pass
1762                         continue
1763
1764                 if k.endswith('DEPEND'):
1765                         token_class = Atom
1766                 else:
1767                         token_class = None
1768
1769                 v = use_reduce(v, uselist=use, token_class=token_class)
1770                 v = paren_enclose(v)
1771                 if not v:
1772                         try:
1773                                 os.unlink(filename)
1774                         except OSError:
1775                                 pass
1776                         continue
1777                 f = io.open(_unicode_encode(os.path.join(build_info_dir,
1778                         k), encoding=_encodings['fs'], errors='strict'),
1779                         mode='w', encoding=_encodings['repo.content'],
1780                         errors='strict')
1781                 f.write(_unicode_decode(v + '\n'))
1782                 f.close()
1783
1784         _reapply_bsdflags_to_image(mysettings)
1785
1786 def _reapply_bsdflags_to_image(mysettings):
1787         """
1788         Reapply flags saved and removed by _preinst_bsdflags.
1789         """
1790         if bsd_chflags:
1791                 os.system("mtree -e -p %s -U -k flags < %s > /dev/null" % \
1792                         (_shell_quote(mysettings["D"]),
1793                         _shell_quote(os.path.join(mysettings["T"], "bsdflags.mtree"))))
1794
1795 def _post_src_install_soname_symlinks(mysettings, out):
1796         """
1797         Check that libraries in $D have corresponding soname symlinks.
1798         If symlinks are missing then create them and trigger a QA Notice.
1799         This requires $PORTAGE_BUILDDIR/build-info/NEEDED.ELF.2 for
1800         operation.
1801         """
1802
1803         image_dir = mysettings["D"]
1804         needed_filename = os.path.join(mysettings["PORTAGE_BUILDDIR"],
1805                 "build-info", "NEEDED.ELF.2")
1806
1807         f = None
1808         try:
1809                 f = io.open(_unicode_encode(needed_filename,
1810                         encoding=_encodings['fs'], errors='strict'),
1811                         mode='r', encoding=_encodings['repo.content'],
1812                         errors='replace')
1813                 lines = f.readlines()
1814         except IOError as e:
1815                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1816                         raise
1817                 return
1818         finally:
1819                 if f is not None:
1820                         f.close()
1821
1822         qa_no_symlink = ""
1823         f = None
1824         try:
1825                 f = io.open(_unicode_encode(os.path.join(
1826                         mysettings["PORTAGE_BUILDDIR"],
1827                         "build-info", "QA_SONAME_NO_SYMLINK"),
1828                         encoding=_encodings['fs'], errors='strict'),
1829                         mode='r', encoding=_encodings['repo.content'],
1830                         errors='replace')
1831                 qa_no_symlink = f.read()
1832         except IOError as e:
1833                 if e.errno not in (errno.ENOENT, errno.ESTALE):
1834                         raise
1835         finally:
1836                 if f is not None:
1837                         f.close()
1838
1839         qa_no_symlink = qa_no_symlink.split()
1840         if qa_no_symlink:
1841                 if len(qa_no_symlink) > 1:
1842                         qa_no_symlink = "|".join("(%s)" % x for x in qa_no_symlink)
1843                         qa_no_symlink = "^(%s)$" % qa_no_symlink
1844                 else:
1845                         qa_no_symlink = "^%s$" % qa_no_symlink[0]
1846                 qa_no_symlink = re.compile(qa_no_symlink)
1847
1848         libpaths = set(portage.util.getlibpaths(
1849                 mysettings["ROOT"], env=mysettings))
1850         libpath_inodes = set()
1851         for libpath in libpaths:
1852                 libdir = os.path.join(mysettings["ROOT"], libpath.lstrip(os.sep))
1853                 try:
1854                         s = os.stat(libdir)
1855                 except OSError:
1856                         continue
1857                 else:
1858                         libpath_inodes.add((s.st_dev, s.st_ino))
1859
1860         is_libdir_cache = {}
1861
1862         def is_libdir(obj_parent):
1863                 try:
1864                         return is_libdir_cache[obj_parent]
1865                 except KeyError:
1866                         pass
1867
1868                 rval = False
1869                 if obj_parent in libpaths:
1870                         rval = True
1871                 else:
1872                         parent_path = os.path.join(mysettings["ROOT"],
1873                                 obj_parent.lstrip(os.sep))
1874                         try:
1875                                 s = os.stat(parent_path)
1876                         except OSError:
1877                                 pass
1878                         else:
1879                                 if (s.st_dev, s.st_ino) in libpath_inodes:
1880                                         rval = True
1881
1882                 is_libdir_cache[obj_parent] = rval
1883                 return rval
1884
1885         missing_symlinks = []
1886
1887         # Parse NEEDED.ELF.2 like LinkageMapELF.rebuild() does.
1888         for l in lines:
1889                 l = l.rstrip("\n")
1890                 if not l:
1891                         continue
1892                 fields = l.split(";")
1893                 if len(fields) < 5:
1894                         portage.util.writemsg_level(_("\nWrong number of fields " \
1895                                 "in %s: %s\n\n") % (needed_filename, l),
1896                                 level=logging.ERROR, noiselevel=-1)
1897                         continue
1898
1899                 obj, soname = fields[1:3]
1900                 if not soname:
1901                         continue
1902                 if not is_libdir(os.path.dirname(obj)):
1903                         continue
1904                 if qa_no_symlink and qa_no_symlink.match(obj.strip(os.sep)) is not None:
1905                         continue
1906
1907                 obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep))
1908                 sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname)
1909                 try:
1910                         os.lstat(sym_file_path)
1911                 except OSError as e:
1912                         if e.errno not in (errno.ENOENT, errno.ESTALE):
1913                                 raise
1914                 else:
1915                         continue
1916
1917                 missing_symlinks.append((obj, soname))
1918
1919         if not missing_symlinks:
1920                 return
1921
1922         qa_msg = ["QA Notice: Missing soname symlink(s):"]
1923         qa_msg.append("")
1924         qa_msg.extend("\t%s -> %s" % (os.path.join(
1925                 os.path.dirname(obj).lstrip(os.sep), soname),
1926                 os.path.basename(obj))
1927                 for obj, soname in missing_symlinks)
1928         qa_msg.append("")
1929         for line in qa_msg:
1930                 eqawarn(line, key=mysettings.mycpv, out=out)
1931
1932 def _merge_unicode_error(errors):
1933         lines = []
1934
1935         msg = _("This package installs one or more file names containing "
1936                 "characters that do not match your current locale "
1937                 "settings. The current setting for filesystem encoding is '%s'.") \
1938                 % _encodings['merge']
1939         lines.extend(wrap(msg, 72))
1940
1941         lines.append("")
1942         errors.sort()
1943         lines.extend("\t" + x for x in errors)
1944         lines.append("")
1945
1946         if _encodings['merge'].lower().replace('_', '').replace('-', '') != 'utf8':
1947                 msg = _("For best results, UTF-8 encoding is recommended. See "
1948                         "the Gentoo Linux Localization Guide for instructions "
1949                         "about how to configure your locale for UTF-8 encoding:")
1950                 lines.extend(wrap(msg, 72))
1951                 lines.append("")
1952                 lines.append("\t" + \
1953                         "http://www.gentoo.org/doc/en/guide-localization.xml")
1954                 lines.append("")
1955
1956         return lines