bin/egencache

   1 #!/usr/bin/python
   2 # Copyright 2009-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 # unicode_literals for compat with TextIOWrapper in Python 2
   6 from __future__ import print_function, unicode_literals
   7
   8 import platform
   9 import signal
  10 import sys
  11 # This block ensures that ^C interrupts are handled quietly.
  12 try:
  13
  14         def exithandler(signum,frame):
  15                 signal.signal(signal.SIGINT, signal.SIG_IGN)
  16                 signal.signal(signal.SIGTERM, signal.SIG_IGN)
  17                 sys.exit(128 + signum)
  18
  19         signal.signal(signal.SIGINT, exithandler)
  20         signal.signal(signal.SIGTERM, exithandler)
  21
  22 except KeyboardInterrupt:
  23         sys.exit(128 + signal.SIGINT)
  24
  25 def debug_signal(signum, frame):
  26         import pdb
  27         pdb.set_trace()
  28
  29 if platform.python_implementation() == 'Jython':
  30         debug_signum = signal.SIGUSR2 # bug #424259
  31 else:
  32         debug_signum = signal.SIGUSR1
  33
  34 signal.signal(debug_signum, debug_signal)
  35
  36 import io
  37 import logging
  38 import optparse
  39 import subprocess
  40 import time
  41 import textwrap
  42 import re
  43
  44 from os import path as osp
  45 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  46 sys.path.insert(0, pym_path)
  47 import portage
  48 portage._internal_caller = True
  49 from portage import os, _encodings, _unicode_encode, _unicode_decode
  50 from _emerge.MetadataRegen import MetadataRegen
  51 from portage.cache.cache_errors import CacheError, StatCollision
  52 from portage.manifest import guessManifestFileType
  53 from portage.package.ebuild._parallel_manifest.ManifestScheduler import ManifestScheduler
  54 from portage.util import cmp_sort_key, writemsg_level
  55 from portage.util._async.run_main_scheduler import run_main_scheduler
  56 from portage.util._eventloop.global_event_loop import global_event_loop
  57 from portage import cpv_getkey
  58 from portage.dep import Atom, isjustname
  59 from portage.versions import pkgsplit, vercmp
  60
  61 try:
  62         from xml.etree import ElementTree
  63 except ImportError:
  64         pass
  65 else:
  66         try:
  67                 from xml.parsers.expat import ExpatError
  68         except ImportError:
  69                 pass
  70         else:
  71                 from repoman.utilities import parse_metadata_use
  72
  73 from repoman.utilities import FindVCS
  74
  75 if sys.hexversion >= 0x3000000:
  76         long = int
  77
  78 def parse_args(args):
  79         usage = "egencache [options] <action> ... [atom] ..."
  80         parser = optparse.OptionParser(usage=usage)
  81
  82         actions = optparse.OptionGroup(parser, 'Actions')
  83         actions.add_option("--update",
  84                 action="store_true",
  85                 help="update metadata/md5-cache/ (generate as necessary)")
  86         actions.add_option("--update-use-local-desc",
  87                 action="store_true",
  88                 help="update the use.local.desc file from metadata.xml")
  89         actions.add_option("--update-changelogs",
  90                 action="store_true",
  91                 help="update the ChangeLog files from SCM logs")
  92         actions.add_option("--update-manifests",
  93                 action="store_true",
  94                 help="update manifests")
  95         parser.add_option_group(actions)
  96
  97         common = optparse.OptionGroup(parser, 'Common options')
  98         common.add_option("--repo",
  99                 action="store",
 100                 help="name of repo to operate on")
 101         common.add_option("--config-root",
 102                 help="location of portage config files",
 103                 dest="portage_configroot")
 104         common.add_option("--gpg-dir",
 105                 help="override the PORTAGE_GPG_DIR variable",
 106                 dest="gpg_dir")
 107         common.add_option("--gpg-key",
 108                 help="override the PORTAGE_GPG_KEY variable",
 109                 dest="gpg_key")
 110         common.add_option("--portdir",
 111                 help="override the PORTDIR variable (deprecated in favor of --repositories-configuration)",
 112                 dest="portdir")
 113         common.add_option("--portdir-overlay",
 114                 help="override the PORTDIR_OVERLAY variable (deprecated in favor of --repositories-configuration)",
 115                 dest="portdir_overlay")
 116         common.add_option("--repositories-configuration",
 117                 help="override configuration of repositories (in format of repos.conf)",
 118                 dest="repositories_configuration")
 119         common.add_option("--sign-manifests",
 120                 type="choice",
 121                 choices=('y', 'n'),
 122                 metavar="<y|n>",
 123                 help="manually override layout.conf sign-manifests setting")
 124         common.add_option("--strict-manifests",
 125                 type="choice",
 126                 choices=('y', 'n'),
 127                 metavar="<y|n>",
 128                 help="manually override \"strict\" FEATURES setting")
 129         common.add_option("--thin-manifests",
 130                 type="choice",
 131                 choices=('y', 'n'),
 132                 metavar="<y|n>",
 133                 help="manually override layout.conf thin-manifests setting")
 134         common.add_option("--tolerant",
 135                 action="store_true",
 136                 help="exit successfully if only minor errors occurred")
 137         common.add_option("--ignore-default-opts",
 138                 action="store_true",
 139                 help="do not use the EGENCACHE_DEFAULT_OPTS environment variable")
 140         parser.add_option_group(common)
 141
 142         update = optparse.OptionGroup(parser, '--update options')
 143         update.add_option("--cache-dir",
 144                 help="location of the metadata cache",
 145                 dest="cache_dir")
 146         update.add_option("-j", "--jobs",
 147                 action="store",
 148                 help="max ebuild processes to spawn")
 149         update.add_option("--load-average",
 150                 action="store",
 151                 help="max load allowed when spawning multiple jobs",
 152                 dest="load_average")
 153         update.add_option("--rsync",
 154                 action="store_true",
 155                 help="enable rsync stat collision workaround " + \
 156                         "for bug 139134 (use with --update)")
 157         parser.add_option_group(update)
 158
 159         uld = optparse.OptionGroup(parser, '--update-use-local-desc options')
 160         uld.add_option("--preserve-comments",
 161                 action="store_true",
 162                 help="preserve the comments from the existing use.local.desc file")
 163         uld.add_option("--use-local-desc-output",
 164                 help="output file for use.local.desc data (or '-' for stdout)",
 165                 dest="uld_output")
 166         parser.add_option_group(uld)
 167
 168         options, args = parser.parse_args(args)
 169
 170         if options.jobs:
 171                 jobs = None
 172                 try:
 173                         jobs = int(options.jobs)
 174                 except ValueError:
 175                         jobs = -1
 176
 177                 if jobs < 1:
 178                         parser.error("Invalid: --jobs='%s'" % \
 179                                 (options.jobs,))
 180
 181                 options.jobs = jobs
 182
 183         else:
 184                 options.jobs = None
 185
 186         if options.load_average:
 187                 try:
 188                         load_average = float(options.load_average)
 189                 except ValueError:
 190                         load_average = 0.0
 191
 192                 if load_average <= 0.0:
 193                         parser.error("Invalid: --load-average='%s'" % \
 194                                 (options.load_average,))
 195
 196                 options.load_average = load_average
 197
 198         else:
 199                 options.load_average = None
 200
 201         options.config_root = options.portage_configroot
 202         if options.config_root is not None and \
 203                 not os.path.isdir(options.config_root):
 204                 parser.error("Not a directory: --config-root='%s'" % \
 205                         (options.config_root,))
 206
 207         if options.cache_dir is not None:
 208                 if not os.path.isdir(options.cache_dir):
 209                         parser.error("Not a directory: --cache-dir='%s'" % \
 210                                 (options.cache_dir,))
 211                 if not os.access(options.cache_dir, os.W_OK):
 212                         parser.error("Write access denied: --cache-dir='%s'" % \
 213                                 (options.cache_dir,))
 214
 215         if options.repo is None:
 216                 parser.error("--repo option is required")
 217
 218         if options.portdir is not None:
 219                 writemsg_level("egencache: warning: --portdir option is deprecated in favor of --repositories-configuration option\n",
 220                         level=logging.WARNING, noiselevel=-1)
 221         if options.portdir_overlay is not None:
 222                 writemsg_level("egencache: warning: --portdir-overlay option is deprecated in favor of --repositories-configuration option\n",
 223                         level=logging.WARNING, noiselevel=-1)
 224
 225         for atom in args:
 226                 try:
 227                         atom = portage.dep.Atom(atom)
 228                 except portage.exception.InvalidAtom:
 229                         parser.error('Invalid atom: %s' % (atom,))
 230
 231                 if not isjustname(atom):
 232                         parser.error('Atom is too specific: %s' % (atom,))
 233
 234         if options.update_use_local_desc:
 235                 try:
 236                         ElementTree
 237                         ExpatError
 238                 except NameError:
 239                         parser.error('--update-use-local-desc requires python with USE=xml!')
 240
 241         if options.uld_output == '-' and options.preserve_comments:
 242                 parser.error('--preserve-comments can not be used when outputting to stdout')
 243
 244         return parser, options, args
 245
 246 class GenCache(object):
 247         def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None,
 248                 rsync=False):
 249                 # The caller must set portdb.porttrees in order to constrain
 250                 # findname, cp_list, and cpv_list to the desired tree.
 251                 tree = portdb.porttrees[0]
 252                 self._portdb = portdb
 253                 self._eclass_db = portdb.repositories.get_repo_for_location(tree).eclass_db
 254                 self._auxdbkeys = portdb._known_keys
 255                 # We can globally cleanse stale cache only if we
 256                 # iterate over every single cp.
 257                 self._global_cleanse = cp_iter is None
 258                 if cp_iter is not None:
 259                         self._cp_set = set(cp_iter)
 260                         cp_iter = iter(self._cp_set)
 261                         self._cp_missing = self._cp_set.copy()
 262                 else:
 263                         self._cp_set = None
 264                         self._cp_missing = set()
 265                 write_auxdb = "metadata-transfer" in portdb.settings.features
 266                 self._regen = MetadataRegen(portdb, cp_iter=cp_iter,
 267                         consumer=self._metadata_callback,
 268                         max_jobs=max_jobs, max_load=max_load,
 269                         write_auxdb=write_auxdb, main=True)
 270                 self.returncode = os.EX_OK
 271                 conf = portdb.repositories.get_repo_for_location(tree)
 272                 self._trg_caches = tuple(conf.iter_pregenerated_caches(
 273                         self._auxdbkeys, force=True, readonly=False))
 274                 if not self._trg_caches:
 275                         raise Exception("cache formats '%s' aren't supported" %
 276                                 (" ".join(conf.cache_formats),))
 277
 278                 if rsync:
 279                         for trg_cache in self._trg_caches:
 280                                 if hasattr(trg_cache, 'raise_stat_collision'):
 281                                         trg_cache.raise_stat_collision = True
 282                                         # Make _metadata_callback write this cache first, in case
 283                                         # it raises a StatCollision and triggers mtime
 284                                         # modification.
 285                                         self._trg_caches = tuple([trg_cache] +
 286                                                 [x for x in self._trg_caches if x is not trg_cache])
 287
 288                 self._existing_nodes = set()
 289
 290         def _metadata_callback(self, cpv, repo_path, metadata,
 291                 ebuild_hash, eapi_supported):
 292                 self._existing_nodes.add(cpv)
 293                 self._cp_missing.discard(cpv_getkey(cpv))
 294
 295                 # Since we're supposed to be able to efficiently obtain the
 296                 # EAPI from _parse_eapi_ebuild_head, we don't write cache
 297                 # entries for unsupported EAPIs.
 298                 if metadata is not None and eapi_supported:
 299                         if metadata.get('EAPI') == '0':
 300                                 del metadata['EAPI']
 301                         for trg_cache in self._trg_caches:
 302                                 self._write_cache(trg_cache,
 303                                         cpv, repo_path, metadata, ebuild_hash)
 304
 305         def _write_cache(self, trg_cache, cpv, repo_path, metadata, ebuild_hash):
 306
 307                         if not hasattr(trg_cache, 'raise_stat_collision'):
 308                                 # This cache does not avoid redundant writes automatically,
 309                                 # so check for an identical existing entry before writing.
 310                                 # This prevents unnecessary disk writes and can also prevent
 311                                 # unnecessary rsync transfers.
 312                                 try:
 313                                         dest = trg_cache[cpv]
 314                                 except (KeyError, CacheError):
 315                                         pass
 316                                 else:
 317                                         if trg_cache.validate_entry(dest,
 318                                                 ebuild_hash, self._eclass_db):
 319                                                 identical = True
 320                                                 for k in self._auxdbkeys:
 321                                                         if dest.get(k, '') != metadata.get(k, ''):
 322                                                                 identical = False
 323                                                                 break
 324                                                 if identical:
 325                                                         return
 326
 327                         try:
 328                                 chf = trg_cache.validation_chf
 329                                 metadata['_%s_' % chf] = getattr(ebuild_hash, chf)
 330                                 try:
 331                                         trg_cache[cpv] = metadata
 332                                 except StatCollision as sc:
 333                                         # If the content of a cache entry changes and neither the
 334                                         # file mtime nor size changes, it will prevent rsync from
 335                                         # detecting changes. Cache backends may raise this
 336                                         # exception from _setitem() if they detect this type of stat
 337                                         # collision. These exceptions are handled by bumping the
 338                                         # mtime on the ebuild (and the corresponding cache entry).
 339                                         # See bug #139134. It is convenient to include checks for
 340                                         # redundant writes along with the internal StatCollision
 341                                         # detection code, so for caches with the
 342                                         # raise_stat_collision attribute, we do not need to
 343                                         # explicitly check for redundant writes like we do for the
 344                                         # other cache types above.
 345                                         max_mtime = sc.mtime
 346                                         for ec, ec_hash in metadata['_eclasses_'].items():
 347                                                 if max_mtime < ec_hash.mtime:
 348                                                         max_mtime = ec_hash.mtime
 349                                         if max_mtime == sc.mtime:
 350                                                 max_mtime += 1
 351                                         max_mtime = long(max_mtime)
 352                                         try:
 353                                                 os.utime(ebuild_hash.location, (max_mtime, max_mtime))
 354                                         except OSError as e:
 355                                                 self.returncode |= 1
 356                                                 writemsg_level(
 357                                                         "%s writing target: %s\n" % (cpv, e),
 358                                                         level=logging.ERROR, noiselevel=-1)
 359                                         else:
 360                                                 ebuild_hash.mtime = max_mtime
 361                                                 metadata['_mtime_'] = max_mtime
 362                                                 trg_cache[cpv] = metadata
 363                                                 self._portdb.auxdb[repo_path][cpv] = metadata
 364
 365                         except CacheError as ce:
 366                                 self.returncode |= 1
 367                                 writemsg_level(
 368                                         "%s writing target: %s\n" % (cpv, ce),
 369                                         level=logging.ERROR, noiselevel=-1)
 370
 371         def run(self):
 372                 signum = run_main_scheduler(self._regen)
 373                 if signum is not None:
 374                         sys.exit(128 + signum)
 375
 376                 self.returncode |= self._regen.returncode
 377
 378                 for trg_cache in self._trg_caches:
 379                         self._cleanse_cache(trg_cache)
 380
 381         def _cleanse_cache(self, trg_cache):
 382                 cp_missing = self._cp_missing
 383                 dead_nodes = set()
 384                 if self._global_cleanse:
 385                         try:
 386                                 for cpv in trg_cache:
 387                                         cp = cpv_getkey(cpv)
 388                                         if cp is None:
 389                                                 self.returncode |= 1
 390                                                 writemsg_level(
 391                                                         "Unable to parse cp for '%s'\n"  % (cpv,),
 392                                                         level=logging.ERROR, noiselevel=-1)
 393                                         else:
 394                                                 dead_nodes.add(cpv)
 395                         except CacheError as ce:
 396                                 self.returncode |= 1
 397                                 writemsg_level(
 398                                         "Error listing cache entries for " + \
 399                                         "'%s': %s, continuing...\n" % \
 400                                         (trg_cache.location, ce),
 401                                         level=logging.ERROR, noiselevel=-1)
 402
 403                 else:
 404                         cp_set = self._cp_set
 405                         try:
 406                                 for cpv in trg_cache:
 407                                         cp = cpv_getkey(cpv)
 408                                         if cp is None:
 409                                                 self.returncode |= 1
 410                                                 writemsg_level(
 411                                                         "Unable to parse cp for '%s'\n"  % (cpv,),
 412                                                         level=logging.ERROR, noiselevel=-1)
 413                                         else:
 414                                                 cp_missing.discard(cp)
 415                                                 if cp in cp_set:
 416                                                         dead_nodes.add(cpv)
 417                         except CacheError as ce:
 418                                 self.returncode |= 1
 419                                 writemsg_level(
 420                                         "Error listing cache entries for " + \
 421                                         "'%s': %s, continuing...\n" % \
 422                                         (trg_cache.location, ce),
 423                                         level=logging.ERROR, noiselevel=-1)
 424
 425                 if cp_missing:
 426                         self.returncode |= 1
 427                         for cp in sorted(cp_missing):
 428                                 writemsg_level(
 429                                         "No ebuilds or cache entries found for '%s'\n"  % (cp,),
 430                                         level=logging.ERROR, noiselevel=-1)
 431
 432                 if dead_nodes:
 433                         dead_nodes.difference_update(self._existing_nodes)
 434                         for k in dead_nodes:
 435                                 try:
 436                                         del trg_cache[k]
 437                                 except KeyError:
 438                                         pass
 439                                 except CacheError as ce:
 440                                         self.returncode |= 1
 441                                         writemsg_level(
 442                                                 "%s deleting stale cache: %s\n" % (k, ce),
 443                                                 level=logging.ERROR, noiselevel=-1)
 444
 445                 if not trg_cache.autocommits:
 446                         try:
 447                                 trg_cache.commit()
 448                         except CacheError as ce:
 449                                 self.returncode |= 1
 450                                 writemsg_level(
 451                                         "committing target: %s\n" % (ce,),
 452                                         level=logging.ERROR, noiselevel=-1)
 453
 454                 if hasattr(trg_cache, '_prune_empty_dirs'):
 455                         trg_cache._prune_empty_dirs()
 456
 457 class GenUseLocalDesc(object):
 458         def __init__(self, portdb, output=None,
 459                         preserve_comments=False):
 460                 self.returncode = os.EX_OK
 461                 self._portdb = portdb
 462                 self._output = output
 463                 self._preserve_comments = preserve_comments
 464
 465         def run(self):
 466                 repo_path = self._portdb.porttrees[0]
 467                 ops = {'<':0, '<=':1, '=':2, '>=':3, '>':4}
 468
 469                 if self._output is None or self._output != '-':
 470                         if self._output is None:
 471                                 prof_path = os.path.join(repo_path, 'profiles')
 472                                 desc_path = os.path.join(prof_path, 'use.local.desc')
 473                                 try:
 474                                         os.mkdir(prof_path)
 475                                 except OSError:
 476                                         pass
 477                         else:
 478                                 desc_path = self._output
 479
 480                         try:
 481                                 if self._preserve_comments:
 482                                         # Probe in binary mode, in order to avoid
 483                                         # potential character encoding issues.
 484                                         output = open(_unicode_encode(desc_path,
 485                                                 encoding=_encodings['fs'], errors='strict'), 'r+b')
 486                                 else:
 487                                         output = io.open(_unicode_encode(desc_path,
 488                                                 encoding=_encodings['fs'], errors='strict'),
 489                                                 mode='w', encoding=_encodings['repo.content'],
 490                                                 errors='backslashreplace')
 491                         except IOError as e:
 492                                 if not self._preserve_comments or \
 493                                         os.path.isfile(desc_path):
 494                                         writemsg_level(
 495                                                 "ERROR: failed to open output file %s: %s\n" \
 496                                                 % (desc_path, e), level=logging.ERROR, noiselevel=-1)
 497                                         self.returncode |= 2
 498                                         return
 499
 500                                 # Open in r+b mode failed because the file doesn't
 501                                 # exist yet. We can probably recover if we disable
 502                                 # preserve_comments mode now.
 503                                 writemsg_level(
 504                                         "WARNING: --preserve-comments enabled, but " + \
 505                                         "output file not found: %s\n" % (desc_path,),
 506                                         level=logging.WARNING, noiselevel=-1)
 507                                 self._preserve_comments = False
 508                                 try:
 509                                         output = io.open(_unicode_encode(desc_path,
 510                                                 encoding=_encodings['fs'], errors='strict'),
 511                                                 mode='w', encoding=_encodings['repo.content'],
 512                                                 errors='backslashreplace')
 513                                 except IOError as e:
 514                                         writemsg_level(
 515                                                 "ERROR: failed to open output file %s: %s\n" \
 516                                                 % (desc_path, e), level=logging.ERROR, noiselevel=-1)
 517                                         self.returncode |= 2
 518                                         return
 519                 else:
 520                         output = sys.stdout
 521
 522                 if self._preserve_comments:
 523                         while True:
 524                                 pos = output.tell()
 525                                 if not output.readline().startswith(b'#'):
 526                                         break
 527                         output.seek(pos)
 528                         output.truncate()
 529                         output.close()
 530
 531                         # Finished probing comments in binary mode, now append
 532                         # in text mode.
 533                         output = io.open(_unicode_encode(desc_path,
 534                                 encoding=_encodings['fs'], errors='strict'),
 535                                 mode='a', encoding=_encodings['repo.content'],
 536                                 errors='backslashreplace')
 537                         output.write('\n')
 538                 else:
 539                         output.write(textwrap.dedent('''\
 540                                 # This file is deprecated as per GLEP 56 in favor of metadata.xml. Please add
 541                                 # your descriptions to your package's metadata.xml ONLY.
 542                                 # * generated automatically using egencache *
 543
 544                                 '''))
 545
 546                 # The cmp function no longer exists in python3, so we'll
 547                 # implement our own here under a slightly different name
 548                 # since we don't want any confusion given that we never
 549                 # want to rely on the builtin cmp function.
 550                 def cmp_func(a, b):
 551                         if a is None or b is None:
 552                                 # None can't be compared with other types in python3.
 553                                 if a is None and b is None:
 554                                         return 0
 555                                 elif a is None:
 556                                         return -1
 557                                 else:
 558                                         return 1
 559                         return (a > b) - (a < b)
 560
 561                 class _MetadataTreeBuilder(ElementTree.TreeBuilder):
 562                         """
 563                         Implements doctype() as required to avoid deprecation warnings
 564                         since Python >=2.7
 565                         """
 566                         def doctype(self, name, pubid, system):
 567                                 pass
 568
 569                 for cp in self._portdb.cp_all():
 570                         metadata_path = os.path.join(repo_path, cp, 'metadata.xml')
 571                         try:
 572                                 metadata = ElementTree.parse(_unicode_encode(metadata_path,
 573                                         encoding=_encodings['fs'], errors='strict'),
 574                                         parser=ElementTree.XMLParser(
 575                                         target=_MetadataTreeBuilder()))
 576                         except IOError:
 577                                 pass
 578                         except (ExpatError, EnvironmentError) as e:
 579                                 writemsg_level(
 580                                         "ERROR: failed parsing %s/metadata.xml: %s\n" % (cp, e),
 581                                         level=logging.ERROR, noiselevel=-1)
 582                                 self.returncode |= 1
 583                         else:
 584                                 try:
 585                                         usedict = parse_metadata_use(metadata)
 586                                 except portage.exception.ParseError as e:
 587                                         writemsg_level(
 588                                                 "ERROR: failed parsing %s/metadata.xml: %s\n" % (cp, e),
 589                                                 level=logging.ERROR, noiselevel=-1)
 590                                         self.returncode |= 1
 591                                 else:
 592                                         for flag in sorted(usedict):
 593                                                 def atomcmp(atoma, atomb):
 594                                                         # None is better than an atom, that's why we reverse the args
 595                                                         if atoma is None or atomb is None:
 596                                                                 return cmp_func(atomb, atoma)
 597                                                         # Same for plain PNs (.operator is None then)
 598                                                         elif atoma.operator is None or atomb.operator is None:
 599                                                                 return cmp_func(atomb.operator, atoma.operator)
 600                                                         # Version matching
 601                                                         elif atoma.cpv != atomb.cpv:
 602                                                                 return vercmp(atoma.version, atomb.version)
 603                                                         # Versions match, let's fallback to operator matching
 604                                                         else:
 605                                                                 return cmp_func(ops.get(atoma.operator, -1),
 606                                                                         ops.get(atomb.operator, -1))
 607
 608                                                 def _Atom(key):
 609                                                         if key is not None:
 610                                                                 return Atom(key)
 611                                                         return None
 612
 613                                                 resdict = usedict[flag]
 614                                                 if len(resdict) == 1:
 615                                                         resdesc = next(iter(resdict.items()))[1]
 616                                                 else:
 617                                                         try:
 618                                                                 reskeys = dict((_Atom(k), k) for k in resdict)
 619                                                         except portage.exception.InvalidAtom as e:
 620                                                                 writemsg_level(
 621                                                                         "ERROR: failed parsing %s/metadata.xml: %s\n" % (cp, e),
 622                                                                         level=logging.ERROR, noiselevel=-1)
 623                                                                 self.returncode |= 1
 624                                                                 resdesc = next(iter(resdict.items()))[1]
 625                                                         else:
 626                                                                 resatoms = sorted(reskeys, key=cmp_sort_key(atomcmp))
 627                                                                 resdesc = resdict[reskeys[resatoms[-1]]]
 628
 629                                                 output.write('%s:%s - %s\n' % (cp, flag, resdesc))
 630
 631                 output.close()
 632
 633 if sys.hexversion < 0x3000000:
 634         _filename_base = unicode
 635 else:
 636         _filename_base = str
 637
 638 class _special_filename(_filename_base):
 639         """
 640         Helps to sort file names by file type and other criteria.
 641         """
 642         def __new__(cls, status_change, file_name):
 643                 return _filename_base.__new__(cls, status_change + file_name)
 644
 645         def __init__(self, status_change, file_name):
 646                 _filename_base.__init__(status_change + file_name)
 647                 self.status_change = status_change
 648                 self.file_name = file_name
 649                 self.file_type = guessManifestFileType(file_name)
 650
 651         def file_type_lt(self, a, b):
 652                 """
 653                 Defines an ordering between file types.
 654                 """
 655                 first = a.file_type
 656                 second = b.file_type
 657                 if first == second:
 658                         return False
 659
 660                 if first == "EBUILD":
 661                         return True
 662                 elif first == "MISC":
 663                         return second in ("EBUILD",)
 664                 elif first == "AUX":
 665                         return second in ("EBUILD", "MISC")
 666                 elif first == "DIST":
 667                         return second in ("EBUILD", "MISC", "AUX")
 668                 elif first is None:
 669                         return False
 670                 else:
 671                         raise ValueError("Unknown file type '%s'" % first)
 672
 673         def __lt__(self, other):
 674                 """
 675                 Compare different file names, first by file type and then
 676                 for ebuilds by version and lexicographically for others.
 677                 EBUILD < MISC < AUX < DIST < None
 678                 """
 679                 if self.__class__ != other.__class__:
 680                         raise NotImplementedError
 681
 682                 # Sort by file type as defined by file_type_lt().
 683                 if self.file_type_lt(self, other):
 684                         return True
 685                 elif self.file_type_lt(other, self):
 686                         return False
 687
 688                 # Files have the same type.
 689                 if self.file_type == "EBUILD":
 690                         # Sort by version. Lowest first.
 691                         ver = "-".join(pkgsplit(self.file_name[:-7])[1:3])
 692                         other_ver = "-".join(pkgsplit(other.file_name[:-7])[1:3])
 693                         return vercmp(ver, other_ver) < 0
 694                 else:
 695                         # Sort lexicographically.
 696                         return self.file_name < other.file_name
 697
 698 class GenChangeLogs(object):
 699         def __init__(self, portdb):
 700                 self.returncode = os.EX_OK
 701                 self._portdb = portdb
 702                 self._wrapper = textwrap.TextWrapper(
 703                                 width = 78,
 704                                 initial_indent = '  ',
 705                                 subsequent_indent = '  '
 706                         )
 707
 708         @staticmethod
 709         def grab(cmd):
 710                 p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
 711                 return _unicode_decode(p.communicate()[0],
 712                                 encoding=_encodings['stdio'], errors='strict')
 713
 714         def generate_changelog(self, cp):
 715                 try:
 716                         output = io.open('ChangeLog',
 717                                 mode='w', encoding=_encodings['repo.content'],
 718                                 errors='backslashreplace')
 719                 except IOError as e:
 720                         writemsg_level(
 721                                 "ERROR: failed to open ChangeLog for %s: %s\n" % (cp,e,),
 722                                 level=logging.ERROR, noiselevel=-1)
 723                         self.returncode |= 2
 724                         return
 725
 726                 output.write(textwrap.dedent('''\
 727                         # ChangeLog for %s
 728                         # Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2
 729                         # $Header: $
 730
 731                         ''' % (cp, time.strftime('%Y'))))
 732
 733                 # now grab all the commits
 734                 commits = self.grab(['git', 'rev-list', 'HEAD', '--', '.']).split()
 735
 736                 for c in commits:
 737                         # Explaining the arguments:
 738                         # --name-status to get a list of added/removed files
 739                         # --no-renames to avoid getting more complex records on the list
 740                         # --format to get the timestamp, author and commit description
 741                         # --root to make it work fine even with the initial commit
 742                         # --relative to get paths relative to ebuilddir
 743                         # -r (recursive) to get per-file changes
 744                         # then the commit-id and path.
 745
 746                         cinfo = self.grab(['git', 'diff-tree', '--name-status', '--no-renames',
 747                                         '--format=%ct %cN <%cE>%n%B', '--root', '--relative', '-r',
 748                                         c, '--', '.']).rstrip('\n').split('\n')
 749
 750                         # Expected output:
 751                         # timestamp Author Name <author@email>
 752                         # commit message l1
 753                         # ...
 754                         # commit message ln
 755                         #
 756                         # status1       filename1
 757                         # ...
 758                         # statusn       filenamen
 759
 760                         changed = []
 761                         for n, l in enumerate(reversed(cinfo)):
 762                                 if not l:
 763                                         body = cinfo[1:-n-1]
 764                                         break
 765                                 else:
 766                                         f = l.split()
 767                                         if f[1] == 'Manifest':
 768                                                 pass # XXX: remanifest commits?
 769                                         elif f[1] == 'ChangeLog':
 770                                                 pass
 771                                         elif f[0].startswith('A'):
 772                                                 changed.append(_special_filename("+", f[1]))
 773                                         elif f[0].startswith('D'):
 774                                                 changed.append(_special_filename("-", f[1]))
 775                                         elif f[0].startswith('M'):
 776                                                 changed.append(_special_filename("", f[1]))
 777                                         else:
 778                                                 writemsg_level(
 779                                                         "ERROR: unexpected git file status for %s: %s\n" % (cp,f,),
 780                                                         level=logging.ERROR, noiselevel=-1)
 781                                                 self.returncode |= 1
 782
 783                         if not changed:
 784                                 continue
 785
 786                         (ts, author) = cinfo[0].split(' ', 1)
 787                         date = time.strftime('%d %b %Y', time.gmtime(float(ts)))
 788
 789                         changed = [str(x) for x in sorted(changed)]
 790
 791                         wroteheader = False
 792                         # Reverse the sort order for headers.
 793                         for c in reversed(changed):
 794                                 if c.startswith('+') and c.endswith('.ebuild'):
 795                                         output.write('*%s (%s)\n' % (c[1:-7], date))
 796                                         wroteheader = True
 797                         if wroteheader:
 798                                 output.write('\n')
 799
 800                         # strip '<cp>: ', '[<cp>] ', and similar
 801                         body[0] = re.sub(r'^\W*' + re.escape(cp) + r'\W+', '', body[0])
 802                         # strip trailing newline
 803                         if not body[-1]:
 804                                 body = body[:-1]
 805                         # strip git-svn id
 806                         if body[-1].startswith('git-svn-id:') and not body[-2]:
 807                                 body = body[:-2]
 808                         # strip the repoman version/manifest note
 809                         if body[-1] == ' (Signed Manifest commit)' or body[-1] == ' (Unsigned Manifest commit)':
 810                                 body = body[:-1]
 811                         if body[-1].startswith('(Portage version:') and body[-1].endswith(')'):
 812                                 body = body[:-1]
 813                                 if not body[-1]:
 814                                         body = body[:-1]
 815
 816                         # don't break filenames on hyphens
 817                         self._wrapper.break_on_hyphens = False
 818                         output.write(self._wrapper.fill(
 819                                 '%s; %s %s:' % (date, author, ', '.join(changed))))
 820                         # but feel free to break commit messages there
 821                         self._wrapper.break_on_hyphens = True
 822                         output.write(
 823                                 '\n%s\n\n' % '\n'.join(self._wrapper.fill(x) for x in body))
 824
 825                 output.close()
 826
 827         def run(self):
 828                 repo_path = self._portdb.porttrees[0]
 829                 os.chdir(repo_path)
 830
 831                 if 'git' not in FindVCS():
 832                         writemsg_level(
 833                                 "ERROR: --update-changelogs supported only in git repos\n",
 834                                 level=logging.ERROR, noiselevel=-1)
 835                         self.returncode = 127
 836                         return
 837
 838                 for cp in self._portdb.cp_all():
 839                         os.chdir(os.path.join(repo_path, cp))
 840                         # Determine whether ChangeLog is up-to-date by comparing
 841                         # the newest commit timestamp with the ChangeLog timestamp.
 842                         lmod = self.grab(['git', 'log', '--format=%ct', '-1', '.'])
 843                         if not lmod:
 844                                 # This cp has not been added to the repo.
 845                                 continue
 846
 847                         try:
 848                                 cmod = os.stat('ChangeLog').st_mtime
 849                         except OSError:
 850                                 cmod = 0
 851
 852                         if float(cmod) < float(lmod):
 853                                 self.generate_changelog(cp)
 854
 855 def egencache_main(args):
 856
 857         # The calling environment is ignored, so the program is
 858         # completely controlled by commandline arguments.
 859         env = {}
 860
 861         if not sys.stdout.isatty():
 862                 portage.output.nocolor()
 863                 env['NOCOLOR'] = 'true'
 864
 865         parser, options, atoms = parse_args(args)
 866
 867         config_root = options.config_root
 868
 869         if options.repositories_configuration is not None:
 870                 env['PORTAGE_REPOSITORIES'] = options.repositories_configuration
 871         elif options.portdir_overlay:
 872                 env['PORTDIR_OVERLAY'] = options.portdir_overlay
 873
 874         if options.cache_dir is not None:
 875                 env['PORTAGE_DEPCACHEDIR'] = options.cache_dir
 876
 877         if options.portdir is not None:
 878                 env['PORTDIR'] = options.portdir
 879
 880         settings = portage.config(config_root=config_root,
 881                 local_config=False, env=env)
 882
 883         default_opts = None
 884         if not options.ignore_default_opts:
 885                 default_opts = portage.util.shlex_split(
 886                         settings.get('EGENCACHE_DEFAULT_OPTS', ''))
 887
 888         if default_opts:
 889                 parser, options, args = parse_args(default_opts + args)
 890
 891                 if options.cache_dir is not None:
 892                         env['PORTAGE_DEPCACHEDIR'] = options.cache_dir
 893
 894                 settings = portage.config(config_root=config_root,
 895                         local_config=False, env=env)
 896
 897         if not (options.update or options.update_use_local_desc or
 898                         options.update_changelogs or options.update_manifests):
 899                 parser.error('No action specified')
 900                 return 1
 901
 902         repo_path = settings.repositories.treemap.get(options.repo)
 903         if repo_path is None:
 904                 parser.error("Unable to locate repository named '%s'" % (options.repo,))
 905                 return 1
 906
 907         repo_config = settings.repositories.get_repo_for_location(repo_path)
 908
 909         if options.strict_manifests is not None:
 910                 if options.strict_manifests == "y":
 911                         settings.features.add("strict")
 912                 else:
 913                         settings.features.discard("strict")
 914
 915         if options.update and 'metadata-transfer' not in settings.features:
 916                 # Forcibly enable metadata-transfer if portdbapi has a pregenerated
 917                 # cache that does not support eclass validation.
 918                 cache = repo_config.get_pregenerated_cache(
 919                         portage.dbapi.dbapi._known_keys, readonly=True)
 920                 if cache is not None and not cache.complete_eclass_entries:
 921                         settings.features.add('metadata-transfer')
 922                 cache = None
 923
 924         settings.lock()
 925
 926         portdb = portage.portdbapi(mysettings=settings)
 927
 928         # Limit ebuilds to the specified repo.
 929         portdb.porttrees = [repo_path]
 930
 931         if options.update:
 932                 if options.cache_dir is not None:
 933                         # already validated earlier
 934                         pass
 935                 else:
 936                         # We check write access after the portdbapi constructor
 937                         # has had an opportunity to create it. This ensures that
 938                         # we don't use the cache in the "volatile" mode which is
 939                         # undesirable for egencache.
 940                         if not os.access(settings["PORTAGE_DEPCACHEDIR"], os.W_OK):
 941                                 writemsg_level("ecachegen: error: " + \
 942                                         "write access denied: %s\n" % (settings["PORTAGE_DEPCACHEDIR"],),
 943                                         level=logging.ERROR, noiselevel=-1)
 944                                 return 1
 945
 946         if options.sign_manifests is not None:
 947                 repo_config.sign_manifest = options.sign_manifests == 'y'
 948
 949         if options.thin_manifests is not None:
 950                 repo_config.thin_manifest = options.thin_manifests == 'y'
 951
 952         gpg_cmd = None
 953         gpg_vars = None
 954         force_sign_key = None
 955
 956         if options.update_manifests:
 957                 if repo_config.sign_manifest:
 958
 959                         sign_problem = False
 960                         gpg_dir = None
 961                         gpg_cmd = settings.get("PORTAGE_GPG_SIGNING_COMMAND")
 962                         if gpg_cmd is None:
 963                                 writemsg_level("egencache: error: "
 964                                         "PORTAGE_GPG_SIGNING_COMMAND is unset! "
 965                                         "Is make.globals missing?\n",
 966                                         level=logging.ERROR, noiselevel=-1)
 967                                 sign_problem = True
 968                         elif "${PORTAGE_GPG_KEY}" in gpg_cmd and \
 969                                 options.gpg_key is None and \
 970                                 "PORTAGE_GPG_KEY" not in settings:
 971                                 writemsg_level("egencache: error: "
 972                                         "PORTAGE_GPG_KEY is unset!\n",
 973                                         level=logging.ERROR, noiselevel=-1)
 974                                 sign_problem = True
 975                         elif "${PORTAGE_GPG_DIR}" in gpg_cmd:
 976                                 if options.gpg_dir is not None:
 977                                         gpg_dir = options.gpg_dir
 978                                 elif "PORTAGE_GPG_DIR" not in settings:
 979                                         gpg_dir = os.path.expanduser("~/.gnupg")
 980                                 else:
 981                                         gpg_dir = os.path.expanduser(settings["PORTAGE_GPG_DIR"])
 982                                 if not os.access(gpg_dir, os.X_OK):
 983                                         writemsg_level(("egencache: error: "
 984                                                 "Unable to access directory: "
 985                                                 "PORTAGE_GPG_DIR='%s'\n") % gpg_dir,
 986                                                 level=logging.ERROR, noiselevel=-1)
 987                                         sign_problem = True
 988
 989                         if sign_problem:
 990                                 writemsg_level("egencache: You may disable manifest "
 991                                         "signatures with --sign-manifests=n or by setting "
 992                                         "\"sign-manifests = false\" in metadata/layout.conf\n",
 993                                         level=logging.ERROR, noiselevel=-1)
 994                                 return 1
 995
 996                         gpg_vars = {}
 997                         if gpg_dir is not None:
 998                                 gpg_vars["PORTAGE_GPG_DIR"] = gpg_dir
 999                         gpg_var_names = []
1000                         if options.gpg_key is None:
1001                                 gpg_var_names.append("PORTAGE_GPG_KEY")
1002                         else:
1003                                 gpg_vars["PORTAGE_GPG_KEY"] = options.gpg_key
1004
1005                         for k in gpg_var_names:
1006                                 v = settings.get(k)
1007                                 if v is not None:
1008                                         gpg_vars[k] = v
1009
1010                         force_sign_key = gpg_vars.get("PORTAGE_GPG_KEY")
1011
1012         ret = [os.EX_OK]
1013
1014         if options.update:
1015                 cp_iter = None
1016                 if atoms:
1017                         cp_iter = iter(atoms)
1018
1019                 gen_cache = GenCache(portdb, cp_iter=cp_iter,
1020                         max_jobs=options.jobs,
1021                         max_load=options.load_average,
1022                         rsync=options.rsync)
1023                 gen_cache.run()
1024                 if options.tolerant:
1025                         ret.append(os.EX_OK)
1026                 else:
1027                         ret.append(gen_cache.returncode)
1028
1029         if options.update_manifests:
1030
1031                 cp_iter = None
1032                 if atoms:
1033                         cp_iter = iter(atoms)
1034
1035                 event_loop = global_event_loop()
1036                 scheduler = ManifestScheduler(portdb, cp_iter=cp_iter,
1037                         gpg_cmd=gpg_cmd, gpg_vars=gpg_vars,
1038                         force_sign_key=force_sign_key,
1039                         max_jobs=options.jobs,
1040                         max_load=options.load_average,
1041                         event_loop=event_loop)
1042
1043                 signum = run_main_scheduler(scheduler)
1044                 if signum is not None:
1045                         sys.exit(128 + signum)
1046
1047                 if options.tolerant:
1048                         ret.append(os.EX_OK)
1049                 else:
1050                         ret.append(scheduler.returncode)
1051
1052         if options.update_use_local_desc:
1053                 gen_desc = GenUseLocalDesc(portdb,
1054                         output=options.uld_output,
1055                         preserve_comments=options.preserve_comments)
1056                 gen_desc.run()
1057                 ret.append(gen_desc.returncode)
1058
1059         if options.update_changelogs:
1060                 gen_clogs = GenChangeLogs(portdb)
1061                 gen_clogs.run()
1062                 ret.append(gen_clogs.returncode)
1063
1064         return max(ret)
1065
1066 if __name__ == "__main__":
1067         portage._disable_legacy_globals()
1068         portage.util.noiselimit = -1
1069         sys.exit(egencache_main(sys.argv[1:]))