pym/_emerge/MetadataRegen.py

   1 # Copyright 1999-2009 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3 # $Id$
   4
   5 import portage
   6 from portage import os
   7 from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
   8 from _emerge.PollScheduler import PollScheduler
   9
  10 class MetadataRegen(PollScheduler):
  11
  12         def __init__(self, portdb, cp_iter=None, consumer=None,
  13                 max_jobs=None, max_load=None):
  14                 PollScheduler.__init__(self)
  15                 self._portdb = portdb
  16                 self._global_cleanse = False
  17                 if cp_iter is None:
  18                         cp_iter = self._iter_every_cp()
  19                         # We can globally cleanse stale cache only if we
  20                         # iterate over every single cp.
  21                         self._global_cleanse = True
  22                 self._cp_iter = cp_iter
  23                 self._consumer = consumer
  24
  25                 if max_jobs is None:
  26                         max_jobs = 1
  27
  28                 self._max_jobs = max_jobs
  29                 self._max_load = max_load
  30                 self._sched_iface = self._sched_iface_class(
  31                         register=self._register,
  32                         schedule=self._schedule_wait,
  33                         unregister=self._unregister)
  34
  35                 self._valid_pkgs = set()
  36                 self._cp_set = set()
  37                 self._process_iter = self._iter_metadata_processes()
  38                 self.returncode = os.EX_OK
  39                 self._error_count = 0
  40
  41         def _iter_every_cp(self):
  42                 every_cp = self._portdb.cp_all()
  43                 every_cp.sort(reverse=True)
  44                 try:
  45                         while True:
  46                                 yield every_cp.pop()
  47                 except IndexError:
  48                         pass
  49
  50         def _iter_metadata_processes(self):
  51                 portdb = self._portdb
  52                 valid_pkgs = self._valid_pkgs
  53                 cp_set = self._cp_set
  54                 consumer = self._consumer
  55
  56                 for cp in self._cp_iter:
  57                         cp_set.add(cp)
  58                         portage.writemsg_stdout("Processing %s\n" % cp)
  59                         cpv_list = portdb.cp_list(cp)
  60                         for cpv in cpv_list:
  61                                 valid_pkgs.add(cpv)
  62                                 ebuild_path, repo_path = portdb.findname2(cpv)
  63                                 metadata, st, emtime = portdb._pull_valid_cache(
  64                                         cpv, ebuild_path, repo_path)
  65                                 if metadata is not None:
  66                                         if consumer is not None:
  67                                                 consumer(cpv, ebuild_path,
  68                                                         repo_path, metadata)
  69                                         continue
  70
  71                                 yield EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path,
  72                                         ebuild_mtime=emtime,
  73                                         metadata_callback=portdb._metadata_callback,
  74                                         portdb=portdb, repo_path=repo_path,
  75                                         settings=portdb.doebuild_settings)
  76
  77         def run(self):
  78
  79                 portdb = self._portdb
  80                 from portage.cache.cache_errors import CacheError
  81                 dead_nodes = {}
  82
  83                 while self._schedule():
  84                         self._poll_loop()
  85
  86                 while self._jobs:
  87                         self._poll_loop()
  88
  89                 if self._global_cleanse:
  90                         for mytree in portdb.porttrees:
  91                                 try:
  92                                         dead_nodes[mytree] = set(portdb.auxdb[mytree])
  93                                 except CacheError as e:
  94                                         portage.writemsg("Error listing cache entries for " + \
  95                                                 "'%s': %s, continuing...\n" % (mytree, e),
  96                                                 noiselevel=-1)
  97                                         del e
  98                                         dead_nodes = None
  99                                         break
 100                 else:
 101                         cp_set = self._cp_set
 102                         cpv_getkey = portage.cpv_getkey
 103                         for mytree in portdb.porttrees:
 104                                 try:
 105                                         dead_nodes[mytree] = set(cpv for cpv in \
 106                                                 portdb.auxdb[mytree] \
 107                                                 if cpv_getkey(cpv) in cp_set)
 108                                 except CacheError as e:
 109                                         portage.writemsg("Error listing cache entries for " + \
 110                                                 "'%s': %s, continuing...\n" % (mytree, e),
 111                                                 noiselevel=-1)
 112                                         del e
 113                                         dead_nodes = None
 114                                         break
 115
 116                 if dead_nodes:
 117                         for y in self._valid_pkgs:
 118                                 for mytree in portdb.porttrees:
 119                                         if portdb.findname2(y, mytree=mytree)[0]:
 120                                                 dead_nodes[mytree].discard(y)
 121
 122                         for mytree, nodes in dead_nodes.items():
 123                                 auxdb = portdb.auxdb[mytree]
 124                                 for y in nodes:
 125                                         try:
 126                                                 del auxdb[y]
 127                                         except (KeyError, CacheError):
 128                                                 pass
 129
 130         def _schedule_tasks(self):
 131                 """
 132                 @rtype: bool
 133                 @returns: True if there may be remaining tasks to schedule,
 134                         False otherwise.
 135                 """
 136                 while self._can_add_job():
 137                         try:
 138                                 metadata_process = next(self._process_iter)
 139                         except StopIteration:
 140                                 return False
 141
 142                         self._jobs += 1
 143                         metadata_process.scheduler = self._sched_iface
 144                         metadata_process.addExitListener(self._metadata_exit)
 145                         metadata_process.start()
 146                 return True
 147
 148         def _metadata_exit(self, metadata_process):
 149                 self._jobs -= 1
 150                 if metadata_process.returncode != os.EX_OK:
 151                         self.returncode = 1
 152                         self._error_count += 1
 153                         self._valid_pkgs.discard(metadata_process.cpv)
 154                         portage.writemsg("Error processing %s, continuing...\n" % \
 155                                 (metadata_process.cpv,), noiselevel=-1)
 156
 157                 if self._consumer is not None:
 158                         # On failure, still notify the consumer (in this case the metadata
 159                         # argument is None).
 160                         self._consumer(metadata_process.cpv,
 161                                 metadata_process.ebuild_path,
 162                                 metadata_process.repo_path,
 163                                 metadata_process.metadata)
 164
 165                 self._schedule()
 166