2 # Copyright 2009 Gentoo Foundation
3 # Distributed under the terms of the GNU General Public License v2
6 from __future__ import print_function
9 # This block ensures that ^C interrupts are handled quietly.
13 def exithandler(signum,frame):
14 signal.signal(signal.SIGINT, signal.SIG_IGN)
15 signal.signal(signal.SIGTERM, signal.SIG_IGN)
18 signal.signal(signal.SIGINT, exithandler)
19 signal.signal(signal.SIGTERM, exithandler)
21 except KeyboardInterrupt:
30 from os import path as osp
31 sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
34 from portage import os
35 from _emerge.MetadataRegen import MetadataRegen
36 from portage.cache.cache_errors import CacheError, StatCollision
37 from portage.util import writemsg_level
38 from portage import cpv_getkey
39 from portage.dep import isjustname
41 if sys.hexversion >= 0x3000000:
45 usage = "egencache [options] --update [atom] ..."
46 parser = optparse.OptionParser(usage=usage)
47 parser.add_option("--update",
49 help="update metadata/cache/ (generate as necessary)")
50 parser.add_option("--repo",
52 help="name of repo to operate on (default repo is located at $PORTDIR)")
53 parser.add_option("--cache-dir",
54 help="location of the metadata cache",
56 parser.add_option("--config-root",
57 help="location of portage config files",
59 parser.add_option("--jobs",
61 help="max ebuild processes to spawn")
62 parser.add_option("--load-average",
64 help="max load allowed when spawning multiple jobs",
66 parser.add_option("--rsync",
68 help="enable rsync stat collision workaround " + \
69 "for bug 139134 (use with --update)")
70 parser.add_option("--ignore-default-opts",
72 help="do not use the EGENCACHE_DEFAULT_OPTS environment variable")
73 options, args = parser.parse_args(args)
78 jobs = int(options.jobs)
83 parser.error("Invalid: --jobs='%s'" % \
91 if options.load_average:
93 load_average = float(options.load_average)
97 if load_average <= 0.0:
98 parser.error("Invalid: --load-average='%s'" % \
99 (options.load_average,))
101 options.load_average = load_average
104 options.load_average = None
106 if options.config_root is not None and \
107 not os.path.isdir(options.config_root):
108 parser.error("Not a directory: --config-root='%s'" % \
109 (options.config_root,))
111 if options.cache_dir is not None and not os.path.isdir(options.cache_dir):
112 parser.error("Not a directory: --cache-dir='%s'" % \
113 (options.cache_dir,))
117 atom = portage.dep.Atom(atom)
118 except portage.exception.InvalidAtom:
119 parser.error('Invalid atom: %s' % (atom,))
121 if not isjustname(atom):
122 parser.error('Atom is too specific: %s' % (atom,))
124 return parser, options, args
126 class GenCache(object):
127 def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None,
129 self._portdb = portdb
130 # We can globally cleanse stale cache only if we
131 # iterate over every single cp.
132 self._global_cleanse = cp_iter is None
133 if cp_iter is not None:
134 self._cp_set = set(cp_iter)
135 cp_iter = iter(self._cp_set)
136 self._cp_missing = self._cp_set.copy()
139 self._cp_missing = set()
140 self._regen = MetadataRegen(portdb, cp_iter=cp_iter,
141 consumer=self._metadata_callback,
142 max_jobs=max_jobs, max_load=max_load)
143 self.returncode = os.EX_OK
144 metadbmodule = portdb.mysettings.load_best_module("portdbapi.metadbmodule")
145 self._trg_cache = metadbmodule(portdb.porttrees[0],
146 "metadata/cache", portage.auxdbkeys[:])
148 self._trg_cache.raise_stat_collision = True
150 self._trg_cache.ec = \
151 portdb._repo_info[portdb.porttrees[0]].eclass_db
152 except AttributeError:
154 self._existing_nodes = set()
156 def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata):
157 self._existing_nodes.add(cpv)
158 self._cp_missing.discard(cpv_getkey(cpv))
159 if metadata is not None:
160 if metadata.get('EAPI') == '0':
164 self._trg_cache[cpv] = metadata
165 except StatCollision as sc:
166 # If the content of a cache entry changes and neither the
167 # file mtime nor size changes, it will prevent rsync from
168 # detecting changes. Cache backends may raise this
169 # exception from _setitem() if they detect this type of stat
170 # collision. These exceptions are handled by bumping the
171 # mtime on the ebuild (and the corresponding cache entry).
174 for ec, (loc, ec_mtime) in metadata['_eclasses_'].items():
175 if max_mtime < ec_mtime:
177 if max_mtime == sc.mtime:
179 max_mtime = long(max_mtime)
181 os.utime(ebuild_path, (max_mtime, max_mtime))
185 "%s writing target: %s\n" % (cpv, e),
186 level=logging.ERROR, noiselevel=-1)
188 metadata['_mtime_'] = max_mtime
189 self._trg_cache[cpv] = metadata
190 self._portdb.auxdb[repo_path][cpv] = metadata
192 except CacheError as ce:
195 "%s writing target: %s\n" % (cpv, ce),
196 level=logging.ERROR, noiselevel=-1)
200 self.returncode |= self._regen.returncode
201 cp_missing = self._cp_missing
203 trg_cache = self._trg_cache
205 if self._global_cleanse:
207 for cpv in trg_cache.keys():
212 "Unable to parse cp for '%s'\n" % (cpv,),
213 level=logging.ERROR, noiselevel=-1)
216 except CacheError as ce:
219 "Error listing cache entries for " + \
220 "'%s/metadata/cache': %s, continuing...\n" % \
221 (self._portdb.porttree_root, ce),
222 level=logging.ERROR, noiselevel=-1)
225 cp_set = self._cp_set
227 for cpv in trg_cache.keys():
232 "Unable to parse cp for '%s'\n" % (cpv,),
233 level=logging.ERROR, noiselevel=-1)
235 cp_missing.discard(cp)
238 except CacheError as ce:
241 "Error listing cache entries for " + \
242 "'%s/metadata/cache': %s, continuing...\n" % \
243 (self._portdb.porttree_root, ce),
244 level=logging.ERROR, noiselevel=-1)
248 for cp in sorted(cp_missing):
250 "No ebuilds or cache entries found for '%s'\n" % (cp,),
251 level=logging.ERROR, noiselevel=-1)
254 dead_nodes.difference_update(self._existing_nodes)
260 except CacheError as ce:
263 "%s deleting stale cache: %s\n" % (k, ce),
264 level=logging.ERROR, noiselevel=-1)
266 if not trg_cache.autocommits:
269 except CacheError as ce:
272 "committing target: %s\n" % (ce,),
273 level=logging.ERROR, noiselevel=-1)
275 def egencache_main(args):
276 parser, options, atoms = parse_args(args)
278 config_root = options.config_root
279 if config_root is None:
282 # The calling environment is ignored, so the program is
283 # completely controlled by commandline arguments.
286 if options.repo is None:
287 env['PORTDIR_OVERLAY'] = ''
289 if options.cache_dir is not None:
290 env['PORTAGE_DEPCACHEDIR'] = options.cache_dir
292 settings = portage.config(config_root=config_root,
293 target_root='/', local_config=False, env=env)
296 if not options.ignore_default_opts:
297 default_opts = settings.get('EGENCACHE_DEFAULT_OPTS', '').split()
300 parser, options, args = parse_args(default_opts + args)
302 if options.config_root is not None:
303 config_root = options.config_root
305 if options.cache_dir is not None:
306 env['PORTAGE_DEPCACHEDIR'] = options.cache_dir
308 settings = portage.config(config_root=config_root,
309 target_root='/', local_config=False, env=env)
311 if not options.update:
312 parser.error('No action specified (--update ' + \
313 'is the only available action)')
316 if 'metadata-transfer' not in settings.features:
317 writemsg_level("ecachegen: warning: " + \
318 "automatically enabling FEATURES=metadata-transfer\n",
319 level=logging.WARNING, noiselevel=-1)
320 settings.features.add('metadata-transfer')
321 settings['FEATURES'] = ' '.join(sorted(settings.features))
322 settings.backup_changes('FEATURES')
326 portdb = portage.portdbapi(settings["PORTDIR"], mysettings=settings)
327 if options.repo is not None:
328 repo_path = portdb.getRepositoryPath(options.repo)
329 if repo_path is None:
330 parser.error("Unable to locate repository named '%s'" % \
334 # Limit ebuilds to the specified repo.
335 portdb.porttrees = [repo_path]
339 cp_iter = iter(atoms)
341 gen_cache = GenCache(portdb, cp_iter=cp_iter,
342 max_jobs=options.jobs,
343 max_load=options.load_average,
346 return gen_cache.returncode
348 if __name__ == "__main__":
349 portage._disable_legacy_globals()
350 portage.util.noiselimit = -1
351 sys.exit(egencache_main(sys.argv[1:]))