1 # Copyright 2010-2011 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
10 from configparser import ParsingError
11 if sys.hexversion >= 0x3020000:
12 from configparser import ConfigParser as SafeConfigParser
14 from configparser import SafeConfigParser
16 from ConfigParser import SafeConfigParser, ParsingError
17 from portage import os
18 from portage.const import USER_CONFIG_PATH, REPO_NAME_LOC
19 from portage.env.loaders import KeyValuePairFileLoader
20 from portage.util import normalize_path, writemsg, writemsg_level, shlex_split
21 from portage.localization import _
22 from portage import _unicode_encode
23 from portage import _encodings
24 from portage import manifest
26 _repo_name_sub_re = re.compile(r'[^\w-]')
28 def _gen_valid_repo(name):
30 Substitute hyphen in place of characters that don't conform to PMS 3.1.5,
31 and strip hyphen from left side if necessary. This returns None if the
32 given name contains no valid characters.
34 name = _repo_name_sub_re.sub(' ', name.strip())
35 name = '-'.join(name.split())
36 name = name.lstrip('-')
41 class RepoConfig(object):
42 """Stores config of one repository"""
44 __slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo',
45 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'sign_manifest', 'thin_manifest',
46 'allow_missing_manifest', 'create_manifest', 'disable_manifest', 'cache_is_authoritative',
47 'trust_authoritative_cache', 'manifest_hash_flags']
49 def __init__(self, name, repo_opts):
50 """Build a RepoConfig with options in repo_opts
51 Try to read repo_name in repository location, but if
52 it is not found use variable name as repository name"""
53 aliases = repo_opts.get('aliases')
54 if aliases is not None:
55 aliases = tuple(aliases.split())
56 self.aliases = aliases
58 eclass_overrides = repo_opts.get('eclass-overrides')
59 if eclass_overrides is not None:
60 eclass_overrides = tuple(eclass_overrides.split())
61 self.eclass_overrides = eclass_overrides
62 #Locations are computed later.
63 self.eclass_locations = None
65 #Masters are only read from layout.conf.
68 #The main-repo key makes only sense for the 'DEFAULT' section.
69 self.main_repo = repo_opts.get('main-repo')
71 priority = repo_opts.get('priority')
72 if priority is not None:
74 priority = int(priority)
77 self.priority = priority
79 sync = repo_opts.get('sync')
84 format = repo_opts.get('format')
85 if format is not None:
86 format = format.strip()
89 location = repo_opts.get('location')
90 self.user_location = location
91 if location is not None and location.strip():
92 if os.path.isdir(location):
93 location = os.path.realpath(location)
96 self.location = location
99 if self.location is not None:
100 name, missing = self._read_repo_name(self.location)
101 # We must ensure that the name conforms to PMS 3.1.5
102 # in order to avoid InvalidAtom exceptions when we
103 # use it to generate atoms.
104 name = _gen_valid_repo(name)
106 # name only contains invalid characters
107 name = "x-" + os.path.basename(self.location)
108 name = _gen_valid_repo(name)
109 # If basename only contains whitespace then the
110 # end result is name = 'x-'.
112 elif name == "DEFAULT":
115 self.missing_repo_name = missing
116 self.sign_manifest = True
117 self.thin_manifest = False
118 self.allow_missing_manifest = False
119 self.create_manifest = True
120 self.disable_manifest = False
121 self.manifest_hash_flags = {}
123 self.cache_is_authoritative = False
125 trust_authoritative_cache = repo_opts.get('trust-authoritative-cache')
126 if trust_authoritative_cache is not None:
127 trust_authoritative_cache = trust_authoritative_cache.lower() == 'true'
128 self.trust_authoritative_cache = trust_authoritative_cache
130 def load_manifest(self, *args, **kwds):
131 kwds['thin'] = self.thin_manifest
132 kwds['allow_missing'] = self.allow_missing_manifest
133 kwds['allow_create'] = self.create_manifest
134 kwds['hash_flags'] = self.manifest_hash_flags
135 if self.disable_manifest:
136 kwds['from_scratch'] = True
137 return manifest.Manifest(*args, **kwds)
139 def update(self, new_repo):
140 """Update repository with options in another RepoConfig"""
141 if new_repo.aliases is not None:
142 self.aliases = new_repo.aliases
143 if new_repo.eclass_overrides is not None:
144 self.eclass_overrides = new_repo.eclass_overrides
145 if new_repo.masters is not None:
146 self.masters = new_repo.masters
147 if new_repo.trust_authoritative_cache is not None:
148 self.trust_authoritative_cache = new_repo.trust_authoritative_cache
149 if new_repo.name is not None:
150 self.name = new_repo.name
151 self.missing_repo_name = new_repo.missing_repo_name
152 if new_repo.user_location is not None:
153 self.user_location = new_repo.user_location
154 if new_repo.location is not None:
155 self.location = new_repo.location
156 if new_repo.priority is not None:
157 self.priority = new_repo.priority
158 if new_repo.sync is not None:
159 self.sync = new_repo.sync
161 def _read_repo_name(self, repo_path):
163 Read repo_name from repo_path.
164 Returns repo_name, missing.
166 repo_name_path = os.path.join(repo_path, REPO_NAME_LOC)
170 _unicode_encode(repo_name_path,
171 encoding=_encodings['fs'], errors='strict'),
172 mode='r', encoding=_encodings['repo.content'],
174 return f.readline().strip(), False
175 except EnvironmentError:
176 return "x-" + os.path.basename(repo_path), True
181 def info_string(self):
183 Returns a formatted string containing informations about the repository.
184 Used by emerge --info.
188 repo_msg.append(self.name)
190 repo_msg.append(indent + "format: " + self.format)
191 if self.user_location:
192 repo_msg.append(indent + "location: " + self.user_location)
194 repo_msg.append(indent + "sync: " + self.sync)
196 repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters))
197 if self.priority is not None:
198 repo_msg.append(indent + "priority: " + str(self.priority))
200 repo_msg.append(indent + "aliases: " + " ".join(self.aliases))
201 if self.eclass_overrides:
202 repo_msg.append(indent + "eclass_overrides: " + \
203 " ".join(self.eclass_overrides))
205 return "\n".join(repo_msg)
207 class RepoConfigLoader(object):
208 """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf"""
211 def _add_overlays(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map):
212 """Add overlays in PORTDIR_OVERLAY as repositories"""
215 portdir = normalize_path(portdir)
216 overlays.append(portdir)
218 port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
219 except ValueError as e:
220 #File "/usr/lib/python3.2/shlex.py", line 168, in read_token
221 # raise ValueError("No closing quotation")
222 writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
223 " %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
225 overlays.extend(port_ov)
226 default_repo_opts = {}
227 if prepos['DEFAULT'].aliases is not None:
228 default_repo_opts['aliases'] = \
229 ' '.join(prepos['DEFAULT'].aliases)
230 if prepos['DEFAULT'].eclass_overrides is not None:
231 default_repo_opts['eclass-overrides'] = \
232 ' '.join(prepos['DEFAULT'].eclass_overrides)
233 if prepos['DEFAULT'].masters is not None:
234 default_repo_opts['masters'] = \
235 ' '.join(prepos['DEFAULT'].masters)
236 if prepos['DEFAULT'].trust_authoritative_cache is not None:
237 if prepos['DEFAULT'].trust_authoritative_cache:
238 default_repo_opts['trust-authoritative-cache'] = 'true'
240 default_repo_opts['trust-authoritative-cache'] = 'false'
243 #overlay priority is negative because we want them to be looked before any other repo
246 if os.path.isdir(ov):
247 repo_opts = default_repo_opts.copy()
248 repo_opts['location'] = ov
249 repo = RepoConfig(None, repo_opts)
250 repo_conf_opts = prepos.get(repo.name)
251 if repo_conf_opts is not None:
252 if repo_conf_opts.aliases is not None:
253 repo_opts['aliases'] = \
254 ' '.join(repo_conf_opts.aliases)
255 if repo_conf_opts.eclass_overrides is not None:
256 repo_opts['eclass-overrides'] = \
257 ' '.join(repo_conf_opts.eclass_overrides)
258 if repo_conf_opts.masters is not None:
259 repo_opts['masters'] = \
260 ' '.join(repo_conf_opts.masters)
261 if repo_conf_opts.trust_authoritative_cache is not None:
262 if repo_conf_opts.trust_authoritative_cache:
263 repo_opts['trust-authoritative-cache'] = 'true'
265 repo_opts['trust-authoritative-cache'] = 'false'
267 repo = RepoConfig(repo.name, repo_opts)
268 if repo.name in prepos:
269 old_location = prepos[repo.name].location
270 if old_location is not None and old_location != repo.location:
271 ignored_map.setdefault(repo.name, []).append(old_location)
272 ignored_location_map[old_location] = repo.name
273 if old_location == portdir:
274 portdir = repo.user_location
275 prepos[repo.name].update(repo)
276 repo = prepos[repo.name]
278 prepos[repo.name] = repo
280 if ov == portdir and portdir not in port_ov:
281 repo.priority = -1000
283 repo.priority = base_priority
287 writemsg(_("!!! Invalid PORTDIR_OVERLAY"
288 " (not a dir): '%s'\n") % ov, noiselevel=-1)
293 def _parse(paths, prepos, ignored_map, ignored_location_map):
294 """Parse files in paths to load config"""
295 parser = SafeConfigParser()
298 except ParsingError as e:
299 writemsg(_("!!! Error while reading repo config file: %s\n") % e, noiselevel=-1)
300 prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults())
301 for sname in parser.sections():
303 for oname in parser.options(sname):
304 optdict[oname] = parser.get(sname, oname)
306 repo = RepoConfig(sname, optdict)
307 if repo.location and not os.path.exists(repo.location):
308 writemsg(_("!!! Invalid repos.conf entry '%s'"
309 " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1)
312 if repo.name in prepos:
313 old_location = prepos[repo.name].location
314 if old_location is not None and repo.location is not None and old_location != repo.location:
315 ignored_map.setdefault(repo.name, []).append(old_location)
316 ignored_location_map[old_location] = repo.name
317 prepos[repo.name].update(repo)
319 prepos[repo.name] = repo
321 def __init__(self, paths, settings):
322 """Load config from files in paths"""
328 ignored_location_map = {}
330 portdir = settings.get('PORTDIR', '')
331 portdir_overlay = settings.get('PORTDIR_OVERLAY', '')
333 self._parse(paths, prepos, ignored_map, ignored_location_map)
335 # If PORTDIR_OVERLAY contains a repo with the same repo_name as
336 # PORTDIR, then PORTDIR is overridden.
337 portdir = self._add_overlays(portdir, portdir_overlay, prepos,
338 ignored_map, ignored_location_map)
339 if portdir and portdir.strip():
340 portdir = os.path.realpath(portdir)
342 ignored_repos = tuple((repo_name, tuple(paths)) \
343 for repo_name, paths in ignored_map.items())
345 self.missing_repo_names = frozenset(repo.location
346 for repo in prepos.values()
347 if repo.location is not None and repo.missing_repo_name)
349 #Parse layout.conf and read masters key.
350 for repo in prepos.values():
351 if not repo.location:
353 layout_filename = os.path.join(repo.location, "metadata", "layout.conf")
354 layout_file = KeyValuePairFileLoader(layout_filename, None, None)
355 layout_data, layout_errors = layout_file.load()
357 masters = layout_data.get('masters')
358 if masters and masters.strip():
359 masters = masters.split()
362 repo.masters = masters
364 aliases = layout_data.get('aliases')
365 if aliases and aliases.strip():
366 aliases = aliases.split()
371 aliases.extend(repo.aliases)
372 repo.aliases = tuple(sorted(set(aliases)))
374 if layout_data.get('sign-manifests', '').lower() == 'false':
375 repo.sign_manifest = False
377 if layout_data.get('thin-manifests', '').lower() == 'true':
378 repo.thin_manifest = True
380 manifest_policy = layout_data.get('use-manifests', 'strict').lower()
381 repo.allow_missing_manifest = manifest_policy != 'strict'
382 repo.create_manifest = manifest_policy != 'false'
383 repo.disable_manifest = manifest_policy == 'false'
385 if 'manifest-rmd160' in layout_data:
386 repo.manifest_hash_flags["RMD160"] = \
387 layout_data['manifest-rmd160'].lower() == 'true'
389 if 'manifest-sha1' in layout_data:
390 repo.manifest_hash_flags["SHA1"] = \
391 layout_data['manifest-sha1'].lower() == 'true'
393 if 'manifest-sha256' in layout_data:
394 repo.manifest_hash_flags["SHA256"] = \
395 layout_data['manifest-sha256'].lower() == 'true'
397 if 'manifest-whirlpool' in layout_data:
398 repo.manifest_hash_flags["WHIRLPOOL"] = \
399 layout_data['manifest-whirlpool'].lower() == 'true'
401 repo.cache_is_authoritative = layout_data.get('authoritative-cache', 'false').lower() == 'true'
402 if not repo.trust_authoritative_cache:
403 repo.cache_is_authoritative = False
405 #Take aliases into account.
407 for repo_name, repo in prepos.items():
411 names.update(repo.aliases)
414 if name in new_prepos:
415 writemsg_level(_("!!! Repository name or alias '%s', " + \
416 "defined for repository '%s', overrides " + \
417 "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1)
418 new_prepos[name] = repo
421 for (name, r) in prepos.items():
422 if r.location is not None:
423 location_map[r.location] = name
424 treemap[name] = r.location
426 # filter duplicates from aliases, by only including
427 # items where repo.name == key
429 prepos_order = sorted(prepos.items(), key=lambda r:r[1].priority or 0)
431 prepos_order = [repo.name for (key, repo) in prepos_order
432 if repo.name == key and repo.location is not None]
434 if portdir in location_map:
435 portdir_repo = prepos[location_map[portdir]]
436 portdir_sync = settings.get('SYNC', '')
437 #if SYNC variable is set and not overwritten by repos.conf
438 if portdir_sync and not portdir_repo.sync:
439 portdir_repo.sync = portdir_sync
441 if prepos['DEFAULT'].main_repo is None or \
442 prepos['DEFAULT'].main_repo not in prepos:
443 #setting main_repo if it was not set in repos.conf
444 if portdir in location_map:
445 prepos['DEFAULT'].main_repo = location_map[portdir]
446 elif portdir in ignored_location_map:
447 prepos['DEFAULT'].main_repo = ignored_location_map[portdir]
449 prepos['DEFAULT'].main_repo = None
450 writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty. \n"), noiselevel=-1)
453 self.prepos_order = prepos_order
454 self.ignored_repos = ignored_repos
455 self.location_map = location_map
456 self.treemap = treemap
457 self._prepos_changed = True
458 self._repo_location_list = []
460 #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig.
461 for repo_name, repo in prepos.items():
462 if repo_name == "DEFAULT":
464 if repo.masters is None:
465 if self.mainRepo() and repo_name != self.mainRepo().name:
466 repo.masters = self.mainRepo(),
470 if repo.masters and isinstance(repo.masters[0], RepoConfig):
471 # This one has already been processed
472 # because it has an alias.
475 for master_name in repo.masters:
476 if master_name not in prepos:
477 layout_filename = os.path.join(repo.user_location,
478 "metadata", "layout.conf")
479 writemsg_level(_("Unavailable repository '%s' " \
480 "referenced by masters entry in '%s'\n") % \
481 (master_name, layout_filename),
482 level=logging.ERROR, noiselevel=-1)
484 master_repos.append(prepos[master_name])
485 repo.masters = tuple(master_repos)
487 #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths.
488 for repo_name, repo in prepos.items():
489 if repo_name == "DEFAULT":
492 eclass_locations = []
493 eclass_locations.extend(master_repo.location for master_repo in repo.masters)
494 eclass_locations.append(repo.location)
496 if repo.eclass_overrides:
497 for other_repo_name in repo.eclass_overrides:
498 if other_repo_name in self.treemap:
499 eclass_locations.append(self.get_location_for_name(other_repo_name))
501 writemsg_level(_("Unavailable repository '%s' " \
502 "referenced by eclass-overrides entry for " \
503 "'%s'\n") % (other_repo_name, repo_name), \
504 level=logging.ERROR, noiselevel=-1)
505 repo.eclass_locations = tuple(eclass_locations)
507 self._prepos_changed = True
508 self._repo_location_list = []
510 self._check_locations()
512 def repoLocationList(self):
513 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
514 if self._prepos_changed:
515 _repo_location_list = []
516 for repo in self.prepos_order:
517 if self.prepos[repo].location is not None:
518 _repo_location_list.append(self.prepos[repo].location)
519 self._repo_location_list = tuple(_repo_location_list)
521 self._prepos_changed = False
522 return self._repo_location_list
524 def repoUserLocationList(self):
525 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
526 user_location_list = []
527 for repo in self.prepos_order:
528 if self.prepos[repo].location is not None:
529 user_location_list.append(self.prepos[repo].user_location)
530 return tuple(user_location_list)
532 def mainRepoLocation(self):
533 """Returns the location of main repo"""
534 main_repo = self.prepos['DEFAULT'].main_repo
535 if main_repo is not None and main_repo in self.prepos:
536 return self.prepos[main_repo].location
541 """Returns the main repo"""
542 maid_repo = self.prepos['DEFAULT'].main_repo
543 if maid_repo is None:
545 return self.prepos[maid_repo]
547 def _check_locations(self):
548 """Check if repositories location are correct and show a warning message if not"""
549 for (name, r) in self.prepos.items():
550 if name != 'DEFAULT':
551 if r.location is None:
552 writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1)
554 if not os.path.isdir(r.location):
555 self.prepos_order.remove(name)
556 writemsg(_("!!! Invalid Repository Location"
557 " (not a dir): '%s'\n") % r.location, noiselevel=-1)
559 def repos_with_profiles(self):
560 for repo_name in self.prepos_order:
561 repo = self.prepos[repo_name]
562 if repo.format != "unavailable":
565 def get_name_for_location(self, location):
566 return self.location_map[location]
568 def get_location_for_name(self, repo_name):
569 if repo_name is None:
570 # This simplifies code in places where
571 # we want to be able to pass in Atom.repo
572 # even if it is None.
574 return self.treemap[repo_name]
576 def get_repo_for_location(self, location):
577 return self.prepos[self.get_name_for_location(location)]
579 def __getitem__(self, repo_name):
580 return self.prepos[repo_name]
583 for repo_name in self.prepos_order:
584 yield self.prepos[repo_name]
586 def load_repository_config(settings):
587 #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")]
589 if settings.local_config:
590 repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
591 USER_CONFIG_PATH, "repos.conf"))
592 return RepoConfigLoader(repoconfigpaths, settings)