Implement GLEP 59 with control via layout.conf.
[portage.git] / pym / portage / repository / config.py
1 # Copyright 2010-2011 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3
4 import io
5 import logging
6 import sys
7 import re
8
9 try:
10         from configparser import ParsingError
11         if sys.hexversion >= 0x3020000:
12                 from configparser import ConfigParser as SafeConfigParser
13         else:
14                 from configparser import SafeConfigParser
15 except ImportError:
16         from ConfigParser import SafeConfigParser, ParsingError
17 from portage import os
18 from portage.const import USER_CONFIG_PATH, REPO_NAME_LOC
19 from portage.env.loaders import KeyValuePairFileLoader
20 from portage.util import normalize_path, writemsg, writemsg_level, shlex_split
21 from portage.localization import _
22 from portage import _unicode_encode
23 from portage import _encodings
24 from portage import manifest
25
26 _repo_name_sub_re = re.compile(r'[^\w-]')
27
28 def _gen_valid_repo(name):
29         """
30         Substitute hyphen in place of characters that don't conform to PMS 3.1.5,
31         and strip hyphen from left side if necessary. This returns None if the
32         given name contains no valid characters.
33         """
34         name = _repo_name_sub_re.sub(' ', name.strip())
35         name = '-'.join(name.split())
36         name = name.lstrip('-')
37         if not name:
38                 name = None
39         return name
40
41 class RepoConfig(object):
42         """Stores config of one repository"""
43
44         __slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo',
45                 'missing_repo_name', 'name', 'priority', 'sync', 'format', 'sign_manifest', 'thin_manifest',
46                 'allow_missing_manifest', 'create_manifest', 'disable_manifest', 'cache_is_authoritative',
47                 'trust_authoritative_cache', 'manifest_hash_flags']
48
49         def __init__(self, name, repo_opts):
50                 """Build a RepoConfig with options in repo_opts
51                    Try to read repo_name in repository location, but if
52                    it is not found use variable name as repository name"""
53                 aliases = repo_opts.get('aliases')
54                 if aliases is not None:
55                         aliases = tuple(aliases.split())
56                 self.aliases = aliases
57
58                 eclass_overrides = repo_opts.get('eclass-overrides')
59                 if eclass_overrides is not None:
60                         eclass_overrides = tuple(eclass_overrides.split())
61                 self.eclass_overrides = eclass_overrides
62                 #Locations are computed later.
63                 self.eclass_locations = None
64
65                 #Masters are only read from layout.conf.
66                 self.masters = None
67
68                 #The main-repo key makes only sense for the 'DEFAULT' section.
69                 self.main_repo = repo_opts.get('main-repo')
70
71                 priority = repo_opts.get('priority')
72                 if priority is not None:
73                         try:
74                                 priority = int(priority)
75                         except ValueError:
76                                 priority = None
77                 self.priority = priority
78
79                 sync = repo_opts.get('sync')
80                 if sync is not None:
81                         sync = sync.strip()
82                 self.sync = sync
83
84                 format = repo_opts.get('format')
85                 if format is not None:
86                         format = format.strip()
87                 self.format = format
88
89                 location = repo_opts.get('location')
90                 self.user_location = location
91                 if location is not None and location.strip():
92                         if os.path.isdir(location):
93                                 location = os.path.realpath(location)
94                 else:
95                         location = None
96                 self.location = location
97
98                 missing = True
99                 if self.location is not None:
100                         name, missing = self._read_repo_name(self.location)
101                         # We must ensure that the name conforms to PMS 3.1.5
102                         # in order to avoid InvalidAtom exceptions when we
103                         # use it to generate atoms.
104                         name = _gen_valid_repo(name)
105                         if not name:
106                                 # name only contains invalid characters
107                                 name = "x-" + os.path.basename(self.location)
108                                 name = _gen_valid_repo(name)
109                                 # If basename only contains whitespace then the
110                                 # end result is name = 'x-'.
111
112                 elif name == "DEFAULT": 
113                         missing = False
114                 self.name = name
115                 self.missing_repo_name = missing
116                 self.sign_manifest = True
117                 self.thin_manifest = False
118                 self.allow_missing_manifest = False
119                 self.create_manifest = True
120                 self.disable_manifest = False
121                 self.manifest_hash_flags = {}
122
123                 self.cache_is_authoritative = False
124
125                 trust_authoritative_cache = repo_opts.get('trust-authoritative-cache')
126                 if trust_authoritative_cache is not None:
127                         trust_authoritative_cache = trust_authoritative_cache.lower() == 'true'
128                 self.trust_authoritative_cache = trust_authoritative_cache
129
130         def load_manifest(self, *args, **kwds):
131                 kwds['thin'] = self.thin_manifest
132                 kwds['allow_missing'] = self.allow_missing_manifest
133                 kwds['allow_create'] = self.create_manifest
134                 kwds['hash_flags'] = self.manifest_hash_flags
135                 if self.disable_manifest:
136                         kwds['from_scratch'] = True
137                 return manifest.Manifest(*args, **kwds)
138
139         def update(self, new_repo):
140                 """Update repository with options in another RepoConfig"""
141                 if new_repo.aliases is not None:
142                         self.aliases = new_repo.aliases
143                 if new_repo.eclass_overrides is not None:
144                         self.eclass_overrides = new_repo.eclass_overrides
145                 if new_repo.masters is not None:
146                         self.masters = new_repo.masters
147                 if new_repo.trust_authoritative_cache is not None:
148                         self.trust_authoritative_cache = new_repo.trust_authoritative_cache
149                 if new_repo.name is not None:
150                         self.name = new_repo.name
151                         self.missing_repo_name = new_repo.missing_repo_name
152                 if new_repo.user_location is not None:
153                         self.user_location = new_repo.user_location
154                 if new_repo.location is not None:
155                         self.location = new_repo.location
156                 if new_repo.priority is not None:
157                         self.priority = new_repo.priority
158                 if new_repo.sync is not None:
159                         self.sync = new_repo.sync
160
161         def _read_repo_name(self, repo_path):
162                 """
163                 Read repo_name from repo_path.
164                 Returns repo_name, missing.
165                 """
166                 repo_name_path = os.path.join(repo_path, REPO_NAME_LOC)
167                 f = None
168                 try:
169                         f = io.open(
170                                 _unicode_encode(repo_name_path,
171                                 encoding=_encodings['fs'], errors='strict'),
172                                 mode='r', encoding=_encodings['repo.content'],
173                                 errors='replace')
174                         return f.readline().strip(), False
175                 except EnvironmentError:
176                         return "x-" + os.path.basename(repo_path), True
177                 finally:
178                         if f is not None:
179                                 f.close()
180
181         def info_string(self):
182                 """
183                 Returns a formatted string containing informations about the repository.
184                 Used by emerge --info.
185                 """
186                 indent = " " * 4
187                 repo_msg = []
188                 repo_msg.append(self.name)
189                 if self.format:
190                         repo_msg.append(indent + "format: " + self.format)
191                 if self.user_location:
192                         repo_msg.append(indent + "location: " + self.user_location)
193                 if self.sync:
194                         repo_msg.append(indent + "sync: " + self.sync)
195                 if self.masters:
196                         repo_msg.append(indent + "masters: " + " ".join(master.name for master in self.masters))
197                 if self.priority is not None:
198                         repo_msg.append(indent + "priority: " + str(self.priority))
199                 if self.aliases:
200                         repo_msg.append(indent + "aliases: " + " ".join(self.aliases))
201                 if self.eclass_overrides:
202                         repo_msg.append(indent + "eclass_overrides: " + \
203                                 " ".join(self.eclass_overrides))
204                 repo_msg.append("")
205                 return "\n".join(repo_msg)
206
207 class RepoConfigLoader(object):
208         """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf"""
209
210         @staticmethod
211         def _add_overlays(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map):
212                 """Add overlays in PORTDIR_OVERLAY as repositories"""
213                 overlays = []
214                 if portdir:
215                         portdir = normalize_path(portdir)
216                         overlays.append(portdir)
217                 try:
218                         port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)]
219                 except ValueError as e:
220                         #File "/usr/lib/python3.2/shlex.py", line 168, in read_token
221                         #       raise ValueError("No closing quotation")
222                         writemsg(_("!!! Invalid PORTDIR_OVERLAY:"
223                                 " %s: %s\n") % (e, portdir_overlay), noiselevel=-1)
224                         port_ov = []
225                 overlays.extend(port_ov)
226                 default_repo_opts = {}
227                 if prepos['DEFAULT'].aliases is not None:
228                         default_repo_opts['aliases'] = \
229                                 ' '.join(prepos['DEFAULT'].aliases)
230                 if prepos['DEFAULT'].eclass_overrides is not None:
231                         default_repo_opts['eclass-overrides'] = \
232                                 ' '.join(prepos['DEFAULT'].eclass_overrides)
233                 if prepos['DEFAULT'].masters is not None:
234                         default_repo_opts['masters'] = \
235                                 ' '.join(prepos['DEFAULT'].masters)
236                 if prepos['DEFAULT'].trust_authoritative_cache is not None:
237                         if prepos['DEFAULT'].trust_authoritative_cache:
238                                 default_repo_opts['trust-authoritative-cache'] = 'true'
239                         else:
240                                 default_repo_opts['trust-authoritative-cache'] = 'false'
241
242                 if overlays:
243                         #overlay priority is negative because we want them to be looked before any other repo
244                         base_priority = 0
245                         for ov in overlays:
246                                 if os.path.isdir(ov):
247                                         repo_opts = default_repo_opts.copy()
248                                         repo_opts['location'] = ov
249                                         repo = RepoConfig(None, repo_opts)
250                                         repo_conf_opts = prepos.get(repo.name)
251                                         if repo_conf_opts is not None:
252                                                 if repo_conf_opts.aliases is not None:
253                                                         repo_opts['aliases'] = \
254                                                                 ' '.join(repo_conf_opts.aliases)
255                                                 if repo_conf_opts.eclass_overrides is not None:
256                                                         repo_opts['eclass-overrides'] = \
257                                                                 ' '.join(repo_conf_opts.eclass_overrides)
258                                                 if repo_conf_opts.masters is not None:
259                                                         repo_opts['masters'] = \
260                                                                 ' '.join(repo_conf_opts.masters)
261                                                 if repo_conf_opts.trust_authoritative_cache is not None:
262                                                         if repo_conf_opts.trust_authoritative_cache:
263                                                                 repo_opts['trust-authoritative-cache'] = 'true'
264                                                         else:
265                                                                 repo_opts['trust-authoritative-cache'] = 'false'
266
267                                         repo = RepoConfig(repo.name, repo_opts)
268                                         if repo.name in prepos:
269                                                 old_location = prepos[repo.name].location
270                                                 if old_location is not None and old_location != repo.location:
271                                                         ignored_map.setdefault(repo.name, []).append(old_location)
272                                                         ignored_location_map[old_location] = repo.name
273                                                         if old_location == portdir:
274                                                                 portdir = repo.user_location
275                                                 prepos[repo.name].update(repo)
276                                                 repo = prepos[repo.name]
277                                         else:
278                                                 prepos[repo.name] = repo
279
280                                         if ov == portdir and portdir not in port_ov:
281                                                 repo.priority = -1000
282                                         else:
283                                                 repo.priority = base_priority
284                                                 base_priority += 1
285
286                                 else:
287                                         writemsg(_("!!! Invalid PORTDIR_OVERLAY"
288                                                 " (not a dir): '%s'\n") % ov, noiselevel=-1)
289
290                 return portdir
291
292         @staticmethod
293         def _parse(paths, prepos, ignored_map, ignored_location_map):
294                 """Parse files in paths to load config"""
295                 parser = SafeConfigParser()
296                 try:
297                         parser.read(paths)
298                 except ParsingError as e:
299                         writemsg(_("!!! Error while reading repo config file: %s\n") % e, noiselevel=-1)
300                 prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults())
301                 for sname in parser.sections():
302                         optdict = {}
303                         for oname in parser.options(sname):
304                                 optdict[oname] = parser.get(sname, oname)
305
306                         repo = RepoConfig(sname, optdict)
307                         if repo.location and not os.path.exists(repo.location):
308                                 writemsg(_("!!! Invalid repos.conf entry '%s'"
309                                         " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1)
310                                 continue
311
312                         if repo.name in prepos:
313                                 old_location = prepos[repo.name].location
314                                 if old_location is not None and repo.location is not None and old_location != repo.location:
315                                         ignored_map.setdefault(repo.name, []).append(old_location)
316                                         ignored_location_map[old_location] = repo.name
317                                 prepos[repo.name].update(repo)
318                         else:
319                                 prepos[repo.name] = repo
320
321         def __init__(self, paths, settings):
322                 """Load config from files in paths"""
323
324                 prepos = {}
325                 location_map = {}
326                 treemap = {}
327                 ignored_map = {}
328                 ignored_location_map = {}
329
330                 portdir = settings.get('PORTDIR', '')
331                 portdir_overlay = settings.get('PORTDIR_OVERLAY', '')
332
333                 self._parse(paths, prepos, ignored_map, ignored_location_map)
334
335                 # If PORTDIR_OVERLAY contains a repo with the same repo_name as
336                 # PORTDIR, then PORTDIR is overridden.
337                 portdir = self._add_overlays(portdir, portdir_overlay, prepos,
338                         ignored_map, ignored_location_map)
339                 if portdir and portdir.strip():
340                         portdir = os.path.realpath(portdir)
341
342                 ignored_repos = tuple((repo_name, tuple(paths)) \
343                         for repo_name, paths in ignored_map.items())
344
345                 self.missing_repo_names = frozenset(repo.location
346                         for repo in prepos.values()
347                         if repo.location is not None and repo.missing_repo_name)
348
349                 #Parse layout.conf and read masters key.
350                 for repo in prepos.values():
351                         if not repo.location:
352                                 continue
353                         layout_filename = os.path.join(repo.location, "metadata", "layout.conf")
354                         layout_file = KeyValuePairFileLoader(layout_filename, None, None)
355                         layout_data, layout_errors = layout_file.load()
356
357                         masters = layout_data.get('masters')
358                         if masters and masters.strip():
359                                 masters = masters.split()
360                         else:
361                                 masters = None
362                         repo.masters = masters
363
364                         aliases = layout_data.get('aliases')
365                         if aliases and aliases.strip():
366                                 aliases = aliases.split()
367                         else:
368                                 aliases = None
369                         if aliases:
370                                 if repo.aliases:
371                                         aliases.extend(repo.aliases)
372                                 repo.aliases = tuple(sorted(set(aliases)))
373
374                         if layout_data.get('sign-manifests', '').lower() == 'false':
375                                 repo.sign_manifest = False
376
377                         if layout_data.get('thin-manifests', '').lower() == 'true':
378                                 repo.thin_manifest = True
379
380                         manifest_policy = layout_data.get('use-manifests', 'strict').lower()
381                         repo.allow_missing_manifest = manifest_policy != 'strict'
382                         repo.create_manifest = manifest_policy != 'false'
383                         repo.disable_manifest = manifest_policy == 'false'
384
385                         if 'manifest-rmd160' in layout_data:
386                                 repo.manifest_hash_flags["RMD160"] = \
387                                         layout_data['manifest-rmd160'].lower() == 'true'
388
389                         if 'manifest-sha1' in layout_data:
390                                 repo.manifest_hash_flags["SHA1"] = \
391                                         layout_data['manifest-sha1'].lower() == 'true'
392
393                         if 'manifest-sha256' in layout_data:
394                                 repo.manifest_hash_flags["SHA256"] = \
395                                         layout_data['manifest-sha256'].lower() == 'true'
396
397                         if 'manifest-whirlpool' in layout_data:
398                                 repo.manifest_hash_flags["WHIRLPOOL"] = \
399                                         layout_data['manifest-whirlpool'].lower() == 'true'
400
401                         repo.cache_is_authoritative = layout_data.get('authoritative-cache', 'false').lower() == 'true'
402                         if not repo.trust_authoritative_cache:
403                                 repo.cache_is_authoritative = False
404
405                 #Take aliases into account.
406                 new_prepos = {}
407                 for repo_name, repo in prepos.items():
408                         names = set()
409                         names.add(repo_name)
410                         if repo.aliases:
411                                 names.update(repo.aliases)
412
413                         for name in names:
414                                 if name in new_prepos:
415                                         writemsg_level(_("!!! Repository name or alias '%s', " + \
416                                                 "defined for repository '%s', overrides " + \
417                                                 "existing alias or repository.\n") % (name, repo_name), level=logging.WARNING, noiselevel=-1)
418                                 new_prepos[name] = repo
419                 prepos = new_prepos
420
421                 for (name, r) in prepos.items():
422                         if r.location is not None:
423                                 location_map[r.location] = name
424                                 treemap[name] = r.location
425
426                 # filter duplicates from aliases, by only including
427                 # items where repo.name == key
428
429                 prepos_order = sorted(prepos.items(), key=lambda r:r[1].priority or 0)
430
431                 prepos_order = [repo.name for (key, repo) in prepos_order
432                         if repo.name == key and repo.location is not None]
433
434                 if portdir in location_map:
435                         portdir_repo = prepos[location_map[portdir]]
436                         portdir_sync = settings.get('SYNC', '')
437                         #if SYNC variable is set and not overwritten by repos.conf
438                         if portdir_sync and not portdir_repo.sync:
439                                 portdir_repo.sync = portdir_sync
440
441                 if prepos['DEFAULT'].main_repo is None or \
442                         prepos['DEFAULT'].main_repo not in prepos:
443                         #setting main_repo if it was not set in repos.conf
444                         if portdir in location_map:
445                                 prepos['DEFAULT'].main_repo = location_map[portdir]
446                         elif portdir in ignored_location_map:
447                                 prepos['DEFAULT'].main_repo = ignored_location_map[portdir]
448                         else:
449                                 prepos['DEFAULT'].main_repo = None
450                                 writemsg(_("!!! main-repo not set in DEFAULT and PORTDIR is empty. \n"), noiselevel=-1)
451
452                 self.prepos = prepos
453                 self.prepos_order = prepos_order
454                 self.ignored_repos = ignored_repos
455                 self.location_map = location_map
456                 self.treemap = treemap
457                 self._prepos_changed = True
458                 self._repo_location_list = []
459
460                 #The 'masters' key currently contains repo names. Replace them with the matching RepoConfig.
461                 for repo_name, repo in prepos.items():
462                         if repo_name == "DEFAULT":
463                                 continue
464                         if repo.masters is None:
465                                 if self.mainRepo() and repo_name != self.mainRepo().name:
466                                         repo.masters = self.mainRepo(),
467                                 else:
468                                         repo.masters = ()
469                         else:
470                                 if repo.masters and isinstance(repo.masters[0], RepoConfig):
471                                         # This one has already been processed
472                                         # because it has an alias.
473                                         continue
474                                 master_repos = []
475                                 for master_name in repo.masters:
476                                         if master_name not in prepos:
477                                                 layout_filename = os.path.join(repo.user_location,
478                                                         "metadata", "layout.conf")
479                                                 writemsg_level(_("Unavailable repository '%s' " \
480                                                         "referenced by masters entry in '%s'\n") % \
481                                                         (master_name, layout_filename),
482                                                         level=logging.ERROR, noiselevel=-1)
483                                         else:
484                                                 master_repos.append(prepos[master_name])
485                                 repo.masters = tuple(master_repos)
486
487                 #The 'eclass_overrides' key currently contains repo names. Replace them with the matching repo paths.
488                 for repo_name, repo in prepos.items():
489                         if repo_name == "DEFAULT":
490                                 continue
491
492                         eclass_locations = []
493                         eclass_locations.extend(master_repo.location for master_repo in repo.masters)
494                         eclass_locations.append(repo.location)
495
496                         if repo.eclass_overrides:
497                                 for other_repo_name in repo.eclass_overrides:
498                                         if other_repo_name in self.treemap:
499                                                 eclass_locations.append(self.get_location_for_name(other_repo_name))
500                                         else:
501                                                 writemsg_level(_("Unavailable repository '%s' " \
502                                                         "referenced by eclass-overrides entry for " \
503                                                         "'%s'\n") % (other_repo_name, repo_name), \
504                                                         level=logging.ERROR, noiselevel=-1)
505                         repo.eclass_locations = tuple(eclass_locations)
506
507                 self._prepos_changed = True
508                 self._repo_location_list = []
509
510                 self._check_locations()
511
512         def repoLocationList(self):
513                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
514                 if self._prepos_changed:
515                         _repo_location_list = []
516                         for repo in self.prepos_order:
517                                 if self.prepos[repo].location is not None:
518                                         _repo_location_list.append(self.prepos[repo].location)
519                         self._repo_location_list = tuple(_repo_location_list)
520
521                         self._prepos_changed = False
522                 return self._repo_location_list
523
524         def repoUserLocationList(self):
525                 """Get a list of repositories location. Replaces PORTDIR_OVERLAY"""
526                 user_location_list = []
527                 for repo in self.prepos_order:
528                         if self.prepos[repo].location is not None:
529                                 user_location_list.append(self.prepos[repo].user_location)
530                 return tuple(user_location_list)
531
532         def mainRepoLocation(self):
533                 """Returns the location of main repo"""
534                 main_repo = self.prepos['DEFAULT'].main_repo
535                 if main_repo is not None and main_repo in self.prepos:
536                         return self.prepos[main_repo].location
537                 else:
538                         return ''
539
540         def mainRepo(self):
541                 """Returns the main repo"""
542                 maid_repo = self.prepos['DEFAULT'].main_repo
543                 if maid_repo is None:
544                         return None
545                 return self.prepos[maid_repo]
546
547         def _check_locations(self):
548                 """Check if repositories location are correct and show a warning message if not"""
549                 for (name, r) in self.prepos.items():
550                         if name != 'DEFAULT':
551                                 if r.location is None:
552                                         writemsg(_("!!! Location not set for repository %s\n") % name, noiselevel=-1)
553                                 else:
554                                         if not os.path.isdir(r.location):
555                                                 self.prepos_order.remove(name)
556                                                 writemsg(_("!!! Invalid Repository Location"
557                                                         " (not a dir): '%s'\n") % r.location, noiselevel=-1)
558
559         def repos_with_profiles(self):
560                 for repo_name in self.prepos_order:
561                         repo = self.prepos[repo_name]
562                         if repo.format != "unavailable":
563                                 yield repo
564
565         def get_name_for_location(self, location):
566                 return self.location_map[location]
567
568         def get_location_for_name(self, repo_name):
569                 if repo_name is None:
570                         # This simplifies code in places where
571                         # we want to be able to pass in Atom.repo
572                         # even if it is None.
573                         return None
574                 return self.treemap[repo_name]
575
576         def get_repo_for_location(self, location):
577                 return self.prepos[self.get_name_for_location(location)]
578
579         def __getitem__(self, repo_name):
580                 return self.prepos[repo_name]
581
582         def __iter__(self):
583                 for repo_name in self.prepos_order:
584                         yield self.prepos[repo_name]
585
586 def load_repository_config(settings):
587         #~ repoconfigpaths = [os.path.join(settings.global_config_path, "repos.conf")]
588         repoconfigpaths = []
589         if settings.local_config:
590                 repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
591                         USER_CONFIG_PATH, "repos.conf"))
592         return RepoConfigLoader(repoconfigpaths, settings)