2 # Copyright 1999-2006 Gentoo Foundation
3 # Distributed under the terms of the GNU General Public License v2
6 # Next to do: dep syntax checking in mask files
7 # Then, check to make sure deps are satisfiable (to avoid "can't find match for" problems)
8 # that last one is tricky because multiple profiles need to be checked.
10 from __future__ import print_function
14 from subprocess import getstatusoutput as subprocess_getstatusoutput
16 from commands import getstatusoutput as subprocess_getstatusoutput
29 from io import StringIO
30 from itertools import chain
31 from stat import S_ISDIR, ST_CTIME
33 if not hasattr(__builtins__, "set"):
34 from sets import Set as set
39 from os import path as osp
40 sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
42 portage._disable_legacy_globals()
43 from portage import os
44 from portage import _encodings
45 from portage import _unicode_encode
48 from repoman.checks import run_checks
49 from repoman import utilities
51 from os import path as osp
52 sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
53 from repoman.checks import run_checks
54 from repoman import utilities
56 from _emerge.Package import Package
57 from _emerge.RootConfig import RootConfig
58 from portage.sets import load_default_config
60 import portage.checksum
63 portage.dep._dep_check_strict = True
64 import portage.exception
65 from portage import cvstree, normalize_path
66 from portage import util
67 from portage.exception import ParseError
68 from portage.manifest import Manifest
69 from portage.process import find_binary, spawn
70 from portage.output import bold, create_color_func, darkgreen, \
71 green, nocolor, red, turquoise, yellow
72 from portage.output import ConsoleStyleFile, StyleWriter
73 from portage.util import cmp_sort_key, writemsg_level
75 if sys.hexversion >= 0x3000000:
78 util.initialize_logger()
80 # 14 is the length of DESCRIPTION=""
82 allowed_filename_chars="a-zA-Z0-9._-+:"
83 disallowed_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]')
84 bad = create_color_func("BAD")
86 # A sane umask is needed for files that portage creates.
88 repoman_settings = portage.config(local_config=False,
89 config_incrementals=portage.const.INCREMENTALS)
90 repoman_settings.lock()
92 if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \
93 not sys.stdout.isatty():
97 print("repoman: " + txt)
103 def exithandler(signum=None, frame=None):
104 logging.fatal("Interrupted; exiting...")
106 os.kill(0, signal.SIGKILL)
108 signal.signal(signal.SIGINT,exithandler)
110 class RepomanHelpFormatter(optparse.IndentedHelpFormatter):
111 """Repoman needs it's own HelpFormatter for now, because the default ones
112 murder the help text."""
114 def __init__(self, indent_increment=1, max_help_position=24, width=150, short_first=1):
115 optparse.HelpFormatter.__init__(self, indent_increment, max_help_position, width, short_first)
117 def format_description(self, description):
120 class RepomanOptionParser(optparse.OptionParser):
121 """Add the on_tail function, ruby has it, optionParser should too
124 def __init__(self, *args, **kwargs):
125 optparse.OptionParser.__init__(self, *args, **kwargs)
128 def on_tail(self, description):
129 self.tail += description
131 def format_help(self, formatter=None):
132 result = optparse.OptionParser.format_help(self, formatter)
137 def ParseArgs(args, qahelp):
138 """This function uses a customized optionParser to parse command line arguments for repoman
140 args - a sequence of command line arguments
141 qahelp - a dict of qa warning to help message
143 (opts, args), just like a call to parser.parse_args()
147 'commit' : 'Run a scan then commit changes',
148 'ci' : 'Run a scan then commit changes',
149 'fix' : 'Fix simple QA issues (stray digests, missing digests)',
150 'full' : 'Scan directory tree and print all issues (not a summary)',
151 'help' : 'Show this screen',
152 'manifest' : 'Generate a Manifest (fetches files if necessary)',
153 'scan' : 'Scan directory tree for QA issues'
156 mode_keys = list(modes)
159 parser = RepomanOptionParser(formatter=RepomanHelpFormatter(), usage="%prog [options] [mode]")
160 parser.description = green(" ".join((os.path.basename(args[0]), "1.2")))
161 parser.description += "\nCopyright 1999-2007 Gentoo Foundation"
162 parser.description += "\nDistributed under the terms of the GNU General Public License v2"
163 parser.description += "\nmodes: " + " | ".join(map(green,mode_keys))
165 parser.add_option('-m', '--commitmsg', dest='commitmsg',
166 help='specify a commit message on the command line')
168 parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile',
169 help='specify a path to a file that contains a commit message')
171 parser.add_option('-p', '--pretend', dest='pretend', default=False,
172 action='store_true', help='don\'t commit or fix anything; just show what would be done')
174 parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0,
175 help='do not print unnecessary messages')
177 parser.add_option('-f', '--force', dest='force', default=False, action='store_true',
178 help='Commit with QA violations')
180 parser.add_option('-v', '--verbose', dest="verbosity", action='count',
181 help='be very verbose in output', default=0)
183 parser.add_option('-V', '--version', dest='version', action='store_true',
184 help='show version info')
186 parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true',
187 default=False, help='forces the metadata.xml parse check to be carried out')
189 parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true',
190 default=False, help='ignore arch-specific failures (where arch != host)')
192 parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true',
193 default=False, help='ignore masked packages (not allowed with commit mode)')
195 parser.add_option('-d', '--include-dev', dest='include_dev', action='store_true',
196 default=False, help='include dev profiles in dependency checks')
198 parser.add_option('--without-mask', dest='without_mask', action='store_true',
199 default=False, help='behave as if no package.mask entries exist (not allowed with commit mode)')
201 parser.add_option('--mode', type='choice', dest='mode', choices=list(modes),
202 help='specify which mode repoman will run in (default=full)')
204 parser.on_tail("\n " + green("Modes".ljust(20) + " Description\n"))
207 parser.on_tail(" %s %s\n" % (k.ljust(20), modes[k]))
209 parser.on_tail("\n " + green("QA keyword".ljust(20) + " Description\n"))
211 sorted_qa = list(qahelp)
214 parser.on_tail(" %s %s\n" % (k.ljust(20), qahelp[k]))
218 opts, args = parser.parse_args(args)
220 if opts.mode == 'help':
221 parser.print_help(short=False)
232 if opts.mode == 'ci':
233 opts.mode = 'commit' # backwards compat shortcut
235 if opts.mode == 'commit' and not (opts.force or opts.pretend):
236 if opts.ignore_masked:
237 parser.error('Commit mode and --ignore-masked are not compatible')
238 if opts.without_mask:
239 parser.error('Commit mode and --without-mask are not compatible')
241 # Use the verbosity and quiet options to fiddle with the loglevel appropriately
242 for val in range(opts.verbosity):
243 logger = logging.getLogger()
244 logger.setLevel(logger.getEffectiveLevel() - 10)
246 for val in range(opts.quiet):
247 logger = logging.getLogger()
248 logger.setLevel(logger.getEffectiveLevel() + 10)
253 "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file",
254 "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file",
255 "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)",
256 "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory",
257 "changelog.ebuildadded":"An ebuild was added but the ChangeLog was not modified",
258 "changelog.missing":"Missing ChangeLog files",
259 "ebuild.notadded":"Ebuilds that exist but have not been added to cvs",
260 "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety",
261 "changelog.notadded":"ChangeLogs that exist but have not been added to cvs",
262 "filedir.missing":"Package lacks a files directory",
263 "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit",
264 "file.size":"Files in the files directory must be under 20 KiB",
265 "file.size.fatal":"Files in the files directory must be under 60 KiB",
266 "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars,
267 "file.UTF8":"File is not UTF8 compliant",
268 "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf",
269 "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass",
270 "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch",
271 "KEYWORDS.missing":"Ebuilds that have a missing or empty KEYWORDS variable",
272 "KEYWORDS.stable":"Ebuilds that have been added directly with stable KEYWORDS",
273 "KEYWORDS.stupid":"Ebuilds that use KEYWORDS=-* instead of package.mask",
274 "LICENSE.missing":"Ebuilds that have a missing or empty LICENSE variable",
275 "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable",
276 "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len,
277 "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)",
278 "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI",
279 "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)",
280 "SLOT.missing":"Ebuilds that have a missing or empty SLOT variable",
281 "HOMEPAGE.missing":"Ebuilds that have a missing or empty HOMEPAGE variable",
282 "DEPEND.bad":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds)",
283 "RDEPEND.bad":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds)",
284 "PDEPEND.bad":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds)",
285 "DEPEND.badmasked":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds)",
286 "RDEPEND.badmasked":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds)",
287 "PDEPEND.badmasked":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds)",
288 "DEPEND.badindev":"User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch",
289 "RDEPEND.badindev":"User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch",
290 "PDEPEND.badindev":"User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch",
291 "DEPEND.badmaskedindev":"Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch",
292 "RDEPEND.badmaskedindev":"Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch",
293 "PDEPEND.badmaskedindev":"Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch",
294 "PDEPEND.suspect":"PDEPEND contains a package that usually only belongs in DEPEND.",
295 "DEPEND.syntax":"Syntax error in DEPEND (usually an extra/missing space/parenthesis)",
296 "RDEPEND.syntax":"Syntax error in RDEPEND (usually an extra/missing space/parenthesis)",
297 "PDEPEND.syntax":"Syntax error in PDEPEND (usually an extra/missing space/parenthesis)",
298 "DEPEND.badtilde":"DEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
299 "RDEPEND.badtilde":"RDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
300 "PDEPEND.badtilde":"PDEPEND uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)",
301 "LICENSE.syntax":"Syntax error in LICENSE (usually an extra/missing space/parenthesis)",
302 "PROVIDE.syntax":"Syntax error in PROVIDE (usually an extra/missing space/parenthesis)",
303 "PROPERTIES.syntax":"Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)",
304 "RESTRICT.syntax":"Syntax error in RESTRICT (usually an extra/missing space/parenthesis)",
305 "SRC_URI.syntax":"Syntax error in SRC_URI (usually an extra/missing space/parenthesis)",
306 "SRC_URI.mirror":"A uri listed in profiles/thirdpartymirrors is found in SRC_URI",
307 "ebuild.syntax":"Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure",
308 "ebuild.output":"A simple sourcing of the ebuild produces output; this breaks ebuild policy.",
309 "ebuild.nesteddie":"Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild.",
310 "variable.invalidchar":"A variable contains an invalid character that is not part of the ASCII character set",
311 "variable.readonly":"Assigning a readonly variable",
312 "LIVEVCS.stable":"This ebuild is a live checkout from a VCS but has stable keywords.",
313 "LIVEVCS.unmasked":"This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask.",
314 "IUSE.invalid":"This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file",
315 "IUSE.undefined":"This ebuild does not define IUSE (style guideline says to define IUSE even when empty)",
316 "LICENSE.invalid":"This ebuild is listing a license that doesnt exist in portages license/ dir.",
317 "KEYWORDS.invalid":"This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found",
318 "RDEPEND.implicit":"RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment",
319 "RDEPEND.suspect":"RDEPEND contains a package that usually only belongs in DEPEND.",
320 "RESTRICT.invalid":"This ebuild contains invalid RESTRICT values.",
321 "digest.assumed":"Existing digest must be assumed correct (Package level only)",
322 "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest",
323 "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI",
324 "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH",
325 "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)",
326 "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully",
327 "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style",
328 "ebuild.badheader":"This ebuild has a malformed header",
329 "metadata.missing":"Missing metadata.xml files",
330 "metadata.bad":"Bad metadata.xml files",
331 "metadata.warning":"Warnings in metadata.xml files",
332 "virtual.versioned":"PROVIDE contains virtuals with versions",
333 "virtual.exists":"PROVIDE contains existing package names",
334 "virtual.unavailable":"PROVIDE contains a virtual which contains no profile default",
335 "usage.obsolete":"The ebuild makes use of an obsolete construct",
336 "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org"
339 qacats = list(qahelp)
344 "changelog.notadded",
352 "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked",
353 "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev",
354 "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev",
355 "DEPEND.badtilde", "RDEPEND.badtilde", "PDEPEND.badtilde",
356 "DESCRIPTION.toolong",
371 "java.eclassesnotused",
375 "virtual.unavailable",
377 "upstream.workaround",
382 non_ascii_re = re.compile(r'[^\x00-\x7f]')
384 missingvars=["KEYWORDS","LICENSE","DESCRIPTION","HOMEPAGE","SLOT"]
385 allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_"))
386 allvars.update(Package.metadata_keys)
387 allvars = sorted(allvars)
389 for x in missingvars:
392 logging.warn('* missingvars values need to be added to qahelp ("%s")' % x)
396 valid_restrict = frozenset(["binchecks", "bindist",
397 "fetch", "installsources", "mirror",
398 "primaryuri", "strip", "test", "userpriv"])
400 live_eclasses = frozenset([
409 suspect_rdepend = frozenset([
410 "app-arch/cabextract",
411 "app-arch/rpm2targz",
416 "dev-perl/extutils-pkgconfig",
422 "dev-util/gtk-doc-am",
425 "dev-util/pkgconfig",
429 "media-gfx/ebdftopcf",
431 "sys-devel/autoconf",
432 "sys-devel/automake",
444 no_exec = frozenset(["Manifest","ChangeLog","metadata.xml"])
446 options, arguments = ParseArgs(sys.argv, qahelp)
449 print("Portage", portage.VERSION)
452 # Set this to False when an extraordinary issue (generally
453 # something other than a QA issue) makes it impossible to
454 # commit (like if Manifest generation fails).
457 portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings)
461 myreporoot = os.path.basename(portdir_overlay)
462 myreporoot += mydir[len(portdir_overlay):]
465 if os.path.isdir("CVS"):
467 if os.path.isdir(".svn"):
469 elif os.path.isdir(os.path.join(portdir_overlay, ".git")):
472 vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split()
473 vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS")
474 if vcs_global_opts is None:
476 vcs_global_opts = "-q"
479 vcs_global_opts = vcs_global_opts.split()
481 if vcs == "cvs" and \
482 "commit" == options.mode and \
483 "RMD160" not in portage.checksum.hashorigin_map:
484 from portage.util import grablines
485 repo_lines = grablines("./CVS/Repository")
487 "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]:
488 msg = "Please install " \
489 "pycrypto or enable python's ssl USE flag in order " \
490 "to enable RMD160 hash support. See bug #198398 for " \
493 from textwrap import wrap
494 for line in wrap(msg, 70):
499 if options.mode == 'commit' and not options.pretend and not vcs:
500 logging.info("Not in a version controlled repository; enabling pretend mode.")
501 options.pretend = True
503 # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD.
504 repoman_settings = portage.config(local_config=False)
505 repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \
506 (repoman_settings.get('PORTDIR_OVERLAY', ''), portdir_overlay)
507 repoman_settings.backup_changes('PORTDIR_OVERLAY')
511 root : {'porttree' : portage.portagetree(root, settings=repoman_settings)}
513 portdb = trees[root]['porttree'].dbapi
515 # Constrain dependency resolution to the master(s)
516 # that are specified in layout.conf.
517 portdir_overlay = os.path.realpath(portdir_overlay)
518 repo_info = portdb._repo_info[portdir_overlay]
519 portdb.porttrees = list(repo_info.eclass_db.porttrees)
520 portdir = portdb.porttrees[0]
522 # Generate an appropriate PORTDIR_OVERLAY value for passing into the
523 # profile-specific config constructor calls.
524 env = os.environ.copy()
525 env['PORTDIR'] = portdir
526 env['PORTDIR_OVERLAY'] = ' '.join(portdb.porttrees[1:])
528 logging.info('Setting paths:')
529 logging.info('PORTDIR = "' + portdir + '"')
530 logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY'])
533 for path in set([portdir, portdir_overlay]):
534 categories.extend(portage.util.grabfile(
535 os.path.join(path, 'profiles', 'categories')))
536 repoman_settings.categories = tuple(sorted(
537 portage.util.stack_lists([categories], incremental=1)))
540 portdb.mysettings = repoman_settings
541 root_config = RootConfig(repoman_settings, trees[root], None)
542 # We really only need to cache the metadata that's necessary for visibility
543 # filtering. Anything else can be discarded to reduce memory consumption.
544 portdb._aux_cache_keys.clear()
545 portdb._aux_cache_keys.update(["EAPI", "KEYWORDS", "SLOT"])
547 reposplit = myreporoot.split(os.path.sep)
548 repolevel = len(reposplit)
550 # check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting.
551 # Reason for this is if they're trying to commit in just $FILESDIR/*, the Manifest needs updating.
552 # this check ensures that repoman knows where it is, and the manifest recommit is at least possible.
553 if options.mode == 'commit' and repolevel not in [1,2,3]:
554 print(red("***")+" Commit attempts *must* be from within a vcs co, category, or package directory.")
555 print(red("***")+" Attempting to commit from a packages files directory will be blocked for instance.")
556 print(red("***")+" This is intended behaviour, to ensure the manifest is recommited for a package.")
558 err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit))
560 startdir = normalize_path(mydir)
562 for x in range(0, repolevel - 1):
563 repodir = os.path.dirname(repodir)
564 repodir = os.path.realpath(repodir)
567 err(mycat+" is not an official category. Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.")
569 class ProfileDesc(object):
570 __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',)
571 def __init__(self, arch, status, sub_path, tree_path):
574 self.sub_path = normalize_path(sub_path.lstrip(os.sep))
575 self.tree_path = tree_path
576 self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path)
579 valid_profile_types = frozenset(['dev', 'exp', 'stable'])
581 # get lists of valid keywords, licenses, and use
585 global_pmasklines = []
587 for path in portdb.porttrees:
589 liclist.update(os.listdir(os.path.join(path, "licenses")))
592 kwlist.update(portage.grabfile(os.path.join(path,
593 "profiles", "arch.list")))
595 use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc'))
601 expand_desc_dir = os.path.join(path, 'profiles', 'desc')
603 expand_list = os.listdir(expand_desc_dir)
607 for fn in expand_list:
608 if not fn[-5:] == '.desc':
610 use_prefix = fn[:-5].lower() + '_'
611 for x in portage.grabfile(os.path.join(expand_desc_dir, fn)):
614 uselist.add(use_prefix + x[0])
616 global_pmasklines.append(portage.util.grabfile_package(
617 os.path.join(path, 'profiles', 'package.mask'), recursive=1))
619 desc_path = os.path.join(path, 'profiles', 'profiles.desc')
621 desc_file = codecs.open(_unicode_encode(desc_path,
622 encoding=_encodings['fs'], errors='strict'),
623 mode='r', encoding=_encodings['repo.content'], errors='replace')
624 except EnvironmentError:
627 for i, x in enumerate(desc_file):
634 err("wrong format: \"" + bad(x.strip()) + "\" in " + \
635 desc_path + " line %d" % (i+1, ))
636 elif arch[0] not in kwlist:
637 err("invalid arch: \"" + bad(arch[0]) + "\" in " + \
638 desc_path + " line %d" % (i+1, ))
639 elif arch[2] not in valid_profile_types:
640 err("invalid profile type: \"" + bad(arch[2]) + "\" in " + \
641 desc_path + " line %d" % (i+1, ))
642 profile_desc = ProfileDesc(arch[0], arch[2], arch[1], portdir)
643 if not os.path.isdir(profile_desc.abs_path):
645 "Invalid %s profile (%s) for arch %s in %s line %d",
646 arch[2], arch[1], arch[0], desc_path, i+1)
648 profile_list.append(profile_desc)
651 repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist))
652 repoman_settings.backup_changes('PORTAGE_ARCHLIST')
654 global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1)
655 global_pmaskdict = {}
656 for x in global_pmasklines:
657 global_pmaskdict.setdefault(portage.dep_getkey(x), []).append(x)
658 del global_pmasklines
660 def has_global_mask(pkg):
661 mask_atoms = global_pmaskdict.get(pkg.cp)
665 if portage.dep.match_from_list(x, pkg_list):
669 # Ensure that profile sub_path attributes are unique. Process in reverse order
670 # so that profiles with duplicate sub_path from overlays will override
671 # profiles with the same sub_path from parent repos.
673 profile_list.reverse()
674 profile_sub_paths = set()
675 for prof in profile_list:
676 if prof.sub_path in profile_sub_paths:
678 profile_sub_paths.add(prof.sub_path)
679 profiles.setdefault(prof.arch, []).append(prof)
681 for x in repoman_settings.archlist():
684 if x not in profiles:
685 print(red("\""+x+"\" doesn't have a valid profile listed in profiles.desc."))
686 print(red("You need to either \"cvs update\" your profiles dir or follow this"))
687 print(red("up with the "+x+" team."))
691 logging.fatal("Couldn't find licenses?")
695 logging.fatal("Couldn't read KEYWORDS from arch.list")
699 logging.fatal("Couldn't find use.desc?")
704 #we are inside a category directory
706 if catdir not in repoman_settings.categories:
708 mydirlist=os.listdir(startdir)
710 if x == "CVS" or x.startswith("."):
712 if os.path.isdir(startdir+"/"+x):
713 scanlist.append(catdir+"/"+x)
714 repo_subdir = catdir + os.sep
716 for x in repoman_settings.categories:
717 if not os.path.isdir(startdir+"/"+x):
719 for y in os.listdir(startdir+"/"+x):
720 if y == "CVS" or y.startswith("."):
722 if os.path.isdir(startdir+"/"+x+"/"+y):
723 scanlist.append(x+"/"+y)
726 catdir = reposplit[-2]
727 if catdir not in repoman_settings.categories:
729 scanlist.append(catdir+"/"+reposplit[-1])
730 repo_subdir = scanlist[-1] + os.sep
731 repo_subdir_len = len(repo_subdir)
734 logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist))
736 def dev_keywords(profiles):
738 Create a set of KEYWORDS values that exist in 'dev'
739 profiles. These are used
740 to trigger a message notifying the user when they might
741 want to add the --include-dev option.
744 for arch, arch_profiles in profiles.items():
745 for prof in arch_profiles:
746 arch_set = type_arch_map.get(prof.status)
749 type_arch_map[prof.status] = arch_set
752 dev_keywords = type_arch_map.get('dev', set())
753 dev_keywords.update(['~' + arch for arch in dev_keywords])
754 return frozenset(dev_keywords)
756 dev_keywords = dev_keywords(profiles)
761 # provided by the desktop-file-utils package
762 desktop_file_validate = find_binary("desktop-file-validate")
763 desktop_pattern = re.compile(r'.*\.desktop$')
768 xmllint_capable = False
769 metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd')
770 if options.mode == "manifest":
772 elif not find_binary('xmllint'):
773 print(red("!!! xmllint not found. Can't check metadata.xml.\n"))
774 if options.xml_parse or repolevel==3:
775 print(red("!!!")+" sorry, xmllint is needed. failing\n")
778 #hardcoded paths/urls suck. :-/
782 # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd
783 # clock is fscked or it's been a week. time to grab a new one.
784 ct = os.stat(metadata_dtd)[ST_CTIME]
785 if abs(time.time() - ct) > (60*60*24*7):
786 # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug.
791 except (OSError,IOError) as e:
793 print(red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing" % (str(e), portage.CACHE_PATH))
798 print(green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now")
803 os.unlink(metadata_dtd)
805 if e.errno != errno.ENOENT:
808 val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \
811 except SystemExit as e:
812 raise # Need to propogate this
813 except Exception as e:
815 print(red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/metadata.dtd', caught")
816 print(red("!!!")+" exception '%s' though." % str(e))
819 print(red("!!!")+" fetching new metadata.dtd failed, aborting")
821 #this can be problematic if xmllint changes their output
824 if options.mode == 'commit' and vcs:
825 utilities.detect_vcs_conflicts(options, vcs)
827 if options.mode == "manifest":
829 elif options.pretend:
830 print(green("\nRepoMan does a once-over of the neighborhood..."))
832 print(green("\nRepoMan scours the neighborhood..."))
835 modified_changelogs = set()
841 mycvstree = cvstree.getentries("./", recursive=1)
842 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
843 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
845 svnstatus = os.popen("svn status").readlines()
846 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
847 mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
849 strip_levels = repolevel - 1
851 mychanged = os.popen("git diff-index --name-only --diff-filter=M HEAD").readlines()
853 mychanged = [elem[repo_subdir_len:] for elem in mychanged \
854 if elem[:repo_subdir_len] == repo_subdir]
855 mychanged = ["./" + elem[:-1] for elem in mychanged]
857 mynew = os.popen("git diff-index --name-only --diff-filter=A HEAD").readlines()
859 mynew = [elem[repo_subdir_len:] for elem in mynew \
860 if elem[:repo_subdir_len] == repo_subdir]
861 mynew = ["./" + elem[:-1] for elem in mynew]
863 new_ebuilds.update(x for x in mynew if x.endswith(".ebuild"))
864 modified_changelogs.update(x for x in chain(mychanged, mynew) \
865 if os.path.basename(x) == "ChangeLog")
868 have_dev_keywords = False
871 arch_xmatch_caches = {}
872 shared_xmatch_caches = {"cp-list":{}}
874 # Disable the "ebuild.notadded" check when not in commit mode and
875 # running `svn status` in every package dir will be too expensive.
877 check_ebuild_notadded = not \
878 (vcs == "svn" and repolevel < 3 and options.mode != "commit")
880 # Build a regex from thirdpartymirrors for the SRC_URI.mirror check.
881 thirdpartymirrors = portage.flatten(list(repoman_settings.thirdpartymirrors().values()))
884 #ebuilds and digests added to cvs respectively.
885 logging.info("checking package %s" % x)
887 catdir,pkgdir=x.split("/")
888 checkdir=repodir+"/"+x
889 checkdir_relative = ""
891 checkdir_relative = os.path.join(pkgdir, checkdir_relative)
893 checkdir_relative = os.path.join(catdir, checkdir_relative)
894 checkdir_relative = os.path.join(".", checkdir_relative)
896 if options.mode == "manifest" or \
897 options.mode in ('commit', 'fix') and not options.pretend:
899 fetchlist_dict = portage.FetchlistDict(checkdir,
900 repoman_settings, portdb)
901 if options.mode == 'manifest' and options.force:
902 portage._doebuild_manifest_exempt_depend += 1
904 distdir = repoman_settings['DISTDIR']
905 mf = portage.manifest.Manifest(checkdir, distdir,
906 fetchlist_dict=fetchlist_dict)
907 mf.create(requiredDistfiles=None,
908 assumeDistHashesAlways=True)
909 for distfiles in fetchlist_dict.values():
910 for distfile in distfiles:
911 if os.path.isfile(os.path.join(distdir, distfile)):
912 mf.fhashdict['DIST'].pop(distfile, None)
914 auto_assumed.add(distfile)
917 portage._doebuild_manifest_exempt_depend -= 1
919 repoman_settings["O"] = checkdir
920 if not portage.digestgen([], repoman_settings, myportdb=portdb):
921 print("Unable to generate manifest.")
923 if options.mode == "manifest":
924 if not dofail and options.force and auto_assumed and \
925 'assume-digests' in repoman_settings.features:
926 # Show which digests were assumed despite the --force option
927 # being given. This output will already have been shown by
928 # digestgen() if assume-digests is not enabled, so only show
929 # it here if assume-digests is enabled.
930 pkgs = list(fetchlist_dict)
932 portage.writemsg_stdout(" digest.assumed" + \
933 portage.output.colorize("WARN",
934 str(len(auto_assumed)).rjust(18)) + "\n")
936 fetchmap = fetchlist_dict[cpv]
937 pf = portage.catsplit(cpv)[1]
938 for distfile in sorted(fetchmap):
939 if distfile in auto_assumed:
940 portage.writemsg_stdout(
941 " %s::%s\n" % (pf, distfile))
946 checkdirlist=os.listdir(checkdir)
949 for y in checkdirlist:
950 if y in no_exec and \
951 stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111:
952 stats["file.executable"] += 1
953 fails["file.executable"].append(os.path.join(checkdir, y))
954 if y.endswith(".ebuild"):
956 ebuildlist.append(pf)
957 cpv = "%s/%s" % (catdir, pf)
959 myaux = dict(zip(allvars, portdb.aux_get(cpv, allvars)))
961 stats["ebuild.syntax"] += 1
962 fails["ebuild.syntax"].append(os.path.join(x, y))
965 stats["ebuild.output"] += 1
966 fails["ebuild.output"].append(os.path.join(x, y))
968 if not portage.eapi_is_supported(myaux["EAPI"]):
969 stats["EAPI.unsupported"] += 1
970 fails["EAPI.unsupported"].append(os.path.join(x, y))
972 pkgs[pf] = Package(cpv=cpv, metadata=myaux,
973 root_config=root_config)
975 # Sort ebuilds in ascending order for the KEYWORDS.dropped check.
977 for i in range(len(ebuildlist)):
978 ebuild_split = portage.pkgsplit(ebuildlist[i])
979 pkgsplits[ebuild_split] = ebuildlist[i]
980 ebuildlist[i] = ebuild_split
981 ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp))
982 for i in range(len(ebuildlist)):
983 ebuildlist[i] = pkgsplits[ebuildlist[i]]
988 if len(pkgs) != len(ebuildlist):
989 # If we can't access all the metadata then it's totally unsafe to
990 # commit since there's no way to generate a correct Manifest.
991 # Do not try to do any more QA checks on this package since missing
992 # metadata leads to false positives for several checks, and false
993 # positives confuse users.
997 for y in checkdirlist:
998 m = disallowed_filename_chars_re.search(y.strip(os.sep))
1000 stats["file.name"] += 1
1001 fails["file.name"].append("%s/%s: char '%s'" % \
1002 (checkdir, y, m.group(0)))
1004 if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")):
1008 for l in codecs.open(_unicode_encode(os.path.join(checkdir, y),
1009 encoding=_encodings['fs'], errors='strict'),
1010 mode='r', encoding=_encodings['repo.content']):
1012 except UnicodeDecodeError as ue:
1013 stats["file.UTF8"] += 1
1014 s = ue.object[:ue.start]
1018 s = s[s.rfind("\n") + 1:]
1019 fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s))
1021 if vcs == "git" and check_ebuild_notadded:
1022 myf = os.popen("git ls-files --others %s" % \
1023 (portage._shell_quote(checkdir_relative),))
1025 if l[:-1][-7:] == ".ebuild":
1026 stats["ebuild.notadded"] += 1
1027 fails["ebuild.notadded"].append(
1028 os.path.join(x, os.path.basename(l[:-1])))
1031 if vcs in ("cvs", "svn") and check_ebuild_notadded:
1034 myf=open(checkdir+"/CVS/Entries","r")
1036 myf = os.popen("svn status --depth=files --verbose " + checkdir)
1037 myl = myf.readlines()
1043 splitl=l[1:].split("/")
1046 if splitl[0][-7:]==".ebuild":
1047 eadded.append(splitl[0][:-7])
1052 # tree conflict, new in subversion 1.6
1055 if l[-7:] == ".ebuild":
1056 eadded.append(os.path.basename(l[:-7]))
1058 myf = os.popen("svn status " + checkdir)
1063 l = l.rstrip().split(' ')[-1]
1064 if l[-7:] == ".ebuild":
1065 eadded.append(os.path.basename(l[:-7]))
1067 if options.mode == 'commit' and vcs == "cvs":
1068 stats["CVS/Entries.IO_error"] += 1
1069 fails["CVS/Entries.IO_error"].append(checkdir+"/CVS/Entries")
1070 if options.mode == 'commit' and vcs == "svn":
1071 stats["svn.IO_error"] += 1
1072 fails["svn.IO_error"].append(checkdir+"svn info")
1075 mf = Manifest(checkdir, repoman_settings["DISTDIR"])
1076 mydigests=mf.getTypeDigests("DIST")
1078 fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb)
1080 src_uri_error = False
1081 for mykey in fetchlist_dict:
1083 myfiles_all.extend(fetchlist_dict[mykey])
1084 except portage.exception.InvalidDependString as e:
1085 src_uri_error = True
1087 portdb.aux_get(mykey, ["SRC_URI"])
1089 # This will be reported as an "ebuild.syntax" error.
1092 stats["SRC_URI.syntax"] = stats["SRC_URI.syntax"] + 1
1093 fails["SRC_URI.syntax"].append(
1094 "%s.ebuild SRC_URI: %s" % (mykey, e))
1096 if not src_uri_error:
1097 # This test can produce false positives if SRC_URI could not
1098 # be parsed for one or more ebuilds. There's no point in
1099 # producing a false error here since the root cause will
1100 # produce a valid error elsewhere, such as "SRC_URI.syntax"
1101 # or "ebuild.sytax".
1102 myfiles_all = set(myfiles_all)
1103 for entry in mydigests:
1104 if entry not in myfiles_all:
1105 stats["digest.unused"] += 1
1106 fails["digest.unused"].append(checkdir+"::"+entry)
1107 for entry in myfiles_all:
1108 if entry not in mydigests:
1109 stats["digest.missing"] += 1
1110 fails["digest.missing"].append(checkdir+"::"+entry)
1113 if os.path.exists(checkdir+"/files"):
1114 filesdirlist=os.listdir(checkdir+"/files")
1116 # recurse through files directory
1117 # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory.
1119 y = filesdirlist.pop(0)
1120 relative_path = os.path.join(x, "files", y)
1121 full_path = os.path.join(repodir, relative_path)
1123 mystat = os.stat(full_path)
1124 except OSError as oe:
1126 # don't worry about it. it likely was removed via fix above.
1130 if S_ISDIR(mystat.st_mode):
1131 # !!! VCS "portability" alert! Need some function isVcsDir() or alike !!!
1132 if y == "CVS" or y == ".svn":
1134 for z in os.listdir(checkdir+"/files/"+y):
1135 if z == "CVS" or z == ".svn":
1137 filesdirlist.append(y+"/"+z)
1138 # Current policy is no files over 20 KiB, these are the checks. File size between
1139 # 20 KiB and 60 KiB causes a warning, while file size over 60 KiB causes an error.
1140 elif mystat.st_size > 61440:
1141 stats["file.size.fatal"] += 1
1142 fails["file.size.fatal"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1143 elif mystat.st_size > 20480:
1144 stats["file.size"] += 1
1145 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y)
1147 m = disallowed_filename_chars_re.search(
1148 os.path.basename(y.rstrip(os.sep)))
1150 stats["file.name"] += 1
1151 fails["file.name"].append("%s/files/%s: char '%s'" % \
1152 (checkdir, y, m.group(0)))
1154 if desktop_file_validate and desktop_pattern.match(y):
1155 status, cmd_output = subprocess_getstatusoutput(
1156 "'%s' '%s'" % (desktop_file_validate, full_path))
1157 if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK:
1158 # Note: in the future we may want to grab the
1159 # warnings in addition to the errors. We're
1160 # just doing errors now since we don't want
1161 # to generate too much noise at first.
1162 error_re = re.compile(r'.*\s*error:\s*(.*)')
1163 for line in cmd_output.splitlines():
1164 error_match = error_re.match(line)
1165 if error_match is None:
1167 stats["desktop.invalid"] += 1
1168 fails["desktop.invalid"].append(
1169 relative_path + ': %s' % error_match.group(1))
1172 # Note: We don't use ChangeLogs in distributed SCMs.
1173 # It will be generated on server side from scm log,
1174 # before package moves to the rsync server.
1175 # This is needed because we try to avoid merge collisions.
1176 if vcs not in ( "git", ) and "ChangeLog" not in checkdirlist:
1177 stats["changelog.missing"]+=1
1178 fails["changelog.missing"].append(x+"/ChangeLog")
1180 #metadata.xml file check
1183 if "metadata.xml" not in checkdirlist:
1184 stats["metadata.missing"]+=1
1185 fails["metadata.missing"].append(x+"/metadata.xml")
1186 #metadata.xml parse check
1188 metadata_bad = False
1190 # load USE flags from metadata.xml
1192 f = open(os.path.join(checkdir, "metadata.xml"))
1193 utilities.parse_metadata_use(f, muselist)
1195 except (EnvironmentError, ParseError) as e:
1197 stats["metadata.bad"] += 1
1198 fails["metadata.bad"].append("%s/metadata.xml: %s" % (x, e))
1201 #Only carry out if in package directory or check forced
1202 if xmllint_capable and not metadata_bad:
1203 # xmlint can produce garbage output even on success, so only dump
1204 # the ouput when it fails.
1205 st, out = subprocess_getstatusoutput(
1206 "xmllint --nonet --noout --dtdvalid '%s' '%s'" % \
1207 (metadata_dtd, os.path.join(checkdir, "metadata.xml")))
1209 print(red("!!!") + " metadata.xml is invalid:")
1210 for z in out.splitlines():
1211 print(red("!!! ")+z)
1212 stats["metadata.bad"]+=1
1213 fails["metadata.bad"].append(x+"/metadata.xml")
1216 muselist = frozenset(muselist)
1218 changelog_path = os.path.join(checkdir_relative, "ChangeLog")
1219 changelog_modified = changelog_path in modified_changelogs
1222 # detect unused local USE-descriptions
1223 used_useflags = set()
1225 for y in ebuildlist:
1226 relative_path = os.path.join(x, y + ".ebuild")
1227 full_path = os.path.join(repodir, relative_path)
1228 ebuild_path = y + ".ebuild"
1230 ebuild_path = os.path.join(pkgdir, ebuild_path)
1232 ebuild_path = os.path.join(catdir, ebuild_path)
1233 ebuild_path = os.path.join(".", ebuild_path)
1234 if not changelog_modified and ebuild_path in new_ebuilds:
1235 stats['changelog.ebuildadded'] += 1
1236 fails['changelog.ebuildadded'].append(relative_path)
1238 if stat.S_IMODE(os.stat(full_path).st_mode) & 0o111:
1239 stats["file.executable"] += 1
1240 fails["file.executable"].append(x+"/"+y+".ebuild")
1241 if vcs in ("cvs", "svn") and check_ebuild_notadded and y not in eadded:
1242 #ebuild not added to vcs
1243 stats["ebuild.notadded"]=stats["ebuild.notadded"]+1
1244 fails["ebuild.notadded"].append(x+"/"+y+".ebuild")
1245 myesplit=portage.pkgsplit(y)
1246 if myesplit is None or myesplit[0] != x.split("/")[-1]:
1247 stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1
1248 fails["ebuild.invalidname"].append(x+"/"+y+".ebuild")
1250 elif myesplit[0]!=pkgdir:
1251 print(pkgdir,myesplit[0])
1252 stats["ebuild.namenomatch"]=stats["ebuild.namenomatch"]+1
1253 fails["ebuild.namenomatch"].append(x+"/"+y+".ebuild")
1259 for k, msgs in pkg.invalid.items():
1261 stats[k] = stats[k] + 1
1262 fails[k].append("%s %s" % (relative_path, msg))
1265 myaux = pkg.metadata
1266 eapi = myaux["EAPI"]
1267 inherited = pkg.inherited
1268 live_ebuild = live_eclasses.intersection(inherited)
1270 for k, v in myaux.items():
1271 if not isinstance(v, basestring):
1273 m = non_ascii_re.search(v)
1275 stats["variable.invalidchar"] += 1
1276 fails["variable.invalidchar"].append(
1277 ("%s: %s variable contains non-ASCII " + \
1278 "character at position %s") % \
1279 (relative_path, k, m.start() + 1))
1281 if not src_uri_error:
1282 # Check that URIs don't reference a server from thirdpartymirrors.
1283 for uri in portage.flatten(portage.dep.use_reduce(
1284 portage.dep.paren_reduce(myaux["SRC_URI"]), matchall=True)):
1285 contains_mirror = False
1286 for mirror in thirdpartymirrors:
1287 if uri.startswith(mirror):
1288 contains_mirror = True
1290 if not contains_mirror:
1293 stats["SRC_URI.mirror"] += 1
1294 fails["SRC_URI.mirror"].append(
1295 "%s: '%s' found in thirdpartymirrors" % \
1296 (relative_path, mirror))
1298 # The Package class automatically evaluates USE conditionals.
1299 for myprovide in portage.flatten(portage.dep.use_reduce(
1300 portage.dep.paren_reduce(pkg.metadata['PROVIDE']), matchall=1)):
1301 prov_cp = portage.dep_getkey(myprovide)
1302 if prov_cp != myprovide:
1303 stats["virtual.versioned"]+=1
1304 fails["virtual.versioned"].append(x+"/"+y+".ebuild: "+myprovide)
1305 prov_pkg = portage.dep_getkey(
1306 portage.best(portdb.xmatch("match-all", prov_cp)))
1307 if prov_cp == prov_pkg:
1308 stats["virtual.exists"]+=1
1309 fails["virtual.exists"].append(x+"/"+y+".ebuild: "+prov_cp)
1311 for pos, missing_var in enumerate(missingvars):
1312 if not myaux.get(missing_var):
1313 if catdir == "virtual" and \
1314 missing_var in ("HOMEPAGE", "LICENSE"):
1316 if live_ebuild and missing_var == "KEYWORDS":
1318 myqakey=missingvars[pos]+".missing"
1319 stats[myqakey]=stats[myqakey]+1
1320 fails[myqakey].append(x+"/"+y+".ebuild")
1322 # 14 is the length of DESCRIPTION=""
1323 if len(myaux['DESCRIPTION']) > max_desc_len:
1324 stats['DESCRIPTION.toolong'] += 1
1325 fails['DESCRIPTION.toolong'].append(
1326 "%s: DESCRIPTION is %d characters (max %d)" % \
1327 (relative_path, len(myaux['DESCRIPTION']), max_desc_len))
1329 keywords = myaux["KEYWORDS"].split()
1330 stable_keywords = []
1331 for keyword in keywords:
1332 if not keyword.startswith("~") and \
1333 not keyword.startswith("-"):
1334 stable_keywords.append(keyword)
1336 if ebuild_path in new_ebuilds:
1337 stable_keywords.sort()
1338 stats["KEYWORDS.stable"] += 1
1339 fails["KEYWORDS.stable"].append(
1340 x + "/" + y + ".ebuild added with stable keywords: %s" % \
1341 " ".join(stable_keywords))
1343 ebuild_archs = set(kw.lstrip("~") for kw in keywords \
1344 if not kw.startswith("-"))
1346 previous_keywords = slot_keywords.get(myaux["SLOT"])
1347 if previous_keywords is None:
1348 slot_keywords[myaux["SLOT"]] = set()
1349 elif not live_ebuild:
1350 dropped_keywords = previous_keywords.difference(ebuild_archs)
1351 if dropped_keywords:
1352 stats["KEYWORDS.dropped"] += 1
1353 fails["KEYWORDS.dropped"].append(
1354 relative_path + ": %s" % \
1355 " ".join(sorted(dropped_keywords)))
1357 slot_keywords[myaux["SLOT"]].update(ebuild_archs)
1359 # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics
1360 if "-*" in keywords:
1368 stats["KEYWORDS.stupid"] += 1
1369 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild")
1372 Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should
1373 not be allowed to be marked stable
1376 bad_stable_keywords = []
1377 for keyword in keywords:
1378 if not keyword.startswith("~") and \
1379 not keyword.startswith("-"):
1380 bad_stable_keywords.append(keyword)
1382 if bad_stable_keywords:
1383 stats["LIVEVCS.stable"] += 1
1384 fails["LIVEVCS.stable"].append(
1385 x + "/" + y + ".ebuild with stable keywords:%s " % \
1386 bad_stable_keywords)
1387 del bad_stable_keywords
1389 if keywords and not has_global_mask(pkg):
1390 stats["LIVEVCS.unmasked"] += 1
1391 fails["LIVEVCS.unmasked"].append(relative_path)
1393 if options.ignore_arches:
1394 arches = [[repoman_settings["ARCH"], repoman_settings["ARCH"],
1395 repoman_settings["ACCEPT_KEYWORDS"].split()]]
1398 for keyword in myaux["KEYWORDS"].split():
1399 if (keyword[0]=="-"):
1401 elif (keyword[0]=="~"):
1402 arches.append([keyword, keyword[1:], [keyword[1:], keyword]])
1404 arches.append([keyword, keyword, [keyword]])
1407 baddepsyntax = False
1408 badlicsyntax = False
1409 badprovsyntax = False
1410 catpkg = catdir+"/"+y
1411 myiuse = set(repoman_settings.archlist())
1412 for myflag in myaux["IUSE"].split():
1413 if myflag.startswith("+"):
1417 inherited_java_eclass = "java-pkg-2" in inherited or \
1418 "java-pkg-opt-2" in inherited
1419 operator_tokens = set(["||", "(", ")"])
1420 type_list, badsyntax = [], []
1421 for mytype in ("DEPEND", "RDEPEND", "PDEPEND",
1422 "LICENSE", "PROPERTIES", "PROVIDE"):
1423 mydepstr = myaux[mytype]
1425 if mydepstr.find(" ?") != -1:
1426 badsyntax.append("'?' preceded by space")
1429 # Missing closing parenthesis will result in a ValueError
1430 mydeplist = portage.dep.paren_reduce(mydepstr)
1431 # Missing opening parenthesis will result in a final "" element
1432 if "" in mydeplist or "(" in mydeplist:
1435 badsyntax.append("parenthesis mismatch")
1437 except portage.exception.InvalidDependString as e:
1438 badsyntax.append(str(e))
1443 portage.dep.use_reduce(mydeplist, matchall=1)
1444 except portage.exception.InvalidDependString as e:
1445 badsyntax.append(str(e))
1447 for token in operator_tokens:
1448 if mydepstr.startswith(token+" "):
1449 myteststr = mydepstr[len(token):]
1451 myteststr = mydepstr
1452 if myteststr.endswith(" "+token):
1453 myteststr = myteststr[:-len(token)]
1454 while myteststr.find(" "+token+" ") != -1:
1455 myteststr = " ".join(myteststr.split(" "+token+" ", 1))
1456 if myteststr.find(token) != -1:
1457 badsyntax.append("'%s' not separated by space" % (token))
1459 if mytype in ("DEPEND", "RDEPEND", "PDEPEND"):
1460 for token in mydepstr.split():
1461 if token in operator_tokens or \
1463 if token == "test?" and \
1464 mytype in ("RDEPEND", "PDEPEND"):
1465 stats[mytype + '.suspect'] += 1
1466 fails[mytype + '.suspect'].append(relative_path + \
1467 ": 'test?' USE conditional in %s" % mytype)
1470 atom = portage.dep.Atom(token)
1471 except portage.exception.InvalidAtom:
1472 badsyntax.append("'%s' not a valid atom" % token)
1474 is_blocker = atom.blocker
1476 if mytype == "DEPEND" and \
1477 not is_blocker and \
1478 not inherited_java_eclass and \
1479 portage.dep_getkey(atom) == "virtual/jdk":
1480 stats['java.eclassesnotused'] += 1
1481 fails['java.eclassesnotused'].append(relative_path)
1482 elif mytype in ("PDEPEND", "RDEPEND"):
1483 if not is_blocker and \
1484 portage.dep_getkey(atom) in suspect_rdepend:
1485 stats[mytype + '.suspect'] += 1
1486 fails[mytype + '.suspect'].append(
1487 relative_path + ": '%s'" % atom)
1489 if portage.dep.dep_getslot(atom):
1490 stats['EAPI.incompatible'] += 1
1491 fails['EAPI.incompatible'].append(
1492 (relative_path + ": %s slot dependency" + \
1493 " not supported with EAPI='%s':" + \
1494 " '%s'") % (mytype, eapi, atom))
1495 if atom.use and eapi in ("0", "1"):
1496 stats['EAPI.incompatible'] += 1
1497 fails['EAPI.incompatible'].append(
1498 (relative_path + ": %s use dependency" + \
1499 " not supported with EAPI='%s':" + \
1500 " '%s'") % (mytype, eapi, atom))
1501 if atom.blocker and atom.blocker.overlap.forbid \
1502 and eapi in ("0", "1"):
1503 stats['EAPI.incompatible'] += 1
1504 fails['EAPI.incompatible'].append(
1505 (relative_path + ": %s new blocker syntax" + \
1506 " not supported with EAPI='%s':" + \
1507 " '%s'") % (mytype, eapi, atom))
1509 if atom.operator == "~" and \
1510 portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
1511 stats[mytype + '.badtilde'] += 1
1512 fails[mytype + '.badtilde'].append(
1513 (relative_path + ": %s uses the ~ operator"
1514 " with a non-zero revision:" + \
1515 " '%s'") % (mytype, atom))
1517 type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
1519 for m,b in zip(type_list, badsyntax):
1520 stats[m+".syntax"] += 1
1521 fails[m+".syntax"].append(catpkg+".ebuild "+m+": "+b)
1523 badlicsyntax = len([z for z in type_list if z == "LICENSE"])
1524 badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
1525 baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
1526 badlicsyntax = badlicsyntax > 0
1527 badprovsyntax = badprovsyntax > 0
1529 # uselist checks - global
1532 for myflag in myaux["IUSE"].split():
1533 flag_name = myflag.lstrip("+-")
1534 used_useflags.add(flag_name)
1535 if myflag != flag_name:
1536 default_use.append(myflag)
1537 if flag_name not in uselist:
1538 myuse.append(flag_name)
1540 # uselist checks - metadata
1541 for mypos in range(len(myuse)-1,-1,-1):
1542 if myuse[mypos] and (myuse[mypos] in muselist):
1545 if default_use and eapi == "0":
1546 for myflag in default_use:
1547 stats['EAPI.incompatible'] += 1
1548 fails['EAPI.incompatible'].append(
1549 (relative_path + ": IUSE defaults" + \
1550 " not supported with EAPI='%s':" + \
1551 " '%s'") % (eapi, myflag))
1553 for mypos in range(len(myuse)):
1554 stats["IUSE.invalid"]=stats["IUSE.invalid"]+1
1555 fails["IUSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1558 if not badlicsyntax:
1559 myuse = myaux["LICENSE"]
1560 # Parse the LICENSE variable, remove USE conditions and
1562 myuse=portage.dep.use_reduce(portage.dep.paren_reduce(myuse), matchall=1)
1563 myuse=portage.flatten(myuse)
1564 # Check each entry to ensure that it exists in PORTDIR's
1565 # license directory.
1566 for mypos in range(0,len(myuse)):
1567 # Need to check for "||" manually as no portage
1568 # function will remove it without removing values.
1569 if myuse[mypos] not in liclist and myuse[mypos] != "||":
1570 stats["LICENSE.invalid"]=stats["LICENSE.invalid"]+1
1571 fails["LICENSE.invalid"].append(x+"/"+y+".ebuild: %s" % myuse[mypos])
1574 myuse = myaux["KEYWORDS"].split()
1582 if myskey not in kwlist:
1583 stats["KEYWORDS.invalid"] += 1
1584 fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey)
1585 elif myskey not in profiles:
1586 stats["KEYWORDS.invalid"] += 1
1587 fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s (profile invalid)" % mykey)
1592 myrestrict = portage.dep.use_reduce(
1593 portage.dep.paren_reduce(myaux["RESTRICT"]), matchall=1)
1594 except portage.exception.InvalidDependString as e:
1595 stats["RESTRICT.syntax"] = stats["RESTRICT.syntax"] + 1
1596 fails["RESTRICT.syntax"].append(
1597 "%s: RESTRICT: %s" % (relative_path, e))
1600 myrestrict = set(portage.flatten(myrestrict))
1601 mybadrestrict = myrestrict.difference(valid_restrict)
1603 stats["RESTRICT.invalid"] += len(mybadrestrict)
1604 for mybad in mybadrestrict:
1605 fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
1607 relative_path = os.path.join(x, y + ".ebuild")
1608 full_path = os.path.join(repodir, relative_path)
1610 # All ebuilds should have utf_8 encoding.
1611 f = codecs.open(_unicode_encode(full_path,
1612 encoding=_encodings['fs'], errors='strict'),
1613 mode='r', encoding=_encodings['repo.content'])
1615 for check_name, e in run_checks(f, pkg):
1616 stats[check_name] += 1
1617 fails[check_name].append(relative_path + ': %s' % e)
1620 except UnicodeDecodeError:
1621 # A file.UTF8 failure will have already been recorded above.
1625 # The dep_check() calls are the most expensive QA test. If --force
1626 # is enabled, there's no point in wasting time on these since the
1627 # user is intent on forcing the commit anyway.
1630 for keyword,arch,groups in arches:
1632 if arch not in profiles:
1633 # A missing profile will create an error further down
1634 # during the KEYWORDS verification.
1637 for prof in profiles[arch]:
1639 if prof.status not in ("stable", "dev") or \
1640 prof.status == "dev" and not options.include_dev:
1643 dep_settings = arch_caches.get(prof.sub_path)
1644 if dep_settings is None:
1645 dep_settings = portage.config(
1646 config_profile_path=prof.abs_path,
1647 config_incrementals=portage.const.INCREMENTALS,
1650 if options.without_mask:
1651 dep_settings.pmaskdict.clear()
1652 arch_caches[prof.sub_path] = dep_settings
1655 # Protect ACCEPT_KEYWORDS from config.regenerate()
1657 dep_settings.incrementals.remove("ACCEPT_KEYWORDS")
1661 xmatch_cache_key = (prof.sub_path, tuple(groups))
1662 xcache = arch_xmatch_caches.get(xmatch_cache_key)
1666 xcache = portdb.xcache
1667 xcache.update(shared_xmatch_caches)
1668 arch_xmatch_caches[xmatch_cache_key] = xcache
1670 trees["/"]["porttree"].settings = dep_settings
1671 portdb.mysettings = dep_settings
1672 portdb.xcache = xcache
1673 # for package.use.mask support inside dep_check
1674 dep_settings.setcpv(pkg)
1675 dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
1676 # just in case, prevent config.reset() from nuking these.
1677 dep_settings.backup_changes("ACCEPT_KEYWORDS")
1679 for myprovide in myaux["PROVIDE"].split():
1680 prov_cp = portage.dep_getkey(myprovide)
1681 if prov_cp not in dep_settings.getvirtuals():
1682 stats["virtual.unavailable"]+=1
1683 fails["virtual.unavailable"].append("%s: %s(%s) %s" % \
1684 (relative_path, keyword, prof.sub_path, prov_cp))
1686 if not baddepsyntax:
1687 ismasked = os.path.join(catdir, y) not in \
1688 portdb.xmatch("list-visible", x)
1690 if not have_pmasked:
1691 have_pmasked = bool(dep_settings._getMaskAtom(
1692 pkg.cpv, pkg.metadata))
1693 if options.ignore_masked:
1695 #we are testing deps for a masked package; give it some lee-way
1697 matchmode = "minimum-all"
1700 matchmode = "minimum-visible"
1702 if not have_dev_keywords:
1703 have_dev_keywords = \
1704 bool(dev_keywords.intersection(keywords))
1706 if prof.status == "dev":
1707 suffix=suffix+"indev"
1709 for mytype,mypos in [["DEPEND",len(missingvars)],["RDEPEND",len(missingvars)+1],["PDEPEND",len(missingvars)+2]]:
1711 mykey=mytype+".bad"+suffix
1712 myvalue = myaux[mytype]
1716 success, atoms = portage.dep_check(myvalue, portdb,
1717 dep_settings, use="all", mode=matchmode,
1722 #we have some unsolvable deps
1723 #remove ! deps, which always show up as unsatisfiable
1724 atoms = [str(atom) for atom in atoms if not atom.blocker]
1725 #if we emptied out our list, continue:
1728 stats[mykey]=stats[mykey]+1
1729 fails[mykey].append("%s: %s(%s) %s" % \
1730 (relative_path, keyword,
1731 prof.sub_path, repr(atoms)))
1733 stats[mykey]=stats[mykey]+1
1734 fails[mykey].append("%s: %s(%s) %s" % \
1735 (relative_path, keyword,
1736 prof.sub_path, repr(atoms)))
1738 # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped
1739 # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely.
1740 #if not portage.portdb.xmatch("bestmatch-visible",x):
1741 # stats["ebuild.nostable"]+=1
1742 # fails["ebuild.nostable"].append(x)
1743 if allmasked and repolevel == 3:
1744 stats["ebuild.allmasked"]+=1
1745 fails["ebuild.allmasked"].append(x)
1747 # check if there are unused local USE-descriptions in metadata.xml
1748 for myflag in muselist.difference(used_useflags):
1749 stats["metadata.warning"] += 1
1750 fails["metadata.warning"].append(
1751 "%s/metadata.xml: unused local USE-description: '%s'" % \
1754 if options.mode == "manifest":
1757 #dofail will be set to 1 if we have failed in at least one non-warning category
1759 #dowarn will be set to 1 if we tripped any warnings
1761 #dofull will be set if we should print a "repoman full" informational message
1762 dofull = options.mode != 'full'
1768 if x not in qawarnings:
1772 (dowarn and not (options.quiet or options.mode == "scan")):
1775 # Save QA output so that it can be conveniently displayed
1776 # in $EDITOR while the user creates a commit message.
1777 # Otherwise, the user would not be able to see this output
1778 # once the editor has taken over the screen.
1779 qa_output = StringIO()
1780 style_file = ConsoleStyleFile(sys.stdout)
1781 if options.mode == 'commit' and \
1782 (not commitmessage or not commitmessage.strip()):
1783 style_file.write_listener = qa_output
1784 console_writer = StyleWriter(file=style_file, maxcol=9999)
1785 console_writer.style_listener = style_file.new_styles
1787 f = formatter.AbstractFormatter(console_writer)
1789 utilities.format_qa_output(f, stats, fails, dofull, dofail, options, qawarnings)
1792 del console_writer, f, style_file
1793 qa_output = qa_output.getvalue()
1794 qa_output = qa_output.splitlines(True)
1796 def grouplist(mylist,seperator="/"):
1797 """(list,seperator="/") -- Takes a list of elements; groups them into
1798 same initial element categories. Returns a dict of {base:[sublist]}
1799 From: ["blah/foo","spork/spatula","blah/weee/splat"]
1800 To: {"blah":["foo","weee/splat"], "spork":["spatula"]}"""
1803 xs=x.split(seperator)
1806 if xs[0] not in mygroups:
1807 mygroups[xs[0]]=[seperator.join(xs[1:])]
1809 mygroups[xs[0]]+=[seperator.join(xs[1:])]
1812 suggest_ignore_masked = False
1813 suggest_include_dev = False
1815 if have_pmasked and not (options.without_mask or options.ignore_masked):
1816 suggest_ignore_masked = True
1817 if have_dev_keywords and not options.include_dev:
1818 suggest_include_dev = True
1820 if suggest_ignore_masked or suggest_include_dev:
1822 if suggest_ignore_masked:
1823 print(bold("Note: use --without-mask to check " + \
1824 "KEYWORDS on dependencies of masked packages"))
1826 if suggest_include_dev:
1827 print(bold("Note: use --include-dev (-d) to check " + \
1828 "dependencies for 'dev' profiles"))
1831 if options.mode != 'commit':
1833 print(bold("Note: type \"repoman full\" for a complete listing."))
1834 if dowarn and not dofail:
1835 print(green("RepoMan sez:"),"\"You're only giving me a partial QA payment?\n I'll take it this time, but I'm not happy.\"")
1837 print(green("RepoMan sez:"),"\"If everyone were like you, I'd be out of business!\"")
1839 print(turquoise("Please fix these important QA issues first."))
1840 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
1843 if dofail and can_force and options.force and not options.pretend:
1844 print(green("RepoMan sez:") + \
1845 " \"You want to commit even with these QA issues?\n" + \
1846 " I'll take it this time, but I'm not happy.\"\n")
1848 if options.force and not can_force:
1849 print(bad("The --force option has been disabled due to extraordinary issues."))
1850 print(turquoise("Please fix these important QA issues first."))
1851 print(green("RepoMan sez:"),"\"Make your QA payment on time and you'll never see the likes of me.\"\n")
1855 print(green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n")
1860 myvcstree=portage.cvstree.getentries("./",recursive=1)
1861 myunadded=portage.cvstree.findunadded(myvcstree,recursive=1,basedir="./")
1862 except SystemExit as e:
1863 raise # TODO propogate this
1865 err("Error retrieving CVS tree; exiting.")
1868 svnstatus=os.popen("svn status --no-ignore").readlines()
1869 myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ]
1870 except SystemExit as e:
1871 raise # TODO propogate this
1873 err("Error retrieving SVN info; exiting.")
1875 # get list of files not under version control or missing
1876 myf = os.popen("git ls-files --others")
1877 myunadded = [ "./" + elem[:-1] for elem in myf ]
1882 for x in range(len(myunadded)-1,-1,-1):
1883 xs=myunadded[x].split("/")
1885 print("!!! files dir is not added! Please correct this.")
1887 elif xs[-1]=="Manifest":
1888 # It's a manifest... auto add
1889 myautoadd+=[myunadded[x]]
1893 print(">>> Auto-Adding missing Manifest(s)...")
1896 print("(cvs add "+" ".join(myautoadd)+")")
1898 print("(svn add "+" ".join(myautoadd)+")")
1900 print("(git add "+" ".join(myautoadd)+")")
1904 retval=os.system("cvs add "+" ".join(myautoadd))
1906 retval=os.system("svn add "+" ".join(myautoadd))
1908 retval=os.system("git add "+" ".join(myautoadd))
1910 writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \
1911 (vcs, retval), level=logging.ERROR, noiselevel=-1)
1915 print(red("!!! The following files are in your local tree but are not added to the master"))
1916 print(red("!!! tree. Please remove them from the local tree or add them to the master tree."))
1924 mycvstree = cvstree.getentries("./", recursive=1)
1925 mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./")
1926 mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./")
1927 myremoved=portage.cvstree.findremoved(mycvstree,recursive=1,basedir="./")
1928 bin_blob_pattern = re.compile("^-kb$")
1929 no_expansion = set(portage.cvstree.findoption(mycvstree, bin_blob_pattern,
1930 recursive=1, basedir="./"))
1934 svnstatus = os.popen("svn status").readlines()
1935 mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ]
1936 mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ]
1937 myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D") ]
1938 # in contrast to CVS, SVN expands nothing by default.
1939 # bin_blobs historically
1940 # were just there to see what files need to be checked for
1941 # keyword expansion, which is exactly what we do here, so
1942 # slightly change the semantic meaning of "bin_blob"... In the
1943 # future we could store which keyword is expanded.
1944 props = os.popen("svn propget -R svn:keywords").readlines()
1946 # For files with multiple props set, props are delimited by newlines,
1947 # so exclude lines that don't contain " - " since each of those lines
1948 # only a contain props for a file listed on a previous line.
1949 expansion = set("./" + prop.split(" - ")[0] \
1950 for prop in props if " - " in prop)
1953 strip_levels = repolevel - 1
1955 mychanged = os.popen("git diff-index --name-only --diff-filter=M HEAD").readlines()
1957 mychanged = [elem[repo_subdir_len:] for elem in mychanged \
1958 if elem[:repo_subdir_len] == repo_subdir]
1959 mychanged = ["./" + elem[:-1] for elem in mychanged]
1961 mynew = os.popen("git diff-index --name-only --diff-filter=A HEAD").readlines()
1963 mynew = [elem[repo_subdir_len:] for elem in mynew \
1964 if elem[:repo_subdir_len] == repo_subdir]
1965 mynew = ["./" + elem[:-1] for elem in mynew]
1967 myremoved = os.popen("git diff-index --name-only --diff-filter=D HEAD").readlines()
1969 myremoved = [elem[repo_subdir_len:] for elem in myremoved \
1970 if elem[:repo_subdir_len] == repo_subdir]
1971 myremoved = ["./" + elem[:-1] for elem in myremoved]
1974 if not (mychanged or mynew or myremoved):
1975 print(green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"")
1977 print("(Didn't find any changed files...)")
1981 # Manifests need to be regenerated after all other commits, so don't commit
1982 # them now even if they have changed.
1985 for f in mychanged + mynew:
1986 if "Manifest" == os.path.basename(f):
1991 myupdates.difference_update(myremoved)
1992 myupdates = list(myupdates)
1993 mymanifests = list(mymanifests)
1996 headerstring = "'\$(Header|Id)"
1997 headerstring += ".*\$'"
1998 for myfile in myupdates:
2000 # for CVS, no_expansion contains files that are excluded from expansion
2002 if myfile in no_expansion:
2005 # for SVN, expansion contains files that are included in expansion
2007 if myfile not in expansion:
2010 myout = subprocess_getstatusoutput("egrep -q "+headerstring+" "+myfile)
2012 myheaders.append(myfile)
2014 print("* %s files being committed..." % green(str(len(myupdates))), end=' ')
2016 # With git, there's never any keyword expansion, so there's
2017 # no need to regenerate manifests and all files will be
2018 # committed in one big commit at the end.
2021 print("%s have headers that will change." % green(str(len(myheaders))))
2022 print("* Files with headers will cause the " + \
2023 "manifests to be made and recommited.")
2024 logging.info("myupdates: %s", myupdates)
2025 logging.info("myheaders: %s", myheaders)
2027 commitmessage = options.commitmsg
2028 if options.commitmsgfile:
2030 f = open(options.commitmsgfile)
2031 commitmessage = f.read()
2034 except (IOError, OSError) as e:
2035 if e.errno == errno.ENOENT:
2036 portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile)
2039 # We've read the content so the file is no longer needed.
2040 commitmessagefile = None
2041 if not commitmessage or not commitmessage.strip():
2043 editor = os.environ.get("EDITOR")
2044 if editor and utilities.editor_is_executable(editor):
2045 commitmessage = utilities.get_commit_message_with_editor(
2046 editor, message=qa_output)
2048 commitmessage = utilities.get_commit_message_with_stdin()
2049 except KeyboardInterrupt:
2051 if not commitmessage or not commitmessage.strip():
2052 print("* no commit message? aborting commit.")
2054 commitmessage = commitmessage.rstrip()
2055 portage_version = getattr(portage, "VERSION", None)
2056 if portage_version is None:
2057 sys.stderr.write("Failed to insert portage version in message!\n")
2059 portage_version = "Unknown"
2060 unameout = platform.system() + " "
2061 if platform.system() in ["Darwin", "SunOS"]:
2062 unameout += platform.processor()
2064 unameout += platform.machine()
2065 commitmessage += "\n(Portage version: %s/%s/%s" % \
2066 (portage_version, vcs, unameout)
2068 commitmessage += ", RepoMan options: --force"
2069 commitmessage += ")"
2071 if vcs != 'git' and (myupdates or myremoved):
2072 myfiles = myupdates + myremoved
2073 if not myheaders and "sign" not in repoman_settings.features:
2074 myfiles += mymanifests
2075 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2076 mymsg = os.fdopen(fd, "w")
2077 mymsg.write(commitmessage)
2081 print(green("Using commit message:"))
2082 print(green("------------------------------------------------------------------------------"))
2083 print(commitmessage)
2084 print(green("------------------------------------------------------------------------------"))
2087 # Having a leading ./ prefix on file paths can trigger a bug in
2088 # the cvs server when committing files to multiple directories,
2089 # so strip the prefix.
2090 myfiles = [f.lstrip("./") for f in myfiles]
2093 commit_cmd.extend(vcs_global_opts)
2094 commit_cmd.append("commit")
2095 commit_cmd.extend(vcs_local_opts)
2096 commit_cmd.extend(["-F", commitmessagefile])
2097 commit_cmd.extend(myfiles)
2101 print("(%s)" % (" ".join(commit_cmd),))
2103 retval = spawn(commit_cmd, env=os.environ)
2104 if retval != os.EX_OK:
2105 writemsg_level(("!!! Exiting on %s (shell) " + \
2106 "error code: %s\n") % (vcs, retval),
2107 level=logging.ERROR, noiselevel=-1)
2111 os.unlink(commitmessagefile)
2115 # Setup the GPG commands
2116 def gpgsign(filename):
2117 if "PORTAGE_GPG_KEY" not in repoman_settings:
2118 raise portage.exception.MissingParameter("PORTAGE_GPG_KEY is unset!")
2119 if "PORTAGE_GPG_DIR" not in repoman_settings:
2120 if "HOME" in os.environ:
2121 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(os.environ["HOME"], ".gnupg")
2122 logging.info("Automatically setting PORTAGE_GPG_DIR to %s" % repoman_settings["PORTAGE_GPG_DIR"])
2124 raise portage.exception.MissingParameter("PORTAGE_GPG_DIR is unset!")
2125 gpg_dir = repoman_settings["PORTAGE_GPG_DIR"]
2126 if gpg_dir.startswith("~") and "HOME" in os.environ:
2127 repoman_settings["PORTAGE_GPG_DIR"] = os.path.join(
2128 os.environ["HOME"], gpg_dir[1:].lstrip(os.path.sep))
2129 if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK):
2130 raise portage.exception.InvalidLocation(
2131 "Unable to access directory: PORTAGE_GPG_DIR='%s'" % \
2132 repoman_settings["PORTAGE_GPG_DIR"])
2133 gpgcmd = "gpg --sign --clearsign --yes "
2134 gpgcmd+= "--default-key "+repoman_settings["PORTAGE_GPG_KEY"]
2135 if "PORTAGE_GPG_DIR" in repoman_settings:
2136 gpgcmd += " --homedir "+repoman_settings["PORTAGE_GPG_DIR"]
2138 print("("+gpgcmd+" "+filename+")")
2140 rValue = os.system(gpgcmd+" "+filename)
2141 if rValue == os.EX_OK:
2142 os.rename(filename+".asc", filename)
2144 raise portage.exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status")
2146 # When files are removed and re-added, the cvs server will put /Attic/
2147 # inside the $Header path. This code detects the problem and corrects it
2148 # so that the Manifest will generate correctly. See bug #169500.
2149 from portage.util import write_atomic
2150 cvs_header = re.compile(r'^#\s*\$Header.*\$$')
2153 mylines = f.readlines()
2156 for i, line in enumerate(mylines):
2157 if cvs_header.match(line) and "/Attic/" in line:
2158 mylines[i] = line.replace("/Attic/", "/")
2161 write_atomic(x, "".join(mylines))
2163 manifest_commit_required = True
2164 if vcs != 'git' and (myupdates or myremoved):
2165 myfiles = myupdates + myremoved
2166 for x in range(len(myfiles)-1, -1, -1):
2167 if myfiles[x].count("/") < 4-repolevel:
2170 if repolevel==3: # In a package dir
2171 repoman_settings["O"] = startdir
2172 portage.digestgen([], repoman_settings, manifestonly=1,
2174 elif repolevel==2: # In a category dir
2177 if len(xs) < 4-repolevel:
2183 mydone.append(xs[0])
2184 repoman_settings["O"] = os.path.join(startdir, xs[0])
2185 if not os.path.isdir(repoman_settings["O"]):
2187 portage.digestgen([], repoman_settings, manifestonly=1,
2189 elif repolevel==1: # repo-cvsroot
2190 print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2193 if len(xs) < 4-repolevel:
2197 if "/".join(xs[:2]) in mydone:
2199 mydone.append("/".join(xs[:2]))
2200 repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1])
2201 if not os.path.isdir(repoman_settings["O"]):
2203 portage.digestgen([], repoman_settings, manifestonly=1,
2206 print(red("I'm confused... I don't know where I am!"))
2209 # Force an unsigned commit when more than one Manifest needs to be signed.
2210 if repolevel < 3 and "sign" in repoman_settings.features:
2212 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2213 mymsg = os.fdopen(fd, "w")
2214 mymsg.write(commitmessage)
2215 mymsg.write("\n (Unsigned Manifest commit)")
2219 commit_cmd.extend(vcs_global_opts)
2220 commit_cmd.append("commit")
2221 commit_cmd.extend(vcs_local_opts)
2222 commit_cmd.extend(["-F", commitmessagefile])
2223 commit_cmd.extend(f.lstrip("./") for f in mymanifests)
2227 print("(%s)" % (" ".join(commit_cmd),))
2229 retval = spawn(commit_cmd, env=os.environ)
2231 writemsg_level(("!!! Exiting on %s (shell) " + \
2232 "error code: %s\n") % (vcs, retval),
2233 level=logging.ERROR, noiselevel=-1)
2237 os.unlink(commitmessagefile)
2240 manifest_commit_required = False
2243 if "sign" in repoman_settings.features:
2245 myfiles = myupdates + myremoved + mymanifests
2247 if repolevel==3: # In a package dir
2248 repoman_settings["O"] = "."
2249 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2250 elif repolevel==2: # In a category dir
2254 if len(xs) < 4-repolevel:
2260 mydone.append(xs[0])
2261 repoman_settings["O"] = os.path.join(".", xs[0])
2262 if not os.path.isdir(repoman_settings["O"]):
2264 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2265 elif repolevel==1: # repo-cvsroot
2266 print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n")
2270 if len(xs) < 4-repolevel:
2274 if "/".join(xs[:2]) in mydone:
2276 mydone.append("/".join(xs[:2]))
2277 repoman_settings["O"] = os.path.join(".", xs[0], xs[1])
2278 if not os.path.isdir(repoman_settings["O"]):
2280 gpgsign(os.path.join(repoman_settings["O"], "Manifest"))
2281 except portage.exception.PortageException as e:
2282 portage.writemsg("!!! %s\n" % str(e))
2283 portage.writemsg("!!! Disabled FEATURES='sign'\n")
2287 # It's not safe to use the git commit -a option since there might
2288 # be some modified files elsewhere in the working tree that the
2289 # user doesn't want to commit. Therefore, call git update-index
2290 # in order to ensure that the index is updated with the latest
2291 # versions of all new and modified files in the relevant portion
2292 # of the working tree.
2293 myfiles = mymanifests + myupdates
2295 update_index_cmd = ["git", "update-index"]
2296 update_index_cmd.extend(f.lstrip("./") for f in myfiles)
2298 print("(%s)" % (" ".join(update_index_cmd),))
2300 retval = spawn(update_index_cmd, env=os.environ)
2301 if retval != os.EX_OK:
2302 writemsg_level(("!!! Exiting on %s (shell) " + \
2303 "error code: %s\n") % (vcs, retval),
2304 level=logging.ERROR, noiselevel=-1)
2307 if vcs == 'git' or manifest_commit_required or signed:
2309 myfiles = mymanifests[:]
2311 myfiles += myupdates
2312 myfiles += myremoved
2315 fd, commitmessagefile = tempfile.mkstemp(".repoman.msg")
2316 mymsg = os.fdopen(fd, "w")
2317 mymsg.write(commitmessage)
2319 mymsg.write("\n (Signed Manifest commit)")
2321 mymsg.write("\n (Unsigned Manifest commit)")
2325 if options.pretend and vcs is None:
2326 # substitute a bogus value for pretend output
2327 commit_cmd.append("cvs")
2329 commit_cmd.append(vcs)
2330 commit_cmd.extend(vcs_global_opts)
2331 commit_cmd.append("commit")
2332 commit_cmd.extend(vcs_local_opts)
2333 commit_cmd.extend(["-F", commitmessagefile])
2334 commit_cmd.extend(f.lstrip("./") for f in myfiles)
2338 print("(%s)" % (" ".join(commit_cmd),))
2340 retval = spawn(commit_cmd, env=os.environ)
2341 if retval != os.EX_OK:
2342 writemsg_level(("!!! Exiting on %s (shell) " + \
2343 "error code: %s\n") % (vcs, retval),
2344 level=logging.ERROR, noiselevel=-1)
2348 os.unlink(commitmessagefile)
2354 print("Commit complete.")
2356 print("repoman was too scared by not seeing any familiar version control file that he forgot to commit anything")
2357 print(green("RepoMan sez:"), "\"If everyone were like you, I'd be out of business!\"\n")