|7891 |Document interaction between config.setinst() and old-style virtuals |
|zmedico |preferences. |
|7892 |Reimplement portdbapi.visible() so that is works without using self.xmatch().|
|zmedico |This should be 100% compatible with the previous implementation and have |
| |comparable performance. By eliminating the xmatch() dependency, the code |
| |becomes more generically usable, like for visibility filtering of binary |
| |packages. It seems that it will be necessary to move this code outside of the|
| |dbapi since visiblity filtering is really a resolver level function and to |
| |solve things like bug #88613 will require a more flexible approach. since the|
| |current internal |
|7893 |Bug #142508 - Clear the screen each time before calling the "diff" command so|
|zmedico |that any remnants of the previous config are now shown with the current one. |
| |command so that |
|7894 |Add new repoman check classes using StringIO; possibly need some testing, |
|WarnerBro|obviously the old code was faster (1 iteration over the file), here we do one|
| |iteration per check, StringIO was to try and negate this by doing the checks |
| |in memory...how much of a price do we pay here? |
|7895 |Add Donnie's quote check, refactor other ebuild-content checks into classes |
|WarnerBro|and get them out of repoman in an attempt to start moving other checks out as|
| |well. Again wonder at the price of StringIO and N passes, is the readability |
| |worth it? |
|7896 |keep old repoman names so we don't confuse people |
|WarnerBro| |
|7897 |Delete some extra leaky vars, also after some testing I discovered read() is |
|WarnerBro|much faster than readlines(), see |
| |http://dev.gentoo.org/~antarus/projects/portage/stats/StringIOVsFiles.txt |
|7898 |Rename checks as the naming was overkill (they are all obviously checks being|
|WarnerBro|in the check module), add nesteddie check |
|7899 |Apply new checks, minor cleanup |
|WarnerBro| |
|7900 |Bug #194398 - Do not log "exiting successfully" in emerge.log unless the |
|zmedico |status really is successful. Otherwise, log "exiting unsuccessfully". Thanks |
| |to David Watzke. |
|7901 |Filter some false positives out of the EbuildQuote check. It might work well |
|zmedico |enough now so that we don't have to degrade it to a warning. |
svn path=/main/branches/prefix/; revision=7902
newconf = mrgconf
show_new_diff = 0
- while 1:
- if show_new_diff:
- os.system((self.options['diff']) % (conf['new'], mrgconf))
- show_new_diff = 0
- else:
- os.system((self.options['diff']) % (conf['current'], newconf))
+ while 1:
+ clear_screen()
+ if show_new_diff:
+ os.system((self.options['diff']) % (conf['new'], mrgconf))
+ show_new_diff = 0
+ else:
+ os.system((self.options['diff']) % (conf['current'], newconf))
print
print '>> (%i of %i) -- %s' % (count, len(confs), conf ['current'])
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
return ch
+def clear_screen():
+ try:
+ import curses
+ try:
+ curses.setupterm()
+ sys.stdout.write(curses.tigetstr("clear"))
+ sys.stdout.flush()
+ return
+ except curses.error:
+ pass
+ except ImportError:
+ pass
+ os.system("clear 2>/dev/null")
# run
d = dispatch ()
except ImportError:
import pickle
+try:
+ import cStringIO as StringIO
+except ImportError:
+ import StringIO
+
if not hasattr(__builtins__, "set"):
from sets import Set as set
import portage
except ImportError:
from os import path as osp
- sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
+ sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
import portage
del os.environ["PORTAGE_LEGACY_GLOBALS"]
+try:
+ from repoman.checks import EbuildWhitespaceCheck, EbuildHeaderCheck, EbuildQuoteCheck, \
+ EbuildAssignmentCheck
+except ImportError:
+ from os import path as osp
+ sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym'))
+ from repoman.checks import EbuildWhitespace, EbuildHeader, EbuildQuote, \
+ EbuildAssignment, EbuildNestedDie
+
import portage.checksum
import portage.const
import portage.dep
stats["usage.obsolete"] += 1
fails["usage.obsolete"].append("%s/%s.ebuild: not migrated to modular X" % (x, y))
- # this check needs work, it won't catch (\ndie)
- if not os.system("egrep '^[^#]*\([^)]*\<die\>' "+checkdir+"/"+y+".ebuild >/dev/null 2>&1"):
- stats["ebuild.nesteddie"]=stats["ebuild.nesteddie"]+1
- fails["ebuild.nesteddie"].append(x+"/"+y+".ebuild")
# uselist checks - global
myuse = []
default_use = []
stats["RESTRICT.invalid"] += len(mybadrestrict)
for mybad in mybadrestrict:
fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad)
-
- #syntax checks
- myear = time.gmtime(os.stat(checkdir+"/"+y+".ebuild")[ST_MTIME])[0]
- gentoo_copyright = re.compile(r'^# Copyright ((1999|200\d)-)?' + str(myear) + r' Gentoo Foundation')
- gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$')
- cvs_header = re.compile(r'^#\s*\$Header.*\$$')
- ignore_line = re.compile(r'(^$)|(^(\t)*#)')
- leading_spaces = re.compile(r'^[\S\t]')
- trailing_whitespace = re.compile(r'.*([\S]$)')
- readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=')
- line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$')
- linenum=0
- previous_line = None
- for line in input(checkdir+"/"+y+".ebuild"):
- linenum += 1
- # Gentoo copyright check
- if linenum == 1:
- match = gentoo_copyright.match(line)
- if not match:
- myerrormsg = "Copyright header Error. Possibly date related."
- stats["ebuild.badheader"] +=1
- fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
- # Gentoo license check
- elif linenum == 2:
- match = gentoo_license.match(line)
- if not match:
- myerrormsg = "Gentoo License Error."
- stats["ebuild.badheader"] +=1
- fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
- # CVS Header check
- elif linenum == 3:
- match = cvs_header.match(line)
- if not match:
- myerrormsg = "CVS Header Error."
- stats["ebuild.badheader"] +=1
- fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
- else:
- match = ignore_line.match(line)
- if not match:
- # Excluded Blank lines and full line comments. Good!
- # Leading Spaces Check
- match = leading_spaces.match(line)
- if not match:
- #Line has got leading spaces. Bad!
- myerrormsg = "Leading Space Syntax Error. Line %d" % linenum
- stats["ebuild.minorsyn"] +=1
- fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
- # Trailing whitespace check
- match = trailing_whitespace.match(line)
- if not match:
- #Line has got trailing whitespace. Bad!
- myerrormsg = "Trailing whitespace Syntax Error. Line %d" % linenum
- stats["ebuild.minorsyn"] +=1
- fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
- # Readonly variable assignment check
- match = readonly_assignment.match(line)
- # The regex can give a false positive for continued lines,
- # so we check the previous line to see if it was continued.
- if match and (not previous_line or not line_continuation.match(previous_line)):
- # invalid assignment, very bad!
- myerrormsg = "Readonly variable assignment to %s on line %d" % (match.group(2), linenum)
- stats["variable.readonly"] += 1
- fails["variable.readonly"].append(x+"/"+y+".ebuild: %s" % myerrormsg)
- previous_line = line
- del previous_line
+ # Syntax Checks
+ path = checkdir + '/' + y + '.ebuild'
+ myear = time.gmtime(os.stat(path)[ST_MTIME])[0]
+ contents = StringIO.StringIO(open(path, 'rb').read())
+ for check in (EbuildWhitespace, EbuildQuote, EbuildAssignment):
+ c = check(contents)
+ errors = c.Run()
+ for e in errors:
+ stats[c.repoman_check_name] += 1
+ fails[c.repoman_check_name].append(x + '/' + y + '.ebuild: %s' % e[1] % e[0])
+ contents.seek(0) # move fp to the beginning of the StringIO Object
+ del check
+ check = EbuildHeader(contents, str(myear))
+ errors = check.Run()
+ for e in errors:
+ stats[check.repoman_check_name] += 1
+ fails[check.repoman_check_name].append(x + '/' + y + '.ebuild: %s' % e[1] % e[0])
+ contents.seek(0)
+ del check
+ check = EbuildNestedDie(contents)
+ errors = check.Run()
+ for e in errors:
+ stats[check.repoman_check_name] += 1
+ fails[check.repoman_check_name].append(x + '/' + y + '.ebuild: %s' % e)
+ del check, errors, path, contents, myear
if "--force" in myoptions:
# The dep_check() calls are the most expensive QA test. If --force
os.chdir("/")
- emergelog("notitles" not in settings.features,
- " *** exiting successfully.")
+ if retval == os.EX_OK:
+ exit_msg = " *** exiting successfully."
+ else:
+ exit_msg = " *** exiting unsuccessfully with status '%s'." % retval
+ emergelog("notitles" not in settings.features, exit_msg)
from portage.util import normalize_path
# Dump the mod_echo output now so that our other notifications are shown
return ret
def setinst(self,mycpv,mydbapi):
+ """This updates the preferences for old-style virtuals,
+ affecting the behavior of dep_expand() and dep_check()
+ calls. It can change dbapi.match() behavior since that
+ calls dep_expand(). However, dbapi instances have
+ internal match caches that are not invalidated when
+ preferences are updated here. This can potentially
+ lead to some inconsistency (relevant to bug #1343)."""
self.modifying()
if len(self.virtuals) == 0:
self.getvirtuals()
from portage.data import portage_gid, secpass
from portage.dbapi import dbapi
from portage.dep import use_reduce, paren_reduce, dep_getslot, dep_getkey, \
- match_from_list, match_to_list
+ match_from_list, match_to_list, remove_slot
from portage.exception import OperationNotPermitted, PortageException, \
UntrustedSignature, SecurityViolation, InvalidSignature, MissingSignature, \
FileNotFound, InvalidDependString
"""two functions in one. Accepts a list of cpv values and uses the package.mask *and*
packages file to remove invisible entries, returning remaining items. This function assumes
that all entries in mylist have the same category and package name."""
- if (mylist is None) or (len(mylist) == 0):
+ if not mylist:
return []
- newlist = mylist[:]
- #first, we mask out packages in the package.mask file
- mykey = newlist[0]
- cpv = catpkgsplit(mykey)
- if not cpv:
+
+ mysplit = catpkgsplit(mylist[0])
+ if not mysplit:
#invalid cat/pkg-v
- print "visible(): invalid cat/pkg-v:", mykey
+ writemsg("visible(): invalid cat/pkg-v: %s\n" % (mylist[0], ),
+ noiselevel=-1)
return []
- mycp = cpv[0] + "/" + cpv[1]
- maskdict = self.mysettings.pmaskdict
- unmaskdict = self.mysettings.punmaskdict
- if maskdict.has_key(mycp):
- for x in maskdict[mycp]:
- mymatches = self.xmatch("match-all", x)
- if mymatches is None:
- #error in package.mask file; print warning and continue:
- print "visible(): package.mask entry \"" + x + "\" is invalid, ignoring..."
- continue
- for y in mymatches:
- unmask = 0
- if unmaskdict.has_key(mycp):
- for z in unmaskdict[mycp]:
- mymatches_unmask = self.xmatch("match-all",z)
- if y in mymatches_unmask:
- unmask = 1
- break
- if unmask == 0:
- try:
- newlist.remove(y)
- except ValueError:
- pass
-
- profile_atoms = self.mysettings.prevmaskdict.get(mycp)
- if profile_atoms:
- for x in profile_atoms:
- #important: only match against the still-unmasked entries...
- #notice how we pass "newlist" to the xmatch() call below....
- #Without this, ~ deps in the packages files are broken.
- mymatches=self.xmatch("match-list",x,mylist=newlist)
- if mymatches is None:
- #error in packages file; print warning and continue:
- print "emerge: visible(): profile packages entry \""+x+"\" is invalid, ignoring..."
- continue
- newlist = [cpv for cpv in newlist if cpv in mymatches]
+ mycp = "%s/%s" % (mysplit[0], mysplit[1])
- return newlist
+ cpv_slots = []
+ for cpv in mylist:
+ try:
+ myslot = self.aux_get(cpv, ["SLOT"])[0]
+ except KeyError:
+ # masked by corruption
+ continue
+ cpv_slots.append("%s:%s" % (cpv, myslot))
+
+ if cpv_slots:
+ mask_atoms = self.mysettings.pmaskdict.get(mycp)
+ if mask_atoms:
+ unmask_atoms = self.mysettings.punmaskdict.get(mycp)
+ for x in mask_atoms:
+ masked_pkgs = match_from_list(x, cpv_slots)
+ if not masked_pkgs:
+ continue
+ if unmask_atoms:
+ for y in unmask_atoms:
+ unmasked_pkgs = match_from_list(y, masked_pkgs)
+ if unmasked_pkgs:
+ masked_pkgs = [pkg for pkg in masked_pkgs \
+ if pkg not in unmasked_pkgs]
+ if not masked_pkgs:
+ break
+ if masked_pkgs:
+ cpv_slots = [pkg for pkg in cpv_slots \
+ if pkg not in masked_pkgs]
+ if not cpv_slots:
+ break
+
+ if cpv_slots:
+ profile_atoms = self.mysettings.prevmaskdict.get(mycp)
+ if profile_atoms:
+ for x in profile_atoms:
+ cpv_slots = match_from_list(x.lstrip("*"), cpv_slots)
+ if not cpv_slots:
+ break
+
+ if not cpv_slots:
+ return cpv_slots
+
+ return [remove_slot(pkg) for pkg in cpv_slots]
def gvisible(self,mylist):
"strip out group-masked (not in current group) entries"
--- /dev/null
+# repoman: Checks
+# Copyright 2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+import time
+import re
+import os
+
+from repoman.errors import COPYRIGHT_ERROR, LICENSE_ERROR, CVS_HEADER_ERROR, \
+ LEADING_SPACES_ERROR, READONLY_ASSIGNMENT_ERROR, TRAILING_WHITESPACE_ERROR, \
+ MISSING_QUOTES_ERROR, NESTED_DIE_ERROR
+
+
+class ContentCheckException(Exception):
+ """Parent class for exceptions relating to invalid content"""
+
+ def __init__(self, str):
+ Exception.__init__(self, str)
+
+
+class ContentCheck(object):
+ """Given a file-like object, run checks over it's content
+
+ Args:
+ contents - A file-like object, preferably a StringIO instance
+
+ Raises:
+ ContentCheckException or a subclass
+ """
+ repoman_check_name = None
+
+ def __init__(self, contents):
+ self.contents = contents
+ pass
+
+ def Run(self):
+ """Run the check against the contents, return a sequence of errors
+ of the form ((line, error),...)
+ """
+ pass
+
+
+class EbuildHeader(ContentCheck):
+ """Ensure ebuilds have proper headers
+
+ Args:
+ modification_year - Year the ebuild was last modified
+ """
+
+ repoman_check_name = 'ebuild.badheader'
+
+ gentoo_copyright = r'^# Copyright ((1999|200\d)-)?%s Gentoo Foundation$'
+ # Why a regex here, use a string match
+ # gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$')
+ gentoo_license = r'# Distributed under the terms of the GNU General Public License v2'
+ cvs_header = re.compile(r'^#\s*\$Header.*\$$')
+
+ def __init__(self, contents, modification_year):
+ ContentCheck.__init__(self, contents)
+ self.modification_year = modification_year
+ self.gentoo_copyright_re = re.compile(self.gentoo_copyright % self.modification_year)
+
+ def Run(self):
+ """Locate simple header mistakes in an ebuild
+ Copyright header errors
+ CVS header errors
+ License header errors
+ """
+ errors = []
+ for num, line in enumerate(self.contents):
+ if num == 0:
+ match = self.gentoo_copyright_re.match(line)
+ if not match:
+ errors.append((num + 1, COPYRIGHT_ERROR))
+ if num == 1 and line.strip() != self.gentoo_license:
+ errors.append((num + 1, LICENSE_ERROR))
+ if num == 2:
+ match = self.cvs_header.match(line)
+ if not match:
+ errors.append((num + 1, CVS_HEADER_ERROR))
+ if num > 2:
+ return errors
+ return errors
+
+
+class EbuildWhitespace(ContentCheck):
+ """Ensure ebuilds have proper whitespacing"""
+
+ repoman_check_name = 'ebuild.minorsyn'
+
+ ignore_line = re.compile(r'(^$)|(^(\t)*#)')
+ leading_spaces = re.compile(r'^[\S\t]')
+ trailing_whitespace = re.compile(r'.*([\S]$)')
+
+ def __init__(self, contents):
+ ContentCheck.__init__(self, contents)
+
+ def Run(self):
+ """Locate simple whitespace errors
+ Lines with leading spaces or trailing whitespace
+ """
+ errors = []
+ for num, line in enumerate(self.contents):
+ match = self.ignore_line.match(line)
+ if match:
+ continue
+ match = self.leading_spaces.match(line)
+ if not match:
+ errors.append((num + 1, LEADING_SPACES_ERROR))
+ match = self.trailing_whitespace.match(line)
+ if not match:
+ errors.append((num + 1, TRAILING_WHITESPACE_ERROR))
+ return errors
+
+
+class EbuildQuote(ContentCheck):
+ """Ensure ebuilds have valid quoting around things like D,FILESDIR, etc..."""
+
+ repoman_check_name = 'ebuild.minorsyn'
+ ignore_line = re.compile(r'(^$)|(^\s*#.*)')
+ var_names = r'(D|S|T|ROOT|FILESDIR|WORKDIR)'
+ var_reference = re.compile(r'\$({'+var_names+'}|' + \
+ r'\$' + var_names + '\W)')
+ missing_quotes = re.compile(r'(\s|^)[^"\s]*\${?' + var_names + \
+ r'}?[^"\s]*(\s|$)')
+ var_assignment = re.compile(r'^\s*\w*=.*')
+ cond_begin = re.compile(r'(^|\s+)\[\[($|\\$|\s+)')
+ cond_end = re.compile(r'(^|\s+)\]\]($|\\$|\s+)')
+
+ def __init__(self, contents):
+ ContentCheck.__init__(self, contents)
+
+ def Run(self):
+ """Locate simple errors in ebuilds:
+ Missing quotes around variables that may contain spaces
+ """
+
+ errors = []
+ for num, line in enumerate(self.contents):
+ if self.ignore_line.match(line) is not None:
+ continue
+ if self.var_reference.search(line) is None:
+ continue
+ # There can be multiple matches / violations on a single line. We
+ # have to make sure none of the matches are violators. Once we've
+ # found one violator, any remaining matches on the same line can
+ # be ignored.
+ pos = 0
+ while pos <= len(line) - 1:
+ missing_quotes = self.missing_quotes.search(line, pos)
+ if not missing_quotes:
+ break
+ # If the last character of the previous match is a whitespace
+ # character, that character may be needed for the next
+ # missing_quotes match, so search overlaps by 1 character.
+ group = missing_quotes.group()
+ pos = missing_quotes.end() - 1
+
+ # Filter out some false positives that can
+ # get through the missing_quotes regex.
+ if self.var_reference.search(group) is None:
+ continue
+ if self.var_assignment.search(group) is not None:
+ continue
+
+ # This is an attempt to avoid false positives without getting
+ # too complex, while possibly allowing some (hopefully
+ # unlikely) violations to slip through. We just assume
+ # everything is correct if the there is a ' [[ ' or a ' ]] '
+ # anywhere in the whole line (possibly continued over one
+ # line).
+ if self.cond_begin.search(line) is not None:
+ continue
+ if self.cond_end.search(line) is not None:
+ continue
+
+ errors.append((num + 1, MISSING_QUOTES_ERROR))
+ # Any remaining matches on the same line can be ignored.
+ break
+ return errors
+
+
+class EbuildAssignment(ContentCheck):
+ """Ensure ebuilds don't assign to readonly variables."""
+
+ repoman_check_name = 'variable.readonly'
+
+ readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=')
+ line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$')
+ ignore_line = re.compile(r'(^$)|(^(\t)*#)')
+
+ def __init__(self, contents):
+ ContentCheck.__init__(self, contents)
+
+ def Run(self):
+ """Locate simple errors in ebuilds:
+ Assigning to read-only variables.
+ """
+
+ errors = []
+ previous_line = None
+ # enumerate is 0 indexed, so add one when necesseary
+ for num, line in enumerate(self.contents):
+ match = self.ignore_line.match(line)
+ if match:
+ continue
+ match = self.readonly_assignment.match(line)
+ if match and (not previous_line or not self.line_continuation.match(previous_line)):
+ errors.append((num + 1, READONLY_ASSIGNMENT_ERROR))
+ previous_line = line
+ return errors
+
+class EbuildNestedDie(ContentCheck):
+ """Check ebuild for nested die statements (die statements in subshells"""
+
+ repoman_check_name = 'ebuild.nesteddie'
+ nesteddie_re = re.compile(r'^[^#]*\([^)]*\bdie\b')
+
+ def __init__(self, contents):
+ ContentCheck.__init__(self, contents)
+
+ def Run(self):
+ errors = []
+ for num, line in enumerate(self.contents):
+ match = self.nesteddie_re.match(line)
+ if match:
+ errors.append((num + 1, NESTED_DIE_ERROR))
+ return errors
--- /dev/null
+# repoman: Error Messages
+# Copyright 2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+COPYRIGHT_ERROR = 'Invalid Gentoo Copyright on line: %d'
+LICENSE_ERROR = 'Invalid Gentoo/GPL License on line: %d'
+CVS_HEADER_ERROR = 'Malformed CVS Header on line: %d'
+LEADING_SPACES_ERROR = 'Ebuild contains leading spaces on line: %d'
+TRAILING_WHITESPACE_ERROR = 'Trailing whitespace error on line: %d'
+READONLY_ASSIGNMENT_ERROR = 'Ebuild contains assignment to read-only variable on line: %d'
+MISSING_QUOTES_ERROR = 'Unquoted Variable on line: %d'
+NESTED_DIE_ERROR = 'Ebuild calls die in a subshell'
projects / portage.git / commitdiff