From: Fabian Groffen Date: Tue, 2 Oct 2007 15:56:54 +0000 (-0000) Subject: Merged from trunk 7890:7901 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=15523aa4f0f90920cf29900d62539b40c757432b;p=portage.git Merged from trunk 7890:7901 |7891 |Document interaction between config.setinst() and old-style virtuals | |zmedico |preferences. | |7892 |Reimplement portdbapi.visible() so that is works without using self.xmatch().| |zmedico |This should be 100% compatible with the previous implementation and have | | |comparable performance. By eliminating the xmatch() dependency, the code | | |becomes more generically usable, like for visibility filtering of binary | | |packages. It seems that it will be necessary to move this code outside of the| | |dbapi since visiblity filtering is really a resolver level function and to | | |solve things like bug #88613 will require a more flexible approach. since the| | |current internal | |7893 |Bug #142508 - Clear the screen each time before calling the "diff" command so| |zmedico |that any remnants of the previous config are now shown with the current one. | | |command so that | |7894 |Add new repoman check classes using StringIO; possibly need some testing, | |WarnerBro|obviously the old code was faster (1 iteration over the file), here we do one| | |iteration per check, StringIO was to try and negate this by doing the checks | | |in memory...how much of a price do we pay here? | |7895 |Add Donnie's quote check, refactor other ebuild-content checks into classes | |WarnerBro|and get them out of repoman in an attempt to start moving other checks out as| | |well. Again wonder at the price of StringIO and N passes, is the readability | | |worth it? | |7896 |keep old repoman names so we don't confuse people | |WarnerBro| | |7897 |Delete some extra leaky vars, also after some testing I discovered read() is | |WarnerBro|much faster than readlines(), see | | |http://dev.gentoo.org/~antarus/projects/portage/stats/StringIOVsFiles.txt | |7898 |Rename checks as the naming was overkill (they are all obviously checks being| |WarnerBro|in the check module), add nesteddie check | |7899 |Apply new checks, minor cleanup | |WarnerBro| | |7900 |Bug #194398 - Do not log "exiting successfully" in emerge.log unless the | |zmedico |status really is successful. Otherwise, log "exiting unsuccessfully". Thanks | | |to David Watzke. | |7901 |Filter some false positives out of the EbuildQuote check. It might work well | |zmedico |enough now so that we don't have to degrade it to a warning. | svn path=/main/branches/prefix/; revision=7902 --- diff --git a/bin/dispatch-conf b/bin/dispatch-conf index 3701f587a..713470466 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -213,12 +213,13 @@ class dispatch: newconf = mrgconf show_new_diff = 0 - while 1: - if show_new_diff: - os.system((self.options['diff']) % (conf['new'], mrgconf)) - show_new_diff = 0 - else: - os.system((self.options['diff']) % (conf['current'], newconf)) + while 1: + clear_screen() + if show_new_diff: + os.system((self.options['diff']) % (conf['new'], mrgconf)) + show_new_diff = 0 + else: + os.system((self.options['diff']) % (conf['current'], newconf)) print print '>> (%i of %i) -- %s' % (count, len(confs), conf ['current']) @@ -377,6 +378,19 @@ def getch (): termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) return ch +def clear_screen(): + try: + import curses + try: + curses.setupterm() + sys.stdout.write(curses.tigetstr("clear")) + sys.stdout.flush() + return + except curses.error: + pass + except ImportError: + pass + os.system("clear 2>/dev/null") # run d = dispatch () diff --git a/bin/repoman b/bin/repoman index 964e7e547..001e3289d 100755 --- a/bin/repoman +++ b/bin/repoman @@ -20,6 +20,11 @@ try: except ImportError: import pickle +try: + import cStringIO as StringIO +except ImportError: + import StringIO + if not hasattr(__builtins__, "set"): from sets import Set as set @@ -38,10 +43,19 @@ try: import portage except ImportError: from os import path as osp - sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym')) import portage del os.environ["PORTAGE_LEGACY_GLOBALS"] +try: + from repoman.checks import EbuildWhitespaceCheck, EbuildHeaderCheck, EbuildQuoteCheck, \ + EbuildAssignmentCheck +except ImportError: + from os import path as osp + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), 'pym')) + from repoman.checks import EbuildWhitespace, EbuildHeader, EbuildQuote, \ + EbuildAssignment, EbuildNestedDie + import portage.checksum import portage.const import portage.dep @@ -1315,10 +1329,6 @@ for x in scanlist: stats["usage.obsolete"] += 1 fails["usage.obsolete"].append("%s/%s.ebuild: not migrated to modular X" % (x, y)) - # this check needs work, it won't catch (\ndie) - if not os.system("egrep '^[^#]*\([^)]*\' "+checkdir+"/"+y+".ebuild >/dev/null 2>&1"): - stats["ebuild.nesteddie"]=stats["ebuild.nesteddie"]+1 - fails["ebuild.nesteddie"].append(x+"/"+y+".ebuild") # uselist checks - global myuse = [] default_use = [] @@ -1389,71 +1399,31 @@ for x in scanlist: stats["RESTRICT.invalid"] += len(mybadrestrict) for mybad in mybadrestrict: fails["RESTRICT.invalid"].append(x+"/"+y+".ebuild: %s" % mybad) - - #syntax checks - myear = time.gmtime(os.stat(checkdir+"/"+y+".ebuild")[ST_MTIME])[0] - gentoo_copyright = re.compile(r'^# Copyright ((1999|200\d)-)?' + str(myear) + r' Gentoo Foundation') - gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$') - cvs_header = re.compile(r'^#\s*\$Header.*\$$') - ignore_line = re.compile(r'(^$)|(^(\t)*#)') - leading_spaces = re.compile(r'^[\S\t]') - trailing_whitespace = re.compile(r'.*([\S]$)') - readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=') - line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$') - linenum=0 - previous_line = None - for line in input(checkdir+"/"+y+".ebuild"): - linenum += 1 - # Gentoo copyright check - if linenum == 1: - match = gentoo_copyright.match(line) - if not match: - myerrormsg = "Copyright header Error. Possibly date related." - stats["ebuild.badheader"] +=1 - fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg) - # Gentoo license check - elif linenum == 2: - match = gentoo_license.match(line) - if not match: - myerrormsg = "Gentoo License Error." - stats["ebuild.badheader"] +=1 - fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg) - # CVS Header check - elif linenum == 3: - match = cvs_header.match(line) - if not match: - myerrormsg = "CVS Header Error." - stats["ebuild.badheader"] +=1 - fails["ebuild.badheader"].append(x+"/"+y+".ebuild: %s" % myerrormsg) - else: - match = ignore_line.match(line) - if not match: - # Excluded Blank lines and full line comments. Good! - # Leading Spaces Check - match = leading_spaces.match(line) - if not match: - #Line has got leading spaces. Bad! - myerrormsg = "Leading Space Syntax Error. Line %d" % linenum - stats["ebuild.minorsyn"] +=1 - fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg) - # Trailing whitespace check - match = trailing_whitespace.match(line) - if not match: - #Line has got trailing whitespace. Bad! - myerrormsg = "Trailing whitespace Syntax Error. Line %d" % linenum - stats["ebuild.minorsyn"] +=1 - fails["ebuild.minorsyn"].append(x+"/"+y+".ebuild: %s" % myerrormsg) - # Readonly variable assignment check - match = readonly_assignment.match(line) - # The regex can give a false positive for continued lines, - # so we check the previous line to see if it was continued. - if match and (not previous_line or not line_continuation.match(previous_line)): - # invalid assignment, very bad! - myerrormsg = "Readonly variable assignment to %s on line %d" % (match.group(2), linenum) - stats["variable.readonly"] += 1 - fails["variable.readonly"].append(x+"/"+y+".ebuild: %s" % myerrormsg) - previous_line = line - del previous_line + # Syntax Checks + path = checkdir + '/' + y + '.ebuild' + myear = time.gmtime(os.stat(path)[ST_MTIME])[0] + contents = StringIO.StringIO(open(path, 'rb').read()) + for check in (EbuildWhitespace, EbuildQuote, EbuildAssignment): + c = check(contents) + errors = c.Run() + for e in errors: + stats[c.repoman_check_name] += 1 + fails[c.repoman_check_name].append(x + '/' + y + '.ebuild: %s' % e[1] % e[0]) + contents.seek(0) # move fp to the beginning of the StringIO Object + del check + check = EbuildHeader(contents, str(myear)) + errors = check.Run() + for e in errors: + stats[check.repoman_check_name] += 1 + fails[check.repoman_check_name].append(x + '/' + y + '.ebuild: %s' % e[1] % e[0]) + contents.seek(0) + del check + check = EbuildNestedDie(contents) + errors = check.Run() + for e in errors: + stats[check.repoman_check_name] += 1 + fails[check.repoman_check_name].append(x + '/' + y + '.ebuild: %s' % e) + del check, errors, path, contents, myear if "--force" in myoptions: # The dep_check() calls are the most expensive QA test. If --force diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 33c587d53..faa5ea758 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -4507,8 +4507,11 @@ def post_emerge(trees, mtimedb, retval): os.chdir("/") - emergelog("notitles" not in settings.features, - " *** exiting successfully.") + if retval == os.EX_OK: + exit_msg = " *** exiting successfully." + else: + exit_msg = " *** exiting unsuccessfully with status '%s'." % retval + emergelog("notitles" not in settings.features, exit_msg) from portage.util import normalize_path # Dump the mod_echo output now so that our other notifications are shown diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 13fb4a094..88c197a91 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1905,6 +1905,13 @@ class config(object): return ret def setinst(self,mycpv,mydbapi): + """This updates the preferences for old-style virtuals, + affecting the behavior of dep_expand() and dep_check() + calls. It can change dbapi.match() behavior since that + calls dep_expand(). However, dbapi instances have + internal match caches that are not invalidated when + preferences are updated here. This can potentially + lead to some inconsistency (relevant to bug #1343).""" self.modifying() if len(self.virtuals) == 0: self.getvirtuals() diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 357b17232..dc54efe5c 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -7,7 +7,7 @@ from portage.const import REPO_NAME_LOC from portage.data import portage_gid, secpass from portage.dbapi import dbapi from portage.dep import use_reduce, paren_reduce, dep_getslot, dep_getkey, \ - match_from_list, match_to_list + match_from_list, match_to_list, remove_slot from portage.exception import OperationNotPermitted, PortageException, \ UntrustedSignature, SecurityViolation, InvalidSignature, MissingSignature, \ FileNotFound, InvalidDependString @@ -585,54 +585,60 @@ class portdbapi(dbapi): """two functions in one. Accepts a list of cpv values and uses the package.mask *and* packages file to remove invisible entries, returning remaining items. This function assumes that all entries in mylist have the same category and package name.""" - if (mylist is None) or (len(mylist) == 0): + if not mylist: return [] - newlist = mylist[:] - #first, we mask out packages in the package.mask file - mykey = newlist[0] - cpv = catpkgsplit(mykey) - if not cpv: + + mysplit = catpkgsplit(mylist[0]) + if not mysplit: #invalid cat/pkg-v - print "visible(): invalid cat/pkg-v:", mykey + writemsg("visible(): invalid cat/pkg-v: %s\n" % (mylist[0], ), + noiselevel=-1) return [] - mycp = cpv[0] + "/" + cpv[1] - maskdict = self.mysettings.pmaskdict - unmaskdict = self.mysettings.punmaskdict - if maskdict.has_key(mycp): - for x in maskdict[mycp]: - mymatches = self.xmatch("match-all", x) - if mymatches is None: - #error in package.mask file; print warning and continue: - print "visible(): package.mask entry \"" + x + "\" is invalid, ignoring..." - continue - for y in mymatches: - unmask = 0 - if unmaskdict.has_key(mycp): - for z in unmaskdict[mycp]: - mymatches_unmask = self.xmatch("match-all",z) - if y in mymatches_unmask: - unmask = 1 - break - if unmask == 0: - try: - newlist.remove(y) - except ValueError: - pass - - profile_atoms = self.mysettings.prevmaskdict.get(mycp) - if profile_atoms: - for x in profile_atoms: - #important: only match against the still-unmasked entries... - #notice how we pass "newlist" to the xmatch() call below.... - #Without this, ~ deps in the packages files are broken. - mymatches=self.xmatch("match-list",x,mylist=newlist) - if mymatches is None: - #error in packages file; print warning and continue: - print "emerge: visible(): profile packages entry \""+x+"\" is invalid, ignoring..." - continue - newlist = [cpv for cpv in newlist if cpv in mymatches] + mycp = "%s/%s" % (mysplit[0], mysplit[1]) - return newlist + cpv_slots = [] + for cpv in mylist: + try: + myslot = self.aux_get(cpv, ["SLOT"])[0] + except KeyError: + # masked by corruption + continue + cpv_slots.append("%s:%s" % (cpv, myslot)) + + if cpv_slots: + mask_atoms = self.mysettings.pmaskdict.get(mycp) + if mask_atoms: + unmask_atoms = self.mysettings.punmaskdict.get(mycp) + for x in mask_atoms: + masked_pkgs = match_from_list(x, cpv_slots) + if not masked_pkgs: + continue + if unmask_atoms: + for y in unmask_atoms: + unmasked_pkgs = match_from_list(y, masked_pkgs) + if unmasked_pkgs: + masked_pkgs = [pkg for pkg in masked_pkgs \ + if pkg not in unmasked_pkgs] + if not masked_pkgs: + break + if masked_pkgs: + cpv_slots = [pkg for pkg in cpv_slots \ + if pkg not in masked_pkgs] + if not cpv_slots: + break + + if cpv_slots: + profile_atoms = self.mysettings.prevmaskdict.get(mycp) + if profile_atoms: + for x in profile_atoms: + cpv_slots = match_from_list(x.lstrip("*"), cpv_slots) + if not cpv_slots: + break + + if not cpv_slots: + return cpv_slots + + return [remove_slot(pkg) for pkg in cpv_slots] def gvisible(self,mylist): "strip out group-masked (not in current group) entries" diff --git a/pym/repoman/__init__.py b/pym/repoman/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py new file mode 100644 index 000000000..de7ceee23 --- /dev/null +++ b/pym/repoman/checks.py @@ -0,0 +1,229 @@ +# repoman: Checks +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +import time +import re +import os + +from repoman.errors import COPYRIGHT_ERROR, LICENSE_ERROR, CVS_HEADER_ERROR, \ + LEADING_SPACES_ERROR, READONLY_ASSIGNMENT_ERROR, TRAILING_WHITESPACE_ERROR, \ + MISSING_QUOTES_ERROR, NESTED_DIE_ERROR + + +class ContentCheckException(Exception): + """Parent class for exceptions relating to invalid content""" + + def __init__(self, str): + Exception.__init__(self, str) + + +class ContentCheck(object): + """Given a file-like object, run checks over it's content + + Args: + contents - A file-like object, preferably a StringIO instance + + Raises: + ContentCheckException or a subclass + """ + repoman_check_name = None + + def __init__(self, contents): + self.contents = contents + pass + + def Run(self): + """Run the check against the contents, return a sequence of errors + of the form ((line, error),...) + """ + pass + + +class EbuildHeader(ContentCheck): + """Ensure ebuilds have proper headers + + Args: + modification_year - Year the ebuild was last modified + """ + + repoman_check_name = 'ebuild.badheader' + + gentoo_copyright = r'^# Copyright ((1999|200\d)-)?%s Gentoo Foundation$' + # Why a regex here, use a string match + # gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$') + gentoo_license = r'# Distributed under the terms of the GNU General Public License v2' + cvs_header = re.compile(r'^#\s*\$Header.*\$$') + + def __init__(self, contents, modification_year): + ContentCheck.__init__(self, contents) + self.modification_year = modification_year + self.gentoo_copyright_re = re.compile(self.gentoo_copyright % self.modification_year) + + def Run(self): + """Locate simple header mistakes in an ebuild + Copyright header errors + CVS header errors + License header errors + """ + errors = [] + for num, line in enumerate(self.contents): + if num == 0: + match = self.gentoo_copyright_re.match(line) + if not match: + errors.append((num + 1, COPYRIGHT_ERROR)) + if num == 1 and line.strip() != self.gentoo_license: + errors.append((num + 1, LICENSE_ERROR)) + if num == 2: + match = self.cvs_header.match(line) + if not match: + errors.append((num + 1, CVS_HEADER_ERROR)) + if num > 2: + return errors + return errors + + +class EbuildWhitespace(ContentCheck): + """Ensure ebuilds have proper whitespacing""" + + repoman_check_name = 'ebuild.minorsyn' + + ignore_line = re.compile(r'(^$)|(^(\t)*#)') + leading_spaces = re.compile(r'^[\S\t]') + trailing_whitespace = re.compile(r'.*([\S]$)') + + def __init__(self, contents): + ContentCheck.__init__(self, contents) + + def Run(self): + """Locate simple whitespace errors + Lines with leading spaces or trailing whitespace + """ + errors = [] + for num, line in enumerate(self.contents): + match = self.ignore_line.match(line) + if match: + continue + match = self.leading_spaces.match(line) + if not match: + errors.append((num + 1, LEADING_SPACES_ERROR)) + match = self.trailing_whitespace.match(line) + if not match: + errors.append((num + 1, TRAILING_WHITESPACE_ERROR)) + return errors + + +class EbuildQuote(ContentCheck): + """Ensure ebuilds have valid quoting around things like D,FILESDIR, etc...""" + + repoman_check_name = 'ebuild.minorsyn' + ignore_line = re.compile(r'(^$)|(^\s*#.*)') + var_names = r'(D|S|T|ROOT|FILESDIR|WORKDIR)' + var_reference = re.compile(r'\$({'+var_names+'}|' + \ + r'\$' + var_names + '\W)') + missing_quotes = re.compile(r'(\s|^)[^"\s]*\${?' + var_names + \ + r'}?[^"\s]*(\s|$)') + var_assignment = re.compile(r'^\s*\w*=.*') + cond_begin = re.compile(r'(^|\s+)\[\[($|\\$|\s+)') + cond_end = re.compile(r'(^|\s+)\]\]($|\\$|\s+)') + + def __init__(self, contents): + ContentCheck.__init__(self, contents) + + def Run(self): + """Locate simple errors in ebuilds: + Missing quotes around variables that may contain spaces + """ + + errors = [] + for num, line in enumerate(self.contents): + if self.ignore_line.match(line) is not None: + continue + if self.var_reference.search(line) is None: + continue + # There can be multiple matches / violations on a single line. We + # have to make sure none of the matches are violators. Once we've + # found one violator, any remaining matches on the same line can + # be ignored. + pos = 0 + while pos <= len(line) - 1: + missing_quotes = self.missing_quotes.search(line, pos) + if not missing_quotes: + break + # If the last character of the previous match is a whitespace + # character, that character may be needed for the next + # missing_quotes match, so search overlaps by 1 character. + group = missing_quotes.group() + pos = missing_quotes.end() - 1 + + # Filter out some false positives that can + # get through the missing_quotes regex. + if self.var_reference.search(group) is None: + continue + if self.var_assignment.search(group) is not None: + continue + + # This is an attempt to avoid false positives without getting + # too complex, while possibly allowing some (hopefully + # unlikely) violations to slip through. We just assume + # everything is correct if the there is a ' [[ ' or a ' ]] ' + # anywhere in the whole line (possibly continued over one + # line). + if self.cond_begin.search(line) is not None: + continue + if self.cond_end.search(line) is not None: + continue + + errors.append((num + 1, MISSING_QUOTES_ERROR)) + # Any remaining matches on the same line can be ignored. + break + return errors + + +class EbuildAssignment(ContentCheck): + """Ensure ebuilds don't assign to readonly variables.""" + + repoman_check_name = 'variable.readonly' + + readonly_assignment = re.compile(r'^\s*(export\s+)?(A|CATEGORY|P|PV|PN|PR|PVR|PF|D|WORKDIR|FILESDIR|FEATURES|USE)=') + line_continuation = re.compile(r'([^#]*\S)(\s+|\t)\\$') + ignore_line = re.compile(r'(^$)|(^(\t)*#)') + + def __init__(self, contents): + ContentCheck.__init__(self, contents) + + def Run(self): + """Locate simple errors in ebuilds: + Assigning to read-only variables. + """ + + errors = [] + previous_line = None + # enumerate is 0 indexed, so add one when necesseary + for num, line in enumerate(self.contents): + match = self.ignore_line.match(line) + if match: + continue + match = self.readonly_assignment.match(line) + if match and (not previous_line or not self.line_continuation.match(previous_line)): + errors.append((num + 1, READONLY_ASSIGNMENT_ERROR)) + previous_line = line + return errors + +class EbuildNestedDie(ContentCheck): + """Check ebuild for nested die statements (die statements in subshells""" + + repoman_check_name = 'ebuild.nesteddie' + nesteddie_re = re.compile(r'^[^#]*\([^)]*\bdie\b') + + def __init__(self, contents): + ContentCheck.__init__(self, contents) + + def Run(self): + errors = [] + for num, line in enumerate(self.contents): + match = self.nesteddie_re.match(line) + if match: + errors.append((num + 1, NESTED_DIE_ERROR)) + return errors diff --git a/pym/repoman/errors.py b/pym/repoman/errors.py new file mode 100644 index 000000000..8378ed4d8 --- /dev/null +++ b/pym/repoman/errors.py @@ -0,0 +1,13 @@ +# repoman: Error Messages +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +COPYRIGHT_ERROR = 'Invalid Gentoo Copyright on line: %d' +LICENSE_ERROR = 'Invalid Gentoo/GPL License on line: %d' +CVS_HEADER_ERROR = 'Malformed CVS Header on line: %d' +LEADING_SPACES_ERROR = 'Ebuild contains leading spaces on line: %d' +TRAILING_WHITESPACE_ERROR = 'Trailing whitespace error on line: %d' +READONLY_ASSIGNMENT_ERROR = 'Ebuild contains assignment to read-only variable on line: %d' +MISSING_QUOTES_ERROR = 'Unquoted Variable on line: %d' +NESTED_DIE_ERROR = 'Ebuild calls die in a subshell'