From 6d8d0c02457c2e94c759fe89db0bef196b78158a Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sun, 23 Sep 2012 15:43:04 -0700 Subject: [PATCH] RepoConfig: add find_invalid_path_char method This binds filename validation to the RepoConfig, so that eventually we'll be able to control it via a layout.conf setting as discussed in bug #435934. --- bin/repoman | 23 ++++++++++------------- pym/portage/manifest.py | 24 ++++++++++++------------ pym/portage/repository/config.py | 22 +++++++++++++++++++++- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/bin/repoman b/bin/repoman index bc2ac9be6..b463cbe7c 100755 --- a/bin/repoman +++ b/bin/repoman @@ -71,8 +71,6 @@ from portage import cvstree, normalize_path from portage import util from portage.exception import (FileNotFound, MissingParameter, ParseError, PermissionDenied) -from portage.manifest import _prohibited_filename_chars_re as \ - disallowed_filename_chars_re from portage.process import find_binary, spawn from portage.output import bold, create_color_func, \ green, nocolor, red @@ -1431,19 +1429,19 @@ for x in effective_scanlist: ebuildlist = [pkg.pf for pkg in ebuildlist] for y in checkdirlist: - m = disallowed_filename_chars_re.search(y.strip(os.sep)) - if m is not None: + index = repo_config.find_invalid_path_char(y) + if index != -1: y_relative = os.path.join(checkdir_relative, y) if vcs is not None and not vcs_new_changed(y_relative): # If the file isn't in the VCS new or changed set, then # assume that it's an irrelevant temporary file (Manifest # entries are not generated for file names containing # prohibited characters). See bug #406877. - m = None - if m is not None: + index = -1 + if index != -1: stats["file.name"] += 1 fails["file.name"].append("%s/%s: char '%s'" % \ - (checkdir, y, m.group(0))) + (checkdir, y, y[index])) if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")): continue @@ -1609,20 +1607,19 @@ for x in effective_scanlist: stats["file.size"] += 1 fails["file.size"].append("("+ str(mystat.st_size//1024) + " KiB) "+x+"/files/"+y) - m = disallowed_filename_chars_re.search( - os.path.basename(y.rstrip(os.sep))) - if m is not None: + index = repo_config.find_invalid_path_char(y) + if index != -1: y_relative = os.path.join(checkdir_relative, "files", y) if vcs is not None and not vcs_new_changed(y_relative): # If the file isn't in the VCS new or changed set, then # assume that it's an irrelevant temporary file (Manifest # entries are not generated for file names containing # prohibited characters). See bug #406877. - m = None - if m is not None: + index = -1 + if index != -1: stats["file.name"] += 1 fails["file.name"].append("%s/files/%s: char '%s'" % \ - (checkdir, y, m.group(0))) + (checkdir, y, y[index])) del mydigests if check_changelog and "ChangeLog" not in checkdirlist: diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 25886bb1e..b81b580d5 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -3,7 +3,6 @@ import errno import io -import re import sys import warnings @@ -11,6 +10,7 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes', + 'portage.repository.config:_find_invalid_path_char', 'portage.util:write_atomic', ) @@ -30,9 +30,6 @@ if sys.hexversion >= 0x3000000: else: _unicode = unicode -# Characters prohibited by repoman's file.name check. -_prohibited_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]') - class FileNotInManifestException(PortageException): pass @@ -44,14 +41,10 @@ def manifest2AuxfileFilter(filename): for x in mysplit: if x[:1] == '.': return False - if _prohibited_filename_chars_re.search(x) is not None: - return False return not filename[:7] == 'digest-' def manifest2MiscfileFilter(filename): filename = filename.strip(os.sep) - if _prohibited_filename_chars_re.search(filename) is not None: - return False return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild")) def guessManifestFileType(filename): @@ -126,7 +119,8 @@ class Manifest(object): parsers = (parseManifest2,) def __init__(self, pkgdir, distdir, fetchlist_dict=None, manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, - allow_missing=False, allow_create=True, hashes=None): + allow_missing=False, allow_create=True, hashes=None, + find_invalid_path_char=None): """ Create new Manifest instance for package in pkgdir. Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of @@ -139,6 +133,9 @@ class Manifest(object): "portage.manifest.Manifest constructor is deprecated.", DeprecationWarning, stacklevel=2) + if find_invalid_path_char is None: + find_invalid_path_char = _find_invalid_path_char + self._find_invalid_path_char = find_invalid_path_char self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep self.fhashdict = {} self.hashes = set() @@ -380,7 +377,8 @@ class Manifest(object): self.__init__(self.pkgdir, self.distdir, fetchlist_dict=self.fetchlist_dict, from_scratch=True, thin=self.thin, allow_missing=self.allow_missing, - allow_create=self.allow_create, hashes=self.hashes) + allow_create=self.allow_create, hashes=self.hashes, + find_invalid_path_char=self._find_invalid_path_char) pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() @@ -475,7 +473,8 @@ class Manifest(object): if pf is not None: mytype = "EBUILD" cpvlist.append(pf) - elif manifest2MiscfileFilter(f): + elif self._find_invalid_path_char(f) == -1 and \ + manifest2MiscfileFilter(f): mytype = "MISC" else: continue @@ -494,7 +493,8 @@ class Manifest(object): full_path = os.path.join(parentdir, f) recursive_files.append(full_path[cut_len:]) for f in recursive_files: - if not manifest2AuxfileFilter(f): + if self._find_invalid_path_char(f) != -1 or \ + not manifest2AuxfileFilter(f): continue self.fhashdict["AUX"][f] = perform_multiple_checksums( os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes) diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 77b016d0c..83018b8a6 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -28,6 +28,9 @@ from portage import _unicode_encode from portage import _encodings from portage import manifest +# Characters prohibited by repoman's file.name check. +_invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+:/]') + _valid_profile_formats = frozenset( ['pms', 'portage-1', 'portage-2']) @@ -49,12 +52,27 @@ def _gen_valid_repo(name): name = None return name +def _find_invalid_path_char(path, pos=0, endpos=None): + """ + Returns the position of the first invalid character found in basename, + or -1 if no invalid characters are found. + """ + if endpos is None: + endpos = len(path) + + m = _invalid_path_char_re.search(path, pos=pos, endpos=endpos) + if m is not None: + return m.start() + + return -1 + class RepoConfig(object): """Stores config of one repository""" __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual', 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi', - 'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location', + 'eclass_db', 'eclass_locations', 'eclass_overrides', + 'find_invalid_path_char', 'format', 'location', 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', 'profile_formats', 'sign_commit', 'sign_manifest', 'sync', @@ -138,6 +156,7 @@ class RepoConfig(object): self.cache_formats = None self.portage1_profiles = True self.portage1_profiles_compat = False + self.find_invalid_path_char = _find_invalid_path_char # Parse layout.conf. if self.location: @@ -211,6 +230,7 @@ class RepoConfig(object): kwds['hashes'] = self.manifest_hashes if self.disable_manifest: kwds['from_scratch'] = True + kwds['find_invalid_path_char'] = self.find_invalid_path_char return manifest.Manifest(*args, **kwds) def update(self, new_repo): -- 2.26.2