From 5b19f0592fcc69076df3fc1cbfac95868b907eae Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 5 Mar 2009 18:56:07 +0000 Subject: [PATCH] Merged from trunk -r12715:12733 MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit | 12716 | Fix a case in which a SRC_URI is incorrectly idendified as | | zmedico | invalid, and add a corresponding test case. Thanks to Jacob | | | Floyd for reporting. | | 12717 | Fix test breakage due to change in calling order of | | zmedico | AsynchronousTask start listeners from r12713. | | 12718 | Add a new upstream.workaround check for hardcoded | | zmedico | MAKEOPTS=-j1 (previously only emake -j1 was recognized). | | | Thanks to Maciej Mrozowski for this | | | patch. | | 12719 | Loosen EMakeParallelDisabled.re so it can match more | | zmedico | variations. | | 12720 | Fix PipeReader and PipeReaderTestCase to work with binary | | zmedico | strings in py3k. | | 12721 | Fix CommandOutputSet to decode binary command output in | | zmedico | py3k. | | 12722 | Use unicode() so that 2to3 handles py3k compatibility. | | zmedico | | | 12723 | Open the temp file in text mode for py3k compatibility. | | zmedico | | | 12724 | Open the temp file in text mode for py3k compatibility. | | zmedico | | | 12725 | Use os.fdopen() on file descriptors returned from mkstemp(). | | zmedico | | | 12726 | Open the temp file in text mode for py3k compatibility. | | zmedico | | | 12727 | Open the temp file in text mode for py3k compatibility. | | zmedico | | | 12728 | Open then temp file in text mode for py3k compatibility. | | zmedico | | | 12729 | Use Popen.communicate() to avoid a potential deadlock. | | zmedico | | | 12730 | Make Atom instances consume less memory by implementing str | | zmedico | methods at the class level instead of referencing bound str | | | methods. | | 12731 | Add support to _ensure_default_encoding() for the same ascii | | zmedico | and utf_8 aliases that encodings.aliases has. | | 12732 | In _ensure_default_encoding(), make sure the default | | zmedico | encoding is normalized. | | 12733 | When checking ebuild content, use codecs.open() to force | | zmedico | utf_8 encoding for py3k compatibility (all ebuilds should | | | have utf_8 encoding). | Merged from trunk -r12733:12744 | 12734 | Skip the code from bug #259954 when $ROOT != / since it | | zmedico | shouldn't matter if there are unsatisfied system runtime | | | deps in this case. | | 12735 | * Fix portage.cache.template.database.__getitem__() to | | zmedico | validate the _mtime_ field and raise a CacheCorruption | | | exception if necessary. * Make _mtime_ and _eclasses_ | | | validation code in portdbapi and mirror_cache() assume that | | | these fields are the correct type (otherwise a | | | CacheCorruption exeception should be raised earlier). * Fix | | | the sqlite module to implement _getitem() so that it | | | properly inherits __getitem__() _mtime_ and _eclasses_ | | | handling. | | 12736 | Remove redundant long(_mtime_) conversion since template | | zmedico | handles it now. | | 12737 | Thanks to Petteri Räty for this new | | zmedico | cache module which uses extended attributes (via pyxattr) to | | | attach metadata cache directly to the ebuild files | | | themselves. | | 12738 | Don't give portage group write permissions on the mtimedb | | zmedico | since it's not really needed. | | 12739 | When fetch() is called and there is an existing file with | | zmedico | the wrong size that is smaller than | | | PORTAGE_FETCH_RESUME_MIN_SIZE, rename the file instead of | | | deleting it. This is more friendly for cases in which the | | | user is attempting to regenerate the digest but has not used | | | `ebuild --force foo.ebuild manifest` like they're supposed | | | to. Thanks to Patrick Lauer for reporting. | | 12740 | Use PORTAGE_USE when evaluating LICENSE inside | | zmedico | portdbapi.gvisible(). | | 12741 | When ACCEPT_CHOSTS is set, enable CHOST masking for unbuilt | | zmedico | ebuilds. This behaves as a sanity check to protect people | | | who are cross-compiling from accidentally merging an ebuild | | | with CHOST set incorrectly. Thanks to Ned Ludd | | | for reporting the problem. | | 12742 | Fix KeyValuePairFileLoader to only split on '=' once and | | zmedico | strip whitespace for keys and values. | | 12743 | ACCEPT_CHOSTS applies to unbuilt ebuild now (not just binary | | zmedico | packages). | | 12744 | Make KeyValuePairFileLoader store the values as a strings | | zmedico | instead of a lists. | svn path=/main/branches/prefix/; revision=12763 --- bin/repoman | 18 +- man/emerge.1 | 2 +- man/make.conf.5 | 2 +- pym/_emerge/__init__.py | 23 ++- pym/portage/__init__.py | 45 +++-- pym/portage/cache/ebuild_xattr.py | 163 ++++++++++++++++++ pym/portage/cache/metadata_overlay.py | 2 +- pym/portage/cache/sqlite.py | 6 +- pym/portage/cache/template.py | 10 ++ pym/portage/cache/util.py | 11 +- pym/portage/dbapi/porttree.py | 32 ++-- pym/portage/dep.py | 70 +++++++- pym/portage/env/loaders.py | 6 +- pym/portage/sets/shell.py | 6 +- pym/portage/tests/dep/test_src_uri.py | 1 + .../tests/env/config/test_PackageUseFile.py | 3 +- .../env/config/test_PortageModulesFile.py | 4 +- pym/portage/tests/news/test_NewsItem.py | 6 +- pym/portage/tests/process/test_poll.py | 17 +- .../tests/sets/files/testConfigFileSet.py | 7 +- .../tests/sets/files/testStaticFileSet.py | 5 +- pym/repoman/checks.py | 12 +- pym/repoman/errors.py | 1 + 23 files changed, 363 insertions(+), 89 deletions(-) create mode 100644 pym/portage/cache/ebuild_xattr.py diff --git a/bin/repoman b/bin/repoman index 76557c784..166d3967e 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1444,14 +1444,18 @@ for x in scanlist: # Syntax Checks relative_path = os.path.join(x, y + ".ebuild") full_path = os.path.join(repodir, relative_path) - f = open(full_path) try: - for check_name, e in run_checks(f, pkg): - stats[check_name] += 1 - fails[check_name].append(relative_path + ': %s' % e) - finally: - f.close() - del f + # All ebuilds should have utf_8 encoding. + f = codecs.open(full_path, mode='r', encoding='utf_8') + try: + for check_name, e in run_checks(f, pkg): + stats[check_name] += 1 + fails[check_name].append(relative_path + ': %s' % e) + finally: + f.close() + except UnicodeDecodeError: + # A file.UTF8 failure will have already been recorded above. + pass if options.force: # The dep_check() calls are the most expensive QA test. If --force diff --git a/man/emerge.1 b/man/emerge.1 index 50c525f72..5e2491aa9 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -596,7 +596,7 @@ problems or are known to have issues on different systems. It resides in .TP .BR CHOST Use the \fBACCEPT_CHOSTS\fR variable in \fBmake.conf\fR(5) to control -\fBCHOST\fR acceptance for binary packages. +\fBCHOST\fR acceptance. .TP .BR EAPI The \fBEAPI\fR variable in an \fBebuild\fR(5) file is used to mask packages diff --git a/man/make.conf.5 b/man/make.conf.5 index 07f7e1946..2f9c36509 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -22,7 +22,7 @@ export USE="\-*" .SH "VARIABLES" .TP \fBACCEPT_CHOSTS\fR = \fI[space delimited list of CHOST values]\fR -Specifies acceptable \fBCHOST\fR values for binary packages. Regular +Specifies acceptable \fBCHOST\fR values. Regular expression syntax is supported, so it is necessary to escape \fBCHOST\fR characters if they have special meaning in regular expressions. .br diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 196e9109b..aaa8fd8bb 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1402,8 +1402,8 @@ def visible(pkgsettings, pkg): """ if not pkg.metadata["SLOT"]: return False - if pkg.built and not pkg.installed and "CHOST" in pkg.metadata: - if not pkgsettings._accept_chost(pkg): + if not pkg.installed: + if not pkgsettings._accept_chost(pkg.cpv, pkg.metadata): return False if pkg.built and not pkg.installed: # we can have an old binary which has no EPREFIX information @@ -1436,8 +1436,8 @@ def get_masking_status(pkg, pkgsettings, root_config): pkg, settings=pkgsettings, portdb=root_config.trees["porttree"].dbapi) - if pkg.built and not pkg.installed and "CHOST" in pkg.metadata: - if not pkgsettings._accept_chost(pkg): + if not pkg.installed: + if not pkgsettings._accept_chost(pkg.cpv, pkg.metadata): mreasons.append("CHOST: %s" % \ pkg.metadata["CHOST"]) @@ -1463,6 +1463,7 @@ def get_mask_info(root_config, cpv, pkgsettings, if metadata and not built: pkgsettings.setcpv(cpv, mydb=metadata) metadata["USE"] = pkgsettings["PORTAGE_USE"] + metadata['CHOST'] = pkgsettings.get('CHOST', '') if metadata is None: mreasons = ["corruption"] else: @@ -2006,6 +2007,8 @@ class PipeReader(AbstractPollTask): def getvalue(self): """Retrieve the entire contents""" + if sys.hexversion >= 0x3000000: + return bytes().join(self._read_data) return "".join(self._read_data) def close(self): @@ -5493,6 +5496,7 @@ class depgraph(object): cpv=mykey, metadata=metadata, onlydeps=onlydeps) pkgsettings.setcpv(pkg) pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"] + pkg.metadata['CHOST'] = pkgsettings.get('CHOST', '') self._pkg_cache[pkg] = pkg args.append(PackageArg(arg=x, package=pkg, root_config=root_config)) @@ -6217,6 +6221,8 @@ class depgraph(object): onlydeps=onlydeps, root_config=root_config, type_name=pkg_type) metadata = pkg.metadata + if not built: + metadata['CHOST'] = pkgsettings.get('CHOST', '') if not built and ("?" in metadata["LICENSE"] or \ "?" in metadata["PROVIDE"]): # This is avoided whenever possible because @@ -6550,6 +6556,7 @@ class depgraph(object): settings = self.pkgsettings[root_config.root] settings.setcpv(pkg) pkg.metadata["USE"] = settings["PORTAGE_USE"] + pkg.metadata['CHOST'] = settings.get('CHOST', '') self._pkg_cache[pkg] = pkg return pkg @@ -8730,6 +8737,7 @@ class depgraph(object): pkgsettings = self.pkgsettings[myroot] pkgsettings.setcpv(pkg) pkg.metadata["USE"] = pkgsettings["PORTAGE_USE"] + pkg.metadata['CHOST'] = pkgsettings.get('CHOST', '') self._pkg_cache[pkg] = pkg root_config = self.roots[pkg.root] @@ -10825,6 +10833,12 @@ class Scheduler(PollScheduler): if graph is None: return pkg = merge.merge.pkg + + # Skip this if $ROOT != / since it shouldn't matter if there + # are unsatisfied system runtime deps in this case. + if pkg.root != '/': + return + completed_tasks = self._completed_tasks unsatisfied = self._unsatisfied_system_deps @@ -11473,6 +11487,7 @@ class Scheduler(PollScheduler): settings = self.pkgsettings[root_config.root] settings.setcpv(pkg) pkg.metadata["USE"] = settings["PORTAGE_USE"] + pkg.metadata['CHOST'] = settings.get('CHOST', '') return pkg diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index e75e875f7..5d3be97b0 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2557,7 +2557,7 @@ class config(object): ret.append(element) return ret - def _accept_chost(self, pkg): + def _accept_chost(self, cpv, metadata): """ @return True if pkg CHOST is accepted, False otherwise. """ @@ -2586,7 +2586,7 @@ class config(object): self._accept_chost_re = re.compile("^$") return self._accept_chost_re.match( - pkg.metadata.get("CHOST", "")) is not None + metadata.get('CHOST', '')) is not None def setinst(self,mycpv,mydbapi): """This updates the preferences for old-style virtuals, @@ -3920,15 +3920,23 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", elif distdir_writable: if mystat.st_size < fetch_resume_size and \ mystat.st_size < size: - writemsg((">>> Deleting distfile with size " + \ + # If the file already exists and the size does not + # match the existing digests, it may be that the + # user is attempting to update the digest. In this + # case, the digestgen() function will advise the + # user to use `ebuild --force foo.ebuild manifest` + # in order to force the old digests to be replaced. + # Since the user may want to keep this file, rename + # it instead of deleting it. + writemsg((">>> Renaming distfile with size " + \ "%d (smaller than " "PORTAGE_FETCH_RESU" + \ "ME_MIN_SIZE)\n") % mystat.st_size) - try: - os.unlink(myfile_path) - except OSError, e: - if e.errno != errno.ENOENT: - raise - del e + temp_filename = \ + _checksum_failure_temp_file( + mysettings["DISTDIR"], myfile) + writemsg_stdout("Refetching... " + \ + "File renamed to '%s'\n\n" % \ + temp_filename, noiselevel=-1) elif mystat.st_size >= size: temp_filename = \ _checksum_failure_temp_file( @@ -7598,7 +7606,8 @@ def commit_mtimedb(mydict=None, filename=None): f = atomic_ofstream(filename, mode='wb') pickle.dump(d, f, -1) f.close() - portage.util.apply_secpass_permissions(filename, uid=uid, gid=portage_gid, mode=0664) + portage.util.apply_secpass_permissions(filename, + uid=uid, gid=portage_gid, mode=0644) except (IOError, OSError), e: pass @@ -7630,7 +7639,7 @@ def _gen_missing_encodings(missing_encodings): class AsciiStreamReader(codecs.StreamReader): decode = codecs.ascii_decode - encodings['ascii'] = codecs.CodecInfo( + codec_info = codecs.CodecInfo( name='ascii', encode=codecs.ascii_encode, decode=codecs.ascii_decode, @@ -7640,6 +7649,11 @@ def _gen_missing_encodings(missing_encodings): streamreader=AsciiStreamReader, ) + for alias in ('ascii', '646', 'ansi_x3.4_1968', 'ansi_x3_4_1968', + 'ansi_x3.4_1986', 'cp367', 'csascii', 'ibm367', 'iso646_us', + 'iso_646.irv_1991', 'iso_ir_6', 'us', 'us_ascii'): + encodings[alias] = codec_info + if 'utf_8' in missing_encodings: def utf8decode(input, errors='strict'): @@ -7658,7 +7672,7 @@ def _gen_missing_encodings(missing_encodings): class Utf8StreamReader(codecs.StreamReader): decode = codecs.utf_8_decode - encodings['utf_8'] = codecs.CodecInfo( + codec_info = codecs.CodecInfo( name='utf-8', encode=codecs.utf_8_encode, decode=utf8decode, @@ -7668,6 +7682,9 @@ def _gen_missing_encodings(missing_encodings): streamwriter=Utf8StreamReader, ) + for alias in ('utf_8', 'u8', 'utf', 'utf8', 'utf8_ucs2', 'utf8_ucs4'): + encodings[alias] = codec_info + return encodings def _ensure_default_encoding(): @@ -7685,7 +7702,7 @@ def _ensure_default_encoding(): """ default_fallback = 'utf_8' - default_encoding = sys.getdefaultencoding() + default_encoding = sys.getdefaultencoding().lower().replace('-', '_') required_encodings = set(['ascii', 'utf_8']) required_encodings.add(default_encoding) missing_encodings = set() @@ -7711,6 +7728,8 @@ def _ensure_default_encoding(): encodings[default_encoding] = encodings[default_fallback] def search_function(name): + name = name.lower() + name = name.replace('-', '_') codec_info = encodings.get(name) if codec_info is not None: return codecs.CodecInfo( diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py new file mode 100644 index 000000000..83ed8a783 --- /dev/null +++ b/pym/portage/cache/ebuild_xattr.py @@ -0,0 +1,163 @@ +# -*- coding: UTF8 -*- +# Copyright: 2009 Gentoo Foundation +# Author(s): Petteri Räty (betelgeuse@gentoo.org) +# License: GPL2 +# $Id$ + +__all__ = ['database'] + +from portage.cache import fs_template +from portage.versions import catsplit +from portage import cpv_getkey +from portage.util import writemsg +import os +import xattr +from errno import ENODATA,ENOSPC,E2BIG + +class NoValueException(Exception): + pass + +class database(fs_template.FsBased): + + autocommits = True + + def __init__(self, *args, **config): + super(database,self).__init__(*args, **config) + self.portdir = self.label + self.ns = xattr.NS_USER + '.gentoo.cache' + self.keys = set(self._known_keys) + self.keys.add('_mtime_') + self.keys.add('_eclasses_') + # xattrs have an upper length + self.max_len = self.__get_max() + + def __get_max(self): + path = os.path.join(self.portdir,'profiles/repo_name') + try: + return int(self.__get(path,'value_max_len')) + except NoValueException,e: + max = self.__calc_max(path) + self.__set(path,'value_max_len',str(max)) + return max + + def __calc_max(self,path): + """ Find out max attribute length supported by the file system """ + + hundred = '' + for i in range(100): + hundred+='a' + + s=hundred + + # Could use finally but needs python 2.5 then + try: + while True: + self.__set(path,'test_max',s) + s+=hundred + except IOError,e: + # ext based give wrong errno + # http://bugzilla.kernel.org/show_bug.cgi?id=12793 + if e.errno in (E2BIG,ENOSPC): + result = len(s)-100 + else: + raise e + + try: + self.__remove(path,'test_max') + except IOError,e: + if e.errno is not ENODATA: + raise e + + return result + + def __get_path(self,cpv): + cat,pn = catsplit(cpv_getkey(cpv)) + return os.path.join(self.portdir,cat,pn,os.path.basename(cpv) + ".ebuild") + + def __has_cache(self,path): + try: + self.__get(path,'_mtime_') + except NoValueException,e: + return False + + return True + + def __get(self,path,key,default=None): + try: + return xattr.get(path,key,namespace=self.ns) + except IOError,e: + if not default is None and ENODATA == e.errno: + return default + else: + raise NoValueException() + + def __remove(self,path,key): + xattr.remove(path,key,namespace=self.ns) + + def __set(self,path,key,value): + xattr.set(path,key,value,namespace=self.ns) + + def _getitem(self, cpv): + values = {} + path = self.__get_path(cpv) + all = {} + for tuple in xattr.get_all(path,namespace=self.ns): + key,value = tuple + all[key] = value + + if not '_mtime_' in all: + raise KeyError(cpv) + + # We default to '' like other caches + for key in self.keys: + attr_value = all.get(key,'1:') + parts,sep,value = attr_value.partition(':') + parts = int(parts) + if parts > 1: + for i in range(1,parts): + value += all.get(key+str(i)) + values[key] = value + + return values + + def _setitem(self, cpv, values): + path = self.__get_path(cpv) + max = self.max_len + for key,value in values.iteritems(): + # mtime comes in as long so need to convert to strings + s = str(value) + # We need to split long values + value_len = len(s) + parts = 0 + if value_len > max: + # Find out how many parts we need + parts = value_len/max + if value_len % max > 0: + parts += 1 + + # Only the first entry carries the number of parts + self.__set(path,key,'%s:%s'%(parts,s[0:max])) + + # Write out the rest + for i in range(1,parts): + start = i * max + val = s[start:start+max] + self.__set(path,key+str(i),val) + else: + self.__set(path,key,"%s:%s"%(1,s)) + + def _delitem(self, cpv): + pass # Will be gone with the ebuild + + def __contains__(self, cpv): + return os.path.exists(self.__get_path(cpv)) + + def __iter__(self): + for root,dirs,files in os.walk(self.portdir): + for file in files: + if file[-7:] == '.ebuild': + cat = os.path.basename(os.path.dirname(root)) + pn_pv = file[:-7] + path = os.path.join(root,file) + if self.__has_cache(path): + yield "%s/%s/%s" % (cat,os.path.basename(root),file[:-7]) diff --git a/pym/portage/cache/metadata_overlay.py b/pym/portage/cache/metadata_overlay.py index a12ffd7ba..027693605 100644 --- a/pym/portage/cache/metadata_overlay.py +++ b/pym/portage/cache/metadata_overlay.py @@ -98,6 +98,6 @@ class database(template.database): return False def _are_values_identical(self, value1, value2): - if long(value1["_mtime_"]) != long(value2["_mtime_"]): + if value1['_mtime_'] != value2['_mtime_']: return False return value1["_eclasses_"] == value2["_eclasses_"] diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index f6e8c103c..5a7bcf1f5 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -143,7 +143,7 @@ class database(fs_template.FsBased): if actual_synchronous!=synchronous: raise cache_errors.InitializationError(self.__class__,"actual synchronous = "+actual_synchronous+" does does not match requested value of "+synchronous) - def __getitem__(self, cpv): + def _getitem(self, cpv): cursor = self._db_cursor cursor.execute("select * from %s where %s=%s" % \ (self._db_table["packages"]["table_name"], @@ -169,10 +169,6 @@ class database(fs_template.FsBased): d[k]=str(d[k]) # convert unicode strings to normal except UnicodeEncodeError, e: pass #writemsg("%s: %s\n" % (cpv, str(e))) - if "_eclasses_" in d: - d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"]) - else: - d["_eclasses_"] = {} for x in self._known_keys: d.setdefault(x,'') return d diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py index 35b9efeff..7fce8ef17 100644 --- a/pym/portage/cache/template.py +++ b/pym/portage/cache/template.py @@ -40,6 +40,16 @@ class database(object): d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"]) elif "_eclasses_" not in d: d["_eclasses_"] = {} + mtime = d.get('_mtime_') + if mtime is None: + raise cache_errors.CacheCorruption(cpv, + '_mtime_ field is missing') + try: + mtime = long(mtime) + except ValueError: + raise cache_errors.CacheCorruption(cpv, + '_mtime_ conversion to long failed: %s' % (mtime,)) + d['_mtime_'] = mtime return d def _getitem(self, cpv): diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py index 86f93440e..e152f5081 100644 --- a/pym/portage/cache/util.py +++ b/pym/portage/cache/util.py @@ -45,13 +45,10 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, except (KeyError, cache_errors.CacheError): pass else: - try: - if long(trg["_mtime_"]) == long(entry["_mtime_"]) and \ - eclass_cache.is_eclass_data_valid(trg["_eclasses_"]) and \ - set(trg["_eclasses_"]) == set(entry["_eclasses_"]): - write_it = False - except cache_errors.CacheError: - pass + if trg['_mtime_'] == entry['_mtime_'] and \ + eclass_cache.is_eclass_data_valid(trg['_eclasses_']) and \ + set(trg['_eclasses_']) == set(entry['_eclasses_']): + write_it = False for d in (entry, trg): if d is not None and d.get('EAPI') in ('', '0'): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index ab5e9ba13..7ac281e8b 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -66,6 +66,9 @@ def _src_uri_validate(cpv, eapi, src_uri): "supported with EAPI='%s'") % (cpv, eapi)) operator = x continue + if operator is None: + uri = x + continue if operator is not None: if "/" in x: raise portage.exception.InvalidDependString( @@ -343,19 +346,6 @@ class portdbapi(dbapi): for auxdb in auxdbs: try: metadata = auxdb[cpv] - eapi = metadata.get("EAPI","").strip() - if not eapi: - eapi = "0" - if eapi.startswith("-") and eapi_is_supported(eapi[1:]): - pass - elif emtime != int(metadata.get("_mtime_", 0)): - pass - elif len(metadata.get("_eclasses_", [])) > 0: - if self.eclassdb.is_eclass_data_valid( - metadata["_eclasses_"]): - doregen = False - else: - doregen = False except KeyError: pass except CacheError: @@ -364,6 +354,15 @@ class portdbapi(dbapi): del auxdb[cpv] except KeyError: pass + else: + eapi = metadata.get('EAPI', '').strip() + if not eapi: + eapi = '0' + if not (eapi[:1] == '-' and eapi_is_supported(eapi[1:])) and \ + emtime == metadata['_mtime_'] and \ + self.eclassdb.is_eclass_data_valid(metadata['_eclasses_']): + doregen = False + if not doregen: break @@ -857,6 +856,8 @@ class portdbapi(dbapi): aux_keys = list(self._aux_cache_keys) metadata = {} local_config = self.mysettings.local_config + chost = self.mysettings.get('CHOST', '') + accept_chost = self.mysettings._accept_chost for mycpv in mylist: metadata.clear() try: @@ -877,10 +878,13 @@ class portdbapi(dbapi): if self.mysettings._getMissingKeywords(mycpv, metadata): continue if local_config: + metadata['CHOST'] = chost + if not accept_chost(mycpv, metadata): + continue metadata["USE"] = "" if "?" in metadata["LICENSE"]: self.doebuild_settings.setcpv(mycpv, mydb=metadata) - metadata["USE"] = self.doebuild_settings.get("USE", "") + metadata['USE'] = self.doebuild_settings['PORTAGE_USE'] try: if self.mysettings._getMissingLicenses(mycpv, metadata): continue diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 22fa8bf56..4a9d85e3c 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -506,14 +506,8 @@ class Atom(object): __metaclass__ = _AtomCache _atoms = weakref.WeakValueDictionary() - _str_methods = ("endswith", "find", "index", "lstrip", "replace", - "startswith", "split", "strip", - "rindex", "rfind", "rstrip", "__getitem__", - "__eq__", "__hash__", "__len__", "__lt__", - "__ne__", "__repr__", "__str__") - __slots__ = ("__weakref__", "blocker", "cp", "cpv", "operator", - "slot", "use") + _str_methods + "slot", "use", "_str") class _blocker(object): __slots__ = ("overlap",) @@ -531,8 +525,7 @@ class Atom(object): if not isvalidatom(s, allow_blockers=True): raise InvalidAtom(s) obj_setattr = object.__setattr__ - for x in self._str_methods: - obj_setattr(self, x, getattr(s, x)) + obj_setattr(self, '_str', s) blocker = "!" == s[:1] if blocker: @@ -561,6 +554,65 @@ class Atom(object): raise AttributeError("Atom instances are immutable", self.__class__, name, value) + # Implement some common str methods. + + def __eq__(self, other): + return self._str == other + + def __getitem__(self, key): + return self._str[key] + + def __hash__(self): + return hash(self._str) + + def __len__(self): + return len(self._str) + + def __lt__(self, other): + return self._str < other + + def __ne__(self, other): + return self._str != other + + def __repr__(self): + return repr(self._str) + + def __str__(self): + return str(self._str) + + def endswith(self, *pargs, **kargs): + return self._str.endswith(*pargs, **kargs) + + def find(self, *pargs, **kargs): + return self._str.find(*pargs, **kargs) + + def index(self, *pargs, **kargs): + return self._str.index(*pargs, **kargs) + + def lstrip(self, *pargs, **kargs): + return self._str.lstrip(*pargs, **kargs) + + def replace(self, *pargs, **kargs): + return self._str.replace(*pargs, **kargs) + + def startswith(self, *pargs, **kargs): + return self._str.startswith(*pargs, **kargs) + + def split(self, *pargs, **kargs): + return self._str.split(*pargs, **kargs) + + def strip(self, *pargs, **kargs): + return self._str.strip(*pargs, **kargs) + + def rindex(self, *pargs, **kargs): + return self._str.rindex(*pargs, **kargs) + + def rfind(self, *pargs, **kargs): + return self._str.rfind(*pargs, **kargs) + + def rstrip(self, *pargs, **kargs): + return self._str.rstrip(*pargs, **kargs) + def get_operator(mydep): """ Return the operator used in a depstring. diff --git a/pym/portage/env/loaders.py b/pym/portage/env/loaders.py index b23c7f393..e031ed60f 100644 --- a/pym/portage/env/loaders.py +++ b/pym/portage/env/loaders.py @@ -268,14 +268,14 @@ class KeyValuePairFileLoader(FileLoader): return if not len(line): # skip empty lines return - split = line.split('=') + split = line.split('=', 1) if len(split) < 2: errors.setdefault(self.fname, []).append( "Malformed data at line: %s, data %s" % (line_num + 1, line)) return - key = split[0] - value = split[1:] + key = split[0].strip() + value = split[1].strip() if not key: errors.setdefault(self.fname, []).append( "Malformed key at line: %s, key %s" diff --git a/pym/portage/sets/shell.py b/pym/portage/sets/shell.py index ba1b2422f..43a8fe02f 100644 --- a/pym/portage/sets/shell.py +++ b/pym/portage/sets/shell.py @@ -33,10 +33,10 @@ class CommandOutputSet(PackageSet): def load(self): pipe = subprocess.Popen(self._command, stdout=subprocess.PIPE, shell=True) + stdout, stderr = pipe.communicate() if pipe.wait() == os.EX_OK: - text = pipe.stdout.read() - self._setAtoms(text.split("\n")) - + self._setAtoms(unicode(stdout, errors='replace').splitlines()) + def singleBuilder(self, options, settings, trees): if not "command" in options: raise SetConfigError("no command specified") diff --git a/pym/portage/tests/dep/test_src_uri.py b/pym/portage/tests/dep/test_src_uri.py index 84c9f29b6..e53297968 100644 --- a/pym/portage/tests/dep/test_src_uri.py +++ b/pym/portage/tests/dep/test_src_uri.py @@ -23,6 +23,7 @@ class SrcUri(TestCase): ( "2", "http://foo/bar ->" , False ), ( "2", "foo? ( http://foo/bar -> ) blah.tbz2" , False ), ( "2", "http://foo/bar -> foo/blah.tbz2" , False ), + ( "2", "http://foo.com/foo http://foo/bar -> blah.tbz2" , True ), ] for eapi, src_uri, valid in tests: diff --git a/pym/portage/tests/env/config/test_PackageUseFile.py b/pym/portage/tests/env/config/test_PackageUseFile.py index f6eb38846..aa18364a0 100644 --- a/pym/portage/tests/env/config/test_PackageUseFile.py +++ b/pym/portage/tests/env/config/test_PackageUseFile.py @@ -30,10 +30,9 @@ class PackageUseFileTestCase(TestCase): def BuildFile(self): fd, self.fname = mkstemp() - f = os.fdopen(fd, 'wb') + f = os.fdopen(fd, 'w') f.write("%s %s" % (self.cpv, ' '.join(self.useflags))) f.close() def NukeFile(self): - import os os.unlink(self.fname) diff --git a/pym/portage/tests/env/config/test_PortageModulesFile.py b/pym/portage/tests/env/config/test_PortageModulesFile.py index e4f9e4252..fb164b4a7 100644 --- a/pym/portage/tests/env/config/test_PortageModulesFile.py +++ b/pym/portage/tests/env/config/test_PortageModulesFile.py @@ -22,14 +22,14 @@ class PortageModulesFileTestCase(TestCase): f = PortageModulesFile(self.fname) f.load() for k in self.keys: - self.assertEqual(f[k], [self.items[k]]) + self.assertEqual(f[k], self.items[k]) for ik in self.invalid_keys: self.assertEqual(False, ik in f) self.NukeFile() def BuildFile(self): fd, self.fname = mkstemp() - f = os.fdopen(fd, 'wb') + f = os.fdopen(fd, 'w') for k, v in self.items.iteritems(): f.write('%s=%s\n' % (k,v)) f.close() diff --git a/pym/portage/tests/news/test_NewsItem.py b/pym/portage/tests/news/test_NewsItem.py index f6c59e6a2..f4a073504 100644 --- a/pym/portage/tests/news/test_NewsItem.py +++ b/pym/portage/tests/news/test_NewsItem.py @@ -56,8 +56,8 @@ The revdep-rebuild tool is provided by app-portage/gentoolkit. tmpItem = self.fakeItem[:].replace("#Display-If-Profile:", "Display-If-Profile: %s" % self.profile) + item = self._processItem(tmpItem) try: - item = self._processItem(tmpItem) self.assertTrue(item.isRelevant(self.vardb, self.settings, self.profile), msg="Expected %s to be relevant, but it was not!" % tmpItem) finally: @@ -88,9 +88,9 @@ The revdep-rebuild tool is provided by app-portage/gentoolkit. def _processItem(self, item): filename = None fd, filename = mkstemp() - f = os.fdopen(fd, 'wb') + f = os.fdopen(fd, 'w') f.write(item) - f.close + f.close() try: return NewsItem(filename, 0) except TypeError: diff --git a/pym/portage/tests/process/test_poll.py b/pym/portage/tests/process/test_poll.py index ebcc822ef..e399c479d 100644 --- a/pym/portage/tests/process/test_poll.py +++ b/pym/portage/tests/process/test_poll.py @@ -28,12 +28,18 @@ class PipeReaderTestCase(TestCase): test_string = 2 * "blah blah blah\n" master_fd, slave_fd = self._create_pipe() - master_file = os.fdopen(master_fd, 'r') + master_file = os.fdopen(master_fd, 'rb') task_scheduler = TaskScheduler(max_jobs=2) scheduler = task_scheduler.sched_iface - producer = SpawnProcess( + class Producer(SpawnProcess): + def _spawn(self, args, **kwargs): + rval = SpawnProcess._spawn(self, args, **kwargs) + os.close(kwargs['fd_pipes'][1]) + return rval + + producer = Producer( args=["bash", "-c", "echo -n '%s'" % test_string], fd_pipes={1:slave_fd}, scheduler=scheduler) @@ -44,10 +50,9 @@ class PipeReaderTestCase(TestCase): task_scheduler.add(producer) task_scheduler.add(consumer) - def producer_start_cb(task): - os.close(slave_fd) - - producer.addStartListener(producer_start_cb) task_scheduler.run() + if sys.hexversion >= 0x3000000: + test_string = test_string.encode() + self._assertEqual(test_string, consumer.getvalue()) diff --git a/pym/portage/tests/sets/files/testConfigFileSet.py b/pym/portage/tests/sets/files/testConfigFileSet.py index 5a54e6a78..7ea17fd1b 100644 --- a/pym/portage/tests/sets/files/testConfigFileSet.py +++ b/pym/portage/tests/sets/files/testConfigFileSet.py @@ -13,13 +13,14 @@ class ConfigFileSetTestCase(TestCase): def setUp(self): fd, self.testfile = tempfile.mkstemp(suffix=".testdata", prefix=self.__class__.__name__, text=True) + f = os.fdopen(fd, 'w') for i in range(0, len(test_cps)): atom = test_cps[i] if i % 2 == 0: - os.write(fd, atom+" abc def"+"\n") + f.write(atom + ' abc def\n') else: - os.write(fd, atom+"\n") - os.close(fd) + f.write(atom + '\n') + f.close() def tearDown(self): os.unlink(self.testfile) diff --git a/pym/portage/tests/sets/files/testStaticFileSet.py b/pym/portage/tests/sets/files/testStaticFileSet.py index 57a8c58cd..fff583da1 100644 --- a/pym/portage/tests/sets/files/testStaticFileSet.py +++ b/pym/portage/tests/sets/files/testStaticFileSet.py @@ -15,8 +15,9 @@ class StaticFileSetTestCase(TestCase): def setUp(self): fd, self.testfile = tempfile.mkstemp(suffix=".testdata", prefix=self.__class__.__name__, text=True) - os.write(fd, "\n".join(test_cps)) - os.close(fd) + f = os.fdopen(fd, 'w') + f.write("\n".join(test_cps)) + f.close() def tearDown(self): os.unlink(self.testfile) diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py index aee6d7c97..a9e29d2be 100644 --- a/pym/repoman/checks.py +++ b/pym/repoman/checks.py @@ -319,9 +319,15 @@ class IUseUndefined(LineCheck): class EMakeParallelDisabled(LineCheck): """Check for emake -j1 calls which disable parallelization.""" repoman_check_name = 'upstream.workaround' - re = re.compile(r'^\s*emake\s+-j\s*1\s') + re = re.compile(r'^\s*emake\s+.*-j\s*1\b') error = errors.EMAKE_PARALLEL_DISABLED +class EMakeParallelDisabledViaMAKEOPTS(LineCheck): + """Check for MAKEOPTS=-j1 that disables parallelization.""" + repoman_check_name = 'upstream.workaround' + re = re.compile(r'^\s*MAKEOPTS=(\'|")?.*-j\s*1\b') + error = errors.EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS + class DeprecatedBindnowFlags(LineCheck): """Check for calls to the deprecated bindnow-flags function.""" repoman_check_name = 'ebuild.minorsyn' @@ -354,8 +360,8 @@ _constant_checks = tuple((c() for c in ( EbuildUselessCdS, EbuildNestedDie, EbuildPatches, EbuildQuotedA, IUseUndefined, ImplicitRuntimeDeps, InheritAutotools, - EMakeParallelDisabled, DeprecatedBindnowFlags, - WantAutoDefaultValue, PortageInternal))) + EMakeParallelDisabled, EMakeParallelDisabledViaMAKEOPTS, + DeprecatedBindnowFlags, WantAutoDefaultValue, PortageInternal))) def run_checks(contents, pkg): checks = _constant_checks diff --git a/pym/repoman/errors.py b/pym/repoman/errors.py index 205913951..451e499ad 100644 --- a/pym/repoman/errors.py +++ b/pym/repoman/errors.py @@ -14,5 +14,6 @@ NESTED_DIE_ERROR = 'Ebuild calls die in a subshell on line: %d' PATCHES_ERROR = 'PATCHES is not a bash array on line: %d' REDUNDANT_CD_S_ERROR = 'Ebuild has redundant cd ${S} statement on line: %d' EMAKE_PARALLEL_DISABLED = 'Upstream parallel compilation bug (ebuild calls emake -j1 on line: %d)' +EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS = 'Upstream parallel compilation bug (MAKEOPTS=-j1 on line: %d)' DEPRECATED_BINDNOW_FLAGS = 'Deprecated bindnow-flags call on line: %d' PREPALLDOCS_ERROR = 'prepalldocs called on line: %d' -- 2.26.2