From: Fabian Groffen Date: Wed, 5 Aug 2009 18:11:44 +0000 (-0000) Subject: Merged from trunk -r13909:13919 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=326318e87378653659eec782c21c9e3c16e865c3;p=portage.git Merged from trunk -r13909:13919 | 13910 | Avoid UnicodeEncodeError with unicode package.mask comments. | | zmedico | Thanks to Thanks to Scott Moreau (soreau) for reporting. | | 13911 | Fix bindbapi.aux_get and aux_update to work with | | zmedico | py3k/unicode. | | 13912 | Open all files in binary mode for py3k compatibility. | | zmedico | | | 13913 | Open file in text mode (unicode) where appropriate. | | zmedico | | | 13914 | In _check_build_log(), open the log in text mode (unicode). | | zmedico | | | 13915 | Open ld.so.conf as text (unicode). | | zmedico | | | 13916 | Fix the FEATURES=parse-eapi-ebuild-head regex to handle | | zmedico | comments on the same line, like this: EAPI=2 #foo Thanks to | | | Markus Meier for reporting. | | 13917 | s/utf8/utf_8/ for consistency | | zmedico | | | 13919 | Fix insert_optional_args() to properly handle thinks like | | zmedico | -Dk which require multiple substitutions of default | | | arguments. | svn path=/main/branches/prefix/; revision=13923 --- diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py index 9a11e0835..c538c4e09 100644 --- a/pym/_emerge/Binpkg.py +++ b/pym/_emerge/Binpkg.py @@ -11,6 +11,7 @@ from _emerge.EbuildMerge import EbuildMerge from _emerge.EbuildBuildDir import EbuildBuildDir from portage.util import writemsg # for an explanation on this logic, see pym/_emerge/__init__.py +import codecs import os import sys if os.environ.__contains__("PORTAGE_PYTHONPATH"): @@ -36,7 +37,8 @@ class Binpkg(CompositeTask): log_path = self.settings.get("PORTAGE_LOG_FILE") if log_path is not None: - f = open(log_path, 'a') + f = codecs.open(log_path, mode='a', + encoding='utf_8', errors='replace') try: f.write(msg) finally: @@ -229,7 +231,8 @@ class Binpkg(CompositeTask): else: continue - f = open(os.path.join(infloc, k), 'wb') + f = codecs.open(os.path.join(infloc, k), mode='w', + encoding='utf_8', errors='replace') try: f.write(v + "\n") finally: diff --git a/pym/_emerge/BinpkgVerifier.py b/pym/_emerge/BinpkgVerifier.py index 56190a60b..b4a4b8f42 100644 --- a/pym/_emerge/BinpkgVerifier.py +++ b/pym/_emerge/BinpkgVerifier.py @@ -6,6 +6,7 @@ from _emerge.AsynchronousTask import AsynchronousTask from portage.util import writemsg import sys # for an explanation on this logic, see pym/_emerge/__init__.py +import codecs import os import sys if os.environ.__contains__("PORTAGE_PYTHONPATH"): @@ -31,7 +32,8 @@ class BinpkgVerifier(AsynchronousTask): stderr_orig = sys.stderr log_file = None if self.background and self.logfile is not None: - log_file = open(self.logfile, 'a') + log_file = codecs.open(self.logfile, mode='a', + encoding='utf_8', errors='replace') try: if log_file is not None: sys.stdout = log_file diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index d7e769482..18c7c59f8 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -12,6 +12,7 @@ from _emerge.EbuildFetchonly import EbuildFetchonly from _emerge.EbuildBuildDir import EbuildBuildDir from portage.util import writemsg # for an explanation on this logic, see pym/_emerge/__init__.py +import codecs import os import sys if os.environ.__contains__("PORTAGE_PYTHONPATH"): @@ -188,7 +189,8 @@ class EbuildBuild(CompositeTask): log_path = self.settings.get("PORTAGE_LOG_FILE") if log_path is not None: - log_file = open(log_path, 'a') + log_file = codecs.open(log_path, mode='a', + encoding='utf_8', errors='replace') try: log_file.write(msg) finally: diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index b17484565..a292c6cdd 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -12,6 +12,7 @@ if os.environ.__contains__("PORTAGE_PYTHONPATH"): else: sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym")) import portage +import codecs import os from portage.elog.messages import eerror class EbuildFetcher(SpawnProcess): @@ -92,7 +93,8 @@ class EbuildFetcher(SpawnProcess): elog_out = None if self.logfile is not None: if self.background: - elog_out = open(self.logfile, 'a') + elog_out = codecs.open(self.logfile, mode='a', + encoding='utf_8', errors='replace') msg = "Fetch failed for '%s'" % (self.pkg.cpv,) if self.logfile is not None: msg += ", Log file:" diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index d88f9e77a..3df624859 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -7,6 +7,7 @@ from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask from portage.util import writemsg # for an explanation on this logic, see pym/_emerge/__init__.py +import codecs import os import sys if os.environ.__contains__("PORTAGE_PYTHONPATH"): @@ -36,7 +37,8 @@ class EbuildPhase(CompositeTask): log_path = self.settings.get("PORTAGE_LOG_FILE") log_file = None if self.background and log_path is not None: - log_file = open(log_path, 'a') + log_file = codecs.open(log_path, mode='a', + encoding='utf_8', errors='replace') out = log_file try: portage._check_build_log(self.settings, out=out) diff --git a/pym/_emerge/PackageUninstall.py b/pym/_emerge/PackageUninstall.py index dee26d2f3..d430a4709 100644 --- a/pym/_emerge/PackageUninstall.py +++ b/pym/_emerge/PackageUninstall.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import codecs import logging # for an explanation on this logic, see pym/_emerge/__init__.py import os @@ -46,7 +47,8 @@ class PackageUninstall(AsynchronousTask): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) - f = open(log_path, 'a') + f = codecs.open(log_path, mode='a', + encoding='utf_8', errors='replace') try: f.write(msg) finally: diff --git a/pym/_emerge/changelog.py b/pym/_emerge/changelog.py index 65427b23a..8fb25e01e 100644 --- a/pym/_emerge/changelog.py +++ b/pym/_emerge/changelog.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import codecs import os import re @@ -25,7 +26,8 @@ def calc_changelog(ebuildpath,current,next): next = next[:-3] changelogpath = os.path.join(os.path.split(ebuildpath)[0],'ChangeLog') try: - changelog = open(changelogpath).read() + changelog = codecs.open(changelogpath, mode='r', + encoding='utf_8', errors='replace').read() except SystemExit, e: raise # Needed else can't exit except: diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index f9363df9f..8a5fba520 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -4356,6 +4356,14 @@ class depgraph(object): if "--changelog" in self._frozen_config.myopts: print for revision,text in changelogs: + + if sys.hexversion < 0x3000000: + # avoid potential UnicodeEncodeError + if isinstance(revision, unicode): + revision = revision.encode('utf_8', 'replace') + if isinstance(text, unicode): + text = text.encode('utf_8', 'replace') + print bold('*'+revision) sys.stdout.write(text) @@ -5113,6 +5121,11 @@ def show_masked_packages(masked_packages): pass print "- "+cpv+" (masked by: "+", ".join(mreasons)+")" + + if sys.hexversion < 0x3000000 and isinstance(comment, unicode): + # avoid potential UnicodeEncodeError + comment = comment.encode('utf_8', 'replace') + if comment and comment not in shown_comments: print filename+":" print comment diff --git a/pym/_emerge/emergelog.py b/pym/_emerge/emergelog.py index 4c8f54719..67aa3f00e 100644 --- a/pym/_emerge/emergelog.py +++ b/pym/_emerge/emergelog.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +import codecs import os import sys import time @@ -27,7 +28,8 @@ def emergelog(xterm_titles, mystr, short_msg=None): xtermTitle(short_msg) try: file_path = os.path.join(_emerge_log_dir, 'emerge.log') - mylogfile = open(file_path, "a") + mylogfile = codecs.open(file_path, mode='a', + encoding='utf_8', errors='replace') portage.util.apply_secpass_permissions(file_path, uid=portage.portage_uid, gid=portage.portage_gid, mode=0660) diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 2113c1b6e..5edb4c5ab 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -487,7 +487,9 @@ def insert_optional_args(args): new_args.append(opt_arg) if saved_opts is not None: - new_args.append("-" + saved_opts) + # Recycle these on arg_stack since they + # might contain another match. + arg_stack.append("-" + saved_opts) return new_args diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index d884f8cb8..873b3eb33 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -727,7 +727,8 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, ldsoconf_path = os.path.join(target_root, EPREFIX_LSTRIP, "etc", "ld.so.conf") try: - myld = open(ldsoconf_path) + myld = codecs.open(ldsoconf_path, mode='r', + encoding='utf_8', errors='replace') myldlines=myld.readlines() myld.close() oldld=[] @@ -5034,7 +5035,8 @@ def _check_build_log(mysettings, out=None): if logfile is None: return try: - f = open(logfile) + f = codecs.open(logfile, mode='r', + encoding='utf_8', errors='replace') except EnvironmentError: return @@ -5295,7 +5297,7 @@ def eapi_is_supported(eapi): # the ebuild. _validate_cache_for_unsupported_eapis = True -_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"]*)') +_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"#]*)') _parse_eapi_ebuild_head_max_lines = 30 def _parse_eapi_ebuild_head(f): diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 6bd47ec1f..313a41598 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -25,6 +25,7 @@ from portage import dep_expand, listdir, _check_distfile, _movefile import codecs import os, errno, stat import re +import sys from itertools import chain, izip class bindbapi(fakedbapi): @@ -68,7 +69,12 @@ class bindbapi(fakedbapi): tbz2_path = self.bintree.getname(mycpv) if not os.path.exists(tbz2_path): raise KeyError(mycpv) - getitem = portage.xpak.tbz2(tbz2_path).getfile + tbz2 = portage.xpak.tbz2(tbz2_path) + def getitem(k): + v = tbz2.getfile(k) + if v is not None and not isinstance(v, unicode): + v = unicode(v, encoding='utf_8', errors='replace') + return v else: getitem = self.bintree._remotepkgs[mycpv].get mydata = {} @@ -81,15 +87,16 @@ class bindbapi(fakedbapi): # or the tbz2 is corrupt. if myval: mydata[x] = " ".join(myval.split()) - if "EAPI" in mykeys: - if not mydata.setdefault("EAPI", "0"): - mydata["EAPI"] = "0" + + if not mydata.setdefault('EAPI', u'0'): + mydata['EAPI'] = u'0' + if cache_me: aux_cache = self._aux_cache_slot_dict() for x in self._aux_cache_keys: - aux_cache[x] = mydata.get(x, "") + aux_cache[x] = mydata.get(x, u'') self._aux_cache[mycpv] = aux_cache - return [mydata.get(x, "") for x in wants] + return [mydata.get(x, u'') for x in wants] def aux_update(self, cpv, values): if not self.bintree.populated: @@ -99,7 +106,23 @@ class bindbapi(fakedbapi): raise KeyError(cpv) mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() - mydata.update(values) + + if sys.hexversion < 0x3000000: + for k, v in values.iteritems(): + if isinstance(k, unicode): + k = k.encode('utf_8', 'replace') + if isinstance(v, unicode): + v = v.encode('utf_8', 'replace') + mydata[k] = v + + else: + for k, v in values.iteritems(): + if isinstance(k, str): + k = k.encode('utf_8', 'replace') + if isinstance(v, str): + v = v.encode('utf_8', 'replace') + mydata[k] = v + for k, v in mydata.items(): if not v: del mydata[k] @@ -1083,7 +1106,7 @@ class binarytree(object): pkgindex = self._new_pkgindex() try: f = codecs.open(self._pkgindex_file, - encoding='utf8', errors='replace') + encoding='utf_8', errors='replace') except EnvironmentError: pass else: diff --git a/pym/portage/xpak.py b/pym/portage/xpak.py index e24da3010..a1516e033 100644 --- a/pym/portage/xpak.py +++ b/pym/portage/xpak.py @@ -68,14 +68,14 @@ def xpak(rootdir,outfile=None): mylist.sort() mydata = {} for x in mylist: - a = open(x, "r") + a = open(x, 'rb') mydata[x] = a.read() a.close() os.chdir(origdir) xpak_segment = xpak_mem(mydata) if outfile: - outf = open(outfile, "w") + outf = open(outfile, 'wb') outf.write(xpak_segment) outf.close() else: @@ -104,7 +104,7 @@ def xsplit(infile): """(infile) -- Splits the infile into two files. 'infile.index' contains the index segment. 'infile.dat' contails the data segment.""" - myfile=open(infile,"r") + myfile = open(infile, 'rb') mydat=myfile.read() myfile.close() @@ -112,10 +112,10 @@ def xsplit(infile): if not splits: return False - myfile=open(infile+".index","w") + myfile = open(infile + '.index', 'wb') myfile.write(splits[0]) myfile.close() - myfile=open(infile+".dat","w") + myfile = open(infile + '.dat', 'wb') myfile.write(splits[1]) myfile.close() return True @@ -130,7 +130,7 @@ def xsplit_mem(mydat): def getindex(infile): """(infile) -- grabs the index segment from the infile and returns it.""" - myfile=open(infile,"r") + myfile = open(infile, 'rb') myheader=myfile.read(16) if myheader[0:8]!="XPAKPACK": myfile.close() @@ -143,7 +143,7 @@ def getindex(infile): def getboth(infile): """(infile) -- grabs the index and data segments from the infile. Returns an array [indexSegment,dataSegment]""" - myfile=open(infile,"r") + myfile = open(infile, 'rb') myheader=myfile.read(16) if myheader[0:8]!="XPAKPACK": myfile.close() @@ -217,7 +217,7 @@ def xpand(myid,mydest): if dirname: if not os.path.exists(dirname): os.makedirs(dirname) - mydat=open(myname,"w") + mydat = open(myname, 'wb') mydat.write(mydata[datapos:datapos+datalen]) mydat.close() startpos=startpos+namelen+12 @@ -262,7 +262,7 @@ class tbz2(object): def recompose_mem(self, xpdata): self.scan() # Don't care about condition... We'll rewrite the data anyway. - myfile=open(self.file,"a+") + myfile = open(self.file, 'ab+') if not myfile: raise IOError myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF. @@ -298,7 +298,7 @@ class tbz2(object): if not changed: return 1 self.filestat=mystat - a=open(self.file,"r") + a = open(self.file, 'rb') a.seek(-16,2) trailer=a.read() self.infosize=0 @@ -341,7 +341,7 @@ class tbz2(object): myresult=searchindex(self.index,myfile) if not myresult: return mydefault - a=open(self.file,"r") + a = open(self.file, 'rb') a.seek(self.datapos+myresult[0],0) myreturn=a.read(myresult[1]) a.close() @@ -365,7 +365,7 @@ class tbz2(object): except: os.chdir("/") origdir="/" - a=open(self.file,"r") + a = open(self.file, 'rb') if not os.path.exists(mydest): os.makedirs(mydest) os.chdir(mydest) @@ -379,7 +379,7 @@ class tbz2(object): if dirname: if not os.path.exists(dirname): os.makedirs(dirname) - mydat=open(myname,"w") + mydat = open(myname, 'wb') a.seek(self.datapos+datapos) mydat.write(a.read(datalen)) mydat.close() @@ -392,7 +392,7 @@ class tbz2(object): """Returns all the files from the dataSegment as a map object.""" if not self.scan(): return 0 - a = open(self.file, "r") + a = open(self.file, 'rb') mydata = {} startpos=0 while ((startpos+8)