From 6de1bf8cdb41ae7c3b920fd5d89ec476d5a8edeb Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 21 Feb 2009 11:08:01 +0000 Subject: [PATCH] Merged from trunk -r12664:12669 | 12665 | When loading vdb_metadata.pickle inside | | zmedico | vardbapi._aux_cache_init(), in order to avoid an extreme | | | performance issue, disable buffering when opening the file | | | under python-3.0. Unfortunately, performance is still poor | | | relative to python-2.x, but buffering makes it much worse. | | 12666 | For python-3.0 compatibility, open files in binary mode | | zmedico | inside SpawnProcess._start(). | | 12667 | Inside summarylist(), it's possible for | | zmedico | myglsa.packages.keys() to contain unicode objects, which | | | trigger and exception like this: File | | | "//usr/lib64/portage/pym/portage/dep.py", line 535, in | | | __init__ obj_setattr(self, x, getattr(s, x)) AttributeError: | | | 'unicode' object has no attribute '__eq__' To solve this | | | problem, convert unicode objects to plain str before passing | | | them to the portage api. | | 12668 | Inside portage.spawn(), open files for logfile handling in | | zmedico | binary mode, for compatibility with python-3.0. | | 12669 | Revert r12639 and use string.ascii_letters for python-3.0 | | zmedico | compatibility. | svn path=/main/branches/prefix/; revision=12671 --- bin/glsa-check | 2 +- pym/_emerge/__init__.py | 6 +++--- pym/portage/__init__.py | 6 +++--- pym/portage/dbapi/vartree.py | 10 +++++++++- pym/portage/util.py | 3 ++- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/bin/glsa-check b/bin/glsa-check index d5ab21f4b..2b8e94db9 100644 --- a/bin/glsa-check +++ b/bin/glsa-check @@ -178,7 +178,7 @@ def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr): fd1.write("... ") else: for pkg in myglsa.packages.keys(): - mylist = vardb.match(portage.dep_getkey(pkg)) + mylist = vardb.match(portage.dep_getkey(str(pkg))) if len(mylist) > 0: pkg = color(" ".join(mylist)) fd1.write(" " + pkg + " ") diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index fae99e04a..d34fbe0e1 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -2301,19 +2301,19 @@ class SpawnProcess(SubProcess): else: fd_pipes[0] = fd_pipes_orig[0] - files.process = os.fdopen(master_fd, 'r') + files.process = os.fdopen(master_fd, 'rb') if logfile is not None: fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd - files.log = open(logfile, "a") + files.log = open(logfile, mode='ab') portage.util.apply_secpass_permissions(logfile, uid=portage.portage_uid, gid=portage.portage_gid, mode=0660) if not self.background: - files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'w') + files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb') output_handler = self._output_handler diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 1f3902294..addb16b63 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -3217,9 +3217,9 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero return mypids if logfile: - log_file = open(logfile, 'a') - stdout_file = os.fdopen(os.dup(fd_pipes_orig[1]), 'w') - master_file = os.fdopen(master_fd, 'r') + log_file = open(logfile, mode='ab') + stdout_file = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb') + master_file = os.fdopen(master_fd, 'rb') iwtd = [master_file] owtd = [] ewtd = [] diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index aa5c7a542..d9916e693 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -35,6 +35,7 @@ from portage.cache.mappings import slot_dict_class import os, re, shutil, stat, errno, copy, subprocess import logging import shlex +import sys from itertools import izip try: @@ -1570,8 +1571,15 @@ class vardbapi(dbapi): def _aux_cache_init(self): aux_cache = None + open_kwargs = {} + if sys.hexversion >= 0x3000000: + # Buffered io triggers extreme performance issues in + # Unpickler.load() (problem observed with python-3.0.1). + # Unfortunately, performance is still poor relative to + # python-2.x, but buffering makes it much worse. + open_kwargs["buffering"] = 0 try: - f = open(self._aux_cache_filename, 'rb') + f = open(self._aux_cache_filename, mode='rb', **open_kwargs) mypickle = pickle.Unpickler(f) aux_cache = mypickle.load() f.close() diff --git a/pym/portage/util.py b/pym/portage/util.py index 1be03e0fb..8039818df 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -589,7 +589,8 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): # only joins relative paths when the infile # attribute is properly set. lex = shlex_class(f, infile=mycfg, posix=True) - lex.wordchars += "~!@#$%*_\:;?,./-+{}" + lex.wordchars = string.digits + string.ascii_letters + \ + "~!@#$%*_\:;?,./-+{}" lex.quotes="\"'" if allow_sourcing: lex.source="source" -- 2.26.2