From: Fabian Groffen Date: Sat, 26 Sep 2009 18:44:55 +0000 (-0000) Subject: Merged from trunk -r14412:14422 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=33de25d55e71c680f3fecee0348f306cd25ad198;p=portage.git Merged from trunk -r14412:14422 | 14413 | Use a runtime check to see if | | zmedico | http://bugs.python.org/issue5380 is fixed. | | 14414 | Make _test_pty_eof() return None if openpty() fails. | | zmedico | | | 14415 | Add a test case for http://bugs.python.org/issue5380 and | | zmedico | mark it as 'todo' since it fails with current versions of | | | python 3. | | 14416 | Fix regression in _getitem() from r14398, since | | zmedico | myf.read().split("\n") yields an empty string at the end | | | which is causes _parse_data() to catch a ValueError and | | | raise CacheCorruption. | | 14417 | Show an appropriate error message in _create_pty_or_pipe() | | zmedico | if openpty() fails inside _test_pty_eof(). | | 14418 | Remove unnecessary dict.has_key() call (fix for python 3). | | zmedico | | | 14419 | Fix ValueError in digestcheck(), reported by Arfrever. | | zmedico | | | 14420 | Fix more. | | zmedico | | | 14421 | Use list comprehensions instead of filter() or map() in | | arfrever | some places for compatibility with Python 3. | | 14422 | Use 'in' instead of has_key() in portage.glsa.Glsa.parse() | | arfrever | when Python 3 is used. | svn path=/main/branches/prefix/; revision=14437 --- diff --git a/bin/dispatch-conf b/bin/dispatch-conf index be02e19f9..4d2c1bc31 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -207,7 +207,7 @@ class dispatch: else: return True - confs = filter (f, confs) + confs = [x for x in confs if f(x)] # # Interactively process remaining diff --git a/bin/emaint b/bin/emaint index 76257eb2c..0e598380a 100755 --- a/bin/emaint +++ b/bin/emaint @@ -82,9 +82,9 @@ class WorldHandler(object): self._check_world(onProgress) errors = [] if self.found: - errors += map(lambda x: "'%s' is not a valid atom" % x, self.invalid) - errors += map(lambda x: "'%s' is not installed" % x, self.not_installed) - errors += map(lambda x: "'%s' has a category that is not listed in /etc/portage/categories" % x, self.invalid_category) + errors += ["'%s' is not a valid atom" % x for x in self.invalid] + errors += ["'%s' is not installed" % x for x in self.not_installed] + errors += ["'%s' has a category that is not listed in /etc/portage/categories" % x for x in self.invalid_category] else: errors.append(self.world_file + " could not be opened for reading") return errors diff --git a/bin/regenworld b/bin/regenworld index f4cd445f4..dffdcc1f0 100755 --- a/bin/regenworld +++ b/bin/regenworld @@ -59,18 +59,16 @@ if len(sys.argv) >= 2 and sys.argv[1] in ["-h", "--help"]: sys.exit(0) worldlist = portage.grabfile(os.path.join(portage.const.EPREFIX, portage.WORLD_FILE)) -syslist = portage.settings.packages -syslist = filter(issyspkg, syslist) +syslist = [x for x in portage.settings.packages if issyspkg(x)] logfile = portage.grabfile(portage.const.EPREFIX+"/var/log/emerge.log") -biglist = filter(iscandidate, logfile) -biglist = map(getpkginfo, biglist) +biglist = [getpkginfo(x) for x in logfile if iscandidate(x)] tmplist = [] for l in biglist: tmplist += l.split() -biglist = filter(isunwanted, tmplist) +biglist = [x for x in tmplist if isunwanted(x)] #for p in biglist: -# print p +# print(p) #sys.exit(0) # resolving virtuals @@ -78,7 +76,7 @@ realsyslist = [] for mykey in syslist: # drop the asterix mykey = mykey[1:] - #print "candidate:",mykey + #print("candidate:",mykey) mylist=portage.db["/"]["vartree"].dbapi.match(mykey) if mylist: mykey=portage.cpv_getkey(mylist[0]) @@ -86,7 +84,7 @@ for mykey in syslist: realsyslist.append(mykey) for mykey in biglist: - #print "checking:",mykey + #print("checking:",mykey) try: mylist=portage.db["/"]["vartree"].dbapi.match(mykey) except (portage.exception.InvalidAtom, KeyError): diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 03f2336c2..7cbb8b998 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -3755,6 +3755,60 @@ class config(object): keys = __iter__ items = iteritems +def _test_pty_eof(): + """ + Returns True if this issues is fixed for the currently + running version of python: http://bugs.python.org/issue5380 + Raises an EnvironmentError from openpty() if it fails. + """ + + import array, pty, termios + test_string = 2 * "blah blah blah\n" + test_string = _unicode_decode(test_string, + encoding='utf_8', errors='strict') + + # may raise EnvironmentError + master_fd, slave_fd = pty.openpty() + + master_file = os.fdopen(master_fd, 'rb') + slave_file = os.fdopen(slave_fd, 'wb') + + # Disable post-processing of output since otherwise weird + # things like \n -> \r\n transformations may occur. + mode = termios.tcgetattr(slave_fd) + mode[1] &= ~termios.OPOST + termios.tcsetattr(slave_fd, termios.TCSANOW, mode) + + # Simulate a subprocess writing some data to the + # slave end of the pipe, and then exiting. + slave_file.write(_unicode_encode(test_string, + encoding='utf_8', errors='strict')) + slave_file.close() + + eof = False + data = [] + + while not eof: + + buf = array.array('B') + try: + buf.fromfile(master_file, 1024) + except EOFError: + eof = True + except IOError: + # This is where data loss occurs. + eof = True + + if not buf: + eof = True + else: + data.append(_unicode_decode(buf.tostring(), + encoding='utf_8', errors='strict')) + + master_file.close() + + return test_string == ''.join(data) + # In some cases, openpty can be slow when it fails. Therefore, # stop trying to use it after the first failure. if platform.system() in ["SunOS"]: @@ -3764,10 +3818,7 @@ if platform.system() in ["SunOS"]: _disable_openpty = True else: _disable_openpty = False - -if sys.hexversion >= 0x3000000: - # This is a temporary workaround for http://bugs.python.org/issue5380. - _disable_openpty = True +_tested_pty = False def _create_pty_or_pipe(copy_term_size=None): """ @@ -3785,7 +3836,18 @@ def _create_pty_or_pipe(copy_term_size=None): got_pty = False - global _disable_openpty + global _disable_openpty, _tested_pty + if not (_tested_pty or _disable_openpty): + try: + if not _test_pty_eof(): + _disable_openpty = True + except EnvironmentError as e: + _disable_openpty = True + writemsg("openpty failed: '%s'\n" % str(e), + noiselevel=-1) + del e + _tested_pty = True + if _disable_openpty: master_fd, slave_fd = os.pipe() else: @@ -5379,6 +5441,7 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): return 0 continue for d in dirs: + d_bytes = d try: d = _unicode_decode(d, encoding=_encodings['fs'], errors='strict') @@ -5391,10 +5454,10 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0): noiselevel=-1) if strict: return 0 - dirs.remove(d) + dirs.remove(d_bytes) continue if d.startswith(".") or d == "CVS": - dirs.remove(d) + dirs.remove(d_bytes) for f in files: try: f = _unicode_decode(f, diff --git a/pym/portage/cache/flat_hash.py b/pym/portage/cache/flat_hash.py index 983055a4b..49d7ce6a0 100644 --- a/pym/portage/cache/flat_hash.py +++ b/pym/portage/cache/flat_hash.py @@ -41,7 +41,10 @@ class database(fs_template.FsBased): mode='r', encoding=_encodings['repo.content'], errors='replace') try: - d = self._parse_data(myf.read().split("\n"), cpv) + lines = myf.read().split("\n") + if not lines[-1]: + lines.pop() + d = self._parse_data(lines, cpv) if '_mtime_' not in d: # Backward compatibility with old cache # that uses mtime mangling. diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index f68d92fd2..eb68ae2b5 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -342,7 +342,7 @@ class portdbapi(dbapi): # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys # ~harring - filtered_auxdbkeys = list(filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys)) + filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")] filtered_auxdbkeys.sort() from portage.cache import metadata_overlay, volatile if not depcachedir_w_ok: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 3bf002a78..38e10c15f 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -388,7 +388,7 @@ class LinkageMap(object): for x in filter(None, fields[3].replace( "${ORIGIN}", os.path.dirname(obj)).replace( "$ORIGIN", os.path.dirname(obj)).split(":"))]) - needed = filter(None, fields[4].split(",")) + needed = [x for x in fields[4].split(",") if x] obj_key = self._obj_key(obj) indexed = True diff --git a/pym/portage/glsa.py b/pym/portage/glsa.py index 4049efc2d..4684ea420 100644 --- a/pym/portage/glsa.py +++ b/pym/portage/glsa.py @@ -511,7 +511,8 @@ class Glsa: # 2007-12-30 revisedEl = myroot.getElementsByTagName("revised")[0] self.revised = getText(revisedEl, format="strip") - if (revisedEl.attributes.has_key("count")): + if ((sys.hexversion >= 0x3000000 and "count" in revisedEl.attributes) or + (sys.hexversion < 0x3000000 and revisedEl.attributes.has_key("count"))): count = revisedEl.getAttribute("count") elif (self.revised.find(":") >= 0): (self.revised, count) = self.revised.split(":") diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index 6e7380409..9a81d22ff 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -176,7 +176,8 @@ class TextTestRunner(unittest.TextTestRunner): self.stream.writeln() if not result.wasSuccessful(): self.stream.write("FAILED (") - failed, errored = map(len, (result.failures, result.errors)) + failed = len(result.failures) + errored = len(result.errors) if failed: self.stream.write("failures=%d" % failed) if errored: diff --git a/pym/portage/tests/ebuild/test_pty_eof.py b/pym/portage/tests/ebuild/test_pty_eof.py new file mode 100644 index 000000000..8723372b1 --- /dev/null +++ b/pym/portage/tests/ebuild/test_pty_eof.py @@ -0,0 +1,22 @@ +# Copyright 1998-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +import pty + +import portage +from portage import os +from portage.tests import TestCase + +class PtyEofTestCase(TestCase): + + def testPtyEof(self): + # This tests if the following python issue is fixed yet: + # http://bugs.python.org/issue5380 + # Since it might not be fixed, mark as todo. + self.todo = True + # The result is only valid if openpty does not raise EnvironmentError. + try: + self.assertEqual(portage._test_pty_eof(), True) + except EnvironmentError: + pass diff --git a/pym/portage/update.py b/pym/portage/update.py index 5e6937d44..c2f8f3006 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -227,8 +227,6 @@ def update_config_files(config_root, protect, protect_mask, update_iter): mode='r', encoding=_encodings['content'], errors='replace').readlines() except IOError: - if file_contents.has_key(x): - del file_contents[x] continue # update /etc/portage/packages.* diff --git a/pym/portage/util.py b/pym/portage/util.py index a3d33fc31..af9cd2db2 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -151,7 +151,7 @@ def map_dictlist_vals(func,myDict): new_dl = {} for key in myDict: new_dl[key] = [] - new_dl[key] = map(func,myDict[key]) + new_dl[key] = [func(x) for x in myDict[key]] return new_dl def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0): diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index 45160cfc8..5ce386737 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -346,7 +346,7 @@ def FindPortdir(settings): Args: settings - portage.config instance, preferably repoman_settings Returns: - tuple(portdir, portdir_overlay, location) + list(portdir, portdir_overlay, location) """ portdir = None @@ -430,4 +430,4 @@ def FindPortdir(settings): if not portdir.endswith('/'): portdir += '/' - return map(normalize_path, (portdir, portdir_overlay, location)) + return [normalize_path(x) for x in (portdir, portdir_overlay, location)]