Merged from trunk -r14412:14422
authorFabian Groffen <grobian@gentoo.org>
Sat, 26 Sep 2009 18:44:55 +0000 (18:44 -0000)
committerFabian Groffen <grobian@gentoo.org>
Sat, 26 Sep 2009 18:44:55 +0000 (18:44 -0000)
   | 14413    | Use a runtime check to see if                               |
   | zmedico  | http://bugs.python.org/issue5380 is fixed.                  |

   | 14414    | Make _test_pty_eof() return None if openpty() fails.        |
   | zmedico  |                                                             |

   | 14415    | Add a test case for http://bugs.python.org/issue5380 and    |
   | zmedico  | mark it as 'todo' since it fails with current versions of   |
   |          | python 3.                                                   |

   | 14416    | Fix regression in _getitem() from r14398, since             |
   | zmedico  | myf.read().split("\n") yields an empty string at the end    |
   |          | which is causes _parse_data() to catch a ValueError and     |
   |          | raise CacheCorruption.                                      |

   | 14417    | Show an appropriate error message in _create_pty_or_pipe()  |
   | zmedico  | if openpty() fails inside _test_pty_eof().                  |

   | 14418    | Remove unnecessary dict.has_key() call (fix for python 3).  |
   | zmedico  |                                                             |

   | 14419    | Fix ValueError in digestcheck(), reported by Arfrever.      |
   | zmedico  |                                                             |

   | 14420    | Fix more.                                                   |
   | zmedico  |                                                             |

   | 14421    | Use list comprehensions instead of filter() or map() in     |
   | arfrever | some places for compatibility with Python 3.                |

   | 14422    | Use 'in' instead of has_key() in portage.glsa.Glsa.parse()  |
   | arfrever | when Python 3 is used.                                      |

svn path=/main/branches/prefix/; revision=14437

13 files changed:
bin/dispatch-conf
bin/emaint
bin/regenworld
pym/portage/__init__.py
pym/portage/cache/flat_hash.py
pym/portage/dbapi/porttree.py
pym/portage/dbapi/vartree.py
pym/portage/glsa.py
pym/portage/tests/__init__.py
pym/portage/tests/ebuild/test_pty_eof.py [new file with mode: 0644]
pym/portage/update.py
pym/portage/util.py
pym/repoman/utilities.py

index be02e19f92323e7df1427ce7f3a6c2728cd065ed..4d2c1bc31e7ab19ba81659355be27fb044c0ba15 100755 (executable)
@@ -207,7 +207,7 @@ class dispatch:
             else:
                 return True
 
-        confs = filter (f, confs)
+        confs = [x for x in confs if f(x)]
 
         #
         # Interactively process remaining
index 76257eb2c05ac44b6810d791f439f3811b8674d7..0e598380a34cf7c33f37a71bd757b136e52b8221 100755 (executable)
@@ -82,9 +82,9 @@ class WorldHandler(object):
                self._check_world(onProgress)
                errors = []
                if self.found:
-                       errors += map(lambda x: "'%s' is not a valid atom" % x, self.invalid)
-                       errors += map(lambda x: "'%s' is not installed" % x, self.not_installed)
-                       errors += map(lambda x: "'%s' has a category that is not listed in /etc/portage/categories" % x, self.invalid_category)
+                       errors += ["'%s' is not a valid atom" % x for x in self.invalid]
+                       errors += ["'%s' is not installed" % x for x in self.not_installed]
+                       errors += ["'%s' has a category that is not listed in /etc/portage/categories" % x for x in self.invalid_category]
                else:
                        errors.append(self.world_file + " could not be opened for reading")
                return errors
index f4cd445f4938e498afc293b4393679efba6e209d..dffdcc1f0ec57bd617e7f41473a562eab3833a68 100755 (executable)
@@ -59,18 +59,16 @@ if len(sys.argv) >= 2 and sys.argv[1] in ["-h", "--help"]:
        sys.exit(0)
 
 worldlist = portage.grabfile(os.path.join(portage.const.EPREFIX, portage.WORLD_FILE))
-syslist = portage.settings.packages
-syslist = filter(issyspkg, syslist)
+syslist = [x for x in portage.settings.packages if issyspkg(x)]
 
 logfile = portage.grabfile(portage.const.EPREFIX+"/var/log/emerge.log")
-biglist = filter(iscandidate, logfile)
-biglist = map(getpkginfo, biglist)
+biglist = [getpkginfo(x) for x in logfile if iscandidate(x)]
 tmplist = []
 for l in biglist:
        tmplist += l.split()
-biglist = filter(isunwanted, tmplist)
+biglist = [x for x in tmplist if isunwanted(x)]
 #for p in biglist:
-#      print p
+#      print(p)
 #sys.exit(0)
 
 # resolving virtuals
@@ -78,7 +76,7 @@ realsyslist = []
 for mykey in syslist:
        # drop the asterix
        mykey = mykey[1:]
-       #print "candidate:",mykey
+       #print("candidate:",mykey)
        mylist=portage.db["/"]["vartree"].dbapi.match(mykey)
        if mylist:
                mykey=portage.cpv_getkey(mylist[0])
@@ -86,7 +84,7 @@ for mykey in syslist:
                        realsyslist.append(mykey)
 
 for mykey in biglist:
-       #print "checking:",mykey
+       #print("checking:",mykey)
        try:
                mylist=portage.db["/"]["vartree"].dbapi.match(mykey)
        except (portage.exception.InvalidAtom, KeyError):
index 03f2336c2dbd3aabe9c85e0a045d28f30edde5a0..7cbb8b998268a8de243243532251adc6742f34d7 100644 (file)
@@ -3755,6 +3755,60 @@ class config(object):
                keys = __iter__
                items = iteritems
 
+def _test_pty_eof():
+       """
+       Returns True if this issues is fixed for the currently
+       running version of python: http://bugs.python.org/issue5380
+       Raises an EnvironmentError from openpty() if it fails.
+       """
+
+       import array, pty, termios
+       test_string = 2 * "blah blah blah\n"
+       test_string = _unicode_decode(test_string,
+               encoding='utf_8', errors='strict')
+
+       # may raise EnvironmentError
+       master_fd, slave_fd = pty.openpty()
+
+       master_file = os.fdopen(master_fd, 'rb')
+       slave_file = os.fdopen(slave_fd, 'wb')
+
+       # Disable post-processing of output since otherwise weird
+       # things like \n -> \r\n transformations may occur.
+       mode = termios.tcgetattr(slave_fd)
+       mode[1] &= ~termios.OPOST
+       termios.tcsetattr(slave_fd, termios.TCSANOW, mode)
+
+       # Simulate a subprocess writing some data to the
+       # slave end of the pipe, and then exiting.
+       slave_file.write(_unicode_encode(test_string,
+               encoding='utf_8', errors='strict'))
+       slave_file.close()
+
+       eof = False
+       data = []
+
+       while not eof:
+
+               buf = array.array('B')
+               try:
+                       buf.fromfile(master_file, 1024)
+               except EOFError:
+                       eof = True
+               except IOError:
+                       # This is where data loss occurs.
+                       eof = True
+
+               if not buf:
+                       eof = True
+               else:
+                       data.append(_unicode_decode(buf.tostring(),
+                               encoding='utf_8', errors='strict'))
+
+       master_file.close()
+
+       return test_string == ''.join(data)
+
 # In some cases, openpty can be slow when it fails. Therefore,
 # stop trying to use it after the first failure.
 if platform.system() in ["SunOS"]:
@@ -3764,10 +3818,7 @@ if platform.system() in ["SunOS"]:
        _disable_openpty = True
 else:
        _disable_openpty = False
-
-if sys.hexversion >= 0x3000000:
-       # This is a temporary workaround for http://bugs.python.org/issue5380.
-       _disable_openpty = True
+_tested_pty = False
 
 def _create_pty_or_pipe(copy_term_size=None):
        """
@@ -3785,7 +3836,18 @@ def _create_pty_or_pipe(copy_term_size=None):
 
        got_pty = False
 
-       global _disable_openpty
+       global _disable_openpty, _tested_pty
+       if not (_tested_pty or _disable_openpty):
+               try:
+                       if not _test_pty_eof():
+                               _disable_openpty = True
+               except EnvironmentError as e:
+                       _disable_openpty = True
+                       writemsg("openpty failed: '%s'\n" % str(e),
+                               noiselevel=-1)
+                       del e
+               _tested_pty = True
+
        if _disable_openpty:
                master_fd, slave_fd = os.pipe()
        else:
@@ -5379,6 +5441,7 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
                                return 0
                        continue
                for d in dirs:
+                       d_bytes = d
                        try:
                                d = _unicode_decode(d,
                                        encoding=_encodings['fs'], errors='strict')
@@ -5391,10 +5454,10 @@ def digestcheck(myfiles, mysettings, strict=0, justmanifest=0):
                                        noiselevel=-1)
                                if strict:
                                        return 0
-                               dirs.remove(d)
+                               dirs.remove(d_bytes)
                                continue
                        if d.startswith(".") or d == "CVS":
-                               dirs.remove(d)
+                               dirs.remove(d_bytes)
                for f in files:
                        try:
                                f = _unicode_decode(f,
index 983055a4b8672775051d3061b49f56b3d9814ac9..49d7ce6a0c756cd4088905fa1b37d21220ddd263 100644 (file)
@@ -41,7 +41,10 @@ class database(fs_template.FsBased):
                                mode='r', encoding=_encodings['repo.content'],
                                errors='replace')
                        try:
-                               d = self._parse_data(myf.read().split("\n"), cpv)
+                               lines = myf.read().split("\n")
+                               if not lines[-1]:
+                                       lines.pop()
+                               d = self._parse_data(lines, cpv)
                                if '_mtime_' not in d:
                                        # Backward compatibility with old cache
                                        # that uses mtime mangling.
index f68d92fd2e34e8d0cd680fa694735a0ea8349742..eb68ae2b57f808e3f8c2d82fd8e877a3fda6c93a 100644 (file)
@@ -342,7 +342,7 @@ class portdbapi(dbapi):
 
                # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
                # ~harring
-               filtered_auxdbkeys = list(filter(lambda x: not x.startswith("UNUSED_0"), auxdbkeys))
+               filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")]
                filtered_auxdbkeys.sort()
                from portage.cache import metadata_overlay, volatile
                if not depcachedir_w_ok:
index 3bf002a7812f2cd8c5067a24a442ac6955ecfec2..38e10c15fc5dbeba3cc0106b00f1d25c608d2f78 100644 (file)
@@ -388,7 +388,7 @@ class LinkageMap(object):
                                for x in filter(None, fields[3].replace(
                                "${ORIGIN}", os.path.dirname(obj)).replace(
                                "$ORIGIN", os.path.dirname(obj)).split(":"))])
-                       needed = filter(None, fields[4].split(","))
+                       needed = [x for x in fields[4].split(",") if x]
 
                        obj_key = self._obj_key(obj)
                        indexed = True
index 4049efc2d7e6f18b6daafb4fdf6677c3bd744a81..4684ea420a77a887f7dbf66813d0f2f2707cc1bd 100644 (file)
@@ -511,7 +511,8 @@ class Glsa:
                # <revised count="2">2007-12-30</revised>
                revisedEl = myroot.getElementsByTagName("revised")[0]
                self.revised = getText(revisedEl, format="strip")
-               if (revisedEl.attributes.has_key("count")):
+               if ((sys.hexversion >= 0x3000000 and "count" in revisedEl.attributes) or
+                       (sys.hexversion < 0x3000000 and revisedEl.attributes.has_key("count"))):
                        count = revisedEl.getAttribute("count")
                elif (self.revised.find(":") >= 0):
                        (self.revised, count) = self.revised.split(":")
index 6e7380409949f55673635859470655c5d5ba6654..9a81d22ff3006fb71b342ddaff7ae8f39a724109 100644 (file)
@@ -176,7 +176,8 @@ class TextTestRunner(unittest.TextTestRunner):
                self.stream.writeln()
                if not result.wasSuccessful():
                        self.stream.write("FAILED (")
-                       failed, errored = map(len, (result.failures, result.errors))
+                       failed = len(result.failures)
+                       errored = len(result.errors)
                        if failed:
                                self.stream.write("failures=%d" % failed)
                        if errored:
diff --git a/pym/portage/tests/ebuild/test_pty_eof.py b/pym/portage/tests/ebuild/test_pty_eof.py
new file mode 100644 (file)
index 0000000..8723372
--- /dev/null
@@ -0,0 +1,22 @@
+# Copyright 1998-2007 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Id$
+
+import pty
+
+import portage
+from portage import os
+from portage.tests import TestCase
+
+class PtyEofTestCase(TestCase):
+
+       def testPtyEof(self):
+               # This tests if the following python issue is fixed yet:
+               #   http://bugs.python.org/issue5380
+               # Since it might not be fixed, mark as todo.
+               self.todo = True
+               # The result is only valid if openpty does not raise EnvironmentError.
+               try:
+                       self.assertEqual(portage._test_pty_eof(), True)
+               except EnvironmentError:
+                       pass
index 5e6937d442dac3bc62cee3c1ad8d49b8820db27b..c2f8f300602ed2b3f6b76db7baf2e577ab036f38 100644 (file)
@@ -227,8 +227,6 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
                                mode='r', encoding=_encodings['content'],
                                errors='replace').readlines()
                except IOError:
-                       if file_contents.has_key(x):
-                               del file_contents[x]
                        continue
 
        # update /etc/portage/packages.*
index a3d33fc3161e8d51293e25845dab29a4809b8a23..af9cd2db28f8ce2d487948bf7866e42b2ef39c63 100644 (file)
@@ -151,7 +151,7 @@ def map_dictlist_vals(func,myDict):
        new_dl = {}
        for key in myDict:
                new_dl[key] = []
-               new_dl[key] = map(func,myDict[key])
+               new_dl[key] = [func(x) for x in myDict[key]]
        return new_dl
 
 def stack_dictlist(original_dicts, incremental=0, incrementals=[], ignore_none=0):
index 45160cfc833fb45f4fabea13d5e5c9a38f35de40..5ce3867376b6e5bcc0b84874d9393572054a5c4e 100644 (file)
@@ -346,7 +346,7 @@ def FindPortdir(settings):
        Args:
                settings - portage.config instance, preferably repoman_settings
        Returns:
-               tuple(portdir, portdir_overlay, location)
+               list(portdir, portdir_overlay, location)
        """
 
        portdir = None
@@ -430,4 +430,4 @@ def FindPortdir(settings):
        if not portdir.endswith('/'):
                portdir += '/'
 
-       return map(normalize_path, (portdir, portdir_overlay, location))
+       return [normalize_path(x) for x in (portdir, portdir_overlay, location)]