Merged from trunk -r14067:14077
authorFabian Groffen <grobian@gentoo.org>
Mon, 24 Aug 2009 09:28:51 +0000 (09:28 -0000)
committerFabian Groffen <grobian@gentoo.org>
Mon, 24 Aug 2009 09:28:51 +0000 (09:28 -0000)
   | 14068    | Use elog in _eapi0_pkg_nofetch().                           |
   | zmedico  |                                                             |

   | 14069    | sets/files.py cleanPackages function stop calling lock and  |
   | volkmar  | load and requires the caller to do that changing unmerge to |
   |          | reflect this change                                         |

   | 14070    | Scheduler is now able to clean world set when removing a    |
   | volkmar  | package. world_atom function has been updated and           |
   |          | PackageUninstall is calling it after unmerge.               |

   | 14071    | Use a clean listener system for portage.elog instead of     |
   | volkmar  | _emerge_elog_listener                                       |

   | 14072    | Use _content_encoding and _fs_encoding for unicode          |
   | zmedico  | encoding/decoding.                                          |

   | 14073    | When _unicode_func_wrapper() decodes a string in a returned |
   | zmedico  | list (typically from os.listdir), discard values with       |
   |          | invalid encoding. This insures that all names returned from |
   |          | all os.listdir() calls are valid.                           |

   | 14074    | Use portage.os and _fs_encoding where appropriate, and fix  |
   | zmedico  | binary string handling for py3k compat.                     |

   | 14075    | Print a warning when nonexistent files have been passed to  |
   | arfrever | dohtml.                                                     |

   | 14076    | Add 'return False' which was missing from the previous      |
   | arfrever | commit.                                                     |

   | 14077    | Test the edge case.                                         |
   | zmedico  |                                                             |

svn path=/main/branches/prefix/; revision=14138

16 files changed:
bin/ebuild-helpers/dohtml
bin/ebuild.sh
pym/_emerge/MergeListItem.py
pym/_emerge/PackageUninstall.py
pym/_emerge/Scheduler.py
pym/_emerge/unmerge.py
pym/portage/__init__.py
pym/portage/cache/ebuild_xattr.py
pym/portage/elog/__init__.py
pym/portage/env/loaders.py
pym/portage/sets/files.py
pym/portage/tests/__init__.py
pym/portage/tests/xpak/test_decodeint.py
pym/portage/update.py
pym/portage/util.py
pym/portage/xpak.py

index 69286a741bcb0fc62894f15266c9b7fca5416824..9db02910c9266568fb37ce013647389d995963cd 100755 (executable)
@@ -59,7 +59,10 @@ def install(basename, dirname, options, prefix=""):
        else:
                destdir = options.ED + "/usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix
 
-       if os.path.isfile(fullpath):
+       if not os.path.exists(fullpath):
+               sys.stderr.write("!!! dohtml: %s does not exist\n" % fullpath)
+               return False
+       elif os.path.isfile(fullpath):
                ext = os.path.splitext(basename)[1]
                if (len(ext) and ext[1:] in options.allowed_exts) or basename in options.allowed_files:
                        dodir(destdir)
index 989bcb5c2326b6510dd7fc00908fe846f4d06825..16c3c192c753d69396a67fda2a88b3cb66fb2d95 100755 (executable)
@@ -588,10 +588,10 @@ einstall() {
 _eapi0_pkg_nofetch() {
        [ -z "${SRC_URI}" ] && return
 
-       echo "!!! The following are listed in SRC_URI for ${PN}:"
+       elog "The following are listed in SRC_URI for ${PN}:"
        local x
        for x in $(echo ${SRC_URI}); do
-               echo "!!!   ${x}"
+               elog "   ${x}"
        done
 }
 
index 7e4556ec2fc3aecbaf1aa6273f2cebd4232cee1f..0205e659528d9b634dac7d61afce85dd660fcf9e 100644 (file)
@@ -132,7 +132,8 @@ class MergeListItem(CompositeTask):
 
                                uninstall = PackageUninstall(background=self.background,
                                        ldpath_mtimes=ldpath_mtimes, opts=self.emerge_opts,
-                                       pkg=pkg, scheduler=scheduler, settings=settings)
+                                       pkg=pkg, scheduler=scheduler, settings=settings,
+                                       world_atom=world_atom)
 
                                uninstall.start()
                                retval = uninstall.wait()
index ff1b5e189ef4a32558c4f8c7ff992b99be8fc9df..d86947c4aa7be09dae9b76d33c8b639e19638876 100644 (file)
@@ -12,11 +12,12 @@ from _emerge.UninstallFailure import UninstallFailure
 
 class PackageUninstall(AsynchronousTask):
 
-       __slots__ = ("ldpath_mtimes", "opts", "pkg", "scheduler", "settings")
+       __slots__ = ("world_atom", "ldpath_mtimes", "opts",
+                       "pkg", "scheduler", "settings")
 
        def _start(self):
                try:
-                       unmerge(self.pkg.root_config, self.opts, "unmerge",
+                       retval = unmerge(self.pkg.root_config, self.opts, "unmerge",
                                [self.pkg.cpv], self.ldpath_mtimes, clean_world=0,
                                clean_delay=0, raise_on_error=1, scheduler=self.scheduler,
                                writemsg_level=self._writemsg_level)
@@ -24,6 +25,10 @@ class PackageUninstall(AsynchronousTask):
                        self.returncode = e.status
                else:
                        self.returncode = os.EX_OK
+
+               if retval == 1:
+                       self.world_atom(self.pkg)
+
                self.wait()
 
        def _writemsg_level(self, msg, level=0, noiselevel=0):
index cef4ee6ca05115769d5bb3c9b294926e367362ad..7e0c35f8940b605a72eef85adbc34de34537ea23 100644 (file)
@@ -1108,7 +1108,7 @@ class Scheduler(PollScheduler):
                pkg_queue = self._pkg_queue
                failed_pkgs = self._failed_pkgs
                portage.locks._quiet = self._background
-               portage.elog._emerge_elog_listener = self._elog_listener
+               portage.elog.add_listener(self._elog_listener)
                rval = os.EX_OK
 
                try:
@@ -1116,7 +1116,7 @@ class Scheduler(PollScheduler):
                finally:
                        self._main_loop_cleanup()
                        portage.locks._quiet = False
-                       portage.elog._emerge_elog_listener = None
+                       portage.elog.remove_listener(self._elog_listener)
                        if failed_pkgs:
                                rval = failed_pkgs[-1].returncode
 
@@ -1566,8 +1566,8 @@ class Scheduler(PollScheduler):
 
        def _world_atom(self, pkg):
                """
-               Add the package to the world file, but only if
-               it's supposed to be added. Otherwise, do nothing.
+               Add or remove the package to the world file, but only if
+               it's supposed to be added or removed. Otherwise, do nothing.
                """
 
                if set(("--buildpkgonly", "--fetchonly",
@@ -1596,17 +1596,25 @@ class Scheduler(PollScheduler):
                        if hasattr(world_set, "load"):
                                world_set.load() # maybe it's changed on disk
 
-                       atom = create_world_atom(pkg, args_set, root_config)
-                       if atom:
-                               if hasattr(world_set, "add"):
-                                       self._status_msg(('Recording %s in "world" ' + \
-                                               'favorites file...') % atom)
-                                       logger.log(" === (%s of %s) Updating world file (%s)" % \
-                                               (pkg_count.curval, pkg_count.maxval, pkg.cpv))
-                                       world_set.add(atom)
-                               else:
-                                       writemsg_level('\n!!! Unable to record %s in "world"\n' % \
-                                               (atom,), level=logging.WARN, noiselevel=-1)
+                       if pkg.operation == "uninstall":
+                               if hasattr(world_set, "cleanPackage"):
+                                       world_set.cleanPackage(pkg.root_config.trees["vartree"].dbapi,
+                                                       pkg.cpv)
+                               if hasattr(world_set, "remove"):
+                                       for s in pkg.root_config.setconfig.active:
+                                               world_set.remove(SETPREFIX+s)
+                       else:
+                               atom = create_world_atom(pkg, args_set, root_config)
+                               if atom:
+                                       if hasattr(world_set, "add"):
+                                               self._status_msg(('Recording %s in "world" ' + \
+                                                       'favorites file...') % atom)
+                                               logger.log(" === (%s of %s) Updating world file (%s)" % \
+                                                       (pkg_count.curval, pkg_count.maxval, pkg.cpv))
+                                               world_set.add(atom)
+                                       else:
+                                               writemsg_level('\n!!! Unable to record %s in "world"\n' % \
+                                                       (atom,), level=logging.WARN, noiselevel=-1)
                finally:
                        if world_locked:
                                world_set.unlock()
index 25c3e4e4f66d9be485c447e97e72e69d07d37b2b..2c9e7576e9b5db31a0ff329db44d1fb2a264eeff 100644 (file)
@@ -510,11 +510,22 @@ def unmerge(root_config, myopts, unmerge_action,
                                        raise UninstallFailure(retval)
                                sys.exit(retval)
                        else:
-                               if clean_world and hasattr(sets["world"], "cleanPackage"):
+                               if clean_world and hasattr(sets["world"], "cleanPackage")\
+                                               and hasattr(sets["world"], "lock"):
+                                       sets["world"].lock()
+                                       if hasattr(sets["world"], "load"):
+                                               sets["world"].load()
                                        sets["world"].cleanPackage(vartree.dbapi, y)
+                                       sets["world"].unlock()
                                emergelog(xterm_titles, " >>> unmerge success: "+y)
-       if clean_world and hasattr(sets["world"], "remove"):
+
+       if clean_world and hasattr(sets["world"], "remove")\
+                       and hasattr(sets["world"], "lock"):
+               sets["world"].lock()
+               # load is called inside remove()
                for s in root_config.setconfig.active:
                        sets["world"].remove(SETPREFIX+s)
+               sets["world"].unlock()
+
        return 1
 
index c7f27523868f34179b304f24240fa993210c1898..390c1de96d0dfd8a74035ed204da3d651917e768 100644 (file)
@@ -169,11 +169,20 @@ class _unicode_func_wrapper(object):
                if isinstance(rval, (basestring, list, tuple)):
                        if isinstance(rval, basestring):
                                rval = _unicode_decode(rval, encoding=encoding)
-                       elif isinstance(rval, list):
-                               rval = [_unicode_decode(x, encoding=encoding) for x in rval]
-                       elif isinstance(rval, tuple):
-                               rval = tuple(_unicode_decode(x, encoding=encoding) \
-                                       for x in rval)
+                       else:
+                               decoded_rval = []
+                               for x in rval:
+                                       try:
+                                               x = _unicode_decode(x, encoding=encoding, errors='strict')
+                                       except UnicodeDecodeError:
+                                               pass
+                                       else:
+                                               decoded_rval.append(x)
+
+                               if isinstance(rval, tuple):
+                                       rval = tuple(decoded_rval)
+                               else:
+                                       rval = decoded_rval
 
                return rval
 
index baba94321b801b1cff78c7ac1f46743761e34741..bcaf30640e86d7d681651d89ecfe489853ba1dfa 100644 (file)
@@ -10,6 +10,7 @@ from portage.cache import fs_template
 from portage.versions import catsplit
 from portage import cpv_getkey
 from portage import os
+from portage import _fs_encoding
 from portage import _unicode_decode
 import xattr
 from errno import ENODATA,ENOSPC,E2BIG
@@ -156,7 +157,11 @@ class database(fs_template.FsBased):
 
                for root, dirs, files in os.walk(self.portdir):
                        for file in files:
-                               file = _unicode_decode(file)
+                               try:
+                                       file = _unicode_decode(file,
+                                               encoding=_fs_encoding, errors='strict')
+                               except UnicodeDecodeError:
+                                       continue
                                if file[-7:] == '.ebuild':
                                        cat = os.path.basename(os.path.dirname(root))
                                        pn_pv = file[:-7]
index 1ebc027c5c314c7fc9688c867a2029d7f1600e31..7bd567ceecc435eacaf23d72f2d46f42671b45d6 100644 (file)
@@ -56,11 +56,23 @@ def _load_mod(name):
                _elog_mod_imports[name] = m
        return m
 
-_emerge_elog_listener = None
+_elog_listeners = []
+def add_listener(listener):
+       '''
+       Listeners should accept four arguments: settings, key, logentries and logtext
+       '''
+       _elog_listeners.append(listener)
+
+def remove_listener(listener):
+       '''
+       Remove previously added listener
+       '''
+       _elog_listeners.remove(listener)
+
 _elog_atexit_handlers = []
 _preserve_logentries = {}
 def elog_process(cpv, mysettings, phasefilter=None):
-       global _elog_atexit_handlers, _emerge_elog_listener, _preserve_logentries
+       global _elog_atexit_handlers, _preserve_logentries
        
        logsystems = mysettings.get("PORTAGE_ELOG_SYSTEM","").split()
        for s in logsystems:
@@ -123,9 +135,9 @@ def elog_process(cpv, mysettings, phasefilter=None):
 
                default_fulllog = _combine_logentries(default_logentries)
 
-               if _emerge_elog_listener is not None:
-                       _emerge_elog_listener(mysettings, str(key),
-                               default_logentries, default_fulllog)
+               # call listeners
+               for listener in _elog_listeners:
+                       listener(mysettings, str(key), default_logentries, default_fulllog)
 
                # pass the processing to the individual modules
                for s, levels in logsystems.iteritems():
index 854304125f331d8794d8483fd72d90287a95be75..e878ba449c64ce6a3645afb7a2d4415abf1b1570 100644 (file)
@@ -4,8 +4,13 @@
 # $Id$
 
 import codecs
-import os
+import errno
 import stat
+from portage import os
+from portage import _content_encoding
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
 from portage.localization import _
 
 class LoaderError(Exception):
@@ -40,11 +45,6 @@ def RecursiveFileLoader(filename):
        @returns: List of files to process
        """
 
-       if isinstance(filename, unicode):
-               # Avoid UnicodeDecodeError raised from
-               # os.path.join when called by os.walk.
-               filename = filename.encode('utf_8', 'replace')
-
        try:
                st = os.stat(filename)
        except OSError:
@@ -55,6 +55,11 @@ def RecursiveFileLoader(filename):
                                if d[:1] == '.' or d == 'CVS':
                                        dirs.remove(d)
                        for f in files:
+                               try:
+                                       f = _unicode_decode(f,
+                                               encoding=_fs_encoding, errors='strict')
+                               except UnicodeDecodeError:
+                                       continue
                                if f[:1] == '.' or f[-1:] == '~':
                                        continue
                                yield os.path.join(root, f)
@@ -145,9 +150,18 @@ class FileLoader(DataLoader):
                # once, which may be expensive due to digging in child classes.
                func = self.lineParser
                for fn in RecursiveFileLoader(self.fname):
-                       f = codecs.open(fn, mode='r', encoding='utf_8', errors='replace')
+                       try:
+                               f = codecs.open(_unicode_encode(fn,
+                                       encoding=_fs_encoding, errors='strict'), mode='r',
+                                       encoding=_content_encoding, errors='replace')
+                       except EnvironmentError, e:
+                               if e.errno not in (errno.ENOENT, errno.ESTALE):
+                                       raise
+                               del e
+                               continue
                        for line_num, line in enumerate(f):
                                func(line, line_num, data, errors)
+                       f.close()
                return (data, errors)
 
        def lineParser(self, line, line_num, data, errors):
index ae004356c81d6f08219b91e2a78020fcca1705c8..d39087e1e49487e2c8842e7b0f3370b1c7f3824c 100644 (file)
@@ -293,8 +293,13 @@ class WorldSet(EditablePackageSet):
                self._lock = None
 
        def cleanPackage(self, vardb, cpv):
-               self.lock()
-               self._load() # loads latest from disk
+               '''
+               Before calling this function you should call lock and load.
+               After calling this function you should call unlock.
+               '''
+               if not self._lock:
+                       raise AssertionError('cleanPackage needs the set to be locked')
+
                worldlist = list(self._atoms)
                mykey = cpv_getkey(cpv)
                newworldlist = []
@@ -316,7 +321,6 @@ class WorldSet(EditablePackageSet):
 
                newworldlist.extend(self._nonatoms)
                self.replace(newworldlist)
-               self.unlock()
 
        def singleBuilder(self, options, settings, trees):
                return WorldSet(settings["ROOT"])
index 8676c6ae25253b2cadeeb1602fc578e615b40c93..4a5ced8a35b8dbbd60f1ed31f03907f0185f9787 100644 (file)
@@ -3,14 +3,21 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import os
 import sys
 import time
 import unittest
 
+from portage import os
+from portage import _fs_encoding
+from portage import _unicode_encode
+from portage import _unicode_decode
+
 def main():
 
-       TEST_FILE = '__test__'
+       TEST_FILE = _unicode_encode('__test__',
+               encoding=_fs_encoding, errors='strict')
+       svn_dirname = _unicode_encode('.svn',
+               encoding=_fs_encoding, errors='strict')
        suite = unittest.TestSuite()
        basedir = os.path.dirname(os.path.realpath(__file__))
        testDirs = []
@@ -19,8 +26,14 @@ def main():
        # I was tired of adding dirs to the list, so now we add __test__
        # to each dir we want tested.
        for root, dirs, files in os.walk(basedir):
-               if ".svn" in dirs:
-                       dirs.remove('.svn')
+               if svn_dirname in dirs:
+                       dirs.remove(svn_dirname)
+               try:
+                       root = _unicode_decode(root,
+                               encoding=_fs_encoding, errors='strict')
+               except UnicodeDecodeError:
+                       continue
+
                if TEST_FILE in files:
                        testDirs.append(root)
 
index c0f3264dbd1cc4aef23eafe9c5fbaf54cf5b38f2..27e8ab7cf841abc1a4edbe7eb488140fa6b355cb 100644 (file)
@@ -12,3 +12,6 @@ class testDecodeIntTestCase(TestCase):
                
                for n in xrange(1000):
                        self.assertEqual(decodeint(encodeint(n)), n)
+
+               for n in (2 ** 32 - 1,):
+                       self.assertEqual(decodeint(encodeint(n)), n)
index 4129565915ba924fcc4698b3ac3f067ecef40692..d38ddf942ce60211538f320f604869b7d24505bf 100644 (file)
@@ -2,8 +2,16 @@
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 
-import errno, os, re, sys
-
+import codecs
+import errno
+import re
+import sys
+
+from portage import os
+from portage import _content_encoding
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
        'portage.dep:dep_getkey,get_operator,isvalidatom,isjustname,remove_slot',
@@ -12,7 +20,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
        'portage.versions:ververify'
 )
 
-from portage.const import USER_CONFIG_PATH, WORLD_FILE
+from portage.const import USER_CONFIG_PATH
 from portage.exception import DirectoryNotFound, PortageException
 from portage.localization import _
 
@@ -63,9 +71,10 @@ def fixdbentries(update_iter, dbdir):
        mydata = {}
        for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]:
                file_path = os.path.join(dbdir, myfile)
-               f = open(file_path, "r")
-               mydata[myfile] = f.read()
-               f.close()
+               mydata[myfile] = codecs.open(_unicode_encode(file_path,
+                       encoding=_fs_encoding, errors='strict'),
+                       mode='r', encoding=_content_encoding,
+                       errors='replace').read()
        updated_items = update_dbentries(update_iter, mydata)
        for myfile, mycontent in updated_items.iteritems():
                file_path = os.path.join(dbdir, myfile)
@@ -100,9 +109,9 @@ def grab_updates(updpath, prev_mtimes=None):
                mystat = os.stat(file_path)
                if file_path not in prev_mtimes or \
                long(prev_mtimes[file_path]) != long(mystat.st_mtime):
-                       f = open(file_path)
-                       content = f.read()
-                       f.close()
+                       content = codecs.open(_unicode_encode(file_path,
+                               encoding=_fs_encoding, errors='strict'),
+                               mode='r', encoding=_content_encoding, errors='replace').read()
                        update_data.append((file_path, mystat, content))
        return update_data
 
@@ -142,17 +151,12 @@ def parse_updates(mycontent):
        return myupd, errors
 
 def update_config_files(config_root, protect, protect_mask, update_iter):
-       """Perform global updates on /etc/portage/package.* and the world file.
+       """Perform global updates on /etc/portage/package.*.
        config_root - location of files to update
        protect - list of paths from CONFIG_PROTECT
        protect_mask - list of paths from CONFIG_PROTECT_MASK
        update_iter - list of update commands as returned from parse_updates()"""
 
-       if isinstance(config_root, unicode):
-               # Avoid UnicodeDecodeError raised from
-               # os.path.join when called by os.walk.
-               config_root = config_root.encode('utf_8', 'replace')
-
        config_root = normalize_path(config_root)
        update_files = {}
        file_contents = {}
@@ -166,9 +170,20 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
                if os.path.isdir(config_file):
                        for parent, dirs, files in os.walk(config_file):
                                for y in dirs:
+                                       try:
+                                               y = _unicode_decode(y,
+                                                       encoding=_fs_encoding, errors='strict')
+                                       except UnicodeDecodeError:
+                                               dirs.remove(y)
+                                               continue
                                        if y.startswith("."):
                                                dirs.remove(y)
                                for y in files:
+                                       try:
+                                               y = _unicode_decode(y,
+                                                       encoding=_fs_encoding, errors='strict')
+                                       except UnicodeDecodeError:
+                                               continue
                                        if y.startswith("."):
                                                continue
                                        recursivefiles.append(
@@ -178,9 +193,11 @@ def update_config_files(config_root, protect, protect_mask, update_iter):
        myxfiles = recursivefiles
        for x in myxfiles:
                try:
-                       myfile = open(os.path.join(abs_user_config, x),"r")
-                       file_contents[x] = myfile.readlines()
-                       myfile.close()
+                       file_contents[x] = codecs.open(
+                               _unicode_encode(os.path.join(abs_user_config, x),
+                               encoding=_fs_encoding, errors='strict'),
+                               mode='r', encoding=_content_encoding,
+                               errors='replace').readlines()
                except IOError:
                        if file_contents.has_key(x):
                                del file_contents[x]
index 96d716e81d623a671e2475a45dd853351baa438c..983c08b0a084eaa0cad1b43cb3d9314c891674e4 100644 (file)
@@ -14,7 +14,6 @@ __all__ = ['apply_permissions', 'apply_recursive_permissions',
 
 import commands
 import codecs
-import os
 import errno
 import logging
 import shlex
@@ -24,7 +23,8 @@ import sys
 
 import portage
 from portage import os
-from portage import _merge_encoding
+from portage import _content_encoding
+from portage import _fs_encoding
 from portage import _os_merge
 from portage import _unicode_encode
 from portage import _unicode_decode
@@ -328,8 +328,9 @@ def grablines(myfilename,recursive=0):
                                        os.path.join(myfilename, f), recursive))
        else:
                try:
-                       myfile = codecs.open(_unicode_encode(myfilename),
-                               mode='r', encoding='utf_8', errors='replace')
+                       myfile = codecs.open(_unicode_encode(myfilename,
+                               encoding=_fs_encoding, errors='strict'),
+                               mode='r', encoding=_content_encoding, errors='replace')
                        mylines = myfile.readlines()
                        myfile.close()
                except IOError, e:
@@ -395,10 +396,12 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True):
                # NOTE: shex doesn't seem to support unicode objects
                # (produces spurious \0 characters with python-2.6.2)
                if sys.hexversion < 0x3000000:
-                       content = open(_unicode_encode(mycfg), 'rb').read()
+                       content = open(_unicode_encode(mycfg,
+                               encoding=_fs_encoding, errors='strict'), 'rb').read()
                else:
-                       content = open(_unicode_encode(mycfg), mode='r',
-                               encoding='utf_8', errors='replace').read()
+                       content = open(_unicode_encode(mycfg,
+                               encoding=_fs_encoding, errors='strict'), mode='r',
+                               encoding=_content_encoding, errors='replace').read()
                if content and content[-1] != '\n':
                        content += '\n'
        except IOError, e:
@@ -590,7 +593,8 @@ def pickle_read(filename,default=None,debug=0):
                return default
        data = None
        try:
-               myf = open(_unicode_encode(filename), 'rb')
+               myf = open(_unicode_encode(filename,
+                       encoding=_fs_encoding, errors='strict'), 'rb')
                mypickle = pickle.Unpickler(myf)
                data = mypickle.load()
                myf.close()
@@ -905,7 +909,7 @@ class atomic_ofstream(ObjectProxy):
                        open_func = open
                else:
                        open_func = codecs.open
-                       kargs.setdefault('encoding', 'utf_8')
+                       kargs.setdefault('encoding', _content_encoding)
                        kargs.setdefault('errors', 'replace')
 
                if follow_links:
@@ -914,7 +918,9 @@ class atomic_ofstream(ObjectProxy):
                        tmp_name = "%s.%i" % (canonical_path, os.getpid())
                        try:
                                object.__setattr__(self, '_file',
-                                       open_func(_unicode_encode(tmp_name), mode=mode, **kargs))
+                                       open_func(_unicode_encode(tmp_name,
+                                               encoding=_fs_encoding, errors='strict'),
+                                               mode=mode, **kargs))
                                return
                        except IOError, e:
                                if canonical_path == filename:
@@ -926,7 +932,9 @@ class atomic_ofstream(ObjectProxy):
                object.__setattr__(self, '_real_name', filename)
                tmp_name = "%s.%i" % (filename, os.getpid())
                object.__setattr__(self, '_file',
-                       open_func(_unicode_encode(tmp_name), mode=mode, **kargs))
+                       open_func(_unicode_encode(tmp_name,
+                               encoding=_fs_encoding, errors='strict'),
+                               mode=mode, **kargs))
 
        def _get_target(self):
                return object.__getattribute__(self, '_file')
index a1516e0338abd4db77a1888bb8fd90fc65329206..da68af59c476578c548d785f0b249e3aa2c4f648 100644 (file)
 # (integer) == encodeint(integer)  ===> 4 characters (big-endian copy)
 # '+' means concatenate the fields ===> All chunks are strings
 
-import sys,os,shutil,errno
-from stat import *
+import array
+import errno
+import shutil
 
-def addtolist(mylist,curdir):
+from portage import os
+from portage import normalize_path
+from portage import _fs_encoding
+from portage import _unicode_decode
+from portage import _unicode_encode
+
+def addtolist(mylist, curdir):
        """(list, dir) --- Takes an array(list) and appends all files from dir down
        the directory tree. Returns nothing. list is modified."""
-       for x in os.listdir("."):
-               if os.path.isdir(x):
-                       os.chdir(x)
-                       addtolist(mylist,curdir+x+"/")
-                       os.chdir("..")
-               else:
-                       if curdir+x not in mylist:
-                               mylist.append(curdir+x)
+       curdir = normalize_path(_unicode_decode(curdir,
+               encoding=_fs_encoding, errors='strict'))
+       for parent, dirs, files in os.walk(curdir):
+
+               parent = _unicode_decode(parent,
+                       encoding=_fs_encoding, errors='strict')
+               if parent != curdir:
+                       mylist.append(parent[len(curdir) + 1:] + os.sep)
+
+               for x in dirs:
+                       try:
+                               _unicode_decode(x, encoding=_fs_encoding, errors='strict')
+                       except UnicodeDecodeError:
+                               dirs.remove(x)
+
+               for x in files:
+                       try:
+                               x = _unicode_decode(x, encoding=_fs_encoding, errors='strict')
+                       except UnicodeDecodeError:
+                               continue
+                       mylist.append(os.path.join(parent, x)[len(curdir) + 1:])
 
 def encodeint(myint):
        """Takes a 4 byte integer and converts it into a string of 4 characters.
        Returns the characters in a string."""
-       part1=chr((myint >> 24 ) & 0x000000ff)
-       part2=chr((myint >> 16 ) & 0x000000ff)
-       part3=chr((myint >> 8 ) & 0x000000ff)
-       part4=chr(myint & 0x000000ff)
-       return part1+part2+part3+part4
+       a = array.array('B')
+       a.append((myint >> 24 ) & 0xff)
+       a.append((myint >> 16 ) & 0xff)
+       a.append((myint >> 8 ) & 0xff)
+       a.append(myint & 0xff)
+       return a.tostring()
 
 def decodeint(mystring):
        """Takes a 4 byte string and converts it into a 4 byte integer.
@@ -54,28 +75,20 @@ def xpak(rootdir,outfile=None):
        """(rootdir,outfile) -- creates an xpak segment of the directory 'rootdir'
        and under the name 'outfile' if it is specified. Otherwise it returns the
        xpak segment."""
-       try:
-               origdir=os.getcwd()
-       except SystemExit, e:
-               raise
-       except:
-               os.chdir("/")
-               origdir="/"
-       os.chdir(rootdir)
+
        mylist=[]
 
-       addtolist(mylist,"")
+       addtolist(mylist, rootdir)
        mylist.sort()
        mydata = {}
        for x in mylist:
-               a = open(x, 'rb')
-               mydata[x] = a.read()
-               a.close()
-       os.chdir(origdir)
+               x = _unicode_encode(x, encoding=_fs_encoding, errors='strict')
+               mydata[x] = open(os.path.join(rootdir, x), 'rb').read()
 
        xpak_segment = xpak_mem(mydata)
        if outfile:
-               outf = open(outfile, 'wb')
+               outf = open(_unicode_encode(outfile,
+                       encoding=_fs_encoding, errors='strict'), 'wb')
                outf.write(xpak_segment)
                outf.close()
        else:
@@ -83,9 +96,9 @@ def xpak(rootdir,outfile=None):
 
 def xpak_mem(mydata):
        """Create an xpack segement from a map object."""
-       indexglob=""
+       indexglob = _unicode_encode('')
        indexpos=0
-       dataglob=""
+       dataglob = _unicode_encode('')
        datapos=0
        for x, newglob in mydata.iteritems():
                mydatasize=len(newglob)
@@ -93,18 +106,21 @@ def xpak_mem(mydata):
                indexpos=indexpos+4+len(x)+4+4
                dataglob=dataglob+newglob
                datapos=datapos+mydatasize
-       return "XPAKPACK" \
+       return _unicode_encode('XPAKPACK') \
        + encodeint(len(indexglob)) \
        + encodeint(len(dataglob)) \
        + indexglob \
        + dataglob \
-       + "XPAKSTOP"
+       + _unicode_encode('XPAKSTOP')
 
 def xsplit(infile):
        """(infile) -- Splits the infile into two files.
        'infile.index' contains the index segment.
        'infile.dat' contails the data segment."""
-       myfile = open(infile, 'rb')
+       infile = _unicode_decode(infile,
+               encoding=_fs_encoding, errors='strict')
+       myfile = open(_unicode_encode(infile,
+               encoding=_fs_encoding, errors='strict'), 'rb')
        mydat=myfile.read()
        myfile.close()
        
@@ -112,27 +128,30 @@ def xsplit(infile):
        if not splits:
                return False
        
-       myfile = open(infile + '.index', 'wb')
+       myfile = open(_unicode_encode(infile + '.index',
+               encoding=_fs_encoding, errors='strict'), 'wb')
        myfile.write(splits[0])
        myfile.close()
-       myfile = open(infile + '.dat', 'wb')
+       myfile = open(_unicode_encode(infile + '.dat',
+               encoding=_fs_encoding, errors='strict'), 'wb')
        myfile.write(splits[1])
        myfile.close()
        return True
 
 def xsplit_mem(mydat):
-       if mydat[0:8]!="XPAKPACK":
+       if mydat[0:8] != _unicode_encode('XPAKPACK'):
                return None
-       if mydat[-8:]!="XPAKSTOP":
+       if mydat[-8:] != _unicode_encode('XPAKSTOP'):
                return None
        indexsize=decodeint(mydat[8:12])
        return (mydat[16:indexsize+16], mydat[indexsize+16:-8])
 
 def getindex(infile):
        """(infile) -- grabs the index segment from the infile and returns it."""
-       myfile = open(infile, 'rb')
+       myfile = open(_unicode_encode(infile,
+               encoding=_fs_encoding, errors='strict'), 'rb')
        myheader=myfile.read(16)
-       if myheader[0:8]!="XPAKPACK":
+       if myheader[0:8] != _unicode_encode('XPAKPACK'):
                myfile.close()
                return
        indexsize=decodeint(myheader[8:12])
@@ -143,9 +162,10 @@ def getindex(infile):
 def getboth(infile):
        """(infile) -- grabs the index and data segments from the infile.
        Returns an array [indexSegment,dataSegment]"""
-       myfile = open(infile, 'rb')
+       myfile = open(_unicode_encode(infile,
+               encoding=_fs_encoding, errors='strict'), 'rb')
        myheader=myfile.read(16)
-       if myheader[0:8]!="XPAKPACK":
+       if myheader[0:8] != _unicode_encode('XPAKPACK'):
                myfile.close()
                return
        indexsize=decodeint(myheader[8:12])
@@ -217,7 +237,8 @@ def xpand(myid,mydest):
                if dirname:
                        if not os.path.exists(dirname):
                                os.makedirs(dirname)
-               mydat = open(myname, 'wb')
+               mydat = open(_unicode_encode(myname,
+                       encoding=_fs_encoding, errors='strict'), 'wb')
                mydat.write(mydata[datapos:datapos+datalen])
                mydat.close()
                startpos=startpos+namelen+12
@@ -227,7 +248,7 @@ class tbz2(object):
        def __init__(self,myfile):
                self.file=myfile
                self.filestat=None
-               self.index=""
+               self.index = _unicode_encode('')
                self.infosize=0
                self.xpaksize=0
                self.indexsize=None
@@ -262,12 +283,13 @@ class tbz2(object):
 
        def recompose_mem(self, xpdata):
                self.scan() # Don't care about condition... We'll rewrite the data anyway.
-               myfile = open(self.file, 'ab+')
+               myfile = open(_unicode_encode(self.file,
+                       encoding=_fs_encoding, errors='strict'), 'ab+')
                if not myfile:
                        raise IOError
                myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF.
                myfile.truncate()
-               myfile.write(xpdata+encodeint(len(xpdata))+"STOP")
+               myfile.write(xpdata+encodeint(len(xpdata)) + _unicode_encode('STOP'))
                myfile.flush()
                myfile.close()
                return 1
@@ -292,28 +314,30 @@ class tbz2(object):
                        mystat=os.stat(self.file)
                        if self.filestat:
                                changed=0
-                               for x in [ST_SIZE, ST_MTIME, ST_CTIME]:
-                                       if mystat[x] != self.filestat[x]:
-                                               changed=1
+                               if mystat.st_size != self.filestat.st_size \
+                                       or mystat.st_mtime != self.filestat.st_mtime \
+                                       or mystat.st_ctime != self.filestat.st_ctime:
+                                       changed = True
                                if not changed:
                                        return 1
                        self.filestat=mystat
-                       a = open(self.file, 'rb')
+                       a = open(_unicode_encode(self.file,
+                               encoding=_fs_encoding, errors='strict'), 'rb')
                        a.seek(-16,2)
                        trailer=a.read()
                        self.infosize=0
                        self.xpaksize=0
-                       if trailer[-4:]!="STOP":
+                       if trailer[-4:] != _unicode_encode('STOP'):
                                a.close()
                                return 0
-                       if trailer[0:8]!="XPAKSTOP":
+                       if trailer[0:8] != _unicode_encode('XPAKSTOP'):
                                a.close()
                                return 0
                        self.infosize=decodeint(trailer[8:12])
                        self.xpaksize=self.infosize+8
                        a.seek(-(self.xpaksize),2)
                        header=a.read(16)
-                       if header[0:8]!="XPAKPACK":
+                       if header[0:8] != _unicode_encode('XPAKPACK'):
                                a.close()
                                return 0
                        self.indexsize=decodeint(header[8:12])
@@ -341,7 +365,8 @@ class tbz2(object):
                myresult=searchindex(self.index,myfile)
                if not myresult:
                        return mydefault
-               a = open(self.file, 'rb')
+               a = open(_unicode_encode(self.file,
+                       encoding=_fs_encoding, errors='strict'), 'rb')
                a.seek(self.datapos+myresult[0],0)
                myreturn=a.read(myresult[1])
                a.close()
@@ -365,7 +390,8 @@ class tbz2(object):
                except:
                        os.chdir("/")
                        origdir="/"
-               a = open(self.file, 'rb')
+               a = open(_unicode_encode(self.file,
+                       encoding=_fs_encoding, errors='strict'), 'rb')
                if not os.path.exists(mydest):
                        os.makedirs(mydest)
                os.chdir(mydest)
@@ -379,7 +405,8 @@ class tbz2(object):
                        if dirname:
                                if not os.path.exists(dirname):
                                        os.makedirs(dirname)
-                       mydat = open(myname, 'wb')
+                       mydat = open(_unicode_encode(myname,
+                               encoding=_fs_encoding, errors='strict'), 'wb')
                        a.seek(self.datapos+datapos)
                        mydat.write(a.read(datalen))
                        mydat.close()
@@ -392,7 +419,8 @@ class tbz2(object):
                """Returns all the files from the dataSegment as a map object."""
                if not self.scan():
                        return 0
-               a = open(self.file, 'rb')
+               a = open(_unicode_encode(self.file,
+                       encoding=_fs_encoding, errors='strict'), 'rb')
                mydata = {}
                startpos=0
                while ((startpos+8)<self.indexsize):
@@ -411,7 +439,8 @@ class tbz2(object):
                if not self.scan():
                        return None
 
-               a = open(self.file, 'rb')
+               a = open(_unicode_encode(self.file,
+                       encoding=_fs_encoding, errors='strict'), 'rb')
                a.seek(self.datapos)
                mydata =a.read(self.datasize)
                a.close()