From: Fabian Groffen Date: Mon, 24 Aug 2009 09:28:51 +0000 (-0000) Subject: Merged from trunk -r14067:14077 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=0be1738b6de4549b42ea8a5eb96d33874eb6f055;p=portage.git Merged from trunk -r14067:14077 | 14068 | Use elog in _eapi0_pkg_nofetch(). | | zmedico | | | 14069 | sets/files.py cleanPackages function stop calling lock and | | volkmar | load and requires the caller to do that changing unmerge to | | | reflect this change | | 14070 | Scheduler is now able to clean world set when removing a | | volkmar | package. world_atom function has been updated and | | | PackageUninstall is calling it after unmerge. | | 14071 | Use a clean listener system for portage.elog instead of | | volkmar | _emerge_elog_listener | | 14072 | Use _content_encoding and _fs_encoding for unicode | | zmedico | encoding/decoding. | | 14073 | When _unicode_func_wrapper() decodes a string in a returned | | zmedico | list (typically from os.listdir), discard values with | | | invalid encoding. This insures that all names returned from | | | all os.listdir() calls are valid. | | 14074 | Use portage.os and _fs_encoding where appropriate, and fix | | zmedico | binary string handling for py3k compat. | | 14075 | Print a warning when nonexistent files have been passed to | | arfrever | dohtml. | | 14076 | Add 'return False' which was missing from the previous | | arfrever | commit. | | 14077 | Test the edge case. | | zmedico | | svn path=/main/branches/prefix/; revision=14138 --- diff --git a/bin/ebuild-helpers/dohtml b/bin/ebuild-helpers/dohtml index 69286a741..9db02910c 100755 --- a/bin/ebuild-helpers/dohtml +++ b/bin/ebuild-helpers/dohtml @@ -59,7 +59,10 @@ def install(basename, dirname, options, prefix=""): else: destdir = options.ED + "/usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix - if os.path.isfile(fullpath): + if not os.path.exists(fullpath): + sys.stderr.write("!!! dohtml: %s does not exist\n" % fullpath) + return False + elif os.path.isfile(fullpath): ext = os.path.splitext(basename)[1] if (len(ext) and ext[1:] in options.allowed_exts) or basename in options.allowed_files: dodir(destdir) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 989bcb5c2..16c3c192c 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -588,10 +588,10 @@ einstall() { _eapi0_pkg_nofetch() { [ -z "${SRC_URI}" ] && return - echo "!!! The following are listed in SRC_URI for ${PN}:" + elog "The following are listed in SRC_URI for ${PN}:" local x for x in $(echo ${SRC_URI}); do - echo "!!! ${x}" + elog " ${x}" done } diff --git a/pym/_emerge/MergeListItem.py b/pym/_emerge/MergeListItem.py index 7e4556ec2..0205e6595 100644 --- a/pym/_emerge/MergeListItem.py +++ b/pym/_emerge/MergeListItem.py @@ -132,7 +132,8 @@ class MergeListItem(CompositeTask): uninstall = PackageUninstall(background=self.background, ldpath_mtimes=ldpath_mtimes, opts=self.emerge_opts, - pkg=pkg, scheduler=scheduler, settings=settings) + pkg=pkg, scheduler=scheduler, settings=settings, + world_atom=world_atom) uninstall.start() retval = uninstall.wait() diff --git a/pym/_emerge/PackageUninstall.py b/pym/_emerge/PackageUninstall.py index ff1b5e189..d86947c4a 100644 --- a/pym/_emerge/PackageUninstall.py +++ b/pym/_emerge/PackageUninstall.py @@ -12,11 +12,12 @@ from _emerge.UninstallFailure import UninstallFailure class PackageUninstall(AsynchronousTask): - __slots__ = ("ldpath_mtimes", "opts", "pkg", "scheduler", "settings") + __slots__ = ("world_atom", "ldpath_mtimes", "opts", + "pkg", "scheduler", "settings") def _start(self): try: - unmerge(self.pkg.root_config, self.opts, "unmerge", + retval = unmerge(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], self.ldpath_mtimes, clean_world=0, clean_delay=0, raise_on_error=1, scheduler=self.scheduler, writemsg_level=self._writemsg_level) @@ -24,6 +25,10 @@ class PackageUninstall(AsynchronousTask): self.returncode = e.status else: self.returncode = os.EX_OK + + if retval == 1: + self.world_atom(self.pkg) + self.wait() def _writemsg_level(self, msg, level=0, noiselevel=0): diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index cef4ee6ca..7e0c35f89 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -1108,7 +1108,7 @@ class Scheduler(PollScheduler): pkg_queue = self._pkg_queue failed_pkgs = self._failed_pkgs portage.locks._quiet = self._background - portage.elog._emerge_elog_listener = self._elog_listener + portage.elog.add_listener(self._elog_listener) rval = os.EX_OK try: @@ -1116,7 +1116,7 @@ class Scheduler(PollScheduler): finally: self._main_loop_cleanup() portage.locks._quiet = False - portage.elog._emerge_elog_listener = None + portage.elog.remove_listener(self._elog_listener) if failed_pkgs: rval = failed_pkgs[-1].returncode @@ -1566,8 +1566,8 @@ class Scheduler(PollScheduler): def _world_atom(self, pkg): """ - Add the package to the world file, but only if - it's supposed to be added. Otherwise, do nothing. + Add or remove the package to the world file, but only if + it's supposed to be added or removed. Otherwise, do nothing. """ if set(("--buildpkgonly", "--fetchonly", @@ -1596,17 +1596,25 @@ class Scheduler(PollScheduler): if hasattr(world_set, "load"): world_set.load() # maybe it's changed on disk - atom = create_world_atom(pkg, args_set, root_config) - if atom: - if hasattr(world_set, "add"): - self._status_msg(('Recording %s in "world" ' + \ - 'favorites file...') % atom) - logger.log(" === (%s of %s) Updating world file (%s)" % \ - (pkg_count.curval, pkg_count.maxval, pkg.cpv)) - world_set.add(atom) - else: - writemsg_level('\n!!! Unable to record %s in "world"\n' % \ - (atom,), level=logging.WARN, noiselevel=-1) + if pkg.operation == "uninstall": + if hasattr(world_set, "cleanPackage"): + world_set.cleanPackage(pkg.root_config.trees["vartree"].dbapi, + pkg.cpv) + if hasattr(world_set, "remove"): + for s in pkg.root_config.setconfig.active: + world_set.remove(SETPREFIX+s) + else: + atom = create_world_atom(pkg, args_set, root_config) + if atom: + if hasattr(world_set, "add"): + self._status_msg(('Recording %s in "world" ' + \ + 'favorites file...') % atom) + logger.log(" === (%s of %s) Updating world file (%s)" % \ + (pkg_count.curval, pkg_count.maxval, pkg.cpv)) + world_set.add(atom) + else: + writemsg_level('\n!!! Unable to record %s in "world"\n' % \ + (atom,), level=logging.WARN, noiselevel=-1) finally: if world_locked: world_set.unlock() diff --git a/pym/_emerge/unmerge.py b/pym/_emerge/unmerge.py index 25c3e4e4f..2c9e7576e 100644 --- a/pym/_emerge/unmerge.py +++ b/pym/_emerge/unmerge.py @@ -510,11 +510,22 @@ def unmerge(root_config, myopts, unmerge_action, raise UninstallFailure(retval) sys.exit(retval) else: - if clean_world and hasattr(sets["world"], "cleanPackage"): + if clean_world and hasattr(sets["world"], "cleanPackage")\ + and hasattr(sets["world"], "lock"): + sets["world"].lock() + if hasattr(sets["world"], "load"): + sets["world"].load() sets["world"].cleanPackage(vartree.dbapi, y) + sets["world"].unlock() emergelog(xterm_titles, " >>> unmerge success: "+y) - if clean_world and hasattr(sets["world"], "remove"): + + if clean_world and hasattr(sets["world"], "remove")\ + and hasattr(sets["world"], "lock"): + sets["world"].lock() + # load is called inside remove() for s in root_config.setconfig.active: sets["world"].remove(SETPREFIX+s) + sets["world"].unlock() + return 1 diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index c7f275238..390c1de96 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -169,11 +169,20 @@ class _unicode_func_wrapper(object): if isinstance(rval, (basestring, list, tuple)): if isinstance(rval, basestring): rval = _unicode_decode(rval, encoding=encoding) - elif isinstance(rval, list): - rval = [_unicode_decode(x, encoding=encoding) for x in rval] - elif isinstance(rval, tuple): - rval = tuple(_unicode_decode(x, encoding=encoding) \ - for x in rval) + else: + decoded_rval = [] + for x in rval: + try: + x = _unicode_decode(x, encoding=encoding, errors='strict') + except UnicodeDecodeError: + pass + else: + decoded_rval.append(x) + + if isinstance(rval, tuple): + rval = tuple(decoded_rval) + else: + rval = decoded_rval return rval diff --git a/pym/portage/cache/ebuild_xattr.py b/pym/portage/cache/ebuild_xattr.py index baba94321..bcaf30640 100644 --- a/pym/portage/cache/ebuild_xattr.py +++ b/pym/portage/cache/ebuild_xattr.py @@ -10,6 +10,7 @@ from portage.cache import fs_template from portage.versions import catsplit from portage import cpv_getkey from portage import os +from portage import _fs_encoding from portage import _unicode_decode import xattr from errno import ENODATA,ENOSPC,E2BIG @@ -156,7 +157,11 @@ class database(fs_template.FsBased): for root, dirs, files in os.walk(self.portdir): for file in files: - file = _unicode_decode(file) + try: + file = _unicode_decode(file, + encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + continue if file[-7:] == '.ebuild': cat = os.path.basename(os.path.dirname(root)) pn_pv = file[:-7] diff --git a/pym/portage/elog/__init__.py b/pym/portage/elog/__init__.py index 1ebc027c5..7bd567cee 100644 --- a/pym/portage/elog/__init__.py +++ b/pym/portage/elog/__init__.py @@ -56,11 +56,23 @@ def _load_mod(name): _elog_mod_imports[name] = m return m -_emerge_elog_listener = None +_elog_listeners = [] +def add_listener(listener): + ''' + Listeners should accept four arguments: settings, key, logentries and logtext + ''' + _elog_listeners.append(listener) + +def remove_listener(listener): + ''' + Remove previously added listener + ''' + _elog_listeners.remove(listener) + _elog_atexit_handlers = [] _preserve_logentries = {} def elog_process(cpv, mysettings, phasefilter=None): - global _elog_atexit_handlers, _emerge_elog_listener, _preserve_logentries + global _elog_atexit_handlers, _preserve_logentries logsystems = mysettings.get("PORTAGE_ELOG_SYSTEM","").split() for s in logsystems: @@ -123,9 +135,9 @@ def elog_process(cpv, mysettings, phasefilter=None): default_fulllog = _combine_logentries(default_logentries) - if _emerge_elog_listener is not None: - _emerge_elog_listener(mysettings, str(key), - default_logentries, default_fulllog) + # call listeners + for listener in _elog_listeners: + listener(mysettings, str(key), default_logentries, default_fulllog) # pass the processing to the individual modules for s, levels in logsystems.iteritems(): diff --git a/pym/portage/env/loaders.py b/pym/portage/env/loaders.py index 854304125..e878ba449 100644 --- a/pym/portage/env/loaders.py +++ b/pym/portage/env/loaders.py @@ -4,8 +4,13 @@ # $Id$ import codecs -import os +import errno import stat +from portage import os +from portage import _content_encoding +from portage import _fs_encoding +from portage import _unicode_decode +from portage import _unicode_encode from portage.localization import _ class LoaderError(Exception): @@ -40,11 +45,6 @@ def RecursiveFileLoader(filename): @returns: List of files to process """ - if isinstance(filename, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - filename = filename.encode('utf_8', 'replace') - try: st = os.stat(filename) except OSError: @@ -55,6 +55,11 @@ def RecursiveFileLoader(filename): if d[:1] == '.' or d == 'CVS': dirs.remove(d) for f in files: + try: + f = _unicode_decode(f, + encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + continue if f[:1] == '.' or f[-1:] == '~': continue yield os.path.join(root, f) @@ -145,9 +150,18 @@ class FileLoader(DataLoader): # once, which may be expensive due to digging in child classes. func = self.lineParser for fn in RecursiveFileLoader(self.fname): - f = codecs.open(fn, mode='r', encoding='utf_8', errors='replace') + try: + f = codecs.open(_unicode_encode(fn, + encoding=_fs_encoding, errors='strict'), mode='r', + encoding=_content_encoding, errors='replace') + except EnvironmentError, e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + del e + continue for line_num, line in enumerate(f): func(line, line_num, data, errors) + f.close() return (data, errors) def lineParser(self, line, line_num, data, errors): diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index ae004356c..d39087e1e 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -293,8 +293,13 @@ class WorldSet(EditablePackageSet): self._lock = None def cleanPackage(self, vardb, cpv): - self.lock() - self._load() # loads latest from disk + ''' + Before calling this function you should call lock and load. + After calling this function you should call unlock. + ''' + if not self._lock: + raise AssertionError('cleanPackage needs the set to be locked') + worldlist = list(self._atoms) mykey = cpv_getkey(cpv) newworldlist = [] @@ -316,7 +321,6 @@ class WorldSet(EditablePackageSet): newworldlist.extend(self._nonatoms) self.replace(newworldlist) - self.unlock() def singleBuilder(self, options, settings, trees): return WorldSet(settings["ROOT"]) diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index 8676c6ae2..4a5ced8a3 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -3,14 +3,21 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -import os import sys import time import unittest +from portage import os +from portage import _fs_encoding +from portage import _unicode_encode +from portage import _unicode_decode + def main(): - TEST_FILE = '__test__' + TEST_FILE = _unicode_encode('__test__', + encoding=_fs_encoding, errors='strict') + svn_dirname = _unicode_encode('.svn', + encoding=_fs_encoding, errors='strict') suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) testDirs = [] @@ -19,8 +26,14 @@ def main(): # I was tired of adding dirs to the list, so now we add __test__ # to each dir we want tested. for root, dirs, files in os.walk(basedir): - if ".svn" in dirs: - dirs.remove('.svn') + if svn_dirname in dirs: + dirs.remove(svn_dirname) + try: + root = _unicode_decode(root, + encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + continue + if TEST_FILE in files: testDirs.append(root) diff --git a/pym/portage/tests/xpak/test_decodeint.py b/pym/portage/tests/xpak/test_decodeint.py index c0f3264db..27e8ab7cf 100644 --- a/pym/portage/tests/xpak/test_decodeint.py +++ b/pym/portage/tests/xpak/test_decodeint.py @@ -12,3 +12,6 @@ class testDecodeIntTestCase(TestCase): for n in xrange(1000): self.assertEqual(decodeint(encodeint(n)), n) + + for n in (2 ** 32 - 1,): + self.assertEqual(decodeint(encodeint(n)), n) diff --git a/pym/portage/update.py b/pym/portage/update.py index 412956591..d38ddf942 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -2,8 +2,16 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -import errno, os, re, sys - +import codecs +import errno +import re +import sys + +from portage import os +from portage import _content_encoding +from portage import _fs_encoding +from portage import _unicode_decode +from portage import _unicode_encode import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dep:dep_getkey,get_operator,isvalidatom,isjustname,remove_slot', @@ -12,7 +20,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.versions:ververify' ) -from portage.const import USER_CONFIG_PATH, WORLD_FILE +from portage.const import USER_CONFIG_PATH from portage.exception import DirectoryNotFound, PortageException from portage.localization import _ @@ -63,9 +71,10 @@ def fixdbentries(update_iter, dbdir): mydata = {} for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]: file_path = os.path.join(dbdir, myfile) - f = open(file_path, "r") - mydata[myfile] = f.read() - f.close() + mydata[myfile] = codecs.open(_unicode_encode(file_path, + encoding=_fs_encoding, errors='strict'), + mode='r', encoding=_content_encoding, + errors='replace').read() updated_items = update_dbentries(update_iter, mydata) for myfile, mycontent in updated_items.iteritems(): file_path = os.path.join(dbdir, myfile) @@ -100,9 +109,9 @@ def grab_updates(updpath, prev_mtimes=None): mystat = os.stat(file_path) if file_path not in prev_mtimes or \ long(prev_mtimes[file_path]) != long(mystat.st_mtime): - f = open(file_path) - content = f.read() - f.close() + content = codecs.open(_unicode_encode(file_path, + encoding=_fs_encoding, errors='strict'), + mode='r', encoding=_content_encoding, errors='replace').read() update_data.append((file_path, mystat, content)) return update_data @@ -142,17 +151,12 @@ def parse_updates(mycontent): return myupd, errors def update_config_files(config_root, protect, protect_mask, update_iter): - """Perform global updates on /etc/portage/package.* and the world file. + """Perform global updates on /etc/portage/package.*. config_root - location of files to update protect - list of paths from CONFIG_PROTECT protect_mask - list of paths from CONFIG_PROTECT_MASK update_iter - list of update commands as returned from parse_updates()""" - if isinstance(config_root, unicode): - # Avoid UnicodeDecodeError raised from - # os.path.join when called by os.walk. - config_root = config_root.encode('utf_8', 'replace') - config_root = normalize_path(config_root) update_files = {} file_contents = {} @@ -166,9 +170,20 @@ def update_config_files(config_root, protect, protect_mask, update_iter): if os.path.isdir(config_file): for parent, dirs, files in os.walk(config_file): for y in dirs: + try: + y = _unicode_decode(y, + encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + dirs.remove(y) + continue if y.startswith("."): dirs.remove(y) for y in files: + try: + y = _unicode_decode(y, + encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + continue if y.startswith("."): continue recursivefiles.append( @@ -178,9 +193,11 @@ def update_config_files(config_root, protect, protect_mask, update_iter): myxfiles = recursivefiles for x in myxfiles: try: - myfile = open(os.path.join(abs_user_config, x),"r") - file_contents[x] = myfile.readlines() - myfile.close() + file_contents[x] = codecs.open( + _unicode_encode(os.path.join(abs_user_config, x), + encoding=_fs_encoding, errors='strict'), + mode='r', encoding=_content_encoding, + errors='replace').readlines() except IOError: if file_contents.has_key(x): del file_contents[x] diff --git a/pym/portage/util.py b/pym/portage/util.py index 96d716e81..983c08b0a 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -14,7 +14,6 @@ __all__ = ['apply_permissions', 'apply_recursive_permissions', import commands import codecs -import os import errno import logging import shlex @@ -24,7 +23,8 @@ import sys import portage from portage import os -from portage import _merge_encoding +from portage import _content_encoding +from portage import _fs_encoding from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode @@ -328,8 +328,9 @@ def grablines(myfilename,recursive=0): os.path.join(myfilename, f), recursive)) else: try: - myfile = codecs.open(_unicode_encode(myfilename), - mode='r', encoding='utf_8', errors='replace') + myfile = codecs.open(_unicode_encode(myfilename, + encoding=_fs_encoding, errors='strict'), + mode='r', encoding=_content_encoding, errors='replace') mylines = myfile.readlines() myfile.close() except IOError, e: @@ -395,10 +396,12 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): # NOTE: shex doesn't seem to support unicode objects # (produces spurious \0 characters with python-2.6.2) if sys.hexversion < 0x3000000: - content = open(_unicode_encode(mycfg), 'rb').read() + content = open(_unicode_encode(mycfg, + encoding=_fs_encoding, errors='strict'), 'rb').read() else: - content = open(_unicode_encode(mycfg), mode='r', - encoding='utf_8', errors='replace').read() + content = open(_unicode_encode(mycfg, + encoding=_fs_encoding, errors='strict'), mode='r', + encoding=_content_encoding, errors='replace').read() if content and content[-1] != '\n': content += '\n' except IOError, e: @@ -590,7 +593,8 @@ def pickle_read(filename,default=None,debug=0): return default data = None try: - myf = open(_unicode_encode(filename), 'rb') + myf = open(_unicode_encode(filename, + encoding=_fs_encoding, errors='strict'), 'rb') mypickle = pickle.Unpickler(myf) data = mypickle.load() myf.close() @@ -905,7 +909,7 @@ class atomic_ofstream(ObjectProxy): open_func = open else: open_func = codecs.open - kargs.setdefault('encoding', 'utf_8') + kargs.setdefault('encoding', _content_encoding) kargs.setdefault('errors', 'replace') if follow_links: @@ -914,7 +918,9 @@ class atomic_ofstream(ObjectProxy): tmp_name = "%s.%i" % (canonical_path, os.getpid()) try: object.__setattr__(self, '_file', - open_func(_unicode_encode(tmp_name), mode=mode, **kargs)) + open_func(_unicode_encode(tmp_name, + encoding=_fs_encoding, errors='strict'), + mode=mode, **kargs)) return except IOError, e: if canonical_path == filename: @@ -926,7 +932,9 @@ class atomic_ofstream(ObjectProxy): object.__setattr__(self, '_real_name', filename) tmp_name = "%s.%i" % (filename, os.getpid()) object.__setattr__(self, '_file', - open_func(_unicode_encode(tmp_name), mode=mode, **kargs)) + open_func(_unicode_encode(tmp_name, + encoding=_fs_encoding, errors='strict'), + mode=mode, **kargs)) def _get_target(self): return object.__getattribute__(self, '_file') diff --git a/pym/portage/xpak.py b/pym/portage/xpak.py index a1516e033..da68af59c 100644 --- a/pym/portage/xpak.py +++ b/pym/portage/xpak.py @@ -16,29 +16,50 @@ # (integer) == encodeint(integer) ===> 4 characters (big-endian copy) # '+' means concatenate the fields ===> All chunks are strings -import sys,os,shutil,errno -from stat import * +import array +import errno +import shutil -def addtolist(mylist,curdir): +from portage import os +from portage import normalize_path +from portage import _fs_encoding +from portage import _unicode_decode +from portage import _unicode_encode + +def addtolist(mylist, curdir): """(list, dir) --- Takes an array(list) and appends all files from dir down the directory tree. Returns nothing. list is modified.""" - for x in os.listdir("."): - if os.path.isdir(x): - os.chdir(x) - addtolist(mylist,curdir+x+"/") - os.chdir("..") - else: - if curdir+x not in mylist: - mylist.append(curdir+x) + curdir = normalize_path(_unicode_decode(curdir, + encoding=_fs_encoding, errors='strict')) + for parent, dirs, files in os.walk(curdir): + + parent = _unicode_decode(parent, + encoding=_fs_encoding, errors='strict') + if parent != curdir: + mylist.append(parent[len(curdir) + 1:] + os.sep) + + for x in dirs: + try: + _unicode_decode(x, encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + dirs.remove(x) + + for x in files: + try: + x = _unicode_decode(x, encoding=_fs_encoding, errors='strict') + except UnicodeDecodeError: + continue + mylist.append(os.path.join(parent, x)[len(curdir) + 1:]) def encodeint(myint): """Takes a 4 byte integer and converts it into a string of 4 characters. Returns the characters in a string.""" - part1=chr((myint >> 24 ) & 0x000000ff) - part2=chr((myint >> 16 ) & 0x000000ff) - part3=chr((myint >> 8 ) & 0x000000ff) - part4=chr(myint & 0x000000ff) - return part1+part2+part3+part4 + a = array.array('B') + a.append((myint >> 24 ) & 0xff) + a.append((myint >> 16 ) & 0xff) + a.append((myint >> 8 ) & 0xff) + a.append(myint & 0xff) + return a.tostring() def decodeint(mystring): """Takes a 4 byte string and converts it into a 4 byte integer. @@ -54,28 +75,20 @@ def xpak(rootdir,outfile=None): """(rootdir,outfile) -- creates an xpak segment of the directory 'rootdir' and under the name 'outfile' if it is specified. Otherwise it returns the xpak segment.""" - try: - origdir=os.getcwd() - except SystemExit, e: - raise - except: - os.chdir("/") - origdir="/" - os.chdir(rootdir) + mylist=[] - addtolist(mylist,"") + addtolist(mylist, rootdir) mylist.sort() mydata = {} for x in mylist: - a = open(x, 'rb') - mydata[x] = a.read() - a.close() - os.chdir(origdir) + x = _unicode_encode(x, encoding=_fs_encoding, errors='strict') + mydata[x] = open(os.path.join(rootdir, x), 'rb').read() xpak_segment = xpak_mem(mydata) if outfile: - outf = open(outfile, 'wb') + outf = open(_unicode_encode(outfile, + encoding=_fs_encoding, errors='strict'), 'wb') outf.write(xpak_segment) outf.close() else: @@ -83,9 +96,9 @@ def xpak(rootdir,outfile=None): def xpak_mem(mydata): """Create an xpack segement from a map object.""" - indexglob="" + indexglob = _unicode_encode('') indexpos=0 - dataglob="" + dataglob = _unicode_encode('') datapos=0 for x, newglob in mydata.iteritems(): mydatasize=len(newglob) @@ -93,18 +106,21 @@ def xpak_mem(mydata): indexpos=indexpos+4+len(x)+4+4 dataglob=dataglob+newglob datapos=datapos+mydatasize - return "XPAKPACK" \ + return _unicode_encode('XPAKPACK') \ + encodeint(len(indexglob)) \ + encodeint(len(dataglob)) \ + indexglob \ + dataglob \ - + "XPAKSTOP" + + _unicode_encode('XPAKSTOP') def xsplit(infile): """(infile) -- Splits the infile into two files. 'infile.index' contains the index segment. 'infile.dat' contails the data segment.""" - myfile = open(infile, 'rb') + infile = _unicode_decode(infile, + encoding=_fs_encoding, errors='strict') + myfile = open(_unicode_encode(infile, + encoding=_fs_encoding, errors='strict'), 'rb') mydat=myfile.read() myfile.close() @@ -112,27 +128,30 @@ def xsplit(infile): if not splits: return False - myfile = open(infile + '.index', 'wb') + myfile = open(_unicode_encode(infile + '.index', + encoding=_fs_encoding, errors='strict'), 'wb') myfile.write(splits[0]) myfile.close() - myfile = open(infile + '.dat', 'wb') + myfile = open(_unicode_encode(infile + '.dat', + encoding=_fs_encoding, errors='strict'), 'wb') myfile.write(splits[1]) myfile.close() return True def xsplit_mem(mydat): - if mydat[0:8]!="XPAKPACK": + if mydat[0:8] != _unicode_encode('XPAKPACK'): return None - if mydat[-8:]!="XPAKSTOP": + if mydat[-8:] != _unicode_encode('XPAKSTOP'): return None indexsize=decodeint(mydat[8:12]) return (mydat[16:indexsize+16], mydat[indexsize+16:-8]) def getindex(infile): """(infile) -- grabs the index segment from the infile and returns it.""" - myfile = open(infile, 'rb') + myfile = open(_unicode_encode(infile, + encoding=_fs_encoding, errors='strict'), 'rb') myheader=myfile.read(16) - if myheader[0:8]!="XPAKPACK": + if myheader[0:8] != _unicode_encode('XPAKPACK'): myfile.close() return indexsize=decodeint(myheader[8:12]) @@ -143,9 +162,10 @@ def getindex(infile): def getboth(infile): """(infile) -- grabs the index and data segments from the infile. Returns an array [indexSegment,dataSegment]""" - myfile = open(infile, 'rb') + myfile = open(_unicode_encode(infile, + encoding=_fs_encoding, errors='strict'), 'rb') myheader=myfile.read(16) - if myheader[0:8]!="XPAKPACK": + if myheader[0:8] != _unicode_encode('XPAKPACK'): myfile.close() return indexsize=decodeint(myheader[8:12]) @@ -217,7 +237,8 @@ def xpand(myid,mydest): if dirname: if not os.path.exists(dirname): os.makedirs(dirname) - mydat = open(myname, 'wb') + mydat = open(_unicode_encode(myname, + encoding=_fs_encoding, errors='strict'), 'wb') mydat.write(mydata[datapos:datapos+datalen]) mydat.close() startpos=startpos+namelen+12 @@ -227,7 +248,7 @@ class tbz2(object): def __init__(self,myfile): self.file=myfile self.filestat=None - self.index="" + self.index = _unicode_encode('') self.infosize=0 self.xpaksize=0 self.indexsize=None @@ -262,12 +283,13 @@ class tbz2(object): def recompose_mem(self, xpdata): self.scan() # Don't care about condition... We'll rewrite the data anyway. - myfile = open(self.file, 'ab+') + myfile = open(_unicode_encode(self.file, + encoding=_fs_encoding, errors='strict'), 'ab+') if not myfile: raise IOError myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF. myfile.truncate() - myfile.write(xpdata+encodeint(len(xpdata))+"STOP") + myfile.write(xpdata+encodeint(len(xpdata)) + _unicode_encode('STOP')) myfile.flush() myfile.close() return 1 @@ -292,28 +314,30 @@ class tbz2(object): mystat=os.stat(self.file) if self.filestat: changed=0 - for x in [ST_SIZE, ST_MTIME, ST_CTIME]: - if mystat[x] != self.filestat[x]: - changed=1 + if mystat.st_size != self.filestat.st_size \ + or mystat.st_mtime != self.filestat.st_mtime \ + or mystat.st_ctime != self.filestat.st_ctime: + changed = True if not changed: return 1 self.filestat=mystat - a = open(self.file, 'rb') + a = open(_unicode_encode(self.file, + encoding=_fs_encoding, errors='strict'), 'rb') a.seek(-16,2) trailer=a.read() self.infosize=0 self.xpaksize=0 - if trailer[-4:]!="STOP": + if trailer[-4:] != _unicode_encode('STOP'): a.close() return 0 - if trailer[0:8]!="XPAKSTOP": + if trailer[0:8] != _unicode_encode('XPAKSTOP'): a.close() return 0 self.infosize=decodeint(trailer[8:12]) self.xpaksize=self.infosize+8 a.seek(-(self.xpaksize),2) header=a.read(16) - if header[0:8]!="XPAKPACK": + if header[0:8] != _unicode_encode('XPAKPACK'): a.close() return 0 self.indexsize=decodeint(header[8:12]) @@ -341,7 +365,8 @@ class tbz2(object): myresult=searchindex(self.index,myfile) if not myresult: return mydefault - a = open(self.file, 'rb') + a = open(_unicode_encode(self.file, + encoding=_fs_encoding, errors='strict'), 'rb') a.seek(self.datapos+myresult[0],0) myreturn=a.read(myresult[1]) a.close() @@ -365,7 +390,8 @@ class tbz2(object): except: os.chdir("/") origdir="/" - a = open(self.file, 'rb') + a = open(_unicode_encode(self.file, + encoding=_fs_encoding, errors='strict'), 'rb') if not os.path.exists(mydest): os.makedirs(mydest) os.chdir(mydest) @@ -379,7 +405,8 @@ class tbz2(object): if dirname: if not os.path.exists(dirname): os.makedirs(dirname) - mydat = open(myname, 'wb') + mydat = open(_unicode_encode(myname, + encoding=_fs_encoding, errors='strict'), 'wb') a.seek(self.datapos+datapos) mydat.write(a.read(datalen)) mydat.close() @@ -392,7 +419,8 @@ class tbz2(object): """Returns all the files from the dataSegment as a map object.""" if not self.scan(): return 0 - a = open(self.file, 'rb') + a = open(_unicode_encode(self.file, + encoding=_fs_encoding, errors='strict'), 'rb') mydata = {} startpos=0 while ((startpos+8)