From dd3327681147454eeda4664ac675e24f0a5cd98e Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 21 Feb 2009 11:05:26 +0000 Subject: [PATCH] Merged from trunk -r12652:12664 | 12653 | Optimize depgraph._dep_expand() to avoid calling | | zmedico | pordbapi.cp_all(), since it can be more efficient to call | | | cp_list() on each category. Thanks to Marat Radchenko | | | for the suggestion. | | 12654 | Make isvalidatom() identify empty slot as invalid (missing | | zmedico | slot is valid) and add a corresponding test case. | | 12655 | When displaying the list of failed packages, tell the number | | zmedico | that failed. Thanks to Jeremy Olexa for the | | | suggestion. | | 12656 | Add back Rev to svn:keywords (accidentally removed when | | zmedico | setting Id). Thanks to ABCD for reporting. | | 12658 | Try to fix svn:keywords again. | | zmedico | | | 12659 | Try to fix svn:keywords again. | | zmedico | | | 12661 | Add a new portage.utils.lazy_import() function which behaves | | zmedico | similar to the snakeoil.demandload.demandload() function. | | 12662 | Fix StringIO imports so that 2to3 can handle them. Also, | | zmedico | replace shlex + StringIO usage with shlex.split() where | | | appropriate. | | 12663 | In python-3.0, integer division results in a float, so | | zmedico | convert back to int. | | 12664 | Add a cross reference from FEATURES=lmirror docs to the | | zmedico | "local" mirror docs in portage(5). | svn path=/main/branches/prefix/; revision=12670 --- bin/repoman | 6 +- man/make.conf.5 | 3 + pym/_emerge/__init__.py | 54 +++++------ pym/portage/__init__.py | 62 ++++++------- pym/portage/dbapi/__init__.py | 3 +- pym/portage/dbapi/bintree.py | 2 +- pym/portage/dbapi/porttree.py | 2 +- pym/portage/dbapi/vartree.py | 2 +- pym/portage/dep.py | 4 + pym/portage/getbinpkg.py | 7 +- pym/portage/tests/dep/test_isvalidatom.py | 1 + pym/portage/util.py | 107 +++++++++++++++++++++- 12 files changed, 171 insertions(+), 82 deletions(-) diff --git a/bin/repoman b/bin/repoman index eeedf8ee2..76557c784 100755 --- a/bin/repoman +++ b/bin/repoman @@ -26,9 +26,9 @@ from itertools import chain, izip from stat import S_ISDIR, ST_CTIME try: - import cStringIO as StringIO + from cStringIO import StringIO except ImportError: - import StringIO + from StringIO import StringIO if not hasattr(__builtins__, "set"): from sets import Set as set @@ -1614,7 +1614,7 @@ if dofail or \ # in $EDITOR while the user creates a commit message. # Otherwise, the user would not be able to see this output # once the editor has taken over the screen. -qa_output = StringIO.StringIO() +qa_output = StringIO() style_file = ConsoleStyleFile(sys.stdout) if options.mode == 'commit' and \ (not commitmessage or not commitmessage.strip()): diff --git a/man/make.conf.5 b/man/make.conf.5 index 83b7fa3ba..07f7e1946 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -222,6 +222,9 @@ enabled for more than a short period of time. .B lmirror When \fImirror\fR is enabled in \fBFEATURES\fR, fetch files even when \fImirror\fR is also in the \fBebuild\fR(5) \fBRESTRICT\fR variable. +Do \fBNOT\fR use \fIlmirror\fR for clients that need to override \fBRESTRICT\fR +when fetching from a local mirror, but instead use a "local" mirror setting +in \fI/etc/portage/mirrors\fR, as described in \fBportage\fR(5). .TP .B metadata\-transfer Automatically perform a metadata transfer when `emerge \-\-sync` is run. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 33edec974..fae99e04a 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -78,9 +78,9 @@ except ImportError: import pickle try: - import cStringIO as StringIO + from cStringIO import StringIO except ImportError: - import StringIO + from StringIO import StringIO class stdout_spinner(object): scroll_msgs = [ @@ -5301,23 +5301,14 @@ class depgraph(object): cat, atom_pn = portage.catsplit(null_cp) dbs = self._filtered_trees[root_config.root]["dbs"] - cp_set = set() + categories = set() for db, pkg_type, built, installed, db_keys in dbs: - cp_set.update(db.cp_all()) - for cp in list(cp_set): - cat, pn = portage.catsplit(cp) - if pn != atom_pn: - cp_set.discard(cp) + for cat in db.categories: + if db.cp_list("%s/%s" % (cat, atom_pn)): + categories.add(cat) + deps = [] - for cp in cp_set: - have_pkg = False - for db, pkg_type, built, installed, db_keys in dbs: - if db.cp_list(cp): - have_pkg = True - break - if not have_pkg: - continue - cat, pn = portage.catsplit(cp) + for cat in categories: deps.append(insert_category_into_atom( atom_without_category, cat)) return deps @@ -9818,8 +9809,8 @@ class JobStatusDisplay(object): failed_str = str(self.failed) load_avg_str = self._load_avg_str() - color_output = StringIO.StringIO() - plain_output = StringIO.StringIO() + color_output = StringIO() + plain_output = StringIO() style_file = portage.output.ConsoleStyleFile(color_output) style_file.write_listener = plain_output style_writer = portage.output.StyleWriter(file=style_file, maxcol=9999) @@ -10707,8 +10698,13 @@ class Scheduler(PollScheduler): if len(self._failed_pkgs_all) > 1 or \ (self._failed_pkgs_all and "--keep-going" in self.myopts): - msg = "The following packages have " + \ - "failed to build or install:" + if len(self._failed_pkgs_all) > 1: + msg = "The following %d packages have " % \ + len(self._failed_pkgs_all) + \ + "failed to build or install:" + else: + msg = "The following package has " + \ + "failed to build or install:" prefix = bad(" * ") writemsg(prefix + "\n", noiselevel=-1) from textwrap import wrap @@ -12503,7 +12499,6 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): mytimeout=180 rsync_opts = [] - import shlex, StringIO if settings["PORTAGE_RSYNC_OPTS"] == "": portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n") rsync_opts.extend([ @@ -12528,12 +12523,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # defaults. portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) - lexer = shlex.shlex(StringIO.StringIO( - settings.get("PORTAGE_RSYNC_OPTS","")), posix=True) - lexer.whitespace_split = True - rsync_opts.extend(lexer) - del lexer - + rsync_opts.extend( + shlex.split(settings.get("PORTAGE_RSYNC_OPTS",""))) for opt in ("--recursive", "--times"): if opt not in rsync_opts: portage.writemsg(yellow("WARNING:") + " adding required option " + \ @@ -12611,11 +12602,8 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): user_name="" updatecache_flg=True all_rsync_opts = set(rsync_opts) - lexer = shlex.shlex(StringIO.StringIO( - settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")), posix=True) - lexer.whitespace_split = True - extra_rsync_opts = list(lexer) - del lexer + extra_rsync_opts = shlex.split( + settings.get("PORTAGE_RSYNC_EXTRA_OPTS","")) all_rsync_opts.update(extra_rsync_opts) family = socket.AF_INET if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index ee4cf8820..1f3902294 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4,7 +4,7 @@ # $Id$ -VERSION="$Rev: 12601 $"[6:-2] + "-svn" +VERSION="$Rev$"[6:-2] + "-svn" # =========================================================================== # START OF IMPORTS -- START OF IMPORTS -- START OF IMPORTS -- START OF IMPORT @@ -73,17 +73,30 @@ if platform.system() in ["FreeBSD"]: try: from portage.cache.cache_errors import CacheError - import portage.cvstree - import portage.xpak - import portage.getbinpkg - import portage.dep - from portage.dep import dep_getcpv, dep_getkey, get_operator, \ - isjustname, isspecific, isvalidatom, \ - match_from_list, match_to_list, best_match_to_list - - # XXX: This needs to get cleaned up. - import portage.output - from portage.output import bold, colorize, green, red, yellow + import portage.util as util + util.lazy_import(globals(), + 'portage.checksum', + 'portage.checksum:perform_checksum,perform_md5,prelink_capable', + 'portage.cvstree', + 'portage.dep', + 'portage.dep:best_match_to_list,dep_getcpv,dep_getkey,' + \ + 'get_operator,isjustname,isspecific,isvalidatom,' + \ + 'match_from_list,match_to_list', + 'portage.eclass_cache', + 'portage.getbinpkg', + 'portage.locks', + 'portage.locks:lockdir,lockfile,unlockdir,unlockfile', + 'portage.output', + 'portage.output:bold,colorize', + 'portage.process', + 'portage.process:atexit_register,run_exitfuncs', + 'portage.update:dep_transform,fixdbentries,grab_updates,' + \ + 'parse_updates,update_config_files,update_dbentries,' + \ + 'update_dbentry', + 'portage.versions:best,catpkgsplit,catsplit,endversion_keys,' + \ + 'suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify', + 'portage.xpak', + ) import portage.const from portage.const import VDB_PATH, PRIVATE_PATH, CACHE_PATH, DEPCACHE_PATH, \ @@ -100,31 +113,13 @@ try: portage_uid, portage_gid, userpriv_groups from portage.manifest import Manifest - import portage.util from portage.util import atomic_ofstream, apply_secpass_permissions, apply_recursive_permissions, \ dump_traceback, getconfig, grabdict, grabdict_package, grabfile, grabfile_package, \ map_dictlist_vals, new_protect_filename, normalize_path, \ pickle_read, pickle_write, stack_dictlist, stack_dicts, stack_lists, \ unique_array, varexpand, writedict, writemsg, writemsg_stdout, write_atomic import portage.exception - import portage.locks - import portage.process - from portage.process import atexit_register, run_exitfuncs - from portage.locks import unlockfile,unlockdir,lockfile,lockdir - import portage.checksum - from portage.checksum import perform_md5,perform_checksum,prelink_capable - import portage.eclass_cache from portage.localization import _ - from portage.update import dep_transform, fixdbentries, grab_updates, \ - parse_updates, update_config_files, update_dbentries, update_dbentry - - # Need these functions directly in portage namespace to not break every external tool in existence - from portage.versions import best, catpkgsplit, catsplit, pkgcmp, \ - pkgsplit, vercmp, ververify - - # endversion and endversion_keys are for backward compatibility only. - from portage.versions import endversion_keys - from portage.versions import suffix_value as endversion except ImportError, e: sys.stderr.write("\n\n") @@ -4103,10 +4098,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", "URI": loc, "FILE": myfile } - import shlex, StringIO - lexer = shlex.shlex(StringIO.StringIO(locfetch), posix=True) - lexer.whitespace_split = True - myfetch = [varexpand(x, mydict=variables) for x in lexer] + import shlex + myfetch = shlex.split(locfetch) + myfetch = [varexpand(x, mydict=variables) for x in myfetch] myret = -1 try: diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index e5db363d7..dae7ad014 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -38,8 +38,7 @@ class dbapi(object): cat_pattern = re.compile(r'(.*)/.*') for cp in self.cp_all(): categories.add(cat_pattern.match(cp).group(1)) - self._categories = list(categories) - self._categories.sort() + self._categories = tuple(sorted(categories)) return self._categories def close_caches(self): diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index f80c3f554..1cca34ac7 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -17,7 +17,7 @@ from portage.const import EAPI, EAPIPREFIX from portage import dep_expand, listdir, _check_distfile, _movefile -import portage.xpak, portage.getbinpkg +import portage import os, errno, stat import re diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 77e6ae494..d342c22d9 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -94,7 +94,7 @@ class portdbapi(dbapi): from portage import settings self.mysettings = config(clone=settings) self._iuse_implicit = self.mysettings._get_implicit_iuse() - self._categories = set(self.mysettings.categories) + self._categories = tuple(sorted(set(self.mysettings.categories))) # This is strictly for use in aux_get() doebuild calls when metadata # is generated by the depend phase. It's safest to use a clone for # this purpose because doebuild makes many changes to the config diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index e4cdbd696..aa5c7a542 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1868,7 +1868,7 @@ class vardbapi(dbapi): from md5 import new as _new_hash _hash_bits = 16 - _hex_chars = _hash_bits / 4 + _hex_chars = int(_hash_bits / 4) def __init__(self, vardb): self._vardb = vardb diff --git a/pym/portage/dep.py b/pym/portage/dep.py index 04817068c..22fa8bf56 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -773,6 +773,10 @@ def isvalidatom(atom, allow_blockers=False): else: atom = atom[1:] + if dep_getslot(atom) == "": + # empty slot is invalid (None is valid) + return 0 + try: use = dep_getusedeps(atom) if use: diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index d49fcd920..db2b6580a 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -401,12 +401,11 @@ def file_get(baseurl,dest,conn=None,fcmd=None): "URI": baseurl, "FILE": os.path.basename(baseurl) } - import shlex, StringIO + import shlex from portage.util import varexpand from portage.process import spawn - lexer = shlex.shlex(StringIO.StringIO(fcmd), posix=True) - lexer.whitespace_split = True - myfetch = [varexpand(x, mydict=variables) for x in lexer] + myfetch = shlex.split(fcmd) + myfetch = [varexpand(x, mydict=variables) for x in myfetch] fd_pipes= { 0:sys.stdin.fileno(), 1:sys.stdout.fileno(), diff --git a/pym/portage/tests/dep/test_isvalidatom.py b/pym/portage/tests/dep/test_isvalidatom.py index 1364a92a2..5d6b4e73a 100644 --- a/pym/portage/tests/dep/test_isvalidatom.py +++ b/pym/portage/tests/dep/test_isvalidatom.py @@ -25,6 +25,7 @@ class IsValidAtom(TestCase): ( "sys-apps/portage:foo", True ), ( "sys-apps/portage-2.1:foo", False ), ( "sys-apps/portage-2.1:", False ), + ( "sys-apps/portage-2.1:[foo]", False ), ( "=sys-apps/portage-2.2*:foo[bar?,!baz?,!doc=,build=]", True ), ( "=sys-apps/portage-2.2*:foo[doc?]", True ), ( "=sys-apps/portage-2.2*:foo[!doc?]", True ), diff --git a/pym/portage/util.py b/pym/portage/util.py index dd96c48a8..1be03e0fb 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -10,6 +10,7 @@ import shlex import stat import string import sys +import types from portage.exception import PortageException, FileNotFound, \ OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem @@ -23,9 +24,9 @@ except ImportError: import pickle try: - import cStringIO as StringIO + from cStringIO import StringIO except ImportError: - import StringIO + from StringIO import StringIO noiselimit = 0 @@ -342,6 +343,8 @@ class ObjectProxy(object): for implementing lazy initialization. """ + __slots__ = () + def _get_target(self): raise NotImplementedError(self) @@ -393,6 +396,104 @@ class ObjectProxy(object): def __nonzero__(self): return bool(object.__getattribute__(self, '_get_target')()) +class _LazyImport(ObjectProxy): + + __slots__ = ('_scope', '_alias', '_name', '_target') + + def __init__(self, scope, alias, name): + ObjectProxy.__init__(self) + object.__setattr__(self, '_scope', scope) + object.__setattr__(self, '_alias', alias) + object.__setattr__(self, '_name', name) + + def _get_target(self): + try: + return object.__getattribute__(self, '_target') + except AttributeError: + pass + name = object.__getattribute__(self, '_name') + __import__(name) + target = sys.modules[name] + object.__setattr__(self, '_target', target) + object.__getattribute__(self, '_scope')[ + object.__getattribute__(self, '_alias')] = target + return target + +class _LazyImportFrom(_LazyImport): + + __slots__ = () + + def _get_target(self): + try: + return object.__getattribute__(self, '_target') + except AttributeError: + pass + name = object.__getattribute__(self, '_name') + components = name.split('.') + parent_name = '.'.join(components[:-1]) + __import__(parent_name) + target = getattr(sys.modules[parent_name], components[-1]) + object.__setattr__(self, '_target', target) + object.__getattribute__(self, '_scope')[ + object.__getattribute__(self, '_alias')] = target + return target + +def lazy_import(scope, *args): + """ + Create a proxy in the given scope in order to performa a lazy import. + + Syntax Result + foo import foo + foo:bar,baz from foo import bar, baz + foo:bar@baz from foo import bar as baz + + @param scope: the scope in which to place the import, typically globals() + @type myfilename: dict + @param args: module names to import + @type args: strings + """ + + for s in args: + parts = s.split(':', 1) + if len(parts) == 1: + name = s + + if not name or not isinstance(name, basestring): + raise ValueError(name) + + components = name.split('.') + parent_scope = scope + for i in xrange(len(components)): + alias = components[i] + mod = parent_scope.get(alias) + if isinstance(mod, types.ModuleType): + parent_scope = mod.__dict__ + continue + if i < len(components) - 1: + parent_name = ".".join(components[:i+1]) + __import__(parent_name) + mod = sys.modules.get(parent_name) + if not isinstance(mod, types.ModuleType): + # raise an exception + __import__(name) + parent_scope[alias] = mod + parent_scope = mod.__dict__ + continue + parent_scope[alias] = _LazyImport(parent_scope, alias, name) + + else: + name, fromlist = parts + fromlist = fromlist.split(',') + for s in fromlist: + alias = s.split('@', 1) + if len(alias) == 1: + alias = alias[0] + orig = alias + else: + orig, alias = alias + scope[alias] = _LazyImportFrom(scope, alias, + name + '.' + orig) + class _tolerant_shlex(shlex.shlex): def sourcehook(self, newfile): try: @@ -400,7 +501,7 @@ class _tolerant_shlex(shlex.shlex): except EnvironmentError, e: writemsg("!!! Parse error in '%s': source command failed: %s\n" % \ (self.infile, str(e)), noiselevel=-1) - return (newfile, StringIO.StringIO()) + return (newfile, StringIO()) class _insert_newline_eof(ObjectProxy): """ -- 2.26.2