From: Fabian Groffen Date: Thu, 20 Dec 2007 18:35:39 +0000 (-0000) Subject: Merged from trunk 8968:8997 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=2bfd66a8428df29e2da83edde3d3774e04c95209;p=portage.git Merged from trunk 8968:8997 | 8969 | Whitespace only: remove extra indentation. | | zmedico | | | 8971 | Bug #202748 - When merging a binary package, only clean up | | zmedico | temp files on success. Otherwise, leave the temp files | | | intact for debugging purposes. | | 8973 | Make some adjustments so that it's possible to install | | zmedico | binary packages without having a portage tree: * Make | | | portdbapi.aux_get() return early by raising a KeyError if it | | | detects that there is no portage tree. * Move the ARCH and | | | USERLAND sanity check to the last moment in doebuild() and | | | only require these variables if an existing environment | | | (such as environment.bz2) is unavailable. * Make the | | | NewsManager constructor cope with a broken make.profile | | | symlink. | | 8974 | Convert messages about missing portage tree to debug level | | zmedico | warnings since quite often it's possible to function without | | | a portage tree now. | | 8975 | Make binarytree.populate() stop relying on having a | | zmedico | categories list since it may not be available if there is no | | | portage tree. | | 8976 | Pass a valid phase into eerror so that it works. | | zmedico | | | 8978 | Fix a type: eclasses -> eclass. | | zmedico | | | 8980 | Enable emerge search actions for binary packages when there | | zmedico | is no portage tree available. | | 8982 | Make vartree.cpv_all() stop relying on having a categories | | zmedico | list since it may not be available if there is no portage | | | tree. | | 8985 | Fix binarytree._update_pkgindex_header() so that it can cope | | zmedico | with a missing profile. | | 8988 | Make dep_zapdeps() fall back to installed packages in it's | | zmedico | availability test in cases where the package does not appear | | | to be masked. This solves dep selection preference issues | | | when working without a portage tree. | | 8990 | Fix portdbapi.cp_list() so that it can cope with a missing | | zmedico | profile. | | 8992 | For simplicity and consistency, always use a regular | | zmedico | expression for category validation instead of using a list. | | 8993 | Filter out some non-category directories in | | zmedico | portdbapi.cp_all(). | | 8997 | Fix a typo in the _non_category_dirs regex. | | zmedico | | svn path=/main/branches/prefix/; revision=9001 --- diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index b32d8b7a9..1a238775d 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -404,12 +404,77 @@ class search(object): The list of available and installed packages is created at object instantiation. This makes successive searches faster.""" self.settings = root_config.settings - self.portdb = root_config.trees["porttree"].dbapi self.vartree = root_config.trees["vartree"] self.spinner = spinner self.verbose = verbose self.searchdesc = searchdesc self.setconfig = root_config.setconfig + _portdb = root_config.trees["porttree"].dbapi + if _portdb._have_root_eclass_dir: + self.portdb = _portdb + else: + def fake_portdb(): + pass + self.portdb = fake_portdb + self._dbs = [root_config.trees["bintree"].dbapi] + for attrib in ("aux_get", "cp_all", + "xmatch", "findname", "getfetchlist"): + setattr(fake_portdb, attrib, getattr(self, "_"+attrib)) + + def _cp_all(self): + cp_all = set() + for db in self._dbs: + cp_all.update(db.cp_all()) + return list(sorted(cp_all)) + + def _aux_get(self, *args, **kwargs): + for db in self._dbs: + try: + return db.aux_get(*args, **kwargs) + except KeyError: + pass + raise + + def _findname(self, *args, **kwargs): + for db in self._dbs: + func = getattr(db, "findname", None) + if func: + value = func(*args, **kwargs) + if value: + return value + return None + + def _getfetchlist(self, *args, **kwargs): + for db in self._dbs: + func = getattr(db, "getfetchlist", None) + if func: + value = func(*args, **kwargs) + if value: + return value + return None + + def _xmatch(self, level, atom): + if level.startswith("bestmatch-"): + matches = [] + for db in self._dbs: + bestmatch = None + if hasattr(db, "xmatch"): + bestmatch = db.xmatch(level, atom) + else: + bestmatch = portage.best(db.match(atom)) + if bestmatch: + matches.append(bestmatch) + return portage.best(matches) + else: + matches = set() + for db in self._dbs: + if hasattr(db, "xmatch"): + matches.update(db.xmatch(level, atom)) + else: + matches.update(db.match(atom)) + matches = list(matches) + db._cpv_sort_ascending(matches) + return matches def execute(self,searchkey): """Performs the search for the supplied search key""" @@ -478,7 +543,8 @@ class search(object): if self.searchre.search(match_string): self.matches["set"].append([setname, False]) elif self.searchdesc: - if self.searchre.search(sdict[setname].getMetadata("DESCRIPTION")): + if self.searchre.search( + self.sdict[setname].getMetadata("DESCRIPTION")): self.matches["set"].append([setname, False]) self.mlen=0 @@ -528,27 +594,31 @@ class search(object): mypkg = match.split("/")[1] mycpv = match + "-" + myversion myebuild = self.portdb.findname(mycpv) - pkgdir = os.path.dirname(myebuild) - from portage import manifest - mf = manifest.Manifest( - pkgdir, self.settings["DISTDIR"]) - fetchlist = self.portdb.getfetchlist(mycpv, - mysettings=self.settings, all=True)[1] - try: - mysum[0] = mf.getDistfilesSize(fetchlist) - mystr = str(mysum[0]/1024) - mycount=len(mystr) - while (mycount > 3): - mycount-=3 - mystr=mystr[:mycount]+","+mystr[mycount:] - mysum[0]=mystr+" kB" - except KeyError, e: - mysum[0] = "Unknown (missing digest for %s)" % str(e) + if myebuild: + pkgdir = os.path.dirname(myebuild) + from portage import manifest + mf = manifest.Manifest( + pkgdir, self.settings["DISTDIR"]) + fetchlist = self.portdb.getfetchlist(mycpv, + mysettings=self.settings, all=True)[1] + try: + mysum[0] = mf.getDistfilesSize(fetchlist) + mystr = str(mysum[0] / 1024) + mycount = len(mystr) + while (mycount > 3): + mycount -= 3 + mystr = mystr[:mycount] + "," + mystr[mycount:] + mysum[0] = mystr + " kB" + except KeyError, e: + mysum[0] = "Unknown (missing digest for %s)" % \ + str(e) if self.verbose: print " ", darkgreen("Latest version available:"),myversion print " ", self.getInstallationStatus(mycat+'/'+mypkg) - print " ", darkgreen("Size of files:"),mysum[0] + if myebuild: + print " %s %s" % \ + (darkgreen("Size of files:"), mysum[0]) print " ", darkgreen("Homepage:")+" ",homepage print " ", darkgreen("Description:")+" ",desc print " ", darkgreen("License:")+" ",license @@ -6746,18 +6816,7 @@ def parse_opts(tmpcmdline, silent=False): return myaction, myopts, myfiles def validate_ebuild_environment(trees): - for myroot in trees: - mysettings = trees[myroot]["vartree"].settings - for var in "ARCH", "USERLAND": - if mysettings.get(var): - continue - print >> sys.stderr, bad(("\a!!! %s is not set... " % var) + \ - "Are you missing the '%setc/make.profile' symlink?" % \ - mysettings["PORTAGE_CONFIGROOT"]) - print >> sys.stderr, bad("\a!!! Is the symlink correct? " + \ - "Is your portage tree complete?\n") - sys.exit(9) - del myroot, mysettings + pass def load_emerge_config(trees=None): kwargs = {} diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 94c16336e..af3d6a8f3 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1681,7 +1681,8 @@ class config(object): groups = self["ACCEPT_KEYWORDS"].split() archlist = self.archlist() if not archlist: - writemsg("--- 'profiles/arch.list' is empty or not available. Empty portage tree?\n") + writemsg("--- 'profiles/arch.list' is empty or " + \ + "not available. Empty portage tree?\n", noiselevel=1) else: for group in groups: if group not in archlist and \ @@ -2415,47 +2416,46 @@ class config(object): # * Forced flags, such as those from {,package}use.force # * build and bootstrap flags used by bootstrap.sh - if True: - # Do this even when there's no package since setcpv() can - # optimize away regenerate() calls. - iuse_implicit = set(iuse) - - # Flags derived from ARCH. - if arch: - iuse_implicit.add(arch) - iuse_implicit.update(self.get("PORTAGE_ARCHLIST", "").split()) - - # Flags derived from USE_EXPAND_HIDDEN variables - # such as ELIBC, KERNEL, and USERLAND. - use_expand_hidden = self.get("USE_EXPAND_HIDDEN", "").split() - use_expand_hidden_raw = use_expand_hidden - if use_expand_hidden: - use_expand_hidden = re.compile("^(%s)_.*" % \ - ("|".join(x.lower() for x in use_expand_hidden))) - for x in usesplit: - if use_expand_hidden.match(x): - iuse_implicit.add(x) - - # Flags that have been masked or forced. - iuse_implicit.update(self.usemask) - iuse_implicit.update(self.useforce) - - # build and bootstrap flags used by bootstrap.sh - iuse_implicit.add("build") - iuse_implicit.add("bootstrap") - - # prefix flag is used in Prefix - iuse_implicit.add("prefix") - - iuse_grep = iuse_implicit.copy() - if use_expand_hidden_raw: - for x in use_expand_hidden_raw: - iuse_grep.add(x.lower() + "_.*") - if iuse_grep: - iuse_grep = "^(%s)$" % "|".join(sorted(iuse_grep)) - else: - iuse_grep = "" - self["PORTAGE_IUSE"] = iuse_grep + # Do this even when there's no package since setcpv() can + # optimize away regenerate() calls. + iuse_implicit = set(iuse) + + # Flags derived from ARCH. + if arch: + iuse_implicit.add(arch) + iuse_implicit.update(self.get("PORTAGE_ARCHLIST", "").split()) + + # Flags derived from USE_EXPAND_HIDDEN variables + # such as ELIBC, KERNEL, and USERLAND. + use_expand_hidden = self.get("USE_EXPAND_HIDDEN", "").split() + use_expand_hidden_raw = use_expand_hidden + if use_expand_hidden: + use_expand_hidden = re.compile("^(%s)_.*" % \ + ("|".join(x.lower() for x in use_expand_hidden))) + for x in usesplit: + if use_expand_hidden.match(x): + iuse_implicit.add(x) + + # Flags that have been masked or forced. + iuse_implicit.update(self.usemask) + iuse_implicit.update(self.useforce) + + # build and bootstrap flags used by bootstrap.sh + iuse_implicit.add("build") + iuse_implicit.add("bootstrap") + + # prefix flag is used in Prefix + iuse_implicit.add("prefix") + + iuse_grep = iuse_implicit.copy() + if use_expand_hidden_raw: + for x in use_expand_hidden_raw: + iuse_grep.add(x.lower() + "_.*") + if iuse_grep: + iuse_grep = "^(%s)$" % "|".join(sorted(iuse_grep)) + else: + iuse_grep = "" + self["PORTAGE_IUSE"] = iuse_grep usesplit = [x for x in usesplit if \ x not in self.usemask] @@ -4564,6 +4564,22 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, env_stat = None if env_stat: mysettings._filter_calling_env = True + else: + for var in "ARCH", "USERLAND": + if mysettings.get(var): + continue + msg = ("%s is not set... " % var) + \ + ("Are you missing the '%setc/make.profile' symlink? " % \ + mysettings["PORTAGE_CONFIGROOT"]) + \ + "Is the symlink correct? " + \ + "Is your portage tree complete?" + from portage.elog.messages import eerror + from textwrap import wrap + for line in wrap(msg, 70): + eerror(line, phase="setup", key=mysettings.mycpv) + from portage.elog import elog_process + elog_process(mysettings.mycpv, mysettings) + return 1 del env_file, env_stat, saved_env _doebuild_exit_status_unlink( mysettings.get("EBUILD_EXIT_STATUS_FILE")) @@ -5265,7 +5281,8 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): if avail_pkg: avail_slot = "%s:%s" % (dep_getkey(atom), mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) - elif not avail_pkg and use_binaries: + elif not avail_pkg and \ + (use_binaries or not mydbapi.cp_list(dep_getkey(atom))): # With --usepkgonly, count installed packages as "available". # Note that --usepkgonly currently has no package.mask support. # See bug #149816. @@ -5854,6 +5871,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes mycat = None mypkg = None did_merge_phase = False + success = False try: """ Don't lock the tbz2 file because the filesytem could be readonly or shared by a cluster.""" @@ -5964,6 +5982,7 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes retval = mylink.merge(pkgloc, infloc, myroot, myebuild, cleanup=0, mydbapi=mydbapi, prev_mtimes=prev_mtimes) did_merge_phase = True + success = retval == os.EX_OK return retval finally: mysettings.pop("PORTAGE_BINPKG_FILE", None) @@ -5977,7 +5996,8 @@ def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes from portage.elog import elog_process elog_process(mycat + "/" + mypkg, mysettings) try: - shutil.rmtree(builddir) + if success: + shutil.rmtree(builddir) except (IOError, OSError), e: if e.errno != errno.ENOENT: raise @@ -6078,7 +6098,8 @@ def _global_updates(trees, prev_mtimes): else: update_data = grab_updates(updpath, prev_mtimes) except portage.exception.DirectoryNotFound: - writemsg("--- 'profiles/updates' is empty or not available. Empty portage tree?\n") + writemsg("--- 'profiles/updates' is empty or " + \ + "not available. Empty portage tree?\n", noiselevel=1) return myupd = None if len(update_data) > 0: diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index c6318bacb..a1de0695f 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -13,6 +13,7 @@ from portage.versions import catpkgsplit, catsplit, pkgcmp class dbapi(object): + _category_re = re.compile(r'^[+\w][-\.+\w]*$') def __init__(self): pass diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index dfb5f516b..f448c4afc 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -363,8 +363,6 @@ class binarytree(object): if (not os.path.isdir(self.pkgdir) and not getbinpkgs): return 0 - categories = set(self.settings.categories) - if not getbinpkgsonly: pkg_paths = {} self._pkg_paths = pkg_paths @@ -501,7 +499,7 @@ class binarytree(object): if mycpv in pkg_paths: # All is first, so it's preferred. continue - if mycat not in categories: + if not mycat: writemsg(("!!! Binary package has an " + \ "unrecognized category: '%s'\n") % full_path, noiselevel=-1) @@ -664,7 +662,7 @@ class binarytree(object): continue mycat = self.remotepkgs[mypkg]["CATEGORY"].strip() fullpkg = mycat+"/"+mypkg[:-5] - if mycat not in categories: + if not mycat: writemsg(("!!! Remote binary package has an " + \ "unrecognized category: '%s'\n") % fullpkg, noiselevel=-1) @@ -795,10 +793,12 @@ class binarytree(object): def _update_pkgindex_header(self, header): portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"])) profiles_base = os.path.join(portdir, "profiles") + os.path.sep - profile_path = normalize_path(os.path.realpath(self.settings.profile_path)) - if profile_path.startswith(profiles_base): - profile_path = profile_path[len(profiles_base):] - header["PROFILE"] = profile_path + if self.settings.profile_path: + profile_path = normalize_path( + os.path.realpath(self.settings.profile_path)) + if profile_path.startswith(profiles_base): + profile_path = profile_path[len(profiles_base):] + header["PROFILE"] = profile_path header["VERSION"] = str(self._pkgindex_version) base_uri = self.settings.get("PORTAGE_BINHOST_HEADER_URI") if base_uri: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 0ed55fba8..89bc80161 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -21,13 +21,14 @@ import portage.gpg, portage.checksum from portage import eclass_cache, auxdbkeys, auxdbkeylen, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check -import os, stat, sys +import os, re, stat, sys from itertools import izip class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] - + _non_category_dirs = re.compile(r'^(%s)$' % \ + "|".join(["eclass", "profiles", "scripts"])) def __init__(self, porttree_root, mysettings=None): portdbapi.portdbapi_instances.append(self) @@ -68,6 +69,13 @@ class portdbapi(dbapi): self.eclassdb = eclass_cache.cache(self.porttree_root, overlays=self.mysettings["PORTDIR_OVERLAY"].split()) + # This is used as sanity check for aux_get(). If there is no + # root eclass dir, we assume that PORTDIR is invalid or + # missing. This check allows aux_get() to detect a missing + # portage tree and return early by raising a KeyError. + self._have_root_eclass_dir = os.path.isdir( + os.path.join(self.porttree_root, "eclass")) + self.metadbmodule = self.mysettings.load_best_module("portdbapi.metadbmodule") #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) @@ -303,6 +311,8 @@ class portdbapi(dbapi): if doregen: if myebuild in self._broken_ebuilds: raise KeyError(mycpv) + if not self._have_root_eclass_dir: + raise KeyError(mycpv) writemsg("doregen: %s %s\n" % (doregen, mycpv), 2) writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n", 1) @@ -476,8 +486,11 @@ class portdbapi(dbapi): def cp_all(self): "returns a list of all keys in our tree" d = {} - for x in self.mysettings.categories: - for oroot in self.porttrees: + for oroot in self.porttrees: + for x in listdir(oroot, EmptyOnError=1, ignorecvs=1, dirsonly=1): + if not self._category_re.match(x) or \ + self._non_category_dirs.match(x): + continue for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1): d[x+"/"+y] = None l = d.keys() @@ -504,7 +517,7 @@ class portdbapi(dbapi): self.xcache["match-all"][mycp] = cachelist return cachelist[:] mysplit = mycp.split("/") - invalid_category = mysplit[0] not in self._categories + invalid_category = not self._category_re.match(mysplit[0]) d={} if mytree: mytrees = [mytree] diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 81c120ef5..b2610b327 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -27,7 +27,7 @@ from portage.elog import elog_process from portage.elog.messages import ewarn from portage.elog.filtering import filter_mergephases, filter_unmergephases -import os, sys, stat, errno, commands, copy, time +import os, re, sys, stat, errno, commands, copy, time from itertools import izip try: @@ -181,9 +181,9 @@ class vardbapi(dbapi): if settings is None: from portage import settings self.settings = settings - if categories is None: - categories = settings.categories - self.categories = categories[:] + # The categories list is now automatically generated + # from a regular expression. + self.categories = None if vartree is None: from portage import db vartree = db[root]["vartree"] @@ -379,7 +379,9 @@ class vardbapi(dbapi): def cpv_all(self, use_cache=1): returnme = [] basepath = os.path.join(self.root, VDB_PATH) + os.path.sep - for x in self.categories: + for x in listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1): + if not self._category_re.match(x): + continue for y in listdir(basepath + x, EmptyOnError=1): if y.startswith("."): continue diff --git a/pym/portage/news.py b/pym/portage/news.py index 540372e10..da7e159af 100644 --- a/pym/portage/news.py +++ b/pym/portage/news.py @@ -36,10 +36,12 @@ class NewsManager(object): portdir = portdb.porttree_root profiles_base = os.path.join(portdir, 'profiles') + os.path.sep - profile_path = normalize_path( - os.path.realpath(portdb.mysettings.profile_path)) - if profile_path.startswith(profiles_base): - profile_path = profile_path[len(profiles_base):] + profile_path = None + if portdb.mysettings.profile_path: + profile_path = normalize_path( + os.path.realpath(portdb.mysettings.profile_path)) + if profile_path.startswith(profiles_base): + profile_path = profile_path[len(profiles_base):] self._profile_path = profile_path # Ensure that the unread path exists and is writable.