From eabfcf5e872971848ec00d877d9afc78455e0efe Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 20 Sep 2008 08:02:45 +0000 Subject: [PATCH] Implement SRC_URI arrows for EAPI 2. The portdbapi.getfetchlist() method is now deprecated and there is a new getFetchMap() method that returns a dict which maps each file name to a set of alternative URIs. The portage.fetch() function uses introspection to detect when such a dict is passed in and handles it appropriately, while maintaining backward compatibility if a list of uris is passed in. svn path=/main/trunk/; revision=11522 --- pym/_emerge/__init__.py | 11 ++- pym/portage/__init__.py | 55 +++++------- pym/portage/dbapi/porttree.py | 161 ++++++++++++++++++++++++++++------ 3 files changed, 160 insertions(+), 67 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 5c5aaa731..6cfe7eb66 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -431,7 +431,7 @@ class search(object): pass self.portdb = fake_portdb for attrib in ("aux_get", "cp_all", - "xmatch", "findname", "getfetchlist"): + "xmatch", "findname", "getFetchMap"): setattr(fake_portdb, attrib, getattr(self, "_"+attrib)) self._dbs = [] @@ -478,14 +478,14 @@ class search(object): return value return None - def _getfetchlist(self, *args, **kwargs): + def _getFetchMap(self, *args, **kwargs): for db in self._dbs: - func = getattr(db, "getfetchlist", None) + func = getattr(db, "getFetchMap", None) if func: value = func(*args, **kwargs) if value: return value - return [], [] + return {} def _visible(self, db, cpv, metadata): installed = db is self.vartree.dbapi @@ -684,8 +684,7 @@ class search(object): from portage import manifest mf = manifest.Manifest( pkgdir, self.settings["DISTDIR"]) - fetchlist = self.portdb.getfetchlist(mycpv, - mysettings=self.settings, all=True)[1] + fetchlist = self.portdb.getFetchMap(mycpv) try: mysum[0] = mf.getDistfilesSize(fetchlist) except KeyError, e: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 199317c28..207432f35 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -3487,11 +3487,19 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", else: locations = mymirrors + file_uri_tuples = [] + if isinstance(myuris, dict): + for myfile, uri_set in myuris.iteritems(): + for myuri in uri_set: + file_uri_tuples.append((myfile, myuri)) + else: + for myuri in myuris: + file_uri_tuples.append((os.path.basename(myuri), myuri)) + filedict={} primaryuri_indexes={} primaryuri_dict = {} - for myuri in myuris: - myfile=os.path.basename(myuri) + for myfile, myuri in file_uri_tuples: if myfile not in filedict: filedict[myfile]=[] for y in range(0,len(locations)): @@ -4150,11 +4158,8 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None doebuild_environment(myebuild, "fetch", mysettings["ROOT"], fetch_settings, debug, 1, myportdb) - alluris, aalist = myportdb.getfetchlist( - cpv, mytree=mytree, all=True, - mysettings=fetch_settings) - myuris = [uri for uri in alluris \ - if os.path.basename(uri) == myfile] + uri_map = myportdb.getFetchMap(cpv, mytree=mytree) + myuris = {myfile:uri_map[myfile]} fetch_settings["A"] = myfile # for use by pkg_nofetch() if fetch(myuris, fetch_settings): success = True @@ -4191,8 +4196,7 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None writemsg_stdout(" digest.assumed" + portage.output.colorize("WARN", str(len(auto_assumed)).rjust(18)) + "\n") for pkg_key in pkgs: - fetchlist = myportdb.getfetchlist(pkg_key, - mysettings=mysettings, all=True, mytree=mytree)[1] + fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree) pv = pkg_key.split("/")[1] for filename in auto_assumed: if filename in fetchlist: @@ -5580,11 +5584,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Make sure we get the correct tree in case there are overlays. mytree = os.path.realpath( os.path.dirname(os.path.dirname(mysettings["O"]))) + useflags = mysettings["PORTAGE_USE"].split() try: - newuris, alist = mydbapi.getfetchlist( - mycpv, mytree=mytree, mysettings=mysettings) - alluris, aalist = mydbapi.getfetchlist( - mycpv, mytree=mytree, all=True, mysettings=mysettings) + alist = mydbapi.getFetchMap(mycpv, useflags=useflags, mytree=mytree) + aalist = mydbapi.getFetchMap(mycpv, mytree=mytree) except portage.exception.InvalidDependString, e: writemsg("!!! %s\n" % str(e), noiselevel=-1) writemsg("!!! Invalid SRC_URI for '%s'.\n" % mycpv, noiselevel=-1) @@ -5593,26 +5596,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, mysettings["A"] = " ".join(alist) mysettings["AA"] = " ".join(aalist) if ("mirror" in features) or fetchall: - fetchme = alluris[:] - checkme = aalist[:] - elif mydo == "digest": - fetchme = alluris[:] - checkme = aalist[:] - # Skip files that we already have digests for. - mf = Manifest(mysettings["O"], mysettings["DISTDIR"]) - mydigests = mf.getTypeDigests("DIST") - required_hash_types = set() - required_hash_types.add("size") - required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) - for filename, hashes in mydigests.iteritems(): - if not required_hash_types.difference(hashes): - checkme = [i for i in checkme if i != filename] - fetchme = [i for i in fetchme \ - if os.path.basename(i) != filename] - del filename, hashes + fetchme = aalist + checkme = aalist else: - fetchme = newuris[:] - checkme = alist[:] + fetchme = alist + checkme = alist if mydo == "fetch": # Files are already checked inside fetch(), @@ -6871,8 +6859,7 @@ class FetchlistDict(UserDict.DictMixin): self.portdb = mydbapi def __getitem__(self, pkg_key): """Returns the complete fetch list for a given package.""" - return self.portdb.getfetchlist(pkg_key, mysettings=self.settings, - all=True, mytree=self.mytree)[1] + return self.portdb.getFetchMap(pkg_key, mytree=self.mytree).keys() def __contains__(self, cpv): return cpv in self.keys() def has_key(self, pkg_key): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index a1165907c..feade6fc8 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -25,6 +25,52 @@ from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ import os, stat from itertools import izip +def _src_uri_validate(cpv, eapi, src_uri): + """ + Take a SRC_URI structure as returned by paren_reduce or use_reduce + and validate it. Raises InvalidDependString if a problem is detected, + such as missing operand for a -> operator. + """ + uri = None + operator = None + for x in src_uri: + if isinstance(x, list): + if operator is not None: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "right operand") % (cpv,)) + uri = None + _src_uri_validate(cpv, eapi, x) + continue + if x[:-1] == "?": + if operator is not None: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "right operand") % (cpv,)) + uri = None + continue + if uri is None: + if x == "->": + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "left operand") % (cpv,)) + uri = x + continue + if x == "->": + if eapi in ("0", "1"): + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrows are not " + \ + "supported with EAPI='%s'") % (cpv, eapi)) + operator = x + continue + uri = None + operator = None + + if operator is not None: + raise portage.exception.InvalidDependString( + "getFetchMap(): '%s' SRC_URI arrow missing right operand" % \ + (cpv,)) + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] @@ -450,9 +496,24 @@ class portdbapi(dbapi): return returnme - def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None): - if mysettings is None: - mysettings = self.doebuild_settings + def getFetchMap(self, mypkg, useflags=None, mytree=None): + """ + Get the SRC_URI metadata as a dict which maps each file name to a + set of alternative URIs. + + @param mypkg: cpv for an ebuild + @type mypkg: String + @param useflags: a collection of enabled USE flags, for evaluation of + conditionals + @type useflags: set, or None to enable all conditionals + @param mytree: The canonical path of the tree in which the ebuild + is located, or None for automatic lookup + @type mypkg: String + @returns: A dict which maps each file name to a set of alternative + URIs. + @rtype: dict + """ + try: eapi, myuris = self.aux_get(mypkg, ["EAPI", "SRC_URI"], mytree=mytree) @@ -460,33 +521,80 @@ class portdbapi(dbapi): # Convert this to an InvalidDependString exception since callers # already handle it. raise portage.exception.InvalidDependString( - "getfetchlist(): aux_get() error reading "+mypkg+"; aborting.") + "getFetchMap(): aux_get() error reading "+mypkg+"; aborting.") if not eapi_is_supported(eapi): # Convert this to an InvalidDependString exception # since callers already handle it. raise portage.exception.InvalidDependString( - "getfetchlist(): '%s' has unsupported EAPI: '%s'" % \ + "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ (mypkg, eapi.lstrip("-"))) - if not all and useflags is None: + myuris = paren_reduce(myuris) + _src_uri_validate(mypkg, eapi, myuris) + myuris = use_reduce(myuris, uselist=useflags, + matchall=(useflags is None)) + myuris = flatten(myuris) + + uri_map = {} + + uri = None + operator = None + myuris.reverse() + while myuris: + token = myuris.pop() + if uri is None: + uri = token + if myuris: + continue + if token == "->": + if eapi in ("0", "1"): + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrows are not " + \ + "supported with EAPI='%s'") % (mypkg, eapi)) + + operator = token + continue + if operator is None: + distfile = os.path.basename(uri) + if not distfile: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI has no file " + \ + "name: '%s'") % (mypkg, uri)) + else: + distfile = token + if "/" in distfile: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI '/' character in " + \ + "file name: '%s'") % (mypkg, distfile)) + uri_set = uri_map.get(distfile) + if uri_set is None: + uri_set = set() + uri_map[distfile] = uri_set + uri_set.add(uri) + uri = None + operator = None + + return uri_map + + def getfetchlist(self, mypkg, useflags=None, mysettings=None, + all=0, mytree=None): + + writemsg("!!! pordbapi.getfetchlist() is deprecated, " + \ + "use getFetchMap() instead.\n", noiselevel=-1) + + if all: + useflags = None + elif useflags is None: + if mysettings is None: + mysettings = self.doebuild_settings mysettings.setcpv(mypkg, mydb=self) useflags = mysettings["PORTAGE_USE"].split() - - myurilist = paren_reduce(myuris) - myurilist = use_reduce(myurilist, uselist=useflags, matchall=all) - newuris = flatten(myurilist) - - myfiles = [] - for x in newuris: - mya = os.path.basename(x) - if not mya: - raise portage.exception.InvalidDependString( - "getfetchlist(): '%s' SRC_URI has no file name: '%s'" % \ - (mypkg, x)) - if not mya in myfiles: - myfiles.append(mya) - return [newuris, myfiles] + uri_map = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree) + uris = set() + for uri_set in uri_map.itervalues(): + uris.update(uri_set) + return [list(uris), uri_map.keys()] def getfetchsizes(self, mypkg, useflags=None, debug=0): # returns a filename:size dictionnary of remaining downloads @@ -499,10 +607,7 @@ class portdbapi(dbapi): print "[empty/missing/bad digest]: "+mypkg return None filesdict={} - if useflags is None: - myuris, myfiles = self.getfetchlist(mypkg,all=1) - else: - myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags) + myfiles = self.getFetchMap(mypkg, useflags=useflags) #XXX: maybe this should be improved: take partial downloads # into account? check checksums? for myfile in myfiles: @@ -530,10 +635,12 @@ class portdbapi(dbapi): return filesdict def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False): - if not useflags: + if all: + useflags = None + elif useflags is None: if mysettings: useflags = mysettings["USE"].split() - myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all) + myfiles = self.getFetchMap(mypkg, useflags=useflags) myebuild = self.findname(mypkg) pkgdir = os.path.dirname(myebuild) mf = Manifest(pkgdir, self.mysettings["DISTDIR"]) -- 2.26.2