From 76e10cec7621e18965c1d810e6fb509297200f1e Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 1 Jun 2007 18:15:35 +0000 Subject: [PATCH] Merged from trunk 6696:6709 +------------------------------------------------------------------------+ | 6697 | Replace deprecated binarytree.remotepkgs usage with | | zmedico | binarytree._remotepkgs since the former is vulnerable to | | | collisions in the tbz2 filename. | |---------+--------------------------------------------------------------| | 6698 | Make dep_zapdeps() pull SLOT from the correct dbapi instance | | zmedico | when it falls back to the vdb. | |---------+--------------------------------------------------------------| | 6699 | Raise a KeyError from bindbapi.aux_get() when appropriate. | | zmedico | | |---------+--------------------------------------------------------------| | 6704 | Fix typo. | | zmedico | | |---------+--------------------------------------------------------------| | 6705 | Add some additional variables to the Packages header. | | zmedico | | |---------+--------------------------------------------------------------| | 6706 | For bug #149397, raise an InvalidDependString exception if a | | zmedico | USE has no basename. | |---------+--------------------------------------------------------------| | 6707 | Don't repeatedly invoke the depend phase for ebuilds that | | zmedico | are already known to be broken. | |---------+--------------------------------------------------------------| | 6708 | Instead of calling sys.exit() when aux_get() fails inside | | zmedico | getfetchlist(), convert the KeyError to an | | | InvalidDependString exception since callers already handle | | | it. | |---------+--------------------------------------------------------------| | 6709 | Add --getbinpkg digest verification. | | zmedico | | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=6710 --- pym/emerge/__init__.py | 29 ++++++++++++-- pym/portage/__init__.py | 11 ++++-- pym/portage/dbapi/bintree.py | 74 ++++++++++++++++++++++++++--------- pym/portage/dbapi/porttree.py | 12 +++++- 4 files changed, 99 insertions(+), 27 deletions(-) diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 623bf48ca..7ce99f303 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -50,6 +50,7 @@ import portage.exception from portage.const import EPREFIX from portage.data import secpass from portage.util import normalize_path as normpath +from portage.util import writemsg if not hasattr(__builtins__, "set"): from sets import Set as set @@ -2940,6 +2941,8 @@ class MergeTask(object): def merge(self, mylist, favorites, mtimedb): failed_fetches = [] + fetchonly = "--fetchonly" in self.myopts or \ + "--fetch-all-uri" in self.myopts mymergelist=[] ldpath_mtimes = mtimedb["ldpath"] xterm_titles = "notitles" not in self.settings.features @@ -3240,9 +3243,29 @@ class MergeTask(object): short_msg = "emerge: (%s of %s) %s Fetch" % \ (mergecount, len(mymergelist), pkg_key) emergelog(xterm_titles, msg, short_msg=short_msg) - if not self.trees[myroot]["bintree"].gettbz2( - pkg_key): - return 1 + try: + self.trees[myroot]["bintree"].gettbz2(pkg_key) + except portage.exception.FileNotFound: + writemsg("!!! Fetching Binary failed " + \ + "for '%s'\n" % pkg_key, noiselevel=-1) + if not fetchonly: + return 1 + failed_fetches.append(pkg_key) + except portage.exception.DigestException, e: + writemsg("\n!!! Digest verification failed:\n", + noiselevel=-1) + writemsg("!!! %s\n" % e.value[0], + noiselevel=-1) + writemsg("!!! Reason: %s\n" % e.value[1], + noiselevel=-1) + writemsg("!!! Got: %s\n" % e.value[2], + noiselevel=-1) + writemsg("!!! Expected: %s\n" % e.value[3], + noiselevel=-1) + os.unlink(mytbz2) + if not fetchonly: + return 1 + failed_fetches.append(pkg_key) finally: if tbz2_lock: portage.locks.unlockfile(tbz2_lock) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 67a3a709e..c6173f057 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4307,16 +4307,21 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): versions = {} for atom in atoms: avail_pkg = best(mydbapi.match(atom)) - if not avail_pkg and use_binaries: + if avail_pkg: + avail_slot = "%s:%s" % (dep_getkey(atom), + mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) + elif not avail_pkg and use_binaries: # With --usepkgonly, count installed packages as "available". # Note that --usepkgonly currently has no package.mask support. # See bug #149816. avail_pkg = best(vardb.match(atom)) + if avail_pkg: + avail_slot = "%s:%s" % (dep_getkey(atom), + vardb.aux_get(avail_pkg, ["SLOT"])[0]) if not avail_pkg: all_available = False break - avail_slot = "%s:%s" % (dep_getkey(atom), - mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) + versions[avail_slot] = avail_pkg this_choice = (atoms, versions, all_available) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 51d565489..c6cc084c9 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -43,11 +43,14 @@ class bindbapi(fakedbapi): mysplit = mycpv.split("/") mylist = [] tbz2name = mysplit[1]+".tbz2" - if self.bintree and not self.bintree.isremote(mycpv): - tbz2 = portage.xpak.tbz2(self.bintree.getname(mycpv)) - getitem = tbz2.getfile + if not self.bintree._remotepkgs or \ + not self.bintree.isremote(mycpv): + tbz2_path = self.bintree.getname(mycpv) + if not os.path.exists(tbz2_path): + raise KeyError(mycpv) + getitem = portage.xpak.tbz2(tbz2_path).getfile else: - getitem = self.bintree.remotepkgs[tbz2name].get + getitem = self.bintree._remotepkgs[mycpv].get mydata = {} mykeys = wants if cache_me: @@ -132,6 +135,9 @@ class binarytree(object): os.path.join(self.pkgdir, "All")) self._pkgindex_file = os.path.join(self.pkgdir, "Packages") self._pkgindex_keys = set(["CPV", "SLOT", "MTIME", "SIZE"]) + self._pkgindex_header_keys = set(["ACCEPT_KEYWORDS", "CBUILD", + "CHOST", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES", + "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"]) def move_ent(self, mylist): if not self.populated: @@ -535,12 +541,8 @@ class binarytree(object): self._remotepkgs = pkgindex.packages self._remote_has_index = True self.remotepkgs = {} - for cpv, metadata in self._remotepkgs.iteritems(): + for cpv in self._remotepkgs: self.dbapi.cpv_inject(cpv) - cat, pf = catsplit(cpv) - # backward compat - self.remotepkgs[pf+".tbz2"] = metadata - metadata["CATEGORY"] = cat self.populated = 1 return self._remotepkgs = {} @@ -683,6 +685,7 @@ class binarytree(object): else: del d[k] pkgindex.packages[cpv] = d + self._update_pkgindex_header(pkgindex.header) from portage.util import atomic_ofstream f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) try: @@ -693,6 +696,19 @@ class binarytree(object): if pkgindex_lock: unlockfile(pkgindex_lock) + def _update_pkgindex_header(self, header): + portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"])) + profiles_base = os.path.join(portdir, "profiles") + profile_path = normalize_path(os.path.realpath(self.settings.profile_path)) + profile_path = profile_path.lstrip(profiles_base) + header["PROFILE"] = profile_path + for k in self._pkgindex_header_keys: + v = self.settings.get(k, None) + if v: + header[k] = v + else: + header.pop(k, None) + def exists_specific(self, cpv): if not self.populated: self.populate() @@ -739,16 +755,14 @@ class binarytree(object): def isremote(self, pkgname): "Returns true if the package is kept remotely." - mysplit = pkgname.split("/") - remote = (not os.path.exists(self.getname(pkgname))) and self.remotepkgs.has_key(mysplit[1]+".tbz2") + remote = pkgname in self._remotepkgs and \ + not os.path.exists(self.getname(pkgname)) return remote def get_use(self, pkgname): - mysplit=pkgname.split("/") - if self.isremote(pkgname): - return self.remotepkgs[mysplit[1]+".tbz2"]["USE"][:].split() - tbz2=portage.xpak.tbz2(self.getname(pkgname)) - return tbz2.getfile("USE").split() + writemsg("deprecated use of binarytree.get_use()," + \ + " use dbapi.aux_get() instead", noiselevel=-1) + return self.dbapi.aux_get(pkgname, ["USE"])[0].split() def gettbz2(self, pkgname): "fetches the package from a remote site, if necessary." @@ -761,6 +775,7 @@ class binarytree(object): else: writemsg("Resuming download of this tbz2, but it is possible that it is corrupt.\n", noiselevel=-1) + tbz2_path = self.getname(pkgname) mydest = os.path.dirname(self.getname(pkgname)) try: os.makedirs(mydest, 0775) @@ -783,9 +798,30 @@ class binarytree(object): else: url = urljoin(base_url, tbz2name) success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd) - if success: - self.inject(pkgname) - return success + if success and "strict" in self.settings.features: + metadata = self._remotepkgs[pkgname] + digests = {} + if "MD5" in metadata: + digests["MD5"] = self._remotepkgs[pkgname]["MD5"] + if "SIZE" in metadata: + try: + digests["size"] = long(self._remotepkgs[pkgname]["SIZE"]) + except ValueError: + writemsg("!!! Malformed SIZE attribute in remote " + \ + "metadata for '%s'\n" % pkgname) + if digests: + from portage.checksum import verify_all + ok, reason = verify_all(tbz2_path, digests) + if not ok: + raise portage.exception.DigestException( + tuple([tbz2_path]+list(reason))) + if not success: + try: + os.unlink(self.getname(pkgname)) + except OSError: + pass + raise portage.exception.FileNotFound(mydest) + self.inject(pkgname) def getslot(self, mycatpkg): "Get a slot for a catpkg; assume it exists." diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index c99b0193c..6130dd49d 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -111,6 +111,7 @@ class portdbapi(dbapi): # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set(["EAPI", "KEYWORDS", "LICENSE", "SLOT"]) self._aux_cache = {} + self._broken_ebuilds = set() def _init_cache_dirs(self): """Create /var/cache/edb/dep and adjust permissions for the portage @@ -301,6 +302,8 @@ class portdbapi(dbapi): writemsg("auxdb is valid: "+str(not doregen)+" "+str(pkg)+"\n", 2) if doregen: + if myebuild in self._broken_ebuilds: + raise KeyError(mycpv) writemsg("doregen: %s %s\n" % (doregen, mycpv), 2) writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n", 1) @@ -310,6 +313,7 @@ class portdbapi(dbapi): self.doebuild_settings["ROOT"], self.doebuild_settings, dbkey=mydata, tree="porttree", mydbapi=self) if myret != os.EX_OK: + self._broken_ebuilds.add(myebuild) raise KeyError(mycpv) if "EAPI" not in mydata or not mydata["EAPI"].strip(): @@ -358,8 +362,10 @@ class portdbapi(dbapi): try: myuris = self.aux_get(mypkg, ["SRC_URI"], mytree=mytree)[0] except KeyError: - print red("getfetchlist():")+" aux_get() error reading "+mypkg+"; aborting." - sys.exit(1) + # Convert this to an InvalidDependString exception since callers + # already handle it. + raise portage.exception.InvalidDependString( + "getfetchlist(): aux_get() error reading "+mypkg+"; aborting.") if useflags is None: useflags = mysettings["USE"].split() @@ -371,6 +377,8 @@ class portdbapi(dbapi): myfiles = [] for x in newuris: mya = os.path.basename(x) + if not mya: + raise portage.exception.InvalidDependString("URI has no basename: '%s'" % x) if not mya in myfiles: myfiles.append(mya) return [newuris, myfiles] -- 2.26.2