From ef6a86e8832c8a411f947d80401cd0f06a628d10 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 23 May 2007 17:21:58 +0000 Subject: [PATCH] Merged from trunk 6580:6590 +------------------------------------------------------------------------+ | 6580 | Fix namespace for S_ISUID and S_ISGID. | | zmedico | | |---------+--------------------------------------------------------------| | 6581 | More namespace fixes. | | zmedico | | |---------+--------------------------------------------------------------| | 6582 | Punt the portdbapi.getsize() function since it's unused and | | zmedico | broken (mysum referenced before assignment). | |---------+--------------------------------------------------------------| | 6583 | Fix missing red import. | | zmedico | | |---------+--------------------------------------------------------------| | 6584 | Split reading/writing of $PKGDIR/Packages into a | | zmedico | PackageIndex class. | |---------+--------------------------------------------------------------| | 6585 | Discard metadata from the Packages file if there's no MTIME | | zmedico | attribute. | |---------+--------------------------------------------------------------| | 6586 | Update /Packages as each package is built (MD5 is done here | | zmedico | since it won't hurt performance). | |---------+--------------------------------------------------------------| | 6587 | Lock $PKGDIR/Packages during population so that it won't get | | zmedico | out of sync if another process updates it. | |---------+--------------------------------------------------------------| | 6588 | Store the path of $PKGDIR/Packages as | | zmedico | binarytree._pkgindex_file. | |---------+--------------------------------------------------------------| | 6589 | Lock the Packages index while it's being updated with a | | zmedico | packages that was just built. | |---------+--------------------------------------------------------------| | 6590 | Save memory by not holding a reference to the PackageIndex | | zmedico | when it's not being used. | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=6597 --- pym/emerge/__init__.py | 3 +- pym/portage/dbapi/bintree.py | 135 ++++++++++++++++++++++++---------- pym/portage/dbapi/porttree.py | 13 ---- pym/portage/dbapi/vartree.py | 4 +- pym/portage/getbinpkg.py | 32 ++++++++ 5 files changed, 132 insertions(+), 55 deletions(-) diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 5cdda0cfe..e1cfac2e8 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -3164,8 +3164,7 @@ class MergeTask(object): if retval != os.EX_OK: return retval bintree = self.trees[myroot]["bintree"] - if bintree.populated: - bintree.inject(pkg_key) + bintree.inject(pkg_key) if "--buildpkgonly" not in self.myopts: msg = " === (%s of %s) Merging (%s::%s)" % \ (mergecount, len(mymergelist), pkg_key, y) diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 030f03163..dda35ce65 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -115,6 +115,7 @@ class binarytree(object): self._pkg_paths = {} self._all_directory = os.path.isdir( os.path.join(self.pkgdir, "All")) + self._pkgindex_file = os.path.join(self.pkgdir, "Packages") self._pkgindex_keys = set(["CPV", "SLOT", "MTIME", "SIZE"]) def move_ent(self, mylist): @@ -360,6 +361,18 @@ class binarytree(object): def populate(self, getbinpkgs=0, getbinpkgsonly=0): "populates the binarytree" + from portage.locks import lockfile, unlockfile + pkgindex_lock = None + try: + if not self._all_directory and os.access(self.pkgdir, os.W_OK): + pkgindex_lock = lockfile(self._pkgindex_file, + wantnewlockfile=1) + self._populate(getbinpkgs, getbinpkgsonly) + finally: + if pkgindex_lock: + unlockfile(pkgindex_lock) + + def _populate(self, getbinpkgs, getbinpkgsonly): if (not os.path.isdir(self.pkgdir) and not getbinpkgs): return 0 @@ -372,25 +385,16 @@ class binarytree(object): dirs.remove("All") dirs.sort() dirs.insert(0, "All") - pkgfile = os.path.join(self.pkgdir, "Packages") - metadata = {} - header = {} + pkgindex = portage.getbinpkg.PackageIndex() + header = pkgindex.header + metadata = pkgindex.packages try: - f = open(pkgfile) + f = open(self._pkgindex_file) except EnvironmentError: pass else: try: - header = portage.getbinpkg.readpkgindex(f) - while True: - d = portage.getbinpkg.readpkgindex(f) - if not d: - break - mycpv = d.get("CPV") - if not mycpv: - continue - d.setdefault("SLOT", "0") - metadata[mycpv] = d + pkgindex.read(f) finally: f.close() del f @@ -463,17 +467,11 @@ class binarytree(object): update_pkgindex = True d = metadata.get(mycpv, {}) if d: - # Reuse metadata such as MD5, since we won't calculate - # MD5 here due to the performance hit. - mtime = d.get("MTIME") - if mtime: - # genpgkindex really should include the mtime and - # then this mtime check should be forced. - try: - if long(mtime) != long(s.st_mtime): - d.clear() - except ValueError: + try: + if long(d["MTIME"]) != long(s.st_mtime): d.clear() + except (KeyError, ValueError): + d.clear() if d: try: if long(d["SIZE"]) != long(s.st_size): @@ -493,29 +491,24 @@ class binarytree(object): self.dbapi._aux_cache[mycpv] = aux_cache self._pkg_paths = pkg_paths - if update_pkgindex and os.access(self.pkgdir, os.W_OK): + # Do not bother to write the Packages index if $PKGDIR/All/ exists + # since it will provide no benefit due to the need to read CATEGORY + # from xpak. + if update_pkgindex and not self._all_directory and \ + os.access(self.pkgdir, os.W_OK): cpv_all = self._pkg_paths.keys() stale = set(metadata).difference(cpv_all) for cpv in stale: del metadata[cpv] - cpv_all.sort() - import time from portage.util import atomic_ofstream - header["TIMESTAMP"] = str(long(time.time())) - header["PACKAGES"] = str(len(cpv_all)) - f = atomic_ofstream(pkgfile) + f = atomic_ofstream(self._pkgindex_file) try: - portage.getbinpkg.writepkgindex(f, header.iteritems()) - for cpv in cpv_all: - d = metadata[cpv] - if d["SLOT"] == "0": - del d["SLOT"] - portage.getbinpkg.writepkgindex(f, d.iteritems()) + pkgindex.write(f) finally: f.close() if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: - writemsg(red("!!! PORTAGE_BINHOST unset, but use is requested.\n"), + writemsg("!!! PORTAGE_BINHOST unset, but use is requested.\n", noiselevel=-1) if getbinpkgs and \ @@ -565,7 +558,73 @@ class binarytree(object): self.populated=1 def inject(self, cpv): - return self.dbapi.cpv_inject(cpv) + """Add a freshly built package to the database. This updates + $PKGDIR/Packages with the new package metadata (including MD5).""" + if not self.populated and self._all_directory: + # There's nothing to update in this case, since the Packages + # index is not created when $PKGDIR/All/ exists. + return + if not self.populated: + self.populate() + full_path = self.getname(cpv) + try: + s = os.stat(full_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + writemsg("!!! Binary package does not exist: '%s'\n" % full_path, + noiselevel=-1) + return + mytbz2 = portage.xpak.tbz2(full_path) + slot = mytbz2.getfile("SLOT") + if slot is None: + writemsg("!!! Invalid binary package: '%s'\n" % full_path, + noiselevel=-1) + return + slot = slot.strip() + from portage.checksum import perform_md5 + md5 = perform_md5(full_path) + self.dbapi.cpv_inject(cpv) + self.dbapi._aux_cache.pop(cpv, None) + + if self._all_directory: + return + + # Reread the Packages index (in case it's been changed by another + # process) and then updated it, all while holding a lock. + from portage.locks import lockfile, unlockfile + pkgindex_lock = None + try: + pkgindex_lock = lockfile(self._pkgindex_file, + wantnewlockfile=1) + pkgindex = portage.getbinpkg.PackageIndex() + try: + f = open(self._pkgindex_file) + except EnvironmentError: + pass + else: + try: + pkgindex.read(f) + finally: + f.close() + del f + d = {} + d["CPV"] = cpv + d["SLOT"] = slot + d["MTIME"] = str(long(s.st_mtime)) + d["SIZE"] = str(s.st_size) + d["MD5"] = str(md5) + pkgindex.packages[cpv] = d + from portage.util import atomic_ofstream + f = atomic_ofstream(os.path.join(self.pkgdir, "Packages")) + try: + pkgindex.write(f) + finally: + f.close() + finally: + if pkgindex_lock: + unlockfile(pkgindex_lock) def exists_specific(self, cpv): if not self.populated: diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 16e962a85..77aa39e7a 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -441,19 +441,6 @@ class portdbapi(dbapi): return False return True - def getsize(self, mypkg, useflags=None, debug=0): - # returns the total size of remaining downloads - # - # we use getfetchsizes() now, so this function would be obsoleted - # - filesdict = self.getfetchsizes(mypkg, useflags=useflags, debug=debug) - if filesdict is None: - return "[empty/missing/bad digest]" - mysize=0 - for myfile in filesdict.keys(): - mysum += filesdict[myfile] - return mysum - def cpv_exists(self, mykey): "Tells us whether an actual ebuild exists on disk (no masking)" cps2 = mykey.split("/") diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 6183785f3..81e767a9c 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -9,7 +9,7 @@ from portage.elog import elog_process from portage.exception import InvalidPackageName, InvalidAtom, \ UnsupportedAPIException, FileNotFound from portage.locks import lockdir, unlockdir -from portage.output import red, green +from portage.output import bold, red, green from portage.update import fixdbentries from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \ writemsg, writemsg_stdout, write_atomic, atomic_ofstream, writedict, \ @@ -1215,7 +1215,7 @@ class dblink(object): # Always blind chmod 0 before unlinking to avoid race conditions. os.chmod(obj, 0000) if statobj.st_nlink > 1: - portage.writemsg("setXid: "+str(st.st_nlink-1)+ \ + writemsg("setXid: "+str(statobj.st_nlink-1)+ \ " hardlinks to '%s'\n" % obj) os.unlink(obj) except (OSError, IOError), e: diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 3494aaef9..3364db8bb 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -591,3 +591,35 @@ def writepkgindex(pkgfile, items): for k, v in items: pkgfile.write("%s: %s\n" % (k, v)) pkgfile.write("\n") + +class PackageIndex(object): + + def __init__(self): + self.header = {} + self.packages = {} + + def read(self, pkgfile): + self.header.update(readpkgindex(pkgfile)) + while True: + d = readpkgindex(pkgfile) + if not d: + break + mycpv = d.get("CPV") + if not mycpv: + continue + d.setdefault("SLOT", "0") + self.packages[mycpv] = d + + def write(self, pkgfile): + cpv_all = self.packages.keys() + cpv_all.sort() + import time + self.header["TIMESTAMP"] = str(long(time.time())) + self.header["PACKAGES"] = str(len(cpv_all)) + writepkgindex(pkgfile, self.header.iteritems()) + for cpv in cpv_all: + metadata = self.packages[cpv] + if metadata["SLOT"] == "0": + metadata = metadata.copy() + del metadata["SLOT"] + writepkgindex(pkgfile, metadata.iteritems()) -- 2.26.2