From: Fabian Groffen Date: Wed, 23 May 2007 16:35:27 +0000 (-0000) Subject: Merged from trunk 6560:6570 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=89fac57420e799df132dfdb12d4848467766586a;p=portage.git Merged from trunk 6560:6570 +------------------------------------------------------------------------+ | 6560 | For bug #179206, improve efficiency by using a containment | | zmedico | test instead of str.count inside update_dbentry. Thanks to | | | Jason Lai . | |---------+--------------------------------------------------------------| | 6561 | In fetch(), don't try to parse digests unless the "O" | | zmedico | variable is defined. | |---------+--------------------------------------------------------------| | 6562 | For bug #179008, just use metadata.dtd directly from | | zmedico | $DISTDIR. | |---------+--------------------------------------------------------------| | 6563 | - blind chmod 0 before unlinking files to avoid anybody | | solar | doing sneaky stuff via hardlinks | |---------+--------------------------------------------------------------| | 6564 | Add a missing newline to the writemsg output. | | zmedico | | |---------+--------------------------------------------------------------| | 6565 | For bug #179380, make support for ECONF_SOURCE in the | | zmedico | default src_compile() conditional on EAPI != 0. | |---------+--------------------------------------------------------------| | 6566 | Use bash to spawn FETCHCOMMAND under selinux since most | | zmedico | other binaries are forbidden as entrypoints into the fetch | | | domain. Thanks to Justin Heesemann for | | | reporting. | |---------+--------------------------------------------------------------| | 6567 | Fix src_compile() ECONF_SOURCE logic to make sure it behaves | | zmedico | properly. | |---------+--------------------------------------------------------------| | 6568 | When ${PKGDIR}/All/ doesn't exist, put packages in | | zmedico | ${PKGDIR}/${CATEGORY}/ instead. This new layout is backward | | | compatible with portage-2.1.2 and it will be a requirement | | | for new PORTAGE_BINHOST support that is comming. | |---------+--------------------------------------------------------------| | 6569 | Cache xpak metadata in ${PKGDIR}/Packages in order to | | zmedico | minimize disk load when binarytree.populate() is called. | | | This does not do all the keys that genpkgindex does but it | | | will copy metadata from the existing Packages file if it | | | appears valid. The binarytree.populate() code is performance | | | critical, so the MD5 will certainly have to be done | | | elsewhere (like just after a package is built). | |---------+--------------------------------------------------------------| | 6570 | Fix size comparison logic. | | zmedico | | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=6595 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 5d8709087..599d9bf3d 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -578,7 +578,9 @@ src_unpack() { } src_compile() { - if [ -x "${ECONF_SOURCE:-.}/configure" ] ; then + if [ "${EAPI:-0}" == 0 ] ; then + [ -x ./configure ] && econf + elif [ -x "${ECONF_SOURCE:-.}/configure" ] ; then econf fi if [ -f Makefile ] || [ -f GNUmakefile ] || [ -f makefile ]; then diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 5ae72c274..84ee89360 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -549,13 +549,18 @@ preinst_selinux_labels() { dyn_package() { cd "${PORTAGE_BUILDDIR}/image" install_mask "${PORTAGE_BUILDDIR}/image" ${PKG_INSTALL_MASK} - local pkg_dest="${PKGDIR}/All/${PF}.tbz2" - local pkg_tmp="${PKGDIR}/All/${PF}.tbz2.$$" + if [ -d "${PKGDIR}/All" ] ; then + local pkg_dest="${PKGDIR}/All/${PF}.tbz2" + else + local pkg_dest="${PKGDIR}/${CATEGORY}/${PF}.tbz2" + fi + local pkg_tmp="${pkg_dest}.$$" local tar_options="" [ "${PORTAGE_QUIET}" == "1" ] || tar_options="${tar_options} -v" # Sandbox is disabled in case the user wants to use a symlink # for $PKGDIR and/or $PKGDIR/All. export SANDBOX_ON="0" + mkdir -p "${pkg_tmp%/*}" || die "mkdir failed" tar ${tar_options} -cf - . | bzip2 -f > "${pkg_tmp}" || \ die "Failed to create tarball" cd .. @@ -566,7 +571,10 @@ dyn_package() { die "Failed to append metadata to the tbz2 file" fi mv -f "${pkg_tmp}" "${pkg_dest}" || die "Failed to move tbz2 to ${pkg_dest}" - ln -sf "../All/${PF}.tbz2" "${PKGDIR}/${CATEGORY}/${PF}.tbz2" || die "Failed to create symlink in ${PKGDIR}/${CATEGORY}" + if [ -d "${PKGDIR}/All" ] ; then + ln -sf "../All/${PF}.tbz2" "${PKGDIR}/${CATEGORY}/${PF}.tbz2" || \ + die "Failed to create symlink in ${PKGDIR}/${CATEGORY}" + fi vecho ">>> Done." cd "${PORTAGE_BUILDDIR}" touch .packaged || die "Failed to 'touch .packaged' in ${PORTAGE_BUILDDIR}" diff --git a/bin/repoman b/bin/repoman index c4ac3902f..42d539d4c 100755 --- a/bin/repoman +++ b/bin/repoman @@ -735,6 +735,7 @@ for x in qacats: stats[x]=0 fails[x]=[] xmllint_capable = False +metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd') if getstatusoutput('which xmllint')[0] != 0: print red("!!! xmllint not found. Can't check metadata.xml.\n") if "--xmlparse" in myoptions or repolevel==3: @@ -747,7 +748,7 @@ else: try: # if it's been over a week since fetching (or the system clock is fscked), grab an updated copy of metadata.dtd # clock is fscked or it's been a week. time to grab a new one. - ct=os.stat(portage.CACHE_PATH + '/metadata.dtd')[ST_CTIME] + ct = os.stat(metadata_dtd)[ST_CTIME] if abs(time.time() - ct) > (60*60*24*7): # don't trap the exception, we're watching for errno 2 (file not found), anything else is a bug. backup_exists=1 @@ -763,18 +764,16 @@ else: print print green("***")+" the local copy of metadata.dtd needs to be refetched, doing that now" print + val = 0 try: - if os.path.exists(repoman_settings["DISTDIR"]+'/metadata.dtd'): - os.remove(repoman_settings["DISTDIR"]+'/metadata.dtd') + try: + os.unlink(metadata_dtd) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e val=portage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoman_settings,fetchonly=0, \ try_mirrors=0) - if val: - if backup_exists: - os.remove(portage.CACHE_PATH+'/metadata.dtd') - shutil.copy(repoman_settings["DISTDIR"]+'/metadata.dtd',portage.CACHE_PATH+'/metadata.dtd') - os.chown(portage.CACHE_PATH+'/metadata.dtd',os.getuid(),portage.portage_gid) - os.chmod(portage.CACHE_PATH+'/metadata.dtd',0664) - except SystemExit, e: raise # Need to propogate this @@ -1097,10 +1096,10 @@ for x in scanlist: else: #Only carry out if in package directory or check forced if xmllint_capable: - st=getstatusoutput("xmllint --nonet --noout --dtdvalid %s/metadata.dtd %s/metadata.xml" % (portage.CACHE_PATH, checkdir)) - if st[0] != 0: - for z in st[1].split("\n"): - print red("!!! ")+z + st = spawn(["xmllint", "--nonet", "--noout", "--nowarning", "--dtdvalid", + metadata_dtd, os.path.join(checkdir, "metadata.xml")]) + if st != os.EX_OK: + print red("!!! metadata.xml is invalid") stats["metadata.bad"]+=1 fails["metadata.bad"].append(x+"/metadata.xml") diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index fd423dfc8..5eeb0da71 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2414,8 +2414,13 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if try_mirrors: mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x] - mydigests = Manifest( - mysettings["O"], mysettings["DISTDIR"]).getTypeDigests("DIST") + pkgdir = mysettings.get("O") + if pkgdir: + mydigests = Manifest( + pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST") + else: + # no digests because fetch was not called for a specific package + mydigests = {} fsmirrors = [] for x in range(len(mymirrors)-1,-1,-1): @@ -2691,6 +2696,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", con = selinux.getcontext() con = con.replace(mysettings["PORTAGE_T"], mysettings["PORTAGE_FETCH_T"]) selinux.setexec(con) + # bash is an allowed entrypoint, while most binaries are not + myfetch = ["bash", "-c", "exec \"$@\"", myfetch[0]] + myfetch myret = portage.process.spawn(myfetch, env=mysettings.environ(), **spawn_keywords) @@ -3821,11 +3828,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, actionmap[x]["dep"] = ' '.join(actionmap_deps[x]) if mydo in actionmap.keys(): - if mydo=="package": - portage.util.ensure_dirs( - os.path.join(mysettings["PKGDIR"], mysettings["CATEGORY"])) - portage.util.ensure_dirs( - os.path.join(mysettings["PKGDIR"], "All")) retval = spawnebuild(mydo, actionmap, mysettings, debug, logfile=logfile) elif mydo=="qmerge": diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 4a0c70e18..9d9b84005 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -113,6 +113,9 @@ class binarytree(object): self.invalids = [] self.settings = settings self._pkg_paths = {} + self._all_directory = os.path.isdir( + os.path.join(self.pkgdir, "All")) + self._pkgindex_keys = set(["CPV", "SLOT", "MTIME", "SIZE"]) def move_ent(self, mylist): if not self.populated: @@ -285,6 +288,8 @@ class binarytree(object): use for a given cpv. If a collision will occur with an existing package from another category, the existing package will be bumped to ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist.""" + if not self._all_directory: + return if not self.populated: # Try to avoid the population routine when possible, so that # FEATURES=buildpkg doesn't always force population. @@ -357,8 +362,6 @@ class binarytree(object): "populates the binarytree" if (not os.path.isdir(self.pkgdir) and not getbinpkgs): return 0 - if (not os.path.isdir(self.pkgdir+"/All") and not getbinpkgs): - return 0 categories = set(self.settings.categories) @@ -369,20 +372,66 @@ class binarytree(object): dirs.remove("All") dirs.sort() dirs.insert(0, "All") + pkgfile = os.path.join(self.pkgdir, "Packages") + metadata = {} + header = {} + try: + f = open(pkgfile) + except EnvironmentError: + pass + else: + try: + header = portage.getbinpkg.readpkgindex(f) + while True: + d = portage.getbinpkg.readpkgindex(f) + if not d: + break + mycpv = d.get("CPV") + if not mycpv: + continue + d.setdefault("SLOT", "0") + metadata[mycpv] = d + finally: + f.close() + del f + update_pkgindex = False for mydir in dirs: for myfile in listdir(os.path.join(self.pkgdir, mydir)): if not myfile.endswith(".tbz2"): continue mypath = os.path.join(mydir, myfile) full_path = os.path.join(self.pkgdir, mypath) - if os.path.islink(full_path): + s = os.lstat(full_path) + if stat.S_ISLNK(s.st_mode): continue + if mydir != "All": + # Validate data from the package index and try to avoid + # reading the xpak if possible. + mycpv = mydir + "/" + myfile[:-5] + d = metadata.get(mycpv) + skip = False + if d: + try: + if long(d.get("MTIME")) == long(s.st_mtime): + skip = True + except ValueError: + pass + if skip and not self._pkgindex_keys.difference(d): + pkg_paths[mycpv] = mypath + self.dbapi.cpv_inject(mycpv) + if not self.dbapi._aux_cache_keys.difference(d): + aux_cache = {} + for k in self.dbapi._aux_cache_keys: + aux_cache[k] = d[k] + self.dbapi._aux_cache[mycpv] = aux_cache + continue mytbz2 = portage.xpak.tbz2(full_path) # For invalid packages, mycat could be None. mycat = mytbz2.getfile("CATEGORY") mypf = mytbz2.getfile("PF") + slot = mytbz2.getfile("SLOT") mypkg = myfile[:-5] - if not mycat or not mypf: + if not mycat or not mypf or not slot: #old-style or corrupt package writemsg("!!! Invalid binary package: '%s'\n" % full_path, noiselevel=-1) @@ -392,6 +441,7 @@ class binarytree(object): self.invalids.append(mypkg) continue mycat = mycat.strip() + slot = slot.strip() if mycat != mydir and mydir != "All": continue if mypkg != mypf.strip(): @@ -410,7 +460,59 @@ class binarytree(object): continue pkg_paths[mycpv] = mypath self.dbapi.cpv_inject(mycpv) + update_pkgindex = True + d = metadata.get(mycpv, {}) + if d: + # Reuse metadata such as MD5, since we won't calculate + # MD5 here due to the performance hit. + mtime = d.get("MTIME") + if mtime: + # genpgkindex really should include the mtime and + # then this mtime check should be forced. + try: + if long(mtime) != long(s.st_mtime): + d.clear() + except ValueError: + d.clear() + if d: + try: + if long(d["SIZE"]) != long(s.st_size): + d.clear() + except (KeyError, ValueError): + d.clear() + + d["CPV"] = mycpv + d["SLOT"] = slot + d["MTIME"] = str(long(s.st_mtime)) + d["SIZE"] = str(s.st_size) + metadata[mycpv] = d + if not self.dbapi._aux_cache_keys.difference(d): + aux_cache = {} + for k in self.dbapi._aux_cache_keys: + aux_cache[k] = d[k] + self.dbapi._aux_cache[mycpv] = aux_cache + self._pkg_paths = pkg_paths + if update_pkgindex and os.access(self.pkgdir, os.W_OK): + cpv_all = self._pkg_paths.keys() + stale = set(metadata).difference(cpv_all) + for cpv in stale: + del metadata[cpv] + cpv_all.sort() + import time + from portage.util import atomic_ofstream + header["TIMESTAMP"] = str(long(time.time())) + header["PACKAGES"] = str(len(cpv_all)) + f = atomic_ofstream(pkgfile) + try: + portage.getbinpkg.writepkgindex(f, header.iteritems()) + for cpv in cpv_all: + d = metadata[cpv] + if d["SLOT"] == "0": + del d["SLOT"] + portage.getbinpkg.writepkgindex(f, d.iteritems()) + finally: + f.close() if getbinpkgs and not self.settings["PORTAGE_BINHOST"]: writemsg(red("!!! PORTAGE_BINHOST unset, but use is requested.\n"), @@ -500,8 +602,11 @@ class binarytree(object): if mypath: return os.path.join(self.pkgdir, mypath) mycat, mypkg = catsplit(mycpv) - mypath = os.path.join("All", mypkg + ".tbz2") - if mypath in self._pkg_paths.values(): + if self._all_directory: + mypath = os.path.join("All", mypkg + ".tbz2") + if mypath in self._pkg_paths.values(): + mypath = os.path.join(mycat, mypkg + ".tbz2") + else: mypath = os.path.join(mycat, mypkg + ".tbz2") self._pkg_paths[mycpv] = mypath # cache for future lookups return os.path.join(self.pkgdir, mypath) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index be9577c48..640235801 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1211,6 +1211,11 @@ class dblink(object): writemsg_stdout("--- !md5 %s %s\n" % ("obj", obj)) continue try: + if statobj.st_nlink > 1: + portage.writemsg("There are "+str(st.st_nlink-1)+ \ + " hardlinks to '%s'\n" % obj) + # Always blind chmod 0 before unlinking to avoid race conditions. + os.chmod(obj, 0000) os.unlink(obj) except (OSError, IOError), e: pass diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 73a0e7958..3494aaef9 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -571,3 +571,23 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= conn.close() return metadata[baseurl]["data"] + +def readpkgindex(pkgfile): + d = {} + for line in pkgfile: + line = line.rstrip("\n") + if not line: + break + line = line.split(":", 1) + if not len(line) == 2: + continue + k, v = line + if v: + v = v[1:] + d[k] = v + return d + +def writepkgindex(pkgfile, items): + for k, v in items: + pkgfile.write("%s: %s\n" % (k, v)) + pkgfile.write("\n")