From: solar <solar@gentoo.org>
Date: Mon, 5 Feb 2007 01:58:18 +0000 (-0000)
Subject: - add caching backend for tarballs,xpak,checksums. Consecutive runs on 901 pkgs went... 
X-Git-Tag: gentoolkit-0.2.4.3~169
X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=06e6d6ba97d67a9f3fc676fb73602ab549ebdb15;p=gentoolkit.git

- add caching backend for tarballs,xpak,checksums. Consecutive runs on 901 pkgs went from 60s down to 1.5s. initial run went up by 9s. the cache should detect stale .xpak files to and clean up after itself to minimize disk usage

svn path=/; revision=344
---

diff --git a/trunk/src/genpkgindex/genpkgindex b/trunk/src/genpkgindex/genpkgindex
index d984280..dc0b439 100644
--- a/trunk/src/genpkgindex/genpkgindex
+++ b/trunk/src/genpkgindex/genpkgindex
@@ -13,6 +13,7 @@ import xpak, portage, portage_checksum, portage_dep, portage_util
 
 argc=len(sys.argv)
 
+
 if argc >= 2:
 	if (sys.argv[1][0] == "-"):
 		print "Usage:\tgenpkgindex <pkgdir>"
@@ -56,14 +57,43 @@ packages = []
 for pkg in os.listdir('.'):
         if not os.path.basename(pkg).endswith("tbz2"):
                 continue
-        tbz2 = xpak.tbz2(pkg)
-        stuff = tbz2.getboth()
-        if not stuff:
-		print "Not a tbz2: "+str(pkg)
-                continue
-        cat = xpak.getitem(stuff, "CATEGORY")
-        if cat:
-                packages.append((cat, pkg, tbz2, stuff))
+
+	st = os.stat(pkg)
+
+	if not os.path.exists(portage.settings["PORTAGE_TMPDIR"]+"/portage/.xpak/"):
+		os.mkdir(portage.settings["PORTAGE_TMPDIR"]+"/portage/.xpak/")
+
+	fname = portage.settings["PORTAGE_TMPDIR"]+"/portage/.xpak/"+os.path.basename(pkg)[:-5]+".xpak"
+
+	if os.path.exists(fname):
+		if st.st_mtime != os.stat(fname).st_mtime:
+			#print "unlinking "+fname
+			os.unlink(fname)
+
+	if not os.path.exists(fname):
+
+	        tbz2 = xpak.tbz2(pkg)
+		xpdata = xpak.xpak_mem(tbz2.get_data())
+		fp = open(fname, "w")
+		fp.write(xpdata+xpak.encodeint(len(xpdata))+"STOP")
+		fp.close()
+
+		chksum = portage_checksum.perform_md5(pkg)
+		fp = open(fname[:-5]+".md5", "w")
+		fp.write(chksum)
+		fp.close()
+
+		os.utime(fname, (st.st_mtime, st.st_mtime))
+
+	else:
+		if os.path.exists(fname[:-5]+".md5"):
+			chksum = "".join(portage.grabfile(fname[:-5]+".md5"))
+		else:
+			chksum = portage_checksum.perform_md5(pkg)
+
+		tbz2 = xpak.tbz2(fname)
+
+	packages.append((pkg, tbz2, chksum, st))
 
 packages.sort()
 
@@ -122,7 +152,14 @@ def serialize_depset(src, context='and'):
 					l.append("( %s )" % v.strip())
 	return ' '.join(l)
 
-for cat, pkg, tbz2, stuff in packages:
+for pkg, tbz2, chksum, st in packages:
+	stuff = tbz2.getboth()
+	if not stuff:
+		print "Not a tbz2: "+str(pkg)
+		continue
+
+	cat = xpak.getitem(stuff, "CATEGORY")
+
 	use = xpak.getitem(stuff, "USE")
 	if use is None:
 		use = ''
@@ -160,9 +197,8 @@ for cat, pkg, tbz2, stuff in packages:
 		l.sort()
 		fp.write("USE: %s\n" % ' '.join(l))
 
-	st = os.stat(pkg)
 	fp.write("SIZE: "+ str(st[stat.ST_SIZE]) +"\n")
-	fp.write("MD5: "+portage_checksum.perform_md5(pkg)+"\n")
+	fp.write("MD5: "+chksum+"\n")
 	fp.write("\n")
 
 fp.write("\n")
@@ -170,6 +206,16 @@ fp.flush()
 fp.close()
 os.rename(".Packages", "Packages")
 
+os.chdir(portage.settings["PORTAGE_TMPDIR"]+"/portage/.xpak")
+for pkg in os.listdir('.'):
+	p = os.path.basename(pkg)
+	if not p.endswith(".xpak"):
+		continue
+	if not os.path.exists(All + "/" + p[:-5] + ".tbz2"):
+		#print "Stale entry: " + All + "/" + p[:-5] + ".tbz2"
+		os.unlink(p)
+		os.unlink(p[:-5]+".md5")
+
 finish = time.time()
 
 portage.writemsg(portage.green(' * ')+"PKGDIR contains "+ str(len(packages)) + ' packages. (%.01fsec)\n' % (finish - start));