Fix downloading of entire index after PyPI made changes in server, add randomizing...

git-svn-id: http://g-pypi.googlecode.com/svn/trunk@14 118783bc-b352-0410-bbc3-0f610f6f7ae8

diff --git a/tests/entire_index.py b/tests/entire_index.py
index e160f5bac40087c77b46b81dfc7f979229d66b06..79a44e899c042ea1616afa9df388f4597e78e90d 100755 (executable)
--- a/tests/entire_index.py
+++ b/tests/entire_index.py
@@ -3,35 +3,73 @@
 
 """
 
-*** WARNING ***
-*** WARNING ***
-*** WARNING ***
+*** WARNING *** *** WARNING *** *** WARNING *** *** WARNING ***
 
 This will attempt to create an ebuild for every single release on PyPI
-which obviously will take a long time and require a decent amount of bandwidth
+which obviously will take a long time and consume a decent amount of bandwidth
 
-*** WARNING ***
-*** WARNING ***
-*** WARNING ***
+The delay it takes to create an ebuild is long enough to keep PyPI from
+getting hammered too hard. Be nice though.
+
+*** WARNING *** *** WARNING *** *** WARNING *** *** WARNING *** 
+
+Usage:
+
+This script is good to run after making significant changes to g-pypi
+after runing nosetests.
+
+Add an overlay to /etc/make.conf PORTDIR_OVERLAY for '/tmp/overlay'
+This script will create the overlay itself and the profiles dir with repo_name.
+
+TODO:
+
+Command-line option to create only n number of ebuilds. This could be
+a range, 0-20 or 50-100 or random 10 etc.
+
+Allow passing of options to g-pypi such as --debug, --quiet etc.
 
 """
 
 import pickle
 import os
+import sys
+from random import shuffle
 
 from yolk.pypi import CheeseShop
 
+PKG_INDEX = "pkg_index"
+
+#A temporary overlay, gets created if it doesn't exist, and profiles/repo_name
+TMP_OVERLAY = "/tmp/overlay"
+
+REPO_NAME = 'g-pypi-tmp'
 
 cheeseshop = CheeseShop()
-PKG_INDEX = "pkg_index"
+
+if not os.path.exists(TMP_OVERLAY):
+    os.mkdir(TMP_OVERLAY)
+
+if not os.path.exists(TMP_OVERLAY + '/profiles'):
+    os.mkdir(TMP_OVERLAY + '/profiles')
+    fdesc = open(TMP_OVERLAY + '/profiles/repo_name', 'w')
+    fdesc.write(REPO_NAME)
+    fdesc.close()
 
 if os.path.exists(PKG_INDEX):
+    print "Reading PyPI cache... rm %s to re-download full index" % PKG_INDEX
     full_index = pickle.load(open(PKG_INDEX, 'r'))
 else:
-    full_index = cheeseshop.search({"name":"foo"}, "or")
+    print "Downloading entire PyPI index to cache..."
+    full_index = cheeseshop.search({"name":""}, "or")
     pickle.dump(full_index, open(PKG_INDEX, "w"))
 
+print "Randomizing cache..."
+shuffle(full_index)
+
 for pkg in full_index:
-    os.system('echo Testing %s' % pkg['name'].encode('utf-8'))
-    os.system('g-pypi -qo %s' % pkg['name'])
-    #os.system('echo %s' % ('-' * 79))
+    try:
+        os.system('echo Testing %s' % pkg['name'].encode('utf-8'))
+        os.system('g-pypi -l %s -o %s' % (REPO_NAME, pkg['name']))
+    except KeyboardInterrupt:
+        sys.exit()
+