Comment on Terms of Service in ticker scraping script.

diff --git a/posts/ticker/ticker.py b/posts/ticker/ticker.py
index a175fc5da1abbb489f65f8ddf7485f520e650deb..f96aadda7e2da677405af88ee6428ef823f527bb 100755 (executable)
--- a/posts/ticker/ticker.py
+++ b/posts/ticker/ticker.py
@@ -19,6 +19,7 @@
 
 import logging
 from StringIO import StringIO
+import time
 import urllib2
 
 from lxml import etree
@@ -26,8 +27,9 @@ from lxml import etree
 
 class Grabber (object):
     "Base ckass for website-specific quote scrapers."
-    def __init__(self, url):
+    def __init__(self, url, sleep=1):
         self._url = url
+        self._sleep_seconds = sleep
 
     def quote(self, ticker):
         "Floating point quote for the given `ticker` symbol string."
@@ -48,6 +50,7 @@ class Grabber (object):
         info = f.info()
         html = f.read()
         f.close()
+        time.sleep(self._sleep_seconds)
         return (info, html)
 
     def _parse_html(self, html):
@@ -59,7 +62,25 @@ class Grabber (object):
 
 
 class GoogleGrabber (Grabber):
-    "Grab quotes from Google Finance."
+    """Grab quotes from Google Finance.
+
+    From Google's `Terms of Service`_:
+
+      5.3 You agree not to access (or attempt to access) any of the
+      Services by any means other than through the interface that is
+      provided by Google, unless you have been specifically allowed to
+      do so in a separate agreement with Google. You specifically
+      agree not to access (or attempt to access) any of the Services
+      through any automated means (including use of scripts or web
+      crawlers) and shall ensure that you comply with the instructions
+      set out in any robots.txt file present on the Services.
+
+    However, I think the distinction between "browser", which Google
+    clearly does allow, and "script run interactively from the command
+    line" is pretty blurry.
+
+    .. _Terms of Service: http://www.google.com/accounts/TOS?loc=us
+    """
     def __init__(self):
         super(GoogleGrabber, self).__init__(
             url='http://www.google.com/finance?q=%s')
@@ -82,7 +103,14 @@ class GoogleGrabber (Grabber):
 
 
 class YahooGrabber (Grabber):
-    "Grab quotes from Yahoo! Finance."
+    """Grab quotes from Yahoo! Finance.
+
+    Yahoo's `Terms of Service`_ don't seem to have any explicitly
+    relevant terms.
+
+    .. _Terms of Service:
+      http://info.yahoo.com/legal/us/yahoo/utos/utos-173.html
+    """
     def __init__(self):
         super(YahooGrabber, self).__init__(
             url='http://finance.yahoo.com/q?s=%s')
@@ -147,7 +175,7 @@ if __name__ == '__main__':
     p.add_option('-v', '--verbose', dest='verbose', default=0, action='count',
                  help='increment verbosity')
     grabbers = sorted(GRABBERS.keys())
-    p.add_option('-g', '--grabber', dest='grabber', default='google',
+    p.add_option('-g', '--grabber', dest='grabber', default='yahoo',
                  type='choice', choices=grabbers,
                  help='select grabber from %s (%%default)' % grabbers)