Use pickle for .sconsign. (Anthony Roach)

git-svn-id: http://scons.tigris.org/svn/scons/trunk@522 fdb21ef1-2011-0410-befe-b5e4ea1792b1

diff --git a/etc/TestSCons.py b/etc/TestSCons.py
index 997104bc616a839309494d32ba2d2a83dd8fad31..c97fd482baeca80f022b5d0fe433b987329fcdef 100644 (file)
--- a/etc/TestSCons.py
+++ b/etc/TestSCons.py
@@ -125,8 +125,8 @@ class TestSCons(TestCmd.TestCmd):
        if _failed(self, status):
             expect = ''
             if status != 0:
-                expect = " (expected %d)" % status
-            print "%s returned %d%s" % (self.program, _status(self), expect)
+                expect = " (expected %s)" % str(status)
+            print "%s returned %s%s" % (self.program, str(_status(self)), expect)
             print "STDOUT ============"
             print self.stdout()
            print "STDERR ============"

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index e3c4859214d2b42de261c5d1d573cbffb2285578..ee47871bfdb62ec4cbf68794671eb2152b64124e 100644 (file)
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -19,6 +19,9 @@ RELEASE 0.10 - XXX
   - Add SetJobs() and GetJobs() methods to allow configuration of the
     number of default jobs (still overridden by -j).
 
+  - Convert the .sconsign file format from ASCII to a pickled Python
+    data structure.
+
 
 
 RELEASE 0.09 - Thu,  5 Dec 2002 04:48:25 -0600

diff --git a/src/RELEASE.txt b/src/RELEASE.txt
index 36479449834934bd3824d681a3199c9662fe6414..15f4ca9f8c44d89481640ffee0535a046a3e442a 100644 (file)
--- a/src/RELEASE.txt
+++ b/src/RELEASE.txt
@@ -27,7 +27,16 @@ RELEASE 0.10 - XXX
 
   Please note the following important changes since release 0.09:
 
-    - XXX
+    - The .sconsign file format has been changed from ASCII to a pickled
+      Python data structure.  This improves performance and future
+      extensibility, but means that the first time you execute SCons
+      0.10 on an already-existing source tree, for every .sconsign
+      file in the tree it will report:
+
+       SCons warning: Ignoring corrupt .sconsign file: xxx
+
+      These warnings are normal in this situation and can be safely
+      ignored.
 
   Please note the following important changes since release 0.08:
 

diff --git a/src/engine/SCons/Sig/SigTests.py b/src/engine/SCons/Sig/SigTests.py
index 37413aa176bd88d360cbb3df52102d70dd876b6f..ccd58976260eadbbf016900f79546c3d7cc030de 100644 (file)
--- a/src/engine/SCons/Sig/SigTests.py
+++ b/src/engine/SCons/Sig/SigTests.py
@@ -385,101 +385,11 @@ class CalcTestCase(unittest.TestCase):
 class SConsignEntryTestCase(unittest.TestCase):
 
     def runTest(self):
-        class DummyModule:
-            def to_string(self, sig):
-                return str(sig)
-
-            def from_string(self, sig):
-                return int(sig)
-
-        m = DummyModule()
-        e = SCons.Sig.SConsignEntry(m)
-        assert e.timestamp == None
-        assert e.csig == None
-        assert e.bsig == None
-        assert e.get_implicit() == None
-        assert e.render(m) == "- - - -"
-
-        e = SCons.Sig.SConsignEntry(m, "- - - -")
-        assert e.timestamp == None
-        assert e.csig == None
-        assert e.bsig == None
-        assert e.get_implicit() == None
-        assert e.render(m) == "- - - -"
-
-        # Check backward compatability with pre-0.07 format:
-        e = SCons.Sig.SConsignEntry(m, "- - - ")
+        e = SCons.Sig.SConsignEntry()
         assert e.timestamp == None
         assert e.csig == None
         assert e.bsig == None
-        assert e.get_implicit() == []
-        assert e.render(m) == "- - - \0\0\0\0"
-
-        # Check backward compatability with pre-0.07 format:
-        e = SCons.Sig.SConsignEntry(m, "- - -")
-        assert e.timestamp == None
-        assert e.csig == None
-        assert e.bsig == None
-        assert e.get_implicit() == None
-        assert e.render(m) == "- - - -"
-        
-        e = SCons.Sig.SConsignEntry(m, "- - - \0\0\0\0")
-        assert e.timestamp == None
-        assert e.csig == None
-        assert e.bsig == None
-        assert e.get_implicit() == []
-        assert e.render(m) == "- - - \0\0\0\0"
-
-        # Check backward compatability with pre-0.08 format:
-        e = SCons.Sig.SConsignEntry(m, "- - - foo\0bar")
-        assert e.timestamp == None
-        assert e.csig == None
-        assert e.bsig == None
-        assert e.get_implicit() == ['foo', 'bar']
-        assert e.render(m) == "- - - \0\0foo\0bar\0\0"
-
-        e = SCons.Sig.SConsignEntry(m, "- - - \0\0foo\0bar\0\0")
-        assert e.timestamp == None
-        assert e.csig == None
-        assert e.bsig == None
-        assert e.get_implicit() == ['foo', 'bar']
-        assert e.render(m) == "- - - \0\0foo\0bar\0\0"
-
-        e = SCons.Sig.SConsignEntry(m, "123 456 789 \0\0foo bletch\0bar\0\0")
-        assert e.timestamp == 123
-        assert e.bsig == 456
-        assert e.csig == 789
-        assert e.get_implicit() == ['foo bletch', 'bar']
-        assert e.render(m) == "123 456 789 \0\0foo bletch\0bar\0\0"
-
-        # Check backward compatability with pre-0.07 format:
-        e = SCons.Sig.SConsignEntry(m, "987 654 321")
-        assert e.timestamp == 987
-        assert e.bsig == 654
-        assert e.csig == 321
-        assert e.get_implicit() == None
-        assert e.render(m) == "987 654 321 -"
-
-        e.set_implicit(None)
-        assert e.get_implicit() == None, e.get_implicit()
-
-        e.set_implicit('')
-        assert e.get_implicit() == [], e.get_implicit()
-
-        e.set_implicit('foo')
-        assert e.get_implicit() == ['foo'], e.get_implicit()
-
-        e.set_implicit('foo bar')
-        assert e.get_implicit() == ['foo bar'], e.get_implicit()
-
-        e.set_implicit(['foo'])
-        assert e.get_implicit() == ['foo'], e.get_implicit()
-
-        e.set_implicit(['foo bar'])
-        assert e.get_implicit() == ['foo bar'], e.get_implicit()
-
-        e.set_implicit(['foo', 'bar'])
-        assert e.get_implicit() == ['foo', 'bar'], e.get_implicit()
+        assert e.implicit == None
 
 class SConsignFileTestCase(unittest.TestCase):
 

diff --git a/src/engine/SCons/Sig/__init__.py b/src/engine/SCons/Sig/__init__.py
index 9a4c84691ee058834c82ad6e138e1a799919a2a9..5de89fa7ae9ab51c5a8761b2f84cb6b4318c3b8b 100644 (file)
--- a/src/engine/SCons/Sig/__init__.py
+++ b/src/engine/SCons/Sig/__init__.py
@@ -35,6 +35,7 @@ import string
 import SCons.Node
 import time
 import SCons.Warnings
+import cPickle
 
 try:
     import MD5
@@ -56,61 +57,21 @@ def write():
         sig_file.write()
 
 class SConsignEntry:
-    def __init__(self, module, entry=None):
 
-        self.timestamp = self.csig = self.bsig = self.implicit = None
+    """Objects of this type are pickled to the .sconsign file, so it
+    should only contain simple builtin Python datatypes and no methods.
 
-        if not entry is None:
-            arr = map(string.strip, string.split(entry, " ", 3))
-
-            try:
-                if arr[0] == '-': self.timestamp = None
-                else:             self.timestamp = int(arr[0])
-
-                if arr[1] == '-': self.bsig = None
-                else:             self.bsig = module.from_string(arr[1])
-
-                if arr[2] == '-': self.csig = None
-                else:             self.csig = module.from_string(arr[2])
-
-                if len(arr) < 4:        self.implicit = None # pre-0.07 format
-                elif arr[3] == '':      self.implicit = '' # pre-0.08 format
-                elif arr[3] == '-':     self.implicit = None
-                else:                   self.implicit = string.replace(arr[3], '\0\0', '')
-            except IndexError:
-                pass
-
-    def render(self, module):
-        if self.timestamp is None: timestamp = '-'
-        else:                      timestamp = "%d"%self.timestamp
-
-        if self.bsig is None: bsig = '-'
-        else:                 bsig = module.to_string(self.bsig)
-
-        if self.csig is None: csig = '-'
-        else:                 csig = module.to_string(self.csig)
-
-        if self.implicit is None: implicit = '-'
-        else:                     implicit = '\0\0%s\0\0'%self.implicit
-
-        return '%s %s %s %s' % (timestamp, bsig, csig, implicit)
-
-    def get_implicit(self):
-        if self.implicit is None:
-            return None
-        elif self.implicit == '':
-            return []
-        else:
-            return string.split(self.implicit, '\0')
-
-    def set_implicit(self, implicit):
-        if implicit is None:
-            self.implicit = None
-        else:
-            if SCons.Util.is_String(implicit):
-                implicit = [implicit]
-            self.implicit = string.join(map(str, implicit), '\0')
+    This class is used to store cache information about nodes between
+    scons runs for efficiency, and to store the build signature for
+    nodes so that scons can determine if they are out of date. """
 
+    # setup the default value for various attributes:
+    # (We make the class variables so the default values won't get pickled
+    # with the instances, which would waste a lot of space)
+    timestamp = None
+    bsig = None
+    csig = None
+    implicit = None
 
 class SConsignFile:
     """
@@ -131,22 +92,22 @@ class SConsignFile:
             self.module = module
         self.sconsign = os.path.join(dir.path, '.sconsign')
         self.entries = {}
-        self.dirty = None
+        self.dirty = 0
 
         try:
-            file = open(self.sconsign, 'rt')
+            file = open(self.sconsign, 'rb')
         except:
             pass
         else:
-            for line in file.readlines():
-                try:
-                    filename, rest = map(string.strip, string.split(line, ":", 1))
-                    self.entries[filename] = SConsignEntry(self.module, rest)
-                except ValueError:
-                    SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
-                                        "Ignoring corrupt .sconsign file: %s"%self.sconsign)
+            try:
+                self.entries = cPickle.load(file)
+                if type(self.entries) is not type({}):
                     self.entries = {}
-                    
+                    raise TypeError
+            except:
+                SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
+                                    "Ignoring corrupt .sconsign file: %s"%self.sconsign)
+
         global sig_files
         sig_files.append(self)
 
@@ -157,24 +118,25 @@ class SConsignFile:
         filename - the filename whose signature will be returned
         returns - (timestamp, bsig, csig)
         """
-        try:
-            entry = self.entries[filename]
-            return (entry.timestamp, entry.bsig, entry.csig)
-        except KeyError:
-            return (None, None, None)
-    
-    def create_entry(self, filename):
+        entry = self.get_entry(filename)
+        return (entry.timestamp, entry.bsig, entry.csig)
+
+    def get_entry(self, filename):
         """
         Create an entry for the filename and return it, or if one already exists,
         then return it.
         """
         try:
-            entry = self.entries[filename]
-        except KeyError:
-            entry = SConsignEntry(self.module)
-            self.entries[filename] = entry
-            
-        return entry
+            return self.entries[filename]
+        except:
+            return SConsignEntry()
+
+    def set_entry(self, filename, entry):
+        """
+        Set the entry.
+        """
+        self.entries[filename] = entry
+        self.dirty = 1
 
     def set_csig(self, filename, csig):
         """
@@ -184,9 +146,9 @@ class SConsignFile:
         csig - the file's content signature
         """
 
-        entry = self.create_entry(filename)
+        entry = self.get_entry(filename)
         entry.csig = csig
-        self.dirty = 1
+        self.set_entry(filename, entry)
 
     def set_bsig(self, filename, bsig):
         """
@@ -196,9 +158,9 @@ class SConsignFile:
         bsig - the file's built signature
         """
 
-        entry = self.create_entry(filename)
+        entry = self.get_entry(filename)
         entry.bsig = bsig
-        self.dirty = 1
+        self.set_entry(filename, entry)
 
     def set_timestamp(self, filename, timestamp):
         """
@@ -208,23 +170,23 @@ class SConsignFile:
         timestamp - the file's timestamp
         """
 
-        entry = self.create_entry(filename)
+        entry = self.get_entry(filename)
         entry.timestamp = timestamp
-        self.dirty = 1
+        self.set_entry(filename, entry)
 
     def get_implicit(self, filename):
         """Fetch the cached implicit dependencies for 'filename'"""
-        try:
-            entry = self.entries[filename]
-            return entry.get_implicit()
-        except KeyError:
-            return None
+        entry = self.get_entry(filename)
+        return entry.implicit
 
     def set_implicit(self, filename, implicit):
         """Cache the implicit dependencies for 'filename'."""
-        entry = self.create_entry(filename)
-        entry.set_implicit(implicit)
-        self.dirty = 1
+        entry = self.get_entry(filename)
+        if SCons.Util.is_String(implicit):
+            implicit = [implicit]
+        implicit = map(str, implicit)
+        entry.implicit = implicit
+        self.set_entry(filename, entry)
 
     def write(self):
         """
@@ -242,18 +204,15 @@ class SConsignFile:
         if self.dirty:
             temp = os.path.join(self.dir.path, '.scons%d' % os.getpid())
             try:
-                file = open(temp, 'wt')
+                file = open(temp, 'wb')
                 fname = temp
             except:
                 try:
-                    file = open(self.sconsign, 'wt')
+                    file = open(self.sconsign, 'wb')
                     fname = self.sconsign
                 except:
                     return
-            keys = self.entries.keys()
-            keys.sort()
-            for name in keys:
-                file.write("%s: %s\n" % (name, self.entries[name].render(self.module)))
+            cPickle.dump(self.entries, file, 1)
             file.close()
             if fname != self.sconsign:
                 try:

diff --git a/test/sconsign.py b/test/sconsign.py
index 6da67177e4b686e20b527edace7c12072c9f7909..ca8792d31f8b77308d7866225627d7ca4fa696c9 100644 (file)
--- a/test/sconsign.py
+++ b/test/sconsign.py
@@ -27,6 +27,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 import os
 import TestSCons
 import TestCmd
+import cPickle
 
 test = TestSCons.TestSCons(match = TestCmd.match_re)
 
@@ -59,8 +60,8 @@ sub1__sconsign = test.workpath('sub1', '.sconsign')
 sub2__sconsign = test.workpath('sub2', '.sconsign')
 sub3__sconsign = test.workpath('sub3', '.sconsign')
 
-test.write(sub1__sconsign, "")
-test.write(sub2__sconsign, "")
+cPickle.dump({}, open(sub1__sconsign, 'wb'), 1)
+cPickle.dump({}, open(sub2__sconsign, 'wb'), 1)
 
 os.chmod(sub1__sconsign, 0444)
 
@@ -89,14 +90,11 @@ SCons warning: Ignoring corrupt .sconsign file: sub1..sconsign
 
 stdout = test.wrap_stdout('foo.in->sub1.foo.out\n')
 
-test.write(sub1__sconsign, 'garbage')
-test.run(arguments = '.', stderr=stderr, stdout=stdout)
-
 test.write(sub1__sconsign, 'not:a:sconsign:file')
-test.run(arguments = '.', stderr=stderr, stdout=stdout)
+test.run(arguments = '.', stderr=stderr, stdout=stdout, status=2)
 
 test.write(sub1__sconsign, '\0\0\0\0\0\0\0\0\0\0\0\0\0\0')
-test.run(arguments = '.', stderr=stderr, stdout=stdout)
+test.run(arguments = '.', stderr=stderr, stdout=stdout, status=2)
 
 
 test.pass_test()