Cache counter values, avoiding I/O when it doesn't change.

This improves merge times by up to 25%, since looping over the vardb for
each package install is slow.

TEST=Emerge a bunch of packages, notice 25% speed improvement.
BUG=chromium-os:15112

Change-Id: I51dd617219cd1820ceeb702291bd790990995be4

diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 12a0baf55f296f49a52d712650c434d7cd1e0be3..05f45d51ade9a20d82264cc40d1de4f7bb239469 100644 (file)
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -143,6 +143,7 @@ class MergeProcess(SpawnProcess):
                fd_pipes[elog_writer_fd] = elog_writer_fd
                self._elog_reg_id = self.scheduler.register(elog_reader_fd,
                        self._registered_events, self._elog_output_handler)
+               counter = self.vartree.dbapi.counter_tick()
 
                pid = os.fork()
                if pid != 0:
@@ -196,7 +197,7 @@ class MergeProcess(SpawnProcess):
                try:
                        rval = mylink.merge(self.pkgloc, self.infloc,
                                myebuild=self.myebuild, mydbapi=self.mydbapi,
-                               prev_mtimes=self.prev_mtimes)
+                               prev_mtimes=self.prev_mtimes, counter=counter)
                except SystemExit:
                        raise
                except:

diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c81e99b9e30f912b398e0c86238b260305ebbb7c..7c6602752c04baff9130754ba5cd0ae198853fa8 100644 (file)
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -166,6 +166,8 @@ class vardbapi(dbapi):
                        self._linkmap = LinkageMap(self)
                self._owners = self._owners_db(self)
 
+               self._cached_counter = None
+
        def getpath(self, mykey, filename=None):
                # This is an optimized hotspot, so don't use unicode-wrapped
                # os module and don't use os.path.join().
@@ -724,17 +726,6 @@ class vardbapi(dbapi):
                @param myroot: ignored, self._eroot is used instead
                """
                myroot = None
-               cp_list = self.cp_list
-               max_counter = 0
-               for cp in self.cp_all():
-                       for cpv in cp_list(cp):
-                               try:
-                                       counter = int(self.aux_get(cpv, ["COUNTER"])[0])
-                               except (KeyError, OverflowError, ValueError):
-                                       continue
-                               if counter > max_counter:
-                                       max_counter = counter
-
                new_vdb = False
                counter = -1
                try:
@@ -762,16 +753,27 @@ class vardbapi(dbapi):
                                writemsg("!!! %s\n" % str(e), noiselevel=-1)
                                del e
 
-               # We must ensure that we return a counter
-               # value that is at least as large as the
-               # highest one from the installed packages,
-               # since having a corrupt value that is too low
-               # can trigger incorrect AUTOCLEAN behavior due
-               # to newly installed packages having lower
-               # COUNTERs than the previous version in the
-               # same slot.
-               if counter > max_counter:
+               if self._cached_counter == counter:
                        max_counter = counter
+               else:
+                       # We must ensure that we return a counter
+                       # value that is at least as large as the
+                       # highest one from the installed packages,
+                       # since having a corrupt value that is too low
+                       # can trigger incorrect AUTOCLEAN behavior due
+                       # to newly installed packages having lower
+                       # COUNTERs than the previous version in the
+                       # same slot.
+                       cp_list = self.cp_list
+                       max_counter = counter
+                       for cp in self.cp_all():
+                               for cpv in cp_list(cp):
+                                       try:
+                                               pkg_counter = int(self.aux_get(cpv, ["COUNTER"])[0])
+                                       except (KeyError, OverflowError, ValueError):
+                                               continue
+                                       if pkg_counter > max_counter:
+                                               max_counter = pkg_counter
 
                if counter < 0 and not new_vdb:
                        writemsg(_("!!! Initializing COUNTER to " \
@@ -789,18 +791,19 @@ class vardbapi(dbapi):
                """
                myroot = None
                mycpv = None
-
                self.lock()
                try:
                        counter = self.get_counter_tick_core() - 1
-                       if incrementing:
-                               #increment counter
-                               counter += 1
-                               # use same permissions as config._init_dirs()
-                               ensure_dirs(os.path.dirname(self._counter_path),
-                                       gid=portage_gid, mode=0o2750, mask=0o2)
-                               # update new global counter file
-                               write_atomic(self._counter_path, str(counter))
+                       if self._cached_counter != counter:
+                               if incrementing:
+                                       #increment counter
+                                       counter += 1
+                                       # use same permissions as config._init_dirs()
+                                       ensure_dirs(os.path.dirname(self._counter_path),
+                                               gid=portage_gid, mode=0o2750, mask=0o2)
+                                       # update new global counter file
+                                       write_atomic(self._counter_path, str(counter))
+                               self._cached_counter = counter
                finally:
                        self.unlock()
 
@@ -2860,7 +2863,7 @@ class dblink(object):
                                os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
 
        def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
-               mydbapi=None, prev_mtimes=None):
+               mydbapi=None, prev_mtimes=None, counter=None):
                """
                
                This function does the following:
@@ -3276,7 +3279,8 @@ class dblink(object):
                        self.copyfile(inforoot+"/"+x)
 
                # write local package counter for recording
-               counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
+               if counter is not None:
+                       counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv)
                codecs.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'),
                        encoding=_encodings['fs'], errors='strict'),
                        'w', encoding=_encodings['repo.content'], errors='backslashreplace'
@@ -3906,7 +3910,7 @@ class dblink(object):
                                showMessage(zing + " " + mydest + "\n")
 
        def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0,
-               mydbapi=None, prev_mtimes=None):
+               mydbapi=None, prev_mtimes=None, counter=None):
                """
                @param myroot: ignored, self._eroot is used instead
                """
@@ -3918,7 +3922,8 @@ class dblink(object):
                self.vartree.dbapi._bump_mtime(self.mycpv)
                try:
                        retval = self.treewalk(mergeroot, myroot, inforoot, myebuild,
-                               cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes)
+                               cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes,
+                               counter=counter)
 
                        # If PORTAGE_BUILDDIR doesn't exist, then it probably means
                        # fail-clean is enabled, and the success/die hooks have