Update Portage to sync BlockerDB at init.

Currently, Portage syncs the BlockerDB before every merge. This slows
down merges considerably because it requires reading the entire vardb
from disk. Since Portage doesn't merge conflicting packages anyway, we
can optimize this by just reading the vardb at initialization and
caching that.

Change-Id: I6701926f022ef3aa2da10482fc8b09573ae24610

Review URL: http://codereview.chromium.org/6688037

diff --git a/pym/_emerge/BlockerDB.py b/pym/_emerge/BlockerDB.py
index fd4bf5ecebe149821c301234904b2971461dfc8d..f5adc4d6728999854dda44ea99787d45b2627bd6 100644 (file)
--- a/pym/_emerge/BlockerDB.py
+++ b/pym/_emerge/BlockerDB.py
@@ -30,12 +30,7 @@ class BlockerDB(object):
                                "vartree"     :  fake_vartree,
                }}
 
-       def _get_fake_vartree(self, acquire_lock=0):
-               fake_vartree = self._fake_vartree
-               fake_vartree.sync(acquire_lock=acquire_lock)
-               return fake_vartree
-
-       def findInstalledBlockers(self, new_pkg, acquire_lock=0):
+       def findInstalledBlockers(self, new_pkg):
                """
                Search for installed run-time blockers in the root where
                new_pkg is planned to be installed. This ignores build-time
@@ -45,7 +40,7 @@ class BlockerDB(object):
                dep_keys = ["RDEPEND", "PDEPEND"]
                settings = self._vartree.settings
                stale_cache = set(blocker_cache)
-               fake_vartree = self._get_fake_vartree(acquire_lock=acquire_lock)
+               fake_vartree = self._fake_vartree
                dep_check_trees = self._dep_check_trees
                vardb = fake_vartree.dbapi
                installed_pkgs = list(vardb)
@@ -118,3 +113,7 @@ class BlockerDB(object):
 
                return blocking_pkgs
 
+       def discardBlocker(self, pkg):
+               """Discard a package from the list of potential blockers."""
+               self._fake_vartree.cpv_discard(pkg)
+

diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py
index f7f292bf545b24a61931e202172a95557ec7fe20..fa490c2d6e8f880321cb5c525552621faf5ba79a 100644 (file)
--- a/pym/_emerge/FakeVartree.py
+++ b/pym/_emerge/FakeVartree.py
@@ -103,6 +103,16 @@ class FakeVartree(vartree):
                                pkg, self.dbapi, self._global_updates)
                return self._aux_get(pkg, wants)
 
+       def cpv_discard(self, pkg):
+               """
+               Discard a package from the fake vardb if it exists.
+               """
+               old_pkg = self.dbapi.get(pkg)
+               if old_pkg is not None:
+                       self.dbapi.cpv_remove(old_pkg)
+                       self._pkg_cache.pop(old_pkg, None)
+                       self._aux_get_history.discard(old_pkg.cpv)
+
        def sync(self, acquire_lock=1):
                """
                Call this method to synchronize state with the real vardb
@@ -144,9 +154,7 @@ class FakeVartree(vartree):
                # Remove any packages that have been uninstalled.
                for pkg in list(pkg_vardb):
                        if pkg.cpv not in current_cpv_set:
-                               pkg_vardb.cpv_remove(pkg)
-                               pkg_cache.pop(pkg, None)
-                               aux_get_history.discard(pkg.cpv)
+                               self.cpv_discard(pkg)
 
                # Validate counters and timestamps.
                slot_counters = {}
@@ -165,9 +173,7 @@ class FakeVartree(vartree):
 
                                if counter != pkg.counter or \
                                        mtime != pkg.mtime:
-                                       pkg_vardb.cpv_remove(pkg)
-                                       pkg_cache.pop(pkg, None)
-                                       aux_get_history.discard(pkg.cpv)
+                                       self.cpv_discard(pkg)
                                        pkg = None
 
                        if pkg is None:

diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index 9215678bbf7866f4f30b645835388f854250b7a3..7d34d706a58c409c41277f58f2f757b4be02b024 100644 (file)
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -317,10 +317,6 @@ class Scheduler(PollScheduler):
                Initialization structures used for dependency calculations
                involving currently installed packages.
                """
-               # TODO: Replace the BlockerDB with a depgraph of installed packages
-               # that's updated incrementally with each upgrade/uninstall operation
-               # This will be useful for making quick and safe decisions with respect
-               # to aggressive parallelization discussed in bug #279623.
                self._set_graph_config(graph_config)
                self._blocker_db = {}
                for root in self.trees:
@@ -329,6 +325,7 @@ class Scheduler(PollScheduler):
                                        pkg_cache=self._pkg_cache)
                        else:
                                fake_vartree = graph_config.trees[root]['vartree']
+                       fake_vartree.sync()
                        self._blocker_db[root] = BlockerDB(fake_vartree)
 
        def _destroy_graph(self):
@@ -643,27 +640,20 @@ class Scheduler(PollScheduler):
 
        def _find_blockers(self, new_pkg):
                """
-               Returns a callable which should be called only when
-               the vdb lock has been acquired.
+               Returns a callable.
                """
                def get_blockers():
-                       return self._find_blockers_with_lock(new_pkg, acquire_lock=0)
+                       return self._find_blockers_impl(new_pkg)
                return get_blockers
 
-       def _find_blockers_with_lock(self, new_pkg, acquire_lock=0):
+       def _find_blockers_impl(self, new_pkg):
                if self._opts_ignore_blockers.intersection(self.myopts):
                        return None
 
-               # Call gc.collect() here to avoid heap overflow that
-               # triggers 'Cannot allocate memory' errors (reported
-               # with python-2.5).
-               gc.collect()
-
                blocker_db = self._blocker_db[new_pkg.root]
 
                blocker_dblinks = []
-               for blocking_pkg in blocker_db.findInstalledBlockers(
-                       new_pkg, acquire_lock=acquire_lock):
+               for blocking_pkg in blocker_db.findInstalledBlockers(new_pkg):
                        if new_pkg.slot_atom == blocking_pkg.slot_atom:
                                continue
                        if new_pkg.cpv == blocking_pkg.cpv:
@@ -673,8 +663,6 @@ class Scheduler(PollScheduler):
                                self.pkgsettings[blocking_pkg.root], treetype="vartree",
                                vartree=self.trees[blocking_pkg.root]["vartree"]))
 
-               gc.collect()
-
                return blocker_dblinks
 
        def _dblink_pkg(self, pkg_dblink):
@@ -1529,6 +1517,8 @@ class Scheduler(PollScheduler):
                self._completed_tasks.add(pkg)
                self._unsatisfied_system_deps.discard(pkg)
                self._choose_pkg_return_early = False
+               blocker_db = self._blocker_db[pkg.root]
+               blocker_db.discardBlocker(pkg)
 
        def _merge(self):