Implement asynchronous binary package extraction, so that the scheduler

can run while a package is extracting in the background.

svn path=/main/trunk/; revision=10902

diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py
index e5b7ed297723fc719b328ed6e5cfc5495fb37dbb..d650092a0797839a24915fa6cc3c35a9ca3c6f95 100644 (file)
--- a/pym/_emerge/__init__.py
+++ b/pym/_emerge/__init__.py
@@ -24,6 +24,7 @@ import array
 import fcntl
 import select
 import shlex
+import shutil
 import urlparse
 import weakref
 import gc
@@ -1565,7 +1566,7 @@ class SpawnProcess(SubProcess):
 
                kwargs["fd_pipes"] = fd_pipes
                kwargs["returnpid"] = True
-               del kwargs["logfile"]
+               kwargs.pop("logfile", None)
 
                retval = portage.process.spawn(self.args, **kwargs)
 
@@ -2104,6 +2105,8 @@ class Binpkg(SlotObject):
                pkg_count = self.pkg_count
                scheduler = self.scheduler
                settings = self.settings
+               settings.setcpv(pkg)
+               debug = settings.get("PORTAGE_DEBUG") == "1"
 
                # The prefetcher has already completed or it
                # could be running now. If it's running now,
@@ -2167,16 +2170,122 @@ class Binpkg(SlotObject):
                        "portage", pkg.category, pkg.pf)
                build_dir = EbuildBuildDir(dir_path=dir_path,
                        pkg=pkg, settings=settings)
+               image_dir = os.path.join(dir_path, "image")
+               infloc = os.path.join(dir_path, "build-info")
+
+               fd_pipes = {
+                       0 : sys.stdin.fileno(),
+                       1 : sys.stdout.fileno(),
+                       2 : sys.stderr.fileno(),
+               }
 
                try:
                        build_dir.lock()
-                       merge = BinpkgMerge(find_blockers=find_blockers,
-                               ldpath_mtimes=ldpath_mtimes, pkg=pkg, pretend=opts.pretend,
-                               pkg_path=pkg_path, settings=settings)
+
+                       root_config = self.pkg.root_config
+                       ebuild_path = os.path.join(infloc, pkg.pf + ".ebuild")
+                       cleanup = 1
+                       tree = "bintree"
+                       mydbapi = root_config.trees[tree].dbapi
+
+                       retval = portage.doebuild(ebuild_path, "clean",
+                               root_config.root, settings, debug, cleanup=cleanup,
+                               mydbapi=mydbapi, tree=tree)
+                       if retval != os.EX_OK:
+                               return retval
+
+                       try:
+                               shutil.rmtree(dir_path)
+                       except (IOError, OSError), e:
+                               if e.errno != errno.ENOENT:
+                                       raise
+                               del e
+
+                       # This initializes PORTAGE_LOG_FILE.
+                       portage.prepare_build_dirs(root_config.root, settings, cleanup)
+
+                       dir_mode = 0755
+                       for mydir in (dir_path, image_dir, infloc):
+                               portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid,
+                                       gid=portage.data.portage_gid, mode=dir_mode)
+
+                       portage.writemsg_stdout(">>> Extracting info\n")
+
+                       pkg_xpak = portage.xpak.tbz2(pkg_path)
+                       check_missing_metadata = ("CATEGORY", "PF")
+                       missing_metadata = set()
+                       for k in check_missing_metadata:
+                               v = pkg_xpak.getfile(k)
+                               if not v:
+                                       missing_metadata.add(k)
+
+                       pkg_xpak.unpackinfo(infloc)
+                       for k in missing_metadata:
+                               if k == "CATEGORY":
+                                       v = pkg.category
+                               elif k == "PF":
+                                       v = pkg.pf
+                               else:
+                                       continue
+
+                               f = open(os.path.join(infloc, k), 'wb')
+                               try:
+                                       f.write(v + "\n")
+                               finally:
+                                       f.close()
+
+                       # Store the md5sum in the vdb.
+                       f = open(os.path.join(infloc, "BINPKGMD5"), "w")
+                       try:
+                               f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n")
+                       finally:
+                               f.close()
+
+                       # This gives bashrc users an opportunity to do various things
+                       # such as remove binary packages after they're installed.
+                       settings["PORTAGE_BINPKG_FILE"] = pkg_path
+                       settings.backup_changes("PORTAGE_BINPKG_FILE")
+
+                       phase = "setup"
+                       ebuild_phase = EbuildPhase(fd_pipes=fd_pipes,
+                               pkg=pkg, phase=phase, register=scheduler.register,
+                               settings=settings, unregister=scheduler.unregister)
+
+                       ebuild_phase.start()
+                       retval = None
+                       while retval is None:
+                               scheduler.schedule()
+                               retval = ebuild_phase.poll()
+
+                       if retval != os.EX_OK:
+                               return retval
+
+                       extractor = BinpkgExtractorAsync(image_dir=image_dir,
+                               pkg=pkg, pkg_path=pkg_path, register=scheduler.register,
+                               unregister=scheduler.unregister)
+                       portage.writemsg_stdout(">>> Extracting %s\n" % pkg.cpv)
+                       extractor.start()
+                       retval = None
+                       while retval is None:
+                               scheduler.schedule()
+                               retval = extractor.poll()
+
+                       if retval != os.EX_OK:
+                               writemsg("!!! Error Extracting '%s'\n" % pkg_path,
+                                       noiselevel=-1)
+                               return retval
+
+                       merge = EbuildMerge(
+                               find_blockers=find_blockers,
+                               ldpath_mtimes=ldpath_mtimes,
+                               pkg=pkg, settings=settings)
+
                        retval = merge.execute()
                        if retval != os.EX_OK:
                                return retval
+
                finally:
+                       settings.pop("PORTAGE_BINPKG_FILE", None)
                        build_dir.unlock()
                return os.EX_OK
 
@@ -2328,6 +2437,21 @@ class BinpkgFetcherAsync(SpawnProcess):
                self._lock_obj = None
                self.locked = False
 
+class BinpkgExtractorAsync(SpawnProcess):
+
+       __slots__ = ("image_dir", "pkg", "pkg_path")
+
+       _shell_binary = portage.const.BASH_BINARY
+
+       def start(self):
+               self.args = [self._shell_binary, "-c",
+                       "bzip2 -dqc -- %s | tar -xp -C %s -f -" % \
+                       (portage._shell_quote(self.pkg_path),
+                       portage._shell_quote(self.image_dir))]
+
+               self.env = self.pkg.root_config.settings.environ()
+               SpawnProcess.start(self)
+
 class BinpkgMerge(Task):
 
        __slots__ = ("find_blockers", "ldpath_mtimes",

diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index 5fe2b41f3afe7db48821af9747c464023717957f..16760c8d4938e2ae6f4907f88a1057347825307b 100644 (file)
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -4874,6 +4874,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
        noauto = "noauto" in features
        from portage.data import secpass
 
+       clean_phases = ("clean", "cleanrm")
        validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst",
                        "config","info","setup","depend","fetch","digest",
                        "unpack","compile","test","install","rpm","qmerge","merge",
@@ -4890,7 +4891,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                writemsg("\n", noiselevel=-1)
                return 1
 
-       if not os.path.exists(myebuild):
+       if mydo not in clean_phases and not os.path.exists(myebuild):
                writemsg("!!! doebuild: %s not found for %s\n" % (myebuild, mydo),
                        noiselevel=-1)
                return 1
@@ -4987,7 +4988,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0,
                doebuild_environment(myebuild, mydo, myroot, mysettings, debug,
                        use_cache, mydbapi)
 
-               clean_phases = ("clean", "cleanrm")
                if mydo in clean_phases:
                        retval = spawn(_shell_quote(ebuild_sh_binary) + " clean",
                                mysettings, debug=debug, free=1, logfile=None)