class RootConfig(object):
"""This is used internally by depgraph to track information about a
particular $ROOT."""
+
+ pkg_tree_map = {
+ "ebuild" : "porttree",
+ "binary" : "bintree",
+ "installed" : "vartree"
+ }
+
+ tree_pkg_map = {}
+ for k, v in pkg_tree_map.iteritems():
+ tree_pkg_map[v] = k
+
def __init__(self, settings, trees, setconfig):
self.trees = trees
self.settings = settings
if buf:
self._read_data.append(buf.tostring())
else:
- for f in files.values():
- f.close()
self.registered = False
for reg_id in self._reg_ids:
self.scheduler.unregister(reg_id)
+
+ for f in files.values():
+ f.close()
+
self.wait()
return self.registered
self.returncode is None
def _wait(self):
- if self.returncode is not None:
- return self.returncode
if self.registered:
self.scheduler.schedule(self._reg_id)
+ if self.returncode is not None:
+ return self.returncode
try:
wait_retval = os.waitpid(self.pid, 0)
except OSError, e:
buf.tofile(files.out)
files.out.flush()
else:
- fd = files.process.fileno()
- for f in files.values():
- f.flush()
- f.close()
self.registered = False
self.scheduler.unregister(self._reg_id)
+
+ for f in files.values():
+ f.close()
+
self.wait()
return self.registered
if buf:
pass
else:
- fd = files.process.fileno()
- for f in files.values():
- f.close()
self.registered = False
self.scheduler.unregister(self._reg_id)
+
+ for f in files.values():
+ f.close()
+
self.wait()
return self.registered
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg,
pkg_count=pkg_count, pkg_path=ebuild_path,
+ scheduler=self.scheduler,
settings=settings, tree=tree, world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
files = self.files
self._raw_metadata.append(files.ebuild.read())
if not self._raw_metadata[-1]:
- for f in files.values():
- f.close()
self.registered = False
self.scheduler.unregister(self._reg_id)
+
+ for f in files.values():
+ f.close()
+
self.wait()
if self.returncode == os.EX_OK:
buf.tofile(files.log)
files.log.flush()
else:
- fd = files.ebuild.fileno()
- for f in files.values():
- f.close()
self.registered = False
self.scheduler.unregister(self._reg_id)
+
+ for f in files.values():
+ f.close()
+
self.wait()
return self.registered
if buf:
pass
else:
- fd = files.ebuild.fileno()
- for f in files.values():
- f.close()
self.registered = False
self.scheduler.unregister(self._reg_id)
+
+ for f in files.values():
+ f.close()
+
self.wait()
return self.registered
def _set_returncode(self, wait_retval):
SubProcess._set_returncode(self, wait_retval)
- if self.phase != "clean":
- msg = portage._doebuild_exit_status_check(
- self.phase, self.settings)
- if msg:
- self.returncode = 1
- from textwrap import wrap
- from portage.elog.messages import eerror
- for l in wrap(msg, 72):
- eerror(l, phase=self.phase, key=self.pkg.cpv)
settings = self.settings
+ debug = settings.get("PORTAGE_DEBUG") == "1"
+ log_path = settings.get("PORTAGE_LOG_FILE")
+
+ if self.phase != "clean":
+ self.returncode = portage._doebuild_exit_status_check_and_log(
+ settings, self.phase, self.returncode)
+
portage._post_phase_userpriv_perms(settings)
if self.phase == "install":
portage._check_build_log(settings)
if self.returncode == os.EX_OK:
self.returncode = portage._post_src_install_checks(settings)
+ elif self.phase == "preinst":
+
+ if self.returncode == os.EX_OK:
+ portage._doebuild_exit_status_unlink(
+ settings.get("EBUILD_EXIT_STATUS_FILE"))
+ phase_retval = portage.spawn(
+ " ".join(portage._post_pkg_preinst_cmd(settings)),
+ settings, debug=debug, free=1, logfile=log_path)
+ phase_retval = portage._doebuild_exit_status_check_and_log(
+ settings, self.phase, phase_retval)
+ if phase_retval != os.EX_OK:
+ writemsg("!!! post preinst failed; exiting.\n",
+ noiselevel=-1)
+ self.returncode = phase_retval
+
+ elif self.phase == "postinst":
+
+ if self.returncode == os.EX_OK:
+ portage._doebuild_exit_status_unlink(
+ settings.get("EBUILD_EXIT_STATUS_FILE"))
+ phase_retval = portage.spawn(
+ " ".join(portage._post_pkg_postinst_cmd(settings)),
+ settings, debug=debug, free=1, logfile=log_path)
+ phase_retval = portage._doebuild_exit_status_check_and_log(
+ settings, self.phase, phase_retval)
+ if phase_retval != os.EX_OK:
+ writemsg("!!! post postinst failed; exiting.\n",
+ noiselevel=-1)
+ self.returncode = phase_retval
+
class EbuildBinpkg(EbuildPhase):
"""
This assumes that src_install() has successfully completed.
__slots__ = ("find_blockers", "logger", "ldpath_mtimes",
"pkg", "pkg_count", "pkg_path", "pretend",
- "settings", "tree", "world_atom")
+ "scheduler", "settings", "tree", "world_atom")
def execute(self):
root_config = self.pkg.root_config
mytree=self.tree, mydbapi=root_config.trees[self.tree].dbapi,
vartree=root_config.trees["vartree"],
prev_mtimes=self.ldpath_mtimes,
+ scheduler=self.scheduler,
blockers=self.find_blockers)
if retval == os.EX_OK:
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=self.ldpath_mtimes, logger=self.logger,
pkg=self.pkg, pkg_count=self.pkg_count,
- pkg_path=self._pkg_path, settings=settings,
- tree=self._tree, world_atom=self.world_atom)
+ pkg_path=self._pkg_path, scheduler=self.scheduler,
+ settings=settings, tree=self._tree, world_atom=self.world_atom)
try:
retval = merge.execute()
class depgraph(object):
- pkg_tree_map = {
- "ebuild":"porttree",
- "binary":"bintree",
- "installed":"vartree"}
+ pkg_tree_map = RootConfig.pkg_tree_map
_mydbapi_keys = Package.metadata_keys
_fetch_log = EPREFIX + "/var/log/emerge-fetch.log"
class _iface_class(SlotObject):
- __slots__ = ("fetch", "register", "schedule", "unregister")
+ __slots__ = ("dblinkEbuildPhase", "fetch",
+ "register", "schedule", "unregister")
class _fetch_iface_class(SlotObject):
__slots__ = ("log_file", "schedule")
fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
schedule=self._schedule_fetch)
self._sched_iface = self._iface_class(
+ dblinkEbuildPhase=self._dblink_ebuild_phase,
fetch=fetch_iface, register=self._register,
schedule=self._schedule_wait, unregister=self._unregister)
return blocker_dblinks
+ def _dblink_pkg(self, pkg_dblink):
+ cpv = pkg_dblink.mycpv
+ type_name = RootConfig.tree_pkg_map[pkg_dblink.treetype]
+ root_config = self.trees[pkg_dblink.myroot]["root_config"]
+ installed = type_name == "installed"
+ return self._pkg(cpv, type_name, root_config, installed=installed)
+
+ def _append_to_log_path(self, log_path, msg):
+ f = open(log_path, 'a')
+ try:
+ f.write(msg)
+ finally:
+ f.close()
+
+ def _dblink_ebuild_phase(self,
+ pkg_dblink, pkg_dbapi, ebuild_path, phase):
+ """
+ Using this callback for merge phases allows the scheduler
+ to run while these phases execute asynchronously, and allows
+ the scheduler control output handling.
+ """
+
+ scheduler = self._sched_iface
+ settings = pkg_dblink.settings
+ pkg = self._dblink_pkg(pkg_dblink)
+ background = self._max_jobs > 1
+ log_path = settings.get("PORTAGE_LOG_FILE")
+
+ if phase == "preinst":
+ msg = ">>> Merging %s to %s\n" % (pkg.cpv, pkg.root)
+ portage.writemsg_stdout(msg)
+ if log_path is not None:
+ self._append_to_log_path(log_path, msg)
+
+ ebuild_phase = EbuildPhase(background=background,
+ pkg=pkg, phase=phase, scheduler=scheduler,
+ settings=settings, tree=pkg_dblink.treetype)
+ ebuild_phase.start()
+ ebuild_phase.wait()
+
+ if phase == "postinst" and \
+ ebuild_phase.returncode == os.EX_OK:
+ msg = ">>> %s %s\n" % (pkg.cpv, "merged.")
+ portage.writemsg_stdout(msg)
+ if log_path is not None:
+ self._append_to_log_path(log_path, msg)
+
+ return ebuild_phase.returncode
+
def _check_manifests(self):
# Verify all the manifests now so that the user is notified of failure
# as soon as possible.
if self._is_restart_scheduled():
self._set_max_jobs(1)
- while not self._failed_pkgs and \
- self._schedule():
+ merge_queue = self._task_queues.merge
+
+ while self._schedule():
self._poll_loop()
- while self._jobs:
+ while self._jobs or merge_queue:
self._poll_loop()
def _schedule_tasks(self):
mode=mystat.st_mode, stat_cached=mystat,
follow_links=False)
+def _post_pkg_preinst_cmd(mysettings):
+ """
+ Post phase logic and tasks that have been factored out of
+ ebuild.sh. Call preinst_mask last so that INSTALL_MASK can
+ can be used to wipe out any gmon.out files created during
+ previous functions (in case any tools were built with -pg
+ in CFLAGS).
+ """
+
+ portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
+ misc_sh_binary = os.path.join(portage_bin_path,
+ os.path.basename(MISC_SH_BINARY))
+
+ mysettings["EBUILD_PHASE"] = ""
+ myargs = [_shell_quote(misc_sh_binary),
+ "preinst_bsdflags",
+ "preinst_sfperms", "preinst_selinux_labels",
+ "preinst_suid_scan", "preinst_mask"]
+
+ return myargs
+
+def _post_pkg_postinst_cmd(mysettings):
+ """
+ Post phase logic and tasks that have been factored out of
+ build.sh.
+ """
+
+ portage_bin_path = mysettings["PORTAGE_BIN_PATH"]
+ misc_sh_binary = os.path.join(portage_bin_path,
+ os.path.basename(MISC_SH_BINARY))
+
+ mysettings["EBUILD_PHASE"] = ""
+ myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"]
+
+ return myargs
+
def _spawn_misc_sh(mysettings, commands, **kwargs):
"""
@param mysettings: the ebuild config
"errors (bug #200313)."
return msg
+def _doebuild_exit_status_check_and_log(settings, mydo, retval):
+ if retval != os.EX_OK:
+ return retval
+ msg = _doebuild_exit_status_check(mydo, settings)
+ if msg:
+ retval = 1
+ from textwrap import wrap
+ from portage.elog.messages import eerror
+ for l in wrap(msg, 72):
+ eerror(l, phase=mydo, key=settings.mycpv)
+ return retval
+
def _doebuild_exit_status_unlink(exit_status_file):
"""
Double check to make sure it really doesn't exist
elif mydo == "preinst":
phase_retval = spawn(
_shell_quote(ebuild_sh_binary) + " " + mydo,
- mysettings, debug=debug, free=1, logfile=logfile)
+ mysettings, debug=debug, free=1, logfile=logfile,
+ fd_pipes=fd_pipes, returnpid=returnpid)
+
+ if returnpid:
+ return phase_retval
+
phase_retval = exit_status_check(phase_retval)
if phase_retval == os.EX_OK:
- # Post phase logic and tasks that have been factored out of
- # ebuild.sh. Call preinst_mask last so that INSTALL_MASK can
- # can be used to wipe out any gmon.out files created during
- # previous functions (in case any tools were built with -pg
- # in CFLAGS).
- myargs = [_shell_quote(misc_sh_binary),
- "preinst_bsdflags",
- "preinst_sfperms", "preinst_selinux_labels",
- "preinst_suid_scan", "preinst_mask"]
_doebuild_exit_status_unlink(
mysettings.get("EBUILD_EXIT_STATUS_FILE"))
- mysettings["EBUILD_PHASE"] = ""
- phase_retval = spawn(" ".join(myargs),
+ phase_retval = spawn(
+ " ".join(_post_pkg_preinst_cmd(mysettings)),
mysettings, debug=debug, free=1, logfile=logfile)
phase_retval = exit_status_check(phase_retval)
if phase_retval != os.EX_OK:
elif mydo == "postinst":
phase_retval = spawn(
_shell_quote(ebuild_sh_binary) + " " + mydo,
- mysettings, debug=debug, free=1, logfile=logfile)
+ mysettings, debug=debug, free=1, logfile=logfile,
+ fd_pipes=fd_pipes, returnpid=returnpid)
+
+ if returnpid:
+ return phase_retval
+
phase_retval = exit_status_check(phase_retval)
if phase_retval == os.EX_OK:
- # Post phase logic and tasks that have been factored out of
- # ebuild.sh.
- myargs = [_shell_quote(misc_sh_binary), "postinst_bsdflags"]
_doebuild_exit_status_unlink(
mysettings.get("EBUILD_EXIT_STATUS_FILE"))
- mysettings["EBUILD_PHASE"] = ""
- phase_retval = spawn(" ".join(myargs),
+ phase_retval = spawn(" ".join(_post_pkg_postinst_cmd(mysettings)),
mysettings, debug=debug, free=1, logfile=logfile)
phase_retval = exit_status_check(phase_retval)
if phase_retval != os.EX_OK:
elif mydo in ("prerm", "postrm", "config", "info"):
retval = spawn(
_shell_quote(ebuild_sh_binary) + " " + mydo,
- mysettings, debug=debug, free=1, logfile=logfile)
+ mysettings, debug=debug, free=1, logfile=logfile,
+ fd_pipes=fd_pipes, returnpid=returnpid)
+
+ if returnpid:
+ return retval
+
retval = exit_status_check(retval)
return retval
return newmtime
def merge(mycat, mypkg, pkgloc, infloc, myroot, mysettings, myebuild=None,
- mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None):
+ mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None,
+ scheduler=None):
if not os.access(myroot + EPREFIX_LSTRIP, os.W_OK):
writemsg("Permission denied: access('%s', W_OK)\n" %
(myroot + EPREFIX_LSTRIP), noiselevel=-1)
return errno.EACCES
mylink = dblink(mycat, mypkg, myroot, mysettings, treetype=mytree,
- vartree=vartree, blockers=blockers)
+ vartree=vartree, blockers=blockers, scheduler=scheduler)
return mylink.merge(pkgloc, infloc, myroot, myebuild,
mydbapi=mydbapi, prev_mtimes=prev_mtimes)
def _dblink(self, cpv):
category, pf = catsplit(cpv)
return dblink(category, pf, self.root,
- self.settings, vartree=self.vartree)
+ self.settings, vartree=self.vartree, treetype="vartree")
def removeFromContents(self, pkg, paths, relative_paths=True):
"""
}
def __init__(self, cat, pkg, myroot, mysettings, treetype=None,
- vartree=None, blockers=None):
+ vartree=None, blockers=None, scheduler=None):
"""
Creates a DBlink object for a given CPV.
The given CPV may not be present in the database already.
vartree = db[myroot]["vartree"]
self.vartree = vartree
self._blockers = blockers
+ self._scheduler = scheduler
self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH))
self.dbcatdir = self.dbroot+"/"+cat
if cur_cpv == self.mycpv:
continue
others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
- self.vartree.root, self.settings, vartree=self.vartree))
+ self.vartree.root, self.settings, vartree=self.vartree,
+ treetype="vartree"))
+
retval = self._security_check([self] + others_in_slot)
if retval:
return retval
uid=portage_uid, gid=portage_gid, mode=070, mask=0)
builddir_lock = None
catdir_lock = None
+ scheduler = self._scheduler
retval = -1
try:
if myebuildpath:
unlockdir(catdir_lock)
finally:
catdir_lock = None
- # Eventually, we'd like to pass in the saved ebuild env here...
- retval = doebuild(myebuildpath, "prerm", self.myroot,
- self.settings, cleanup=cleanup, use_cache=0,
- mydbapi=self.vartree.dbapi, tree="vartree",
- vartree=self.vartree)
+
+ prepare_build_dirs(self.myroot, self.settings, 1)
+
+ if scheduler is None:
+ retval = doebuild(myebuildpath, ebuild_phase, self.myroot,
+ self.settings, cleanup=cleanup, use_cache=0,
+ mydbapi=self.vartree.dbapi, tree=self.treetype,
+ vartree=self.vartree)
+ else:
+ retval = scheduler.dblinkEbuildPhase(
+ self, self.vartree.dbapi, myebuildpath, ebuild_phase)
+
# XXX: Decide how to handle failures here.
if retval != os.EX_OK:
writemsg("!!! FAILED prerm: %s\n" % retval, noiselevel=-1)
if myebuildpath:
ebuild_phase = "postrm"
- retval = doebuild(myebuildpath, "postrm", self.myroot,
- self.settings, use_cache=0, tree="vartree",
- mydbapi=self.vartree.dbapi, vartree=self.vartree)
+ if scheduler is None:
+ retval = doebuild(myebuildpath, ebuild_phase, self.myroot,
+ self.settings, use_cache=0, tree=self.treetype,
+ mydbapi=self.vartree.dbapi, vartree=self.vartree)
+ else:
+ retval = scheduler.dblinkEbuildPhase(
+ self, self.vartree.dbapi, myebuildpath, ebuild_phase)
# XXX: Decide how to handle failures here.
if retval != os.EX_OK:
continue
others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
self.vartree.root, self.settings,
- vartree=self.vartree))
+ vartree=self.vartree, treetype="vartree"))
+
dest_root = normalize_path(self.vartree.root).rstrip(os.path.sep) + \
os.path.sep
dest_root_len = len(dest_root) - 1
# we need it to have private ${T} etc... for things like elog.
others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
self.vartree.root, config(clone=self.settings),
- vartree=self.vartree))
+ vartree=self.vartree, treetype="vartree",
+ scheduler=self._scheduler))
+
retval = self._security_check(others_in_slot)
if retval:
return retval
if collision_protect:
return 1
- writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot))
-
# The merge process may move files out of the image directory,
# which causes invalidation of the .installed flag.
try:
self.delete()
ensure_dirs(self.dbtmpdir)
+ scheduler = self._scheduler
+
# run preinst script
- a = doebuild(myebuild, "preinst", destroot, self.settings,
- use_cache=0, tree=self.treetype, mydbapi=mydbapi,
- vartree=self.vartree)
+ if scheduler is None:
+ writemsg_stdout(">>> Merging %s to %s\n" % (self.mycpv, destroot))
+ a = doebuild(myebuild, "preinst", destroot, self.settings,
+ use_cache=0, tree=self.treetype, mydbapi=mydbapi,
+ vartree=self.vartree)
+ else:
+ a = scheduler.dblinkEbuildPhase(
+ self, mydbapi, myebuild, "preinst")
# XXX: Decide how to handle failures here.
if a != os.EX_OK:
self.settings["PORTAGE_UPDATE_ENV"] = \
os.path.join(self.dbpkgdir, "environment.bz2")
self.settings.backup_changes("PORTAGE_UPDATE_ENV")
- a = doebuild(myebuild, "postinst", destroot, self.settings, use_cache=0,
- tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree)
- self.settings.pop("PORTAGE_UPDATE_ENV", None)
+ try:
+ if scheduler is None:
+ a = doebuild(myebuild, "postinst", destroot, self.settings,
+ use_cache=0, tree=self.treetype, mydbapi=mydbapi,
+ vartree=self.vartree)
+ if a == os.EX_OK:
+ writemsg_stdout(">>> %s %s\n" % (self.mycpv, "merged."))
+ else:
+ a = scheduler.dblinkEbuildPhase(
+ self, mydbapi, myebuild, "postinst")
+ finally:
+ self.settings.pop("PORTAGE_UPDATE_ENV", None)
# XXX: Decide how to handle failures here.
if a != os.EX_OK:
target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes,
contents=contents, env=self.settings.environ())
- writemsg_stdout(">>> %s %s\n" % (self.mycpv,"merged."))
return os.EX_OK
def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime):
projects / portage.git / commitdiff