From 1e103f58a0e5799ff902533c3da0045d67e047c4 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sun, 27 Jul 2008 09:31:10 +0000 Subject: [PATCH] Merged from trunk 11201:11206 | 11202 | Fix AsynchronousTask.removeStartListener() and | | zmedico | removeExitListener() to return early if there are no | | | listeners. | | 11203 | In PollScheduler._poll(), add a call to _schedule() in order | | zmedico | to ensure that event handlers are registered if possible. If | | | not, raise StopIteration in order to avoid endless blocking | | | in a poll call with no file descriptors registered. Add | | | StopIteration exception handling for all | | | PollScheduler._poll._next_poll_event() calls. | | 11204 | Schedule ebuild pkg_setup phases on the merge queue, in | | zmedico | order to serialize unsandboxed access to the live | | | filesystem. | | 11205 | Bug #233077 - Add QA check which verifies that LDFLAGS are | | zmedico | respected. For now this only works when LDFLAGS contains | | | --hash-style=gnu since in this case the the elf files should | | | not contain .hash sections and we can use scanelf to check | | | whether or not the those sections exist. This adds a | | | QA_DT_HASH variable that is analogous to existing QA control | | | variables documented in the ebuild.5 man page. There is also | | | a complementary QA_STRICT_DT_HASH variable that can be set | | | in make.conf in order to ignore QA_DT_HASH settings in | | | ebuilds. Thanks to Arfrever Frehtes Taifersar Arahesis for | | | this patch which I've tweaked in just a few minor ways. | | 11206 | Temporarily `set -o noglob` when splitting QA_DT_HASH. | | zmedico | | svn path=/main/branches/prefix/; revision=11207 --- bin/misc-functions.sh | 35 +++++++++++++ man/ebuild.5 | 5 ++ man/make.conf.5 | 4 ++ pym/_emerge/__init__.py | 108 ++++++++++++++++++++++++++++------------ 4 files changed, 121 insertions(+), 31 deletions(-) diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 21bce35cb..12cca1401 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -60,6 +60,7 @@ install_qa_check() { if type -P scanelf > /dev/null && ! hasq binchecks ${RESTRICT}; then local qa_var insecure_rpath=0 tmp_quiet=${PORTAGE_QUIET} + local f x # display warnings when using stricter because we die afterwards if has stricter ${FEATURES} ; then @@ -158,6 +159,40 @@ install_qa_check() { sleep 1 fi + # Check for files built without respecting LDFLAGS + if [[ "${LDFLAGS}" == *--hash-style=gnu* ]] && [[ "${PN}" != *-bin ]] ; then + f=$(scanelf -qyRF '%k %p' -k .hash "${D}" | sed -e "s:\.hash ::") + if [[ -n ${f} ]] ; then + echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log + if [ "${QA_STRICT_DT_HASH-unset}" == unset ] ; then + if [[ ${#QA_DT_HASH[@]} -gt 1 ]] ; then + for x in "${QA_DT_HASH[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log + done + else + local shopts=$- + set -o noglob + for x in ${QA_DT_HASH} ; do + sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log + done + set +o noglob + set -${shopts} + fi + fi + sed -e "/^\$/d" -e "s#^#/#" -i "${T}"/scanelf-ignored-LDFLAGS.log + f=$(<"${T}"/scanelf-ignored-LDFLAGS.log) + if [[ -n ${f} ]] ; then + vecho -ne '\a\n' + eqawarn "QA Notice: Files built without respecting LDFLAGS have been detected" + eqawarn " Please include this file in your report:" + eqawarn " ${T}/scanelf-ignored-LDFLAGS.log" + eqawarn "${f}" + vecho -ne '\a\n' + sleep 1 + fi + fi + fi + # Save NEEDED information after removing self-contained providers scanelf -qyRF '%a;%p;%S;%r;%n' "${D}" | { while IFS= read l; do arch=${l%%;*}; l=${l#*;} diff --git a/man/ebuild.5 b/man/ebuild.5 index b99be8cb9..d4af35d17 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -438,6 +438,11 @@ stack (i.e. not those marked to need it which in fact do not). This should contain a list of file paths, relative to the image directory, of files that contain writable and executable segments. These are rare. The paths may contain regular expressions. +.TP +\fBQA_DT_HASH\fR +This should contain a list of file paths, relative to the image directory, of +files that contain .hash sections. The paths may contain regular expressions +with escape\-quoted special characters. .SH "PORTAGE DECLARATIONS" .TP .B inherit diff --git a/man/make.conf.5 b/man/make.conf.5 index 4c291c0da..b1e0693f5 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -552,6 +552,10 @@ settings from ebuilds. See also \fBebuild\fR(5). Set this to cause portage to ignore any \fIQA_TEXTREL\fR override settings from ebuilds. See also \fBebuild\fR(5). .TP +\fBQA_STRICT_DT_HASH = \fI"set"\fR +Set this to cause portage to ignore any \fIQA_DT_HASH\fR override +settings from ebuilds. See also \fBebuild\fR(5). +.TP .B RESUMECOMMAND This variable contains the command used for resuming package sources that have been partially downloaded. It should be defined using the same format diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 2fbc8b4e2..18cb2cf18 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1669,6 +1669,8 @@ class AsynchronousTask(SlotObject): self._start_listeners.append(f) def removeStartListener(self, f): + if self._start_listeners is None: + return self._start_listeners.remove(f) def _start_hook(self): @@ -1688,6 +1690,8 @@ class AsynchronousTask(SlotObject): self._exit_listeners.append(f) def removeExitListener(self, f): + if self._exit_listeners is None: + return self._exit_listeners.remove(f) def _wait_hook(self): @@ -2580,19 +2584,16 @@ class EbuildBuild(CompositeTask): class EbuildExecuter(CompositeTask): - __slots__ = ("pkg", "scheduler", "settings") + __slots__ = ("pkg", "scheduler", "settings") + ("_tree",) - _phases = ("setup", "unpack", "compile", "test", "install") + _phases = ("unpack", "compile", "test", "install") def _start(self): + self._tree = "porttree" pkg = self.pkg - scheduler = self.scheduler - tree = "porttree" - settings = self.settings - phase = "clean" clean_phase = EbuildPhase(background=self.background, pkg=pkg, phase=phase, - scheduler=scheduler, settings=settings, tree=tree) + scheduler=self.scheduler, settings=self.settings, tree=self._tree) self._start_task(clean_phase, self._clean_phase_exit) def _clean_phase_exit(self, clean_phase): @@ -2603,19 +2604,32 @@ class EbuildExecuter(CompositeTask): pkg = self.pkg scheduler = self.scheduler - tree = "porttree" settings = self.settings cleanup = 1 # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(pkg.root, settings, cleanup) - ebuild_phases = TaskSequence(scheduler=scheduler) + setup_phase = EbuildPhase(background=self.background, + pkg=pkg, phase="setup", scheduler=scheduler, + settings=settings, tree=self._tree) + + setup_phase.addExitListener(self._setup_exit) + self._current_task = setup_phase + self.scheduler.scheduleSetup(setup_phase) + + def _setup_exit(self, setup_phase): + + if self._default_exit(setup_phase) != os.EX_OK: + self.wait() + return + + ebuild_phases = TaskSequence(scheduler=self.scheduler) for phase in self._phases: ebuild_phases.add(EbuildPhase(background=self.background, - pkg=pkg, phase=phase, scheduler=scheduler, - settings=settings, tree=tree)) + pkg=self.pkg, phase=phase, scheduler=self.scheduler, + settings=self.settings, tree=self._tree)) self._start_task(ebuild_phases, self._default_final_exit) @@ -3148,11 +3162,13 @@ class Binpkg(CompositeTask): settings.backup_changes("PORTAGE_BINPKG_FILE") phase = "setup" - ebuild_phase = EbuildPhase(background=self.background, + setup_phase = EbuildPhase(background=self.background, pkg=self.pkg, phase=phase, scheduler=self.scheduler, settings=settings, tree=self._tree) - self._start_task(ebuild_phase, self._setup_exit) + setup_phase.addExitListener(self._setup_exit) + self._current_task = setup_phase + self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: @@ -8282,7 +8298,16 @@ class PollScheduler(object): """ All poll() calls pass through here. The poll events are added directly to self._poll_event_queue. - """ + In order to avoid endless blocking, this raises + StopIteration if timeout is None and there are + no file descriptors to poll. + """ + if not self._poll_event_handlers: + self._schedule() + if timeout is None and \ + not self._poll_event_handlers: + raise StopIteration( + "timeout is None and there are no poll() event handlers") self._poll_event_queue.extend(self._poll_obj.poll(timeout)) def _next_poll_event(self, timeout=None): @@ -8290,7 +8315,9 @@ class PollScheduler(object): Since the _schedule_wait() loop is called by event handlers from _poll_loop(), maintain a central event queue for both of them to share events from a single - poll() call. + poll() call. In order to avoid endless blocking, this + raises StopIteration if timeout is None and there are + no file descriptors to poll. """ if not self._poll_event_queue: self._poll(timeout) @@ -8301,10 +8328,13 @@ class PollScheduler(object): event_handlers = self._poll_event_handlers event_handled = False - while event_handlers: - f, event = self._next_poll_event() - handler, reg_id = event_handlers[f] - handler(f, event) + try: + while event_handlers: + f, event = self._next_poll_event() + handler, reg_id = event_handlers[f] + handler(f, event) + event_handled = True + except StopIteration: event_handled = True if not event_handled: @@ -8327,10 +8357,13 @@ class PollScheduler(object): if not self._poll_event_queue: self._poll(0) - while event_handlers and self._poll_event_queue: - f, event = self._next_poll_event() - handler, reg_id = event_handlers[f] - handler(f, event) + try: + while event_handlers and self._poll_event_queue: + f, event = self._next_poll_event() + handler, reg_id = event_handlers[f] + handler(f, event) + events_handled += 1 + except StopIteration: events_handled += 1 return bool(events_handled) @@ -8370,10 +8403,13 @@ class PollScheduler(object): if isinstance(wait_ids, int): wait_ids = frozenset([wait_ids]) - while wait_ids.intersection(handler_ids): - f, event = self._next_poll_event() - handler, reg_id = event_handlers[f] - handler(f, event) + try: + while wait_ids.intersection(handler_ids): + f, event = self._next_poll_event() + handler, reg_id = event_handlers[f] + handler(f, event) + event_handled = True + except StopIteration: event_handled = True return event_handled @@ -8710,7 +8746,7 @@ class Scheduler(PollScheduler): class _iface_class(SlotObject): __slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge", "dblinkElog", "fetch", "register", "schedule", - "scheduleYield", "unregister") + "scheduleSetup", "scheduleYield", "unregister") class _fetch_iface_class(SlotObject): __slots__ = ("log_file", "schedule") @@ -8782,7 +8818,9 @@ class Scheduler(PollScheduler): dblinkDisplayMerge=self._dblink_display_merge, dblinkElog=self._dblink_elog, fetch=fetch_iface, register=self._register, - schedule=self._schedule_wait, scheduleYield=self._schedule_yield, + schedule=self._schedule_wait, + scheduleSetup=self._schedule_setup, + scheduleYield=self._schedule_yield, unregister=self._unregister) self._task_queues = self._task_queues_class() @@ -8961,6 +8999,14 @@ class Scheduler(PollScheduler): """ self._task_queues.fetch.addFront(fetcher) + def _schedule_setup(self, setup_phase): + """ + Schedule a setup phase on the merge queue, in order to + serialize unsandboxed access to the live filesystem. + """ + self._task_queues.merge.addFront(setup_phase) + self._schedule() + def _find_blockers(self, new_pkg): """ Returns a callable which should be called only when @@ -9588,9 +9634,7 @@ class Scheduler(PollScheduler): self._poll_loop() def _schedule_tasks(self): - remaining, state_change = self._schedule_tasks_imp() self._task_queues.merge.schedule() - self._status_display.display() # Cancel prefetchers if they're the only reason # the main poll loop is still running. @@ -9598,6 +9642,8 @@ class Scheduler(PollScheduler): not (self._jobs or self._task_queues.merge): self._task_queues.fetch.clear() + remaining, state_change = self._schedule_tasks_imp() + self._status_display.display() return remaining def _schedule_tasks_imp(self): -- 2.26.2