)
abort="no"
i=0
+ local grep_cmd=grep
+ [[ $PORTAGE_LOG_FILE = *.gz ]] && grep_cmd=zgrep
while [[ -n ${msgs[${i}]} ]] ; do
m=${msgs[$((i++))]}
# force C locale to work around slow unicode locales #160234
- f=$(LC_ALL=C grep "${m}" "${PORTAGE_LOG_FILE}")
+ f=$(LC_ALL=C $grep_cmd "${m}" "${PORTAGE_LOG_FILE}")
if [[ -n ${f} ]] ; then
vecho -ne '\a\n'
eqawarn "QA Notice: Package has poor programming practices which may compile"
abort="yes"
fi
done
+ local cat_cmd=cat
+ [[ $PORTAGE_LOG_FILE = *.gz ]] && cat_cmd=zcat
[[ $reset_debug = 1 ]] && set -x
- f=$(cat "${PORTAGE_LOG_FILE}" | \
+ f=$($cat_cmd "${PORTAGE_LOG_FILE}" | \
"${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/check-implicit-pointer-usage.py || die "check-implicit-pointer-usage.py failed")
if [[ -n ${f} ]] ; then
The \fICOLLISION_IGNORE\fR variable can be used to selectively disable this
feature. Also see the related \fIprotect\-owned\fR feature.
.TP
+.B compress\-build\-logs
+The causes all build logs to be compressed while they are being written.
+Log file names have an extension that is appropriate for the compression
+type. Currently, only \fBgzip\fR(1) compression is supported, so build
+logs will have a '.gz' extension when this feature is enabled.
+.TP
.B digest
Autogenerate digests for packages when running the \fBemerge\fR(1) command. If
the \fIassume\-digests\fR feature is also enabled then existing SRC_URI digests
phase = self.phase
for line in lines:
eerror(line, phase=phase, key=self.settings.mycpv, out=out)
- logfile = self.logfile
- if logfile is None:
- logfile = self.settings.get("PORTAGE_LOG_FILE")
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
- if not self.background:
- writemsg_stdout(msg, noiselevel=-1)
- if logfile is not None:
- log_file = codecs.open(_unicode_encode(logfile,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'],
- errors='backslashreplace')
- log_file.write(msg)
- log_file.close()
+ self.scheduler.output(msg,
+ log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
"_image_dir", "_infloc", "_pkg_path", "_tree", "_verify")
def _writemsg_level(self, msg, level=0, noiselevel=0):
-
- if not self.background:
- portage.util.writemsg_level(msg,
- level=level, noiselevel=noiselevel)
-
- log_path = self.settings.get("PORTAGE_LOG_FILE")
- if log_path is not None:
- f = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'], errors='replace')
- try:
- f.write(msg)
- finally:
- f.close()
+ self.scheduler.output(msg, level=level, noiselevel=noiselevel,
+ log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _start(self):
verifier = None
if self._verify:
- logfile = None
- if self.background:
- logfile = self.settings.get("PORTAGE_LOG_FILE")
+ logfile = self.settings.get("PORTAGE_LOG_FILE")
verifier = BinpkgVerifier(background=self.background,
logfile=logfile, pkg=self.pkg)
self._start_task(verifier, self._verifier_exit)
import portage
from portage import os
from portage import _encodings
+from portage import _unicode_decode
from portage import _unicode_encode
+from portage import StringIO
from portage.package.ebuild.fetch import _checksum_failure_temp_file
import codecs
rval = os.EX_OK
stdout_orig = sys.stdout
stderr_orig = sys.stderr
- log_file = None
- if self.background and self.logfile is not None:
- if sys.hexversion >= 0x3000000:
- # Since we are replacing the sys.std* streams,
- # we need to use the normal open() function
- # so that we get the right class (otherwise our
- # code that expects the 'buffer' attribute
- # will break).
- log_file = open(_unicode_encode(self.logfile,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'],
- errors='backslashreplace')
- else:
- # For python2, sys.std* are expected to be binary streams.
- log_file = open(_unicode_encode(self.logfile,
- encoding=_encodings['fs'], errors='strict'),
- mode='ab')
+ out = portage.StringIO()
try:
- if log_file is not None:
- sys.stdout = log_file
- sys.stderr = log_file
+ sys.stdout = out
+ sys.stderr = out
try:
bintree.digestCheck(pkg)
except portage.exception.FileNotFound:
if rval == os.EX_OK:
# If this was successful, discard the log here since otherwise
# we'll get multiple logs for the same package.
- if log_file is not None:
+ if self.logfile is not None:
try:
os.unlink(self.logfile)
except OSError:
finally:
sys.stdout = stdout_orig
sys.stderr = stderr_orig
- if log_file is not None:
- log_file.close()
+
+ msg = _unicode_decode(out.getvalue(),
+ encoding=_encodings['content'], errors='replace')
+ if msg:
+ self.scheduler.output(msg, log_path=self.logfile)
self.returncode = rval
self.wait()
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
-
- log_path = self.settings.get("PORTAGE_LOG_FILE")
- if log_path is not None:
- log_file = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'], errors='replace')
- try:
- log_file.write(msg)
- finally:
- log_file.close()
-
- if not self.background:
- portage.writemsg_stdout(msg, noiselevel=-1)
+ self.scheduler.output(msg,
+ log_path=self.settings.get("PORTAGE_LOG_FILE"))
packager = EbuildBinpkg(background=self.background, pkg=self.pkg,
scheduler=self.scheduler, settings=self.settings)
out = portage.StringIO()
for line in lines:
eerror(line, phase="unpack", key=self.pkg.cpv, out=out)
- logfile = self.logfile
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
- if not self.background:
- writemsg_stdout(msg, noiselevel=-1)
- if logfile is not None:
- log_file = codecs.open(_unicode_encode(logfile,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'],
- errors='backslashreplace')
- log_file.write(msg)
- log_file.close()
+ self.scheduler.output(msg, log_path=self.logfile)
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self.phase == "install":
out = portage.StringIO()
- log_path = self.settings.get("PORTAGE_LOG_FILE")
- log_file = None
- if log_path is not None:
- log_file = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'], errors='replace')
- try:
- _check_build_log(self.settings, out=out)
- msg = _unicode_decode(out.getvalue(),
- encoding=_encodings['content'], errors='replace')
- if msg:
- if not self.background:
- writemsg_stdout(msg, noiselevel=-1)
- if log_file is not None:
- log_file.write(msg)
- finally:
- if log_file is not None:
- log_file.close()
+ _check_build_log(self.settings, out=out)
+ msg = _unicode_decode(out.getvalue(),
+ encoding=_encodings['content'], errors='replace')
+ self.scheduler.output(msg,
+ log_path=self.settings.get("PORTAGE_LOG_FILE"))
if fail:
self._die_hooks()
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
- if not self.background:
- writemsg_stdout(msg, noiselevel=-1)
- log_path = self.settings.get("PORTAGE_LOG_FILE")
- if log_path is not None:
- log_file = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'], errors='replace')
- log_file.write(msg)
- log_file.close()
+ self.scheduler.output(msg,
+ log_path=self.settings.get("PORTAGE_LOG_FILE"))
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2010 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import portage
self._max_jobs = max_jobs
self._max_load = max_load
- self._sched_iface = self._sched_iface_class(
- register=self._register,
- schedule=self._schedule_wait,
- unregister=self._unregister)
self._valid_pkgs = set()
self._cp_set = set()
portage.util.writemsg_level(msg,
level=level, noiselevel=noiselevel)
else:
- if not background:
- portage.util.writemsg_level(msg,
- level=level, noiselevel=noiselevel)
-
- f = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'], errors='replace')
- try:
- f.write(msg)
- finally:
- f.close()
-
+ self.scheduler.output(msg, level=level, noiselevel=noiselevel)
-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2010 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+import gzip
import logging
import select
import time
+from portage import _encodings
+from portage import _unicode_encode
from portage import os
from portage.util import writemsg_level
class PollScheduler(object):
class _sched_iface_class(SlotObject):
- __slots__ = ("register", "schedule", "unregister")
+ __slots__ = ("output", "register", "schedule", "unregister")
def __init__(self):
self._max_jobs = 1
self._event_handler_id = 0
self._poll_obj = create_poll_instance()
self._scheduling = False
+ self._background = False
+ self.sched_iface = self._sched_iface_class(
+ output=self._task_output,
+ register=self._register,
+ schedule=self._schedule_wait,
+ unregister=self._unregister)
def _schedule(self):
"""
return event_handled
+ def _task_output(self, msg, log_path=None, level=0, noiselevel=-1):
+ """
+ Output msg to stdout if not self._background. If log_path
+ is not None then append msg to the log (appends with
+ compression if the filename extension of log_path
+ corresponds to a supported compression type).
+ """
+
+ if not self._background:
+ writemsg_level(msg, level=level, noiselevel=noiselevel)
+
+ if log_path is not None:
+ f = open(_unicode_encode(log_path,
+ encoding=_encodings['fs'], errors='strict'),
+ mode='ab')
+
+ if log_path.endswith('.gz'):
+ # NOTE: The empty filename argument prevents us from triggering
+ # a bug in python3 which causes GzipFile to raise AttributeError
+ # if fileobj.name is bytes instead of unicode.
+ f = gzip.GzipFile(filename='', mode='ab', fileobj=f)
+
+ f.write(_unicode_encode(msg))
+ f.close()
+
_can_poll_device = None
def can_poll_device():
-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2010 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from _emerge.PollScheduler import PollScheduler
self._max_jobs = max_jobs
self._max_load = max_load
- self.sched_iface = self._sched_iface_class(
- register=self._register,
- schedule=self._schedule_wait,
- unregister=self._unregister)
self._queues = []
self._schedule_listeners = []
from __future__ import print_function
import codecs
+import gzip
import logging
import shutil
import sys
class _iface_class(SlotObject):
__slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge",
- "dblinkElog", "dblinkEmergeLog", "fetch", "register", "schedule",
+ "dblinkElog", "dblinkEmergeLog", "fetch",
+ "output", "register", "schedule",
"scheduleSetup", "scheduleUnpack", "scheduleYield",
"unregister")
dblinkDisplayMerge=self._dblink_display_merge,
dblinkElog=self._dblink_elog,
dblinkEmergeLog=self._dblink_emerge_log,
- fetch=fetch_iface, register=self._register,
+ fetch=fetch_iface, output=self._task_output,
+ register=self._register,
schedule=self._schedule_wait,
scheduleSetup=self._schedule_setup,
scheduleUnpack=self._schedule_unpack,
installed = type_name == "installed"
return self._pkg(cpv, type_name, root_config, installed=installed)
- def _append_to_log_path(self, log_path, msg):
-
- f = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'],
- errors='backslashreplace')
- try:
- f.write(_unicode_decode(msg,
- encoding=_encodings['content'], errors='replace'))
- finally:
- f.close()
-
def _dblink_elog(self, pkg_dblink, phase, func, msgs):
log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE")
- background = self._background
out = StringIO()
for msg in msgs:
out_str = out.getvalue()
- if not background:
- portage.util.writemsg_stdout(out_str, noiselevel=-1)
-
- if log_path is not None:
- self._append_to_log_path(log_path, out_str)
+ self._task_output(out_str, log_path=log_path)
def _dblink_emerge_log(self, msg):
self._logger.log(msg)
portage.util.writemsg_level(msg,
level=level, noiselevel=noiselevel)
else:
- if not background:
- portage.util.writemsg_level(msg,
- level=level, noiselevel=noiselevel)
- self._append_to_log_path(log_path, msg)
+ self._task_output(msg, log_path=log_path)
def _dblink_ebuild_phase(self,
pkg_dblink, pkg_dbapi, ebuild_path, phase):
log_path = self._locate_failure_log(failed_pkg)
if log_path is not None:
try:
- log_file = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='r', encoding=_encodings['content'], errors='replace')
+ log_file = open(_unicode_encode(log_path,
+ encoding=_encodings['fs'], errors='strict'), mode='rb')
except IOError:
pass
+ else:
+ if log_path.endswith('.gz'):
+ log_file = gzip.GzipFile(filename='',
+ mode='rb', fileobj=log_file)
if log_file is not None:
try:
import sys
from portage.cache.mappings import slot_dict_class
import portage
+from portage import _encodings
+from portage import _unicode_encode
from portage import os
import fcntl
import errno
import array
+import gzip
class SpawnProcess(SubProcess):
fd_pipes[1] = slave_fd
fd_pipes[2] = slave_fd
- files.log = open(logfile, mode='ab')
+ files.log = open(_unicode_encode(logfile,
+ encoding=_encodings['fs'], errors='strict'), mode='ab')
+ if logfile.endswith('.gz'):
+ files.log = gzip.GzipFile(filename='', mode='ab',
+ fileobj=files.log)
+
portage.util.apply_secpass_permissions(logfile,
uid=portage.portage_uid, gid=portage.portage_gid,
mode=0o660)
fcntl.fcntl(files.stdout.fileno(),
fcntl.F_GETFL) ^ os.O_NONBLOCK)
- buf.tofile(files.log)
+ try:
+ buf.tofile(files.log)
+ except TypeError:
+ # array.tofile() doesn't work with GzipFile
+ files.log.write(buf.tostring())
files.log.flush()
else:
self._unregister()
"nofetch", "config", "info", "other")
SUPPORTED_FEATURES = frozenset([
"assume-digests", "buildpkg", "buildsyspkg", "ccache",
- "collision-protect", "digest", "distcc", "distlocks",
+ "collision-protect", "compress-build-logs",
+ "digest", "distcc", "distlocks",
"fakeroot", "fail-clean", "fixpackages", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
import codecs
import errno
+import gzip
from itertools import chain
import logging
import os as _os
if logfile is None:
return
try:
- f = codecs.open(_unicode_encode(logfile,
- encoding=_encodings['fs'], errors='strict'),
- mode='r', encoding=_encodings['content'], errors='replace')
+ f = open(_unicode_encode(logfile, encoding=_encodings['fs'],
+ errors='strict'), mode='rb')
except EnvironmentError:
return
+ if logfile.endswith('.gz'):
+ f = gzip.GzipFile(filename='', mode='rb', fileobj=f)
+
am_maintainer_mode = []
bash_command_not_found = []
bash_command_not_found_re = re.compile(
try:
for line in f:
+ line = _unicode_decode(line)
if am_maintainer_mode_re.search(line) is not None and \
am_maintainer_mode_exclude_re.search(line) is None:
am_maintainer_mode.append(line.rstrip("\n"))
__all__ = ['prepare_build_dirs']
-import codecs
import errno
+import gzip
import shutil
import stat
import time
if background and log_path is not None:
try:
- log_file = codecs.open(_unicode_encode(log_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='a', encoding=_encodings['content'], errors='replace')
+ log_file = open(_unicode_encode(log_path,
+ encoding=_encodings['fs'], errors='strict'), mode='ab')
except IOError:
def write(msg):
pass
else:
+ if log_path.endswith('.gz'):
+ log_file = gzip.GzipFile(filename='',
+ mode='ab', fileobj=log_file)
def write(msg):
- log_file.write(_unicode_decode(msg))
+ log_file.write(_unicode_encode(msg))
log_file.flush()
try:
writemsg(_("!!! Disabling logging.\n"), noiselevel=-1)
while "PORT_LOGDIR" in mysettings:
del mysettings["PORT_LOGDIR"]
+
+ compress_log_ext = ''
+ if 'compress-build-logs' in mysettings.features:
+ compress_log_ext = '.gz'
+
if "PORT_LOGDIR" in mysettings and \
os.access(mysettings["PORT_LOGDIR"], os.W_OK):
logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid")
if "split-log" in mysettings.features:
mysettings["PORTAGE_LOG_FILE"] = os.path.join(
- mysettings["PORT_LOGDIR"], "build", "%s/%s:%s.log" % \
- (mysettings["CATEGORY"], mysettings["PF"], logid_time))
+ mysettings["PORT_LOGDIR"], "build", "%s/%s:%s.log%s" % \
+ (mysettings["CATEGORY"], mysettings["PF"], logid_time,
+ compress_log_ext))
else:
mysettings["PORTAGE_LOG_FILE"] = os.path.join(
- mysettings["PORT_LOGDIR"], "%s:%s:%s.log" % \
- (mysettings["CATEGORY"], mysettings["PF"], logid_time))
+ mysettings["PORT_LOGDIR"], "%s:%s:%s.log%s" % \
+ (mysettings["CATEGORY"], mysettings["PF"], logid_time,
+ compress_log_ext))
ensure_dirs(os.path.dirname(mysettings["PORTAGE_LOG_FILE"]))
# current policy will allow it to work when a pty is available, but
# not through a normal pipe. See bug #162404.
mysettings["PORTAGE_LOG_FILE"] = os.path.join(
- mysettings["T"], "build.log")
+ mysettings["T"], "build.log%s" % compress_log_ext)
projects / portage.git / commitdiff