From: Fabian Groffen Date: Wed, 28 Oct 2009 18:36:02 +0000 (-0000) Subject: Merged from trunk -r14724:14735 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=8ac0f6b573f3562eb125a19e87f73ab842094d68;p=portage.git Merged from trunk -r14724:14735 | 14725 | Don't set mtime on downloaded metadata.dtd when using | | zmedico | python3, since the rfc822.parsedate() function is not | | | available. Thanks to Arfrever for reporting. | | 14728 | Use calendar.timegm instead of time.mktime, for correct | | zmedico | timezone handling. | | 14729 | Add a parsedate() function which emulates | | zmedico | rfc822.parsedate(), since python3 doesn't have it. | | 14730 | Add -R as a shortcut for --depclean. Thanks to Jonathan | | zmedico | Callen for the suggestion. | | 14731 | Take the -c option and make it mean --depclean instead of | | zmedico | --clean, since --clean is pretty useless anyway. | | 14732 | Add a reference to bug #141118 inside | | zmedico | _expand_new_virtuals(). | | 14733 | Revert the workaround for bug #288863. This will require a | | zmedico | dependency on >=sys-apps/sandbox-2.2. | | 14734 | Bug #290625 - Manually encode output to stdout in python3, | | zmedico | in order to avoid potential UnicodeEncodeError exceptions. | | 14735 | Bug #290428 - Update mtime of /var/db/pkg and category | | zmedico | subdirectories when stuff inside is modified, so that | | | consumers can use directory mtimes to validate caches. | svn path=/main/branches/prefix/; revision=14743 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index a79e9fd83..b91122fab 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -6,12 +6,6 @@ PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-@PORTAGE_BASE@/bin}" PORTAGE_PYM_PATH="${PORTAGE_PYM_PATH:-@PORTAGE_BASE@/pym}" -# Ensure that /dev/std* streams have appropriate sandbox permission for -# bug #288863. This can be removed after sandbox is fixed and portage -# depends on the fixed version. -export SANDBOX_WRITE="${SANDBOX_WRITE:+${SANDBOX_WRITE}:}/dev/stdout:/dev/stderr" -export SANDBOX_READ="${SANDBOX_READ:+${SANDBOX_READ}:}/dev/stdin" - # Don't use sandbox's BASH_ENV for new shells because it does # 'source /etc/profile' which can interfere with the build # environment by modifying our PATH. diff --git a/bin/repoman b/bin/repoman index 5169da449..18a37df7b 100755 --- a/bin/repoman +++ b/bin/repoman @@ -9,6 +9,7 @@ from __future__ import print_function +import calendar import codecs try: from subprocess import getstatusoutput as subprocess_getstatusoutput @@ -25,7 +26,11 @@ import sys import tempfile import time import platform -import urllib + +try: + from urllib.request import urlopen as urllib_request_urlopen +except ImportError: + from urllib import urlopen as urllib_request_urlopen from io import StringIO from itertools import chain @@ -774,6 +779,33 @@ for x in qacats: xmllint_capable = False metadata_dtd = os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd') +def parsedate(s): + """Parse a RFC 822 date and time string. + This is required for python3 compatibility, since the + rfc822.parsedate() function is not available.""" + + s_split = [] + for x in s.upper().split(): + for y in x.split(','): + if y: + s_split.append(y) + + if len(s_split) != 6: + return None + + # %a, %d %b %Y %H:%M:%S %Z + a, d, b, Y, H_M_S, Z = s_split + + # Convert month to integer, since strptime %w is locale-dependent. + month_map = {'JAN':1, 'FEB':2, 'MAR':3, 'APR':4, 'MAY':5, 'JUN':6, + 'JUL':7, 'AUG':8, 'SEP':9, 'OCT':10, 'NOV':11, 'DEC':12} + m = month_map.get(b) + if m is None: + return None + m = str(m).rjust(2, '0') + + return time.strptime(':'.join((Y, m, d, H_M_S)), '%Y:%m:%d:%H:%M:%S') + def fetch_metadata_dtd(): """ Fetch metadata.dtd if it doesn't exist or the ctime is older than @@ -803,10 +835,13 @@ def fetch_metadata_dtd(): "needs to be refetched, doing that now") print() try: - url_f = urllib.urlopen(metadata_dtd_uri) - last_modified = url_f.info().getdate('last-modified') + url_f = urllib_request_urlopen(metadata_dtd_uri) + msg_info = url_f.info() + last_modified = msg_info.get('last-modified') if last_modified is not None: - last_modified = time.mktime(last_modified) + last_modified = parsedate(last_modified) + if last_modified is not None: + last_modified = calendar.timegm(last_modified) metadata_dtd_tmp = "%s.%s" % (metadata_dtd, os.getpid()) try: diff --git a/man/emerge.1 b/man/emerge.1 index 346d444c2..b9380c7b0 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -93,7 +93,7 @@ option if you want to install a tbz2\fR. The packages are added to the \fBworld\fR file at the end, so that they are considered for later updating. .TP -.BR "\-\-clean " (\fB\-c\fR) +.BR \-\-clean Cleans up the system by examining the installed packages and removing older packages. This is accomplished by looking at each installed package and separating the installed versions by \fBslot\fR. Clean will \fBremove all but the most recently @@ -105,7 +105,7 @@ Run package specific actions needed to be executed after the emerge process has completed. This usually entails configuration file setup or other similar setups that the user may wish to run. .TP -.BR \-\-depclean +.BR "\-\-depclean (-c)" Cleans the system by removing packages that are not associated with explicitly merged packages. Depclean works by creating the full dependency tree from the @world set, diff --git a/pym/_emerge/JobStatusDisplay.py b/pym/_emerge/JobStatusDisplay.py index 1c80c5ffa..bcc682b23 100644 --- a/pym/_emerge/JobStatusDisplay.py +++ b/pym/_emerge/JobStatusDisplay.py @@ -17,6 +17,7 @@ import portage from portage import os from portage import _encodings from portage import _unicode_decode +from portage import _unicode_encode from portage.output import xtermTitle from _emerge.getloadavg import getloadavg @@ -75,11 +76,14 @@ class JobStatusDisplay(object): return sys.stdout def _write(self, s): - if sys.hexversion < 0x3000000 and isinstance(s, unicode): - # avoid potential UnicodeEncodeError - s = s.encode(_encodings['stdio'], 'backslashreplace') - self.out.write(s) - self.out.flush() + # avoid potential UnicodeEncodeError + s = _unicode_encode(s, + encoding=_encodings['stdio'], errors='backslashreplace') + out = self.out + if sys.hexversion >= 0x3000000: + out = out.buffer + out.write(s) + out.flush() def _init_term(self): """ diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index 9d036f05f..94dc17218 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -42,7 +42,7 @@ def help(myopts, havecolor=1): print(" emerge to display detailed help.") print() print(turquoise("Actions:")) - print(" "+green("--clean")+" ("+green("-c")+" short option)") + print(" "+green("--clean")) print(" Cleans the system by removing outdated packages which will not") print(" remove functionalities or prevent your system from working.") print(" The arguments can be in several different formats :") @@ -66,7 +66,7 @@ def help(myopts, havecolor=1): print(" emerge process has completed. This usually entails configuration") print(" file setup or other similar setups that the user may wish to run.") print() - print(" "+green("--depclean")) + print(" "+green("--depclean")+" ("+green("-c")+" short option)") paragraph = "Cleans the system by removing packages that are " + \ "not associated with explicitly merged packages. Depclean works " + \ diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index d69eb61a3..90e08aaa9 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -77,7 +77,8 @@ shortmapping={ "1":"--oneshot", "a":"--ask", "b":"--buildpkg", "B":"--buildpkgonly", -"c":"--clean", "C":"--unmerge", +"c":"--depclean", +"C":"--unmerge", "d":"--debug", "e":"--emptytree", "f":"--fetchonly", "F":"--fetch-all-uri", diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 3d431ff17..165db71b4 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -7671,8 +7671,10 @@ def dep_virtual(mysplit, mysettings): def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", trees=None, use_mask=None, use_force=None, **kwargs): - """Recursively expand new-style virtuals so as to collapse one or more - levels of indirection. In dep_zapdeps, new-style virtuals will be assigned + """ + In order to solve bug #141118, recursively expand new-style virtuals so + as to collapse one or more levels of indirection, generating an expanded + search space. In dep_zapdeps, new-style virtuals will be assigned zero cost regardless of whether or not they are currently installed. Virtual blockers are supported but only when the virtual expands to a single atom because it wouldn't necessarily make sense to block all the components diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 4d94398d8..210a1d553 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -53,6 +53,7 @@ import re, shutil, stat, errno, copy, subprocess import logging import os as _os import sys +import time import warnings try: @@ -1565,6 +1566,19 @@ class vardbapi(dbapi): rValue = _os.path.join(rValue, filename) return rValue + def _bump_mtime(self, cpv): + """ + This is called before an after any modifications, so that consumers + can use directory mtimes to validate caches. See bug #290428. + """ + base = self.root + _os.sep + VDB_PATH + cat = catsplit(cpv)[0] + catdir = base + _os.sep + cat + t = time.time() + t = (t, t) + for x in (catdir, base): + os.utime(x, t) + def cpv_exists(self, mykey): "Tells us whether an actual ebuild exists on disk (no masking)" return os.path.exists(self.getpath(mykey)) @@ -2017,6 +2031,7 @@ class vardbapi(dbapi): return results def aux_update(self, cpv, values): + self._bump_mtime(cpv) cat, pkg = catsplit(cpv) mylink = dblink(cat, pkg, self.root, self.settings, treetype="vartree", vartree=self.vartree) @@ -2030,6 +2045,7 @@ class vardbapi(dbapi): os.unlink(os.path.join(self.getpath(cpv), k)) except EnvironmentError: pass + self._bump_mtime(cpv) def counter_tick(self, myroot, mycpv=None): return self.counter_tick_core(myroot, incrementing=1, mycpv=mycpv) @@ -2151,9 +2167,11 @@ class vardbapi(dbapi): removed += 1 if removed: + self._bump_mtime(pkg.mycpv) f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) write_contents(new_contents, root, f) f.close() + self._bump_mtime(pkg.mycpv) pkg._clear_contents_cache() class _owners_cache(object): @@ -2783,6 +2801,7 @@ class dblink(object): The caller must ensure that lockdb() and unlockdb() are called before and after this method. """ + self.vartree.dbapi._bump_mtime(self.mycpv) showMessage = self._display_merge if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None @@ -2924,6 +2943,7 @@ class dblink(object): self.vartree.dbapi.plib_registry.pruneNonExisting() finally: + self.vartree.dbapi._bump_mtime(self.mycpv) if builddir_lock: try: if myebuildpath: @@ -5066,6 +5086,7 @@ class dblink(object): mydbapi=None, prev_mtimes=None): retval = -1 self.lockdb() + self.vartree.dbapi._bump_mtime(self.mycpv) try: self.vartree.dbapi.plib_registry.load() self.vartree.dbapi.plib_registry.pruneNonExisting() @@ -5091,6 +5112,7 @@ class dblink(object): finally: self.vartree.dbapi.linkmap._clear_cache() self.unlockdb() + self.vartree.dbapi._bump_mtime(self.mycpv) return retval def getstring(self,name): diff --git a/pym/portage/elog/messages.py b/pym/portage/elog/messages.py index a563ad271..b2a2dc4cf 100644 --- a/pym/portage/elog/messages.py +++ b/pym/portage/elog/messages.py @@ -96,11 +96,12 @@ def _elog_base(level, msg, phase="other", key=None, color=None, out=None): formatted_msg = colorize(color, " * ") + msg + "\n" - if sys.hexversion < 0x3000000 and \ - out in (sys.stdout, sys.stderr) and isinstance(formatted_msg, unicode): - # avoid potential UnicodeEncodeError - formatted_msg = formatted_msg.encode( - _encodings['stdio'], 'backslashreplace') + # avoid potential UnicodeEncodeError + if out in (sys.stdout, sys.stderr): + formatted_msg = _unicode_encode(formatted_msg, + encoding=_encodings['stdio'], errors='backslashreplace') + if sys.hexversion >= 0x3000000: + out = out.buffer out.write(formatted_msg) diff --git a/pym/portage/output.py b/pym/portage/output.py index 89374e0b1..d0e555069 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -251,11 +251,15 @@ def xtermTitle(mystr, raw=False): mystr = mystr[:_max_xtermTitle_len] if not raw: mystr = '\x1b]0;%s\x07' % mystr - if sys.hexversion < 0x3000000 and isinstance(mystr, unicode): - # avoid potential UnicodeEncodeError - mystr = mystr.encode(_encodings['stdio'], 'backslashreplace') - sys.stderr.write(mystr) - sys.stderr.flush() + + # avoid potential UnicodeEncodeError + mystr = _unicode_encode(mystr, + encoding=_encodings['stdio'], errors='backslashreplace') + f = sys.stderr + if sys.hexversion >= 0x3000000: + f = f.buffer + f.write(mystr) + f.flush() default_xterm_title = None @@ -374,11 +378,12 @@ class ConsoleStyleFile(object): self._write(self.write_listener, s) def _write(self, f, s): - if sys.hexversion < 0x3000000 and \ - isinstance(s, unicode) and \ - f in (sys.stdout, sys.stderr): - # avoid potential UnicodeEncodeError - s = s.encode(_encodings['stdio'], 'backslashreplace') + # avoid potential UnicodeEncodeError + if f in (sys.stdout, sys.stderr): + s = _unicode_encode(s, + encoding=_encodings['stdio'], errors='backslashreplace') + if sys.hexversion >= 0x3000000: + f = f.buffer f.write(s) def writelines(self, lines): @@ -484,9 +489,12 @@ class EOutput(object): sys.stderr.flush() def _write(self, f, s): - if sys.hexversion < 0x3000000 and isinstance(s, unicode): - # avoid potential UnicodeEncodeError - s = s.encode(_encodings['stdio'], 'backslashreplace') + # avoid potential UnicodeEncodeError + s = _unicode_encode(s, + encoding=_encodings['stdio'], errors='backslashreplace') + f = sys.stderr + if sys.hexversion >= 0x3000000: + f = f.buffer f.write(s) f.flush() diff --git a/pym/portage/util.py b/pym/portage/util.py index af9cd2db2..5a9376666 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -68,10 +68,11 @@ def writemsg(mystr,noiselevel=0,fd=None): if fd is None: fd = sys.stderr if noiselevel <= noiselimit: - if sys.hexversion < 0x3000000: - # avoid potential UnicodeEncodeError - mystr = _unicode_encode(mystr, - encoding=_encodings['stdio'], errors='backslashreplace') + # avoid potential UnicodeEncodeError + mystr = _unicode_encode(mystr, + encoding=_encodings['stdio'], errors='backslashreplace') + if sys.hexversion >= 0x3000000: + fd = fd.buffer fd.write(mystr) fd.flush()