From b597dc38331b2b1f3839e1c357e82a995f6d4d8e Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 19 Jan 2007 19:24:28 +0000 Subject: [PATCH] Merged from trunk 5659:5673 fix minor syntax issues with locking (WarnerBro) Enable repoman to commit a Manifest alone instead of bailing out with "Didn't find any changed files". ferringb pointed out this portroot parameter, ended up not using it, try an os.path.exists to save a very common (and slow) except on missing repo_name files for overlays, use a dict to facility a nicer regex comparison loop for news restrictions. (WarnerBro) document FEATURES=sesandbox #161985 (SpankMan) make.conf is not a manual so only list common FEATURES #161985 (SpankMan) dirty dirty lies (WarnerBro) Make repoman commit removals together with the other changes (instead of with the final Manifest commit). For consistency, just use os.environ. Fix broken commitmessagefile handling. Replace mktemp with mkstemp since the former is deprecated for security reasons. Various glep 42 fixes, fix up the portageq calls to fail properly, rewrite the news checking code to check news for all repos with a name. (WarnerBro) fix a few typos in tests (WarnerBro) If repo/profiles/news doesn't exist, return 0 updates (WarnerBro) switch news timestamp and unread/read storage to '/var/lib/gentoo' like the GLEP states. (WarnerBro) (Prefix adjusted) svn path=/main/branches/prefix/; revision=5726 --- bin/emerge | 24 ++++++++---- bin/portageq | 11 +++++- bin/repoman | 62 ++++++++++++++++++------------ cnf/make.conf | 33 ++-------------- man/make.conf.5 | 3 ++ pym/portage.py | 8 ++-- pym/portage_const.py | 1 + pym/portage_news.py | 39 +++++++++---------- tests/portage_dep/__init__.py | 2 +- tests/portage_news/__init__.py | 4 ++ tests/portage_versions/__init__.py | 2 +- 11 files changed, 98 insertions(+), 91 deletions(-) create mode 100644 tests/portage_news/__init__.py diff --git a/bin/emerge b/bin/emerge index 46783befd..cf3619f0d 100755 --- a/bin/emerge +++ b/bin/emerge @@ -29,7 +29,8 @@ except ImportError: sys.path.insert(0, "@PORTAGE_BASE@/pym") import portage del os.environ["PORTAGE_LEGACY_GLOBALS"] -from portage import digraph +from portage import digraph, portdbapi +from portage_const import NEWS_LIB_PATH import emergehelp, xpak, commands, errno, re, socket, time, types import output @@ -3563,14 +3564,21 @@ def post_emerge(settings, mtimedb, retval): chk_updated_cfg_files(normalize_path(target_root + EPREFIX), config_protect) NEWS_PATH = os.path.join( "metadata", "news" ) - UNREAD_PATH = os.path.join( target_root, portage.PRIVATE_PATH, 'news') - repo_id = 'gentoo' - unreadItems = checkUpdatedNewsItems(target_root, NEWS_PATH, UNREAD_PATH, repo_id) - if unreadItems: - print colorize("WARN", " * IMPORTANT:"), - print "%s news items need reading." % unreadItems + UNREAD_PATH = os.path.join( target_root, NEWS_LIB_PATH, 'news') + porttree = portdbapi( porttree_root = settings["PORTDIR"], mysettings = settings ) + newsReaderDisplay = False + + for repo in porttree.getRepositories(): + unreadItems = checkUpdatedNewsItems(target_root, NEWS_PATH, UNREAD_PATH, repo) + if unreadItems: + print colorize("WARN", " * IMPORTANT:"), + print "%s news items need reading for repository '%s'." % (unreadItems, repo) + newsReaderDisplay = True + + if newsReaderDisplay: print colorize("WARN", " *"), - print "Type something to read the news" + print "Use " + colorize("GOOD", "eselect news") + " to read news items." + mtimedb.commit() sys.exit(retval) diff --git a/bin/portageq b/bin/portageq index 51c50442b..17315fa2c 100755 --- a/bin/portageq +++ b/bin/portageq @@ -264,13 +264,20 @@ def get_repos(argv): """ Returns all repos with names (repo_name file) argv[0] = $ROOT """ + if len(argv) < 1: + print "ERROR: insufficient parameters!" + sys.exit(2) print " ".join(portage.db[argv[0]]["porttree"].dbapi.getRepositories()) def get_repo_path(argv): - """ + """ + Returns the path to the repo named argv[1], argv[0] = $ROOT """ - print portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(argv[1]) + if len(argv) < 2: + print "ERROR: insufficient parameters!" + sys.exit(2) + for arg in arvg[1:] + print portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(argv[1]) #----------------------------------------------------------------------------- # diff --git a/bin/repoman b/bin/repoman index e9e496853..c8d1702cd 100755 --- a/bin/repoman +++ b/bin/repoman @@ -40,6 +40,7 @@ import codecs from portage_manifest import Manifest from portage_exception import ParseError +from portage_exec import spawn from output import bold, darkgreen, darkred, green, nocolor, red, turquoise, yellow @@ -1689,8 +1690,6 @@ else: if isCvs: mycvstree=cvstree.getentries("./",recursive=1) mychanged=cvstree.findchanged(mycvstree,recursive=1,basedir="./") - for manifest in [file for file in mychanged if '/Manifest' in file]: - mychanged.remove(manifest) mynew=cvstree.findnew(mycvstree,recursive=1,basedir="./") myremoved=cvstree.findremoved(mycvstree,recursive=1,basedir="./") if isSvn: @@ -1710,6 +1709,9 @@ else: print sys.exit(0) + # Manifests need to be regenerated after all other commits, so don't commit + # them now even if they have changed. + mychanged = [f for f in mychanged if "Manifest" != os.path.basename(f)] myupdates=mychanged+mynew myheaders=[] mydirty=[] @@ -1726,7 +1728,7 @@ else: print "myupdates:",myupdates print "myheaders:",myheaders print - unlinkfile=0 + if commitmessagefile: try: f = open(commitmessagefile) @@ -1759,12 +1761,10 @@ else: except AttributeError: print "Failed to insert portage version in message!" commitmessage+="\n(Portage version: Unknown)" - if not commitmessagefile: - unlinkfile=1 - commitmessagefile=tempfile.mktemp(".repoman.msg") - if os.path.exists(commitmessagefile): - os.unlink(commitmessagefile) - mymsg=open(commitmessagefile,"w") + if myupdates or myremoved: + myfiles = myupdates + myremoved + fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") + mymsg = os.fdopen(fd, "w") mymsg.write(commitmessage) mymsg.close() @@ -1778,18 +1778,30 @@ else: retval = None if "--pretend" in myoptions: if isCvs: - print "(cvs -q commit -F "+commitmessagefile +" "+ " ".join(myupdates)+")" + print "(cvs -q commit -F %s %s)" % \ + (commitmessagefile, " ".join(myfiles)) if isSvn: - print "(svn commit -F "+commitmessagefile +" "+ " ".join(myupdates)+")" - retval=0 + print "(svn commit -F %s %s)" % \ + (commitmessagefile. " ".join(myfiles)) else: + retval = spawn(["/usr/bin/cvs", "-q", "commit", + "-F", commitmessagefile] + myfiles, + env=os.environ) if isCvs: - retval=os.system("cvs -q commit -F "+commitmessagefile + " " +" ".join(myupdates)) + retval = spawn(["cvs", "-q", "commit", + "-F", commitmessagefile] + myfiles, + env=os.environ) if isSvn: - retval=os.system("svn commit -F "+commitmessagefile + " " +" ".join(myupdates)) - if retval: - print "!!! Exiting on vcs (shell) error code:",retval - sys.exit(retval) + retval = spawn(["svn", "commit", + "-F", commitmessagefile] + myfiles, + env=os.environ) + try: + os.unlink(commitmessagefile) + except OSError: + pass + if retval: + print "!!! Exiting on cvs (shell) error code:",retval + sys.exit(retval) # Setup the GPG commands def gpgsign(filename): @@ -1823,7 +1835,7 @@ else: else: raise portage_exception.PortageException("!!! gpg exited with '" + str(rValue) + "' status") - need_commit = False + manifest_commit_required = True if myheaders or myupdates or myremoved or mynew: myfiles=myheaders+myupdates+myremoved+mynew for x in range(len(myfiles)-1, -1, -1): @@ -1874,8 +1886,7 @@ else: if retval: print "!!! Exiting on cvs (shell) error code:",retval sys.exit(retval) - else: - need_commit = True + manifest_commit_required = False signed = False if "sign" in repoman_settings.features: @@ -1912,14 +1923,15 @@ else: portage.writemsg("!!! Disabled FEATURES='sign'\n") signed = False - if need_commit or signed: + if manifest_commit_required or signed: if "--pretend" in myoptions: if isCvs: print "(cvs -q commit -F "+commitmessagefile+")" if isSvn: print "(svn -q commit -F "+commitmessagefile+")" else: - mymsg=open(commitmessagefile,"w") + fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") + mymsg = os.fdopen(fd, "w") mymsg.write(commitmessage) if signed: mymsg.write("\n (Signed Manifest commit)") @@ -1930,12 +1942,14 @@ else: retval=os.system("cvs -q commit -F "+commitmessagefile) if isSvn: retval=os.system("svn -q commit -F "+commitmessagefile) + try: + os.unlink(commitmessagefile) + except OSError: + pass if retval: print "!!! Exiting on cvs (shell) error code:",retval sys.exit(retval) - if unlinkfile: - os.unlink(commitmessagefile) print if isCvs or isSvn: print "Commit complete." diff --git a/cnf/make.conf b/cnf/make.conf index ea913e36b..c894d7b95 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -1,4 +1,4 @@ -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2007 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id: make.conf 4618 2006-10-08 18:21:05Z zmedico $ # Contains local system settings for Portage system @@ -245,44 +245,21 @@ # # FEATURES are settings that affect the functionality of portage. Most of # these settings are for developer use, but some are available to non- -# developers as well. +# developers as well. # -# 'assume-digests' -# when committing work to cvs with repoman(1), assume that all -# existing SRC_URI digests are correct. This feature also -# affects digest generation via ebuild(1) and emerge(1) (emerge -# generates digests only when the 'digest' feature is enabled). # 'buildpkg' causes binary packages to be created of all packages that # are being merged. +# 'buildsyspkg' only build binary packages for system packages. # 'ccache' enable support for the dev-util/ccache package, which can # noticably decrease the time needed to remerge previously built # packages. -# 'confcache' enable confcache support; speeds up autotool based configure -# calls # 'collision-protect' # prevents packages from overwriting files that are owned by # another package or by no package at all. -# 'cvs' causes portage to enable all cvs features (commits, adds), -# and to apply all USE flags in SRC_URI for digests -- for -# developers only. -# 'digest' autogenerate digests for packages when running the emerge(1) -# command. If the 'assume-digests' feature is also enabled then -# existing SRC_URI digests will be reused whenever they are -# available. # 'distcc' enables distcc support via CC. # 'distlocks' enables distfiles locking using fcntl or hardlinks. This # is enabled by default. Tools exist to help clean the locks # after crashes: /usr/lib/portage/bin/clean_locks. -# 'fixpackages' allows portage to fix binary packages that are stored in -# PKGDIR. This can consume a lot of time. 'fixpackages' is -# also a script that can be run at any given time to force -# the same actions. -# 'gpg' enables basic verification of Manifest files using gpg. -# This features is UNDER DEVELOPMENT and reacts to features -# of strict and severe. Heavy use of gpg sigs is coming. -# 'keeptemp' prevents the clean phase from deleting the temp files ($T) -# from a merge. -# 'keepwork' prevents the clean phase from deleting the WORKDIR. # 'test' causes ebuilds to perform testing phases if they are capable # of it. Some packages support this automatically via makefiles. # 'metadata-transfer' @@ -291,8 +268,6 @@ # 'noauto' causes ebuild to perform only the action requested and # not any other required actions like clean or unpack -- for # debugging purposes only. -# 'noclean' prevents portage from removing the source and temporary files -# after a merge -- for debugging purposes only. # 'nostrip' prevents the stripping of binaries. # 'notitles' disables xterm titlebar updates (which contain status info). # 'parallel-fetch' @@ -309,7 +284,7 @@ # as a security measure. As a side effect this can remove # sandbox access violations for users. # 'usersandbox' enables sandboxing while portage is running under userpriv. -#FEATURES="sandbox buildpkg ccache distcc userpriv usersandbox notitles noclean noauto cvs keeptemp keepwork" +#FEATURES="sandbox buildpkg ccache distcc userpriv usersandbox notitles noauto" #FEATURES="sandbox ccache distcc distlocks" # CCACHE_SIZE and CCACHE_DIR are used to control the behavior of ccache, and diff --git a/man/make.conf.5 b/man/make.conf.5 index 951c6fbad..9251d8a58 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -223,6 +223,9 @@ Fetch in the background while compiling. .B sandbox Enable sandbox\-ing when running \fBemerge\fR(1) and \fBebuild\fR(1). .TP +.B sesandbox +Enable SELinux sandbox\-ing. Do not toggle this \fBFEATURE\fR yourself. +.TP .B severe When checking Manifests, only accept ones that have been signed by a key which you trust. diff --git a/pym/portage.py b/pym/portage.py index 94bce04ff..1add16242 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -4535,13 +4535,10 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): return rValue class portagetree: - def __init__(self, root="/", virtual=None, clone=None, settings=None, portroot=None): + def __init__(self, root="/", virtual=None, clone=None, settings=None): """ Constructor for a PortageTree - Note: Portroot was added for GLEP 42 functionality and defaults to the $PORTDIR - env variable. - @param root: ${ROOT}, defaults to '/', see make.conf(5) @type root: String/Path @param virtual: UNUSED @@ -5557,8 +5554,9 @@ class portdbapi(dbapi): [os.path.realpath(t) for t in self.mysettings["PORTDIR_OVERLAY"].split()] self.treemap = {} for path in self.porttrees: + repo_name_path = os.path.join( path, REPO_NAME_LOC ) try: - repo_name = open( os.path.join( path , REPO_NAME_LOC ) ,'r').readline().rstrip() + repo_name = open( repo_name_path ,'r').readline().strip() self.treemap[repo_name] = path except (OSError,IOError): pass diff --git a/pym/portage_const.py b/pym/portage_const.py index d5cda0111..6051def41 100644 --- a/pym/portage_const.py +++ b/pym/portage_const.py @@ -26,6 +26,7 @@ CUSTOM_PROFILE_PATH = USER_CONFIG_PATH+"/profile" PORTAGE_BASE_PATH = PORTAGE_BASE PORTAGE_BIN_PATH = PORTAGE_BASE_PATH+"/bin" PORTAGE_PYM_PATH = PORTAGE_BASE_PATH+"/pym" +NEWS_LIB_PATH = EPREFIX+"/var/lib/gentoo" PROFILE_PATH = EPREFIX+"/etc/make.profile" LOCALE_DATA_PATH = PORTAGE_BASE_PATH+"/locale" diff --git a/pym/portage_news.py b/pym/portage_news.py index f02b7938f..50da11f8d 100644 --- a/pym/portage_news.py +++ b/pym/portage_news.py @@ -3,7 +3,7 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -from portage_const import PRIVATE_PATH, INCREMENTALS, PROFILE_PATH +from portage_const import INCREMENTALS, PROFILE_PATH, NEWS_LIB_PATH from portage import config, vartree, vardbapi, portdbapi from portage_util import ensure_dirs from portage_data import portage_gid @@ -29,7 +29,7 @@ class NewsManager(object): def __init__( self, root, NEWS_PATH, UNREAD_PATH, LANGUAGE_ID='en' ): self.NEWS_PATH = NEWS_PATH self.UNREAD_PATH = UNREAD_PATH - self.TIMESTAMP_PATH = os.path.join( root, PRIVATE_PATH, NewsManager.TIMESTAMP_FILE ) + self.TIMESTAMP_PATH = os.path.join( root, self.NEWS_LIB_PATH, NewsManager.TIMESTAMP_FILE ) self.target_root = root self.LANGUAGE_ID = LANGUAGE_ID self.config = config( config_root = os.environ.get("PORTAGE_CONFIGROOT", "/"), @@ -60,6 +60,10 @@ class NewsManager(object): timestamp = 0 path = os.path.join( self.portdb.getRepositoryPath( repoid ), self.NEWS_PATH ) + # Skip reading news for repoid if the news dir does not exist. Requested by + # NightMorph :) + if not os.path.exists( path ): + return None news = os.listdir( path ) updates = [] for item in news: @@ -74,13 +78,13 @@ class NewsManager(object): del path path = os.path.join( self.UNREAD_PATH, "news-" + repoid + ".unread" ) - lockfile( path ) + unread_lock = lockfile( path ) unread_file = open( path, "a" ) for item in updates: unread_file.write( item.path + "\n" ) unread_file.close() - unlockfile(path) + unlockfile(unread_lock) # Touch the timestamp file f = open(self.TIMESTAMP_PATH, "w") @@ -98,12 +102,12 @@ class NewsManager(object): self.updateItems( repoid ) unreadfile = os.path.join( self.UNREAD_PATH, "news-"+ repoid +".unread" ) - lockfile(unreadfile) + unread_lock = lockfile(unreadfile) if os.path.exists( unreadfile ): unread = open( unreadfile ).readlines() if len(unread): return len(unread) - unlockfile(unread) + unlockfile(unread_lock) _installedRE = re.compile("Display-If-Installed:(.*)\n") _profileRE = re.compile("Display-If-Profile:(.*)\n") @@ -163,21 +167,14 @@ class NewsItem(object): #will never match if not line.startswith("D"): continue - match = _installedRE.match( line ) - if match: - self.restrictions.append( - DisplayInstalledRestriction( match.groups()[0].strip().rstrip() ) ) - continue - match = _profileRE.match( line ) - if match: - self.restrictions.append( - DisplayProfileRestriction( match.groups()[0].strip().rstrip() ) ) - continue - match = _keywordRE.match( line ) - if match: - self.restrictions.append( - DisplayKeywordRestriction( match.groups()[0].strip().rstrip() ) ) - continue + restricts = { _installedRE : DisplayInstalledRestriction, + _profileRE : DisplayProfileRestriction, + _keywordRE : DisplayKeywordRestriction } + for regex, restriction in restricts.iteritems(): + match = regex.match(line) + if match: + self.restrictions.append( restriction( match.groups()[0].strip() ) ) + continue self._parsed = True def __getattr__( self, attr ): diff --git a/tests/portage_dep/__init__.py b/tests/portage_dep/__init__.py index 920c2f7e3..a258eec60 100644 --- a/tests/portage_dep/__init__.py +++ b/tests/portage_dep/__init__.py @@ -1,4 +1,4 @@ -# tests/portage/__init__.py -- Portage Unit Test functionality +# tests/portage_dep/__init__.py -- Portage Unit Test functionality # Copyright 2006 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ diff --git a/tests/portage_news/__init__.py b/tests/portage_news/__init__.py new file mode 100644 index 000000000..cc7420e1f --- /dev/null +++ b/tests/portage_news/__init__.py @@ -0,0 +1,4 @@ +# tests/portage_news/__init__.py -- Portage Unit Test functionality +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ diff --git a/tests/portage_versions/__init__.py b/tests/portage_versions/__init__.py index 920c2f7e3..76f62f969 100644 --- a/tests/portage_versions/__init__.py +++ b/tests/portage_versions/__init__.py @@ -1,4 +1,4 @@ -# tests/portage/__init__.py -- Portage Unit Test functionality +# tests/portage_versions/__init__.py -- Portage Unit Test functionality # Copyright 2006 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ -- 2.26.2