From bcb1c7a36ee9dde63fe5db07d1d76821dbca92c2 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 19 Jan 2007 19:44:53 +0000 Subject: [PATCH] Merged from trunk 5697:5722 dump emerge's normpath in favor of portage_util's normalize_path (WarnerBro) fix apply_permissions import and use keyworded args in the call, as well as PORTAGE_INST_UID instead of root, also fix some spacies in emerge, thanks for Zac for reporting (WarnerBro) lock file around chown/chmod (WarnerBro) locking, try 100 (WarnerBro) Syntax error (genone) Attribute error (genone) I really meant NEWS_LIB_PATH (/var/lib/gentoo) but decent idea to have a timestamp per repo ;) (WarnerBro) but news lib path is global, not an attr (WarnerBro) more trivial fixes (genone) Allow per-module filtering based on loglevels in addition to the global PORTAGE_ELOG_CLASSES one, add wildcard support for loglevels (genone) update make.conf.example with per-module filtering (genone) Don't load a module if there is nothing to process after filtering (genone) don't force a comparison (WarnerBro) Unified try-except-finally only works with python-2.5 (genone) Remove -- from params so the length check works (genone) Validate ebuild names inside portdbapi.cp_list() in order to prevent problems later on. Thanks to delusions for reporting on irc. Raise an informative InvalidData exception instead of 'TypeError: unsubscriptable object' when pkgsplit returns None. Simplify rsync timestamp.chk handling and print the correct path for bug #50738. Thanks to jeka for reporting on irc. For bug #162404, when sesandbox is enabled, only log if PORT_LOGDIR is explicitly enabled since it is possible that local SELinux security policies do not allow ouput to be piped out of the sesandbox domain. Fix syntax. svn path=/main/branches/prefix/; revision=5728 --- bin/emerge | 107 ++++++++++++++++++-------------------------- bin/portageq | 2 +- cnf/make.conf | 6 ++- pym/portage.py | 86 +++++++++++++++++++++++++---------- pym/portage_dep.py | 2 + pym/portage_news.py | 57 +++++++++++++---------- pym/portage_util.py | 5 +++ 7 files changed, 152 insertions(+), 113 deletions(-) diff --git a/bin/emerge b/bin/emerge index cf3619f0d..14dbd6ef0 100755 --- a/bin/emerge +++ b/bin/emerge @@ -47,6 +47,7 @@ import portage_locks import portage_exception from portage_const import EPREFIX from portage_data import secpass +from portage_util import normalize_path as normpath if not hasattr(__builtins__, "set"): from sets import Set as set @@ -113,19 +114,6 @@ class stdout_spinner(object): def update_quiet(self): return - - -def normpath(mystr): - """ - os.path.normpath("//foo") returns "//foo" instead of "/foo" - We dislike this behavior so we create our own normpath func - to fix it. - """ - if mystr and (mystr[0]=='/'): - return os.path.normpath("///"+mystr) - else: - return os.path.normpath(mystr) - def userquery(prompt, responses=None, colours=None): """Displays a prompt and a set of responses, then waits for a response which is checked against the responses and the first to match is @@ -213,7 +201,7 @@ options=[ "--searchdesc", "--selective", "--skipfirst", "--tree", -"--update", +"--update", "--usepkg", "--usepkgonly", "--verbose", "--version" ] @@ -3799,32 +3787,23 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): rsync_opts.append("--bwlimit=%s" % \ settings["RSYNC_RATELIMIT"]) - servertimestampdir = settings.depcachedir+"/" - servertimestampfile = settings.depcachedir+"/timestamp.chk" - tmpservertimestampdir = settings["PORTAGE_TMPDIR"]+"/" - tmpservertimestampfile = settings["PORTAGE_TMPDIR"]+"/timestamp.chk" - - # We only use the backup if a timestamp exists in the portdir. - content=None - if os.path.exists(myportdir+"/metadata/timestamp.chk"): - content=portage.grabfile(servertimestampfile) - if (not content): - content=portage.grabfile(myportdir+"/metadata/timestamp.chk") + # Real local timestamp file. + servertimestampfile = os.path.join( + myportdir, "metadata", "timestamp.chk") + # Temporary file for remote server timestamp comparison. + tmpservertimestampfile = os.path.join( + settings["PORTAGE_TMPDIR"], "timestamp.chk") - if (content): + content = portage_util.grabfile(servertimestampfile) + mytimestamp = 0 + if content: try: - mytimestamp=time.mktime(time.strptime(content[0], "%a, %d %b %Y %H:%M:%S +0000")) - except ValueError: - mytimestamp=0 - else: - mytimestamp=0 - - if not os.path.exists(servertimestampdir): - os.mkdir(servertimestampdir) - os.chown(servertimestampdir, os.getuid(), portage.portage_gid) - os.chmod(servertimestampdir, 02775) + mytimestamp = time.mktime(time.strptime(content[0], + "%a, %d %b %Y %H:%M:%S +0000")) + except OverflowError, ValueError: + pass + del content - #exitcode=0 try: if settings.has_key("RSYNC_RETRIES"): print yellow("WARNING:")+" usage of RSYNC_RETRIES is deprecated, use PORTAGE_RSYNC_RETRIES instead" @@ -3899,7 +3878,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): (retries,maxretries,dosyncuri)) print "\n\n>>> Starting retry %d of %d with %s" % (retries,maxretries,dosyncuri) - if "--quiet" not in myopts: + if mytimestamp != 0 and "--quiet" not in myopts: print ">>> Checking server timestamp ..." rsynccommand = " ".join([EPREFIX, "/usr/bin/rsync", " ".join(rsync_opts), @@ -3908,18 +3887,32 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if "--debug" in myopts: print rsynccommand - mycommand = " ".join([rsynccommand, - dosyncuri + "/metadata/timestamp.chk", - tmpservertimestampdir]) - exitcode=portage.spawn(mycommand,settings,free=1) - if (exitcode==0): + exitcode = os.EX_OK + servertimestamp = 0 + if mytimestamp != 0: + mycommand = rsynccommand.split() + mycommand.append(dosyncuri.rstrip("/") + \ + "/metadata/timestamp.chk") + mycommand.append(tmpservertimestampfile) + import portage_exec try: - servertimestamp = time.mktime(time.strptime(portage.grabfile(tmpservertimestampfile)[0], "%a, %d %b %Y %H:%M:%S +0000")) - except SystemExit, e: - raise # Needed else can't exit - except: - servertimestamp = 0 - + exitcode = portage_exec.spawn( + mycommand, env=settings.environ()) + content = portage.grabfile(tmpservertimestampfile) + if content: + try: + servertimestamp = time.mktime(time.strptime( + content[0], "%a, %d %b %Y %H:%M:%S +0000")) + except OverflowError, ValueError: + pass + del content + finally: + try: + os.unlink(tmpservertimestampfile) + except OSError: + pass + del mycommand + if exitcode == os.EX_OK: if (servertimestamp != 0) and (servertimestamp == mytimestamp): emergelog(xterm_titles, ">>> Cancelling sync -- Already current.") @@ -3928,7 +3921,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): print ">>> Timestamps on the server and in the local repository are the same." print ">>> Cancelling all further sync action. You are already up to date." print ">>>" - print ">>> In order to force sync, remove '%s'." % tmpservertimestampfile + print ">>> In order to force sync, remove '%s'." % servertimestampfile print ">>>" print sys.exit(0) @@ -3939,7 +3932,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): print ">>>" print ">>> SERVER OUT OF DATE: %s" % dosyncuri print ">>>" - print ">>> In order to force sync, remove '%s'." % tmpservertimestampfile + print ">>> In order to force sync, remove '%s'." % servertimestampfile print ">>>" print elif (servertimestamp == 0) or (servertimestamp > mytimestamp): @@ -3964,18 +3957,6 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if (exitcode==0): emergelog(xterm_titles, "=== Sync completed with %s" % dosyncuri) - # save timestamp.chk for next timestamp check. - try: - if tmpservertimestampfile is not None: - portage.movefile(tmpservertimestampfile, - servertimestampfile, mysettings=settings) - except SystemExit, e: - raise - except Exception, e: - portage.writemsg("!!! Failed to save current timestamp.\n", - noiselevel=-1) - portage.writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e elif (exitcode>0): print if exitcode==1: diff --git a/bin/portageq b/bin/portageq index 17315fa2c..c318365db 100755 --- a/bin/portageq +++ b/bin/portageq @@ -276,7 +276,7 @@ def get_repo_path(argv): if len(argv) < 2: print "ERROR: insufficient parameters!" sys.exit(2) - for arg in arvg[1:] + for arg in arvg[1:]: print portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(argv[1]) #----------------------------------------------------------------------------- diff --git a/cnf/make.conf b/cnf/make.conf index c894d7b95..0fe7efd50 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -310,7 +310,7 @@ # logging related variables: # PORTAGE_ELOG_CLASSES: selects messages to be logged, possible values are: -# info, warn, error, log, qa +# info, warn, error, log, qa, * # Warning: commenting this will disable elog PORTAGE_ELOG_CLASSES="warn error log" @@ -329,6 +329,10 @@ PORTAGE_ELOG_CLASSES="warn error log" # mail_summary (like "mail" but sends all messages in # a single mail when emerge exits) # To use elog you should enable at least one module +# The module name may be followed by a colon and a comma +# separated list of loglevels to override PORTAGE_ELOG_CLASSES +# for this module (e.g. +# PORTAGE_ELOG_SYSTEM="mail:warn,error syslog:* save") #PORTAGE_ELOG_SYSTEM="save mail" # PORTAGE_ELOG_COMMAND: only used with the "custom" logging module. Specifies a command diff --git a/pym/portage.py b/pym/portage.py index 4f4dbf55e..80218cb32 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -467,40 +467,66 @@ def elog_process(cpv, mysettings): return # exploit listdir() file order so we process log entries in chronological order mylogfiles.reverse() - mylogentries = {} - my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split()) + all_logentries = {} for f in mylogfiles: msgfunction, msgtype = f.split(".") - if msgtype.upper() not in my_elog_classes \ - and msgtype.lower() not in my_elog_classes: - continue if msgfunction not in portage_const.EBUILD_PHASES: writemsg("!!! can't process invalid log file: %s\n" % f, noiselevel=-1) continue - if not msgfunction in mylogentries: - mylogentries[msgfunction] = [] + if not msgfunction in all_logentries: + all_logentries[msgfunction] = [] msgcontent = open(mysettings["T"]+"/logging/"+f, "r").readlines() - mylogentries[msgfunction].append((msgtype, msgcontent)) + all_logentries[msgfunction].append((msgtype, msgcontent)) + + def filter_loglevels(logentries, loglevels): + # remove unwanted entries from all logentries + rValue = {} + loglevels = map(str.upper, loglevels) + for phase in logentries.keys(): + for msgtype, msgcontent in logentries[phase]: + if msgtype.upper() in loglevels or "*" in loglevels: + if not rValue.has_key(phase): + rValue[phase] = [] + rValue[phase].append((msgtype, msgcontent)) + return rValue + + my_elog_classes = set(mysettings.get("PORTAGE_ELOG_CLASSES", "").split()) + default_logentries = filter_loglevels(all_logentries, my_elog_classes) - # in case the filters matched all messages - if len(mylogentries) == 0: + # in case the filters matched all messages and no module overrides exist + if len(default_logentries) == 0 and (not ":" in mysettings.get("PORTAGE_ELOG_SYSTEM", "")): return - # generate a single string with all log messages - fulllog = "" - for phase in portage_const.EBUILD_PHASES: - if not phase in mylogentries: - continue - for msgtype,msgcontent in mylogentries[phase]: - fulllog += "%s: %s\n" % (msgtype, phase) - for line in msgcontent: - fulllog += line - fulllog += "\n" + def combine_logentries(logentries): + # generate a single string with all log messages + rValue = "" + for phase in portage_const.EBUILD_PHASES: + if not phase in logentries: + continue + for msgtype,msgcontent in logentries[phase]: + rValue += "%s: %s\n" % (msgtype, phase) + for line in msgcontent: + rValue += line + rValue += "\n" + return rValue + + default_fulllog = combine_logentries(default_logentries) # pass the processing to the individual modules logsystems = mysettings["PORTAGE_ELOG_SYSTEM"].split() for s in logsystems: + # allow per module overrides of PORTAGE_ELOG_CLASSES + if ":" in s: + s, levels = s.split(":", 1) + levels = levels.split(",") + mod_logentries = filter_loglevels(all_logentries, levels) + mod_fulllog = combine_logentries(mod_logentries) + else: + mod_logentries = default_logentries + mod_fulllog = default_fulllog + if len(mod_logentries) == 0: + continue # - is nicer than _ for module names, so allow people to use it. s = s.replace("-", "_") try: @@ -517,7 +543,7 @@ def elog_process(cpv, mysettings): # module gets hung). signal.alarm(60) try: - m.process(mysettings, cpv, mylogentries, fulllog) + m.process(mysettings, cpv, mod_logentries, mod_fulllog) finally: signal.alarm(0) if hasattr(m, "finalize") and not m.finalize in _elog_atexit_handlers: @@ -3210,7 +3236,13 @@ def prepare_build_dirs(myroot, mysettings, cleanup): (mysettings["CATEGORY"], mysettings["PF"], logid_time)) del logid_path, logid_time else: - mysettings["PORTAGE_LOG_FILE"] = os.path.join(mysettings["T"], "build.log") + # When sesandbox is enabled, only log if PORT_LOGDIR is explicitly + # enabled since it is possible that local SELinux security policies + # do not allow ouput to be piped out of the sesandbox domain. + if not (mysettings.selinux_enabled() and \ + "sesandbox" in mysettings.features): + mysettings["PORTAGE_LOG_FILE"] = os.path.join( + mysettings["T"], "build.log") _doebuild_manifest_exempt_depend = 0 _doebuild_manifest_checked = None @@ -6015,8 +6047,14 @@ class portdbapi(dbapi): mytrees = self.porttrees for oroot in mytrees: for x in listdir(oroot+"/"+mycp,EmptyOnError=1,ignorecvs=1): - if x[-7:]==".ebuild": - d[mysplit[0]+"/"+x[:-7]] = None + if x.endswith(".ebuild"): + pf = x[:-7] + ps = pkgsplit(pf) + if not ps: + writemsg("\nInvalid ebuild name: %s\n" % \ + os.path.join(oroot, mycp, x), noiselevel=-1) + continue + d[mysplit[0]+"/"+pf] = None return d.keys() def freeze(self): diff --git a/pym/portage_dep.py b/pym/portage_dep.py index 0b9e512f8..7991c8bf7 100644 --- a/pym/portage_dep.py +++ b/pym/portage_dep.py @@ -607,6 +607,8 @@ def match_from_list(mydep, candidate_list): elif operator == "~": # version, any revision, match for x in candidate_list: xs = catpkgsplit(x) + if xs is None: + raise InvalidData(x) if not cpvequal(xs[0]+"/"+xs[1]+"-"+xs[2], mycpv_cps[0]+"/"+mycpv_cps[1]+"-"+mycpv_cps[2]): continue if xs[2] != ver: diff --git a/pym/portage_news.py b/pym/portage_news.py index c06a726a0..b54261d92 100644 --- a/pym/portage_news.py +++ b/pym/portage_news.py @@ -5,10 +5,10 @@ from portage_const import INCREMENTALS, PROFILE_PATH, NEWS_LIB_PATH from portage import config, vartree, vardbapi, portdbapi -from portage_util import ensure_dirs +from portage_util import ensure_dirs, apply_permissions from portage_data import portage_gid from portage_locks import lockfile, unlockfile, lockdir, unlockdir - +from portage_exception import FileNotFound import os, re class NewsManager(object): @@ -29,7 +29,7 @@ class NewsManager(object): def __init__( self, root, NEWS_PATH, UNREAD_PATH, LANGUAGE_ID='en' ): self.NEWS_PATH = NEWS_PATH self.UNREAD_PATH = UNREAD_PATH - self.TIMESTAMP_PATH = os.path.join( root, self.NEWS_LIB_PATH, NewsManager.TIMESTAMP_FILE ) + self.TIMESTAMP_PATH = os.path.join( root, NEWS_LIB_PATH, NewsManager.TIMESTAMP_FILE ) self.target_root = root self.LANGUAGE_ID = LANGUAGE_ID self.config = config( config_root = os.environ.get("PORTAGE_CONFIGROOT", "/"), @@ -54,16 +54,19 @@ class NewsManager(object): if repoid not in repos: raise ValueError("Invalid repoID: %s" % repoid) - if os.path.exists(self.TIMESTAMP_PATH): + timestamp_file = self.TIMESTAMP_PATH + repoid + if os.path.exists(timestamp_file): # Make sure the timestamp has correct permissions. - apply_permissions( unreadfile, 0, portage_gid, 664 ) - timestamp = os.stat(self.TIMESTAMP_PATH).st_mtime + apply_permissions( filename=timestamp_file, + uid=self.config["PORTAGE_INST_UID"], gid=portage_gid, mode=664 ) + timestamp = os.stat(timestamp_file).st_mtime else: timestamp = 0 path = os.path.join( self.portdb.getRepositoryPath( repoid ), self.NEWS_PATH ) + newsdir_lock = None try: - newsdir_lock = lockdir( self.portdb.getRepositoryPath ) + newsdir_lock = lockdir( self.portdb.getRepositoryPath(repoid) ) # Skip reading news for repoid if the news dir does not exist. Requested by # NightMorph :) if not os.path.exists( path ): @@ -80,26 +83,31 @@ class NewsManager(object): if tmp.isRelevant( profile=os.readlink(PROFILE_PATH), config=config, vardb=self.vdb): updates.append( tmp ) finally: - unlockdir(newsdir_lock) + if newsdir_lock: + unlockdir(newsdir_lock) del path path = os.path.join( self.UNREAD_PATH, "news-" + repoid + ".unread" ) try: unread_lock = lockfile( path ) + if not os.path.exists( path ): + #create the file if it does not exist + open( path, "w" ) + # Ensure correct perms on the unread file. + apply_permissions( filename=path, + uid=self.config["PORTAGE_INST_UID"], gid=portage_gid, mode=664 ) # Make sure we have the correct permissions when created unread_file = open( path, "a" ) - apply_permissions( unreadfile, 0, portage_gid, 664 ) - + for item in updates: unread_file.write( item.path + "\n" ) - unread_file.close() finally: unlockfile(unread_lock) # Touch the timestamp file - f = open(self.TIMESTAMP_PATH, "w") + f = open(timestamp_file, "w") f.close() def getUnreadItems( self, repoid, update=False ): @@ -110,22 +118,23 @@ class NewsManager(object): check for new items. """ - unreadfile = os.path.join( self.UNREAD_PATH, "news-"+ repoid +".unread" ) - # Set correct permissions on the news-repoid.unread file - try: - apply_permissions( unreadfile, 0, portage_gid, 664 ) - except FileNotFound: - pass # It may not exist yet, thats ok. - if update: self.updateItems( repoid ) + unreadfile = os.path.join( self.UNREAD_PATH, "news-"+ repoid +".unread" ) try: - unread_lock = lockfile(unreadfile) - if os.path.exists( unreadfile ): - unread = open( unreadfile ).readlines() - if len(unread): - return len(unread) + try: + unread_lock = lockfile(unreadfile) + # Set correct permissions on the news-repoid.unread file + apply_permissions( filename=unreadfile, + uid=int(self.config["PORTAGE_INST_UID"]), gid=portage_gid, mode=0664 ) + + if os.path.exists( unreadfile ): + unread = open( unreadfile ).readlines() + if len(unread): + return len(unread) + except FileNotFound: + pass # unread file may not exist finally: if unread_lock: unlockfile(unread_lock) diff --git a/pym/portage_util.py b/pym/portage_util.py index 591cd6f8f..875eb8799 100644 --- a/pym/portage_util.py +++ b/pym/portage_util.py @@ -32,6 +32,11 @@ def writemsg_stdout(mystr,noiselevel=0): writemsg(mystr, noiselevel=noiselevel, fd=sys.stdout) def normalize_path(mypath): + """ + os.path.normpath("//foo") returns "//foo" instead of "/foo" + We dislike this behavior so we create our own normpath func + to fix it. + """ if mypath.startswith(os.path.sep): # posixpath.normpath collapses 3 or more leading slashes to just 1. return os.path.normpath(2*os.path.sep + mypath) -- 2.26.2