From b3475906d5f51a21ecaf4ff048002a2f44face52 Mon Sep 17 00:00:00 2001 From: Eric Edgar Date: Tue, 5 Jul 2005 20:49:48 +0000 Subject: [PATCH] add additional logging output. Use standard os redirection methods to log to a file git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@741 d1e1f19c-881f-0410-ab34-b69fee027534 --- ChangeLog | 7 +- modules/catalyst_support.py | 408 ++++++++++++++++++++++++++++---- modules/generic_stage_target.py | 7 +- 3 files changed, 376 insertions(+), 46 deletions(-) diff --git a/ChangeLog b/ChangeLog index 10869d17..409a7e2e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,10 @@ # Copyright 2002-2005 Gentoo Foundation; Distributed under the GPL v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.286 2005/07/05 18:58:20 wolf31o2 Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.287 2005/07/05 20:49:48 rocket Exp $ + + 05 Jul 2005; modules/catalyst_support.py, + modules/generic_stage_target.py: + add additional logging output. Use standard os redirection methods to log to + a file 05 Jul 2005; Chris Gianelloni ChangeLog: Added profile sanity check for bug #97867. diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py index 5c34e1f3..770dd8a6 100644 --- a/modules/catalyst_support.py +++ b/modules/catalyst_support.py @@ -1,8 +1,54 @@ # Copyright 1999-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.47 2005/05/06 17:48:59 rocket Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.48 2005/07/05 20:49:48 rocket Exp $ import sys,string,os,types,re,signal,traceback,md5,time +selinux_capable = False +#userpriv_capable = (os.getuid() == 0) +#fakeroot_capable = False +BASH_BINARY = "/bin/bash" + +try: + import resource + max_fd_limit=resource.getrlimit(RLIMIT_NOFILE) +except SystemExit, e: + raise +except: + # hokay, no resource module. + max_fd_limit=256 + +# pids this process knows of. +spawned_pids = [] + + +def cleanup(pids,block_exceptions=True): + """function to go through and reap the list of pids passed to it""" + global spawned_pids + if type(pids) == int: + pids = [pids] + for x in pids: + try: + os.kill(x,signal.SIGTERM) + if os.waitpid(x,os.WNOHANG)[1] == 0: + # feisty bugger, still alive. + os.kill(x,signal.SIGKILL) + os.waitpid(x,0) + + except OSError, oe: + if block_exceptions: + pass + if oe.errno not in (10,3): + raise oe + except SystemExit: + raise + except Exception: + if block_exceptions: + pass + try: spawned_pids.remove(x) + except IndexError: pass + + + # a function to turn a string of non-printable characters into a string of # hex characters def hexify(str): @@ -104,60 +150,339 @@ def die(msg=None): def warn(msg): print "!!! catalyst: "+msg -def spawn(mystring,debug=0,fd_pipes=None): - """ - apparently, os.system mucks up return values, so this code - should fix that. - Taken from portage.py - thanks to carpaski@gentoo.org - """ - print "Running command \""+mystring+"\"" - myargs=[] - mycommand = "/bin/bash" +def find_binary(myc): + """look through the environmental path for an executable file named whatever myc is""" + # this sucks. badly. + p=os.getenv("PATH") + if p == None: + return None + for x in p.split(":"): + #if it exists, and is executable + if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248: + return "%s/%s" % (x,myc) + return None + + +def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords): + """spawn mycommand as an arguement to bash""" + args=[BASH_BINARY] + if not opt_name: + opt_name=mycommand.split()[0] + if not env.has_key("BASH_ENV"): + env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env" if debug: - myargs=["bash","-x","-c",mystring] - else: - myargs=["bash","-c",mystring] + args.append("-x") + args.append("-c") + args.append(mycommand) + return spawn(args,env=env,opt_name=opt_name,**keywords) + +#def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \ +# collect_fds=[1],fd_pipes=None,**keywords): +def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \ + collect_fds=[1],fd_pipes=None,**keywords): + """call spawn, collecting the output to fd's specified in collect_fds list + emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the + requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the + 'lets let log only stdin and let stderr slide by'. + + emulate_gso was deprecated from the day it was added, so convert your code over. + spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot""" + global selinux_capable + pr,pw=os.pipe() + + #if type(spawn_type) not in [types.FunctionType, types.MethodType]: + # s="spawn_type must be passed a function, not",type(spawn_type),spawn_type + # raise Exception,s + + if fd_pipes==None: + fd_pipes={} + fd_pipes[0] = 0 + + for x in collect_fds: + fd_pipes[x] = pw + keywords["returnpid"]=True + + mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords) + os.close(pw) + if type(mypid) != types.ListType: + os.close(pr) + return [mypid, "%s: No such file or directory" % mycommand.split()[0]] + + fd=os.fdopen(pr,"r") + mydata=fd.readlines() + fd.close() + if emulate_gso: + mydata=string.join(mydata) + if len(mydata) and mydata[-1] == "\n": + mydata=mydata[:-1] + retval=os.waitpid(mypid[0],0)[1] + cleanup(mypid) + if raw_exit_code: + return [retval,mydata] + retval=process_exit_code(retval) + return [retval, mydata] + + +# base spawn function +def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\ + uid=None,gid=None,groups=None,umask=None,logfile=None,path_lookup=True,\ + selinux_context=None, raise_signals=False, func_call=False): + """base fork/execve function. + mycommand is the desired command- if you need a command to execute in a bash/sandbox/fakeroot + environment, use the appropriate spawn call. This is a straight fork/exec code path. + Can either have a tuple, or a string passed in. If uid/gid/groups/umask specified, it changes + the forked process to said value. If path_lookup is on, a non-absolute command will be converted + to an absolute command, otherwise it returns None. - mypid=os.fork() - if mypid==0: - if fd_pipes: - os.dup2(fd_pipes[0], 0) # stdin -- (Read)/Write - os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write) - os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write) - try: - os.execvp(mycommand,myargs) - except Exception, e: - raise CatalystError,myexc + selinux_context is the desired context, dependant on selinux being available. + opt_name controls the name the processor goes by. + fd_pipes controls which file descriptor numbers are left open in the forked process- it's a dict of + current fd's raw fd #, desired #. + + func_call is a boolean for specifying to execute a python function- use spawn_func instead. + raise_signals is questionable. Basically throw an exception if signal'd. No exception is thrown + if raw_input is on. + + logfile overloads the specified fd's to write to a tee process which logs to logfile + returnpid returns the relevant pids (a list, including the logging process if logfile is on). + + non-returnpid calls to spawn will block till the process has exited, returning the exitcode/signal + raw_exit_code controls whether the actual waitpid result is returned, or intrepretted.""" + + + myc='' + if not func_call: + if type(mycommand)==types.StringType: + mycommand=mycommand.split() + myc = mycommand[0] + if not os.access(myc, os.X_OK): + if not path_lookup: + return None + myc = find_binary(myc) + if myc == None: + return None + mypid=[] + if logfile: + pr,pw=os.pipe() + mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2})) + retval=os.waitpid(mypid[-1],os.WNOHANG)[1] + if retval != 0: + # he's dead jim. + if raw_exit_code: + return retval + return process_exit_code(retval) - # If the execve fails, we need to report it, and exit - # *carefully* --- report error here - os._exit(1) - sys.exit(1) - return # should never get reached - try: - retval=os.waitpid(mypid,0)[1] - if (retval & 0xff)==0: - return (retval >> 8) # return exit code - else: - return ((retval & 0xff) << 8) # interrupted by signal + if fd_pipes == None: + fd_pipes={} + fd_pipes[0] = 0 + fd_pipes[1]=pw + fd_pipes[2]=pw + + if not opt_name: + opt_name = mycommand[0] + myargs=[opt_name] + myargs.extend(mycommand[1:]) + global spawned_pids + mypid.append(os.fork()) + if mypid[-1] != 0: + #log the bugger. + spawned_pids.extend(mypid) + + if mypid[-1] == 0: + if func_call: + spawned_pids = [] + + # this may look ugly, but basically it moves file descriptors around to ensure no + # handles that are needed are accidentally closed during the final dup2 calls. + trg_fd=[] + if type(fd_pipes)==types.DictType: + src_fd=[] + k=fd_pipes.keys() + k.sort() + + #build list of which fds will be where, and where they are at currently + for x in k: + trg_fd.append(x) + src_fd.append(fd_pipes[x]) - except: - os.kill(mypid,signal.SIGTERM) - if os.waitpid(mypid,os.WNOHANG)[1] == 0: - # feisty bugger, still alive. - os.kill(mypid,signal.SIGKILL) - raise + # run through said list dup'ing descriptors so that they won't be waxed + # by other dup calls. + for x in range(0,len(trg_fd)): + if trg_fd[x] == src_fd[x]: + continue + if trg_fd[x] in src_fd[x+1:]: + new=os.dup2(trg_fd[x],max(src_fd) + 1) + os.close(trg_fd[x]) + try: + while True: + src_fd[s.index(trg_fd[x])]=new + except SystemExit, e: + raise + except: + pass + + # transfer the fds to their final pre-exec position. + for x in range(0,len(trg_fd)): + if trg_fd[x] != src_fd[x]: + os.dup2(src_fd[x], trg_fd[x]) + else: + trg_fd=[0,1,2] + + # wax all open descriptors that weren't requested be left open. + for x in range(0,max_fd_limit): + if x not in trg_fd: + try: + os.close(x) + except SystemExit, e: + raise + except: + pass + + # note this order must be preserved- can't change gid/groups if you change uid first. + if selinux_capable and selinux_context: + import selinux + selinux.setexec(selinux_context) + if gid: + os.setgid(gid) + if groups: + os.setgroups(groups) + if uid: + os.setuid(uid) + if umask: + os.umask(umask) + + try: + #print "execing", myc, myargs + if func_call: + # either use a passed in func for interpretting the results, or return if no exception. + # note the passed in list, and dict are expanded. + if len(mycommand) == 4: + os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2]))) + try: + mycommand[0](*mycommand[1],**mycommand[2]) + except Exception,e: + print "caught exception",e," in forked func",mycommand[0] + sys.exit(0) + + os.execvp(myc,myargs) + #os.execve(myc,myargs,env) + except SystemExit, e: + raise + except Exception, e: + if not func_call: + raise str(e)+":\n "+myc+" "+string.join(myargs) + print "func call failed" + + # If the execve fails, we need to report it, and exit + # *carefully* --- report error here + os._exit(1) + sys.exit(1) + return # should never get reached + + # if we were logging, kill the pipes. + if logfile: + os.close(pr) + os.close(pw) + + if returnpid: + return mypid + # loop through pids (typically one, unless logging), either waiting on their death, or waxing them + # if the main pid (mycommand) returned badly. + while len(mypid): + retval=os.waitpid(mypid[-1],0)[1] + if retval != 0: + cleanup(mypid[0:-1],block_exceptions=False) + # at this point we've killed all other kid pids generated via this call. + # return now. + if raw_exit_code: + return retval + return process_exit_code(retval,throw_signals=raise_signals) + else: + mypid.pop(-1) + cleanup(mypid) + return 0 + + + + + + + + + + + +#def spawn(mystring,debug=0,fd_pipes=None): +# """ +# apparently, os.system mucks up return values, so this code +# should fix that. +# +# Taken from portage.py - thanks to carpaski@gentoo.org +# """ +# print "Running command \""+mystring+"\"" +# myargs=[] +# mycommand = "/bin/bash" +# if debug: +# myargs=["bash","-x","-c",mystring] +# else: +# myargs=["bash","-c",mystring] +# +# mypid=os.fork() +# if mypid==0: +# if fd_pipes: +# os.dup2(fd_pipes[0], 0) # stdin -- (Read)/Write +# os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write) +# os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write) +# try: +# os.execvp(mycommand,myargs) +# except Exception, e: +# raise CatalystError,myexc +# + # If the execve fails, we need to report it, and exit + # *carefully* --- report error here +# os._exit(1) +# sys.exit(1) +# return # should never get reached +# try: +# retval=os.waitpid(mypid,0)[1] +# if (retval & 0xff)==0: +# return (retval >> 8) # return exit code +# else: +# return ((retval & 0xff) << 8) # interrupted by signal +# +# except: +# os.kill(mypid,signal.SIGTERM) +# if os.waitpid(mypid,os.WNOHANG)[1] == 0: +# # feisty bugger, still alive. +# os.kill(mypid,signal.SIGKILL) +# raise +# def cmd(mycmd,myexc=""): try: - retval=spawn(mycmd) + sys.stdout.flush() + retval=spawn_bash(mycmd) if retval != 0: raise CatalystError,myexc except: raise +def process_exit_code(retval,throw_signals=False): + """process a waitpid returned exit code, returning exit code if it exit'd, or the + signal if it died from signalling + if throw_signals is on, it raises a SystemExit if the process was signaled. + This is intended for usage with threads, although at the moment you can't signal individual + threads in python, only the master thread, so it's a questionable option.""" + if (retval & 0xff)==0: + return retval >> 8 # return exit code + else: + if throw_signals: + #use systemexit, since portage is stupid about exception catching. + raise SystemExit() + return (retval & 0xff) << 8 # interrupted by signal + def file_locate(settings,filelist,expand=1): #if expand=1, non-absolute paths will be accepted and @@ -394,4 +719,3 @@ def countdown(secs=5, doing="Starting"): sys.stdout.flush() time.sleep(1) print - diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py index b14d6cf0..14720cc7 100644 --- a/modules/generic_stage_target.py +++ b/modules/generic_stage_target.py @@ -1,6 +1,6 @@ # Copyright 1999-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.47 2005/06/23 15:42:50 rocket Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.48 2005/07/05 20:49:48 rocket Exp $ """ This class does all of the chroot setup, copying of files, etc. It is @@ -102,7 +102,7 @@ class generic_stage_target(generic_target): # define all of our core variables self.set_target_profile() self.set_target_subpath() - + # set paths self.set_snapshot_path() self.set_source_path() @@ -165,7 +165,7 @@ class generic_stage_target(generic_target): self.mountmap["/var/tmp/ccache"]=ccdir # for the chroot: os.environ["CCACHE_DIR"]="/var/tmp/ccache" - + def override_chost(self): if os.environ.has_key("CHOST"): self.settings["CHOST"] = os.environ["CHOST"] @@ -810,6 +810,7 @@ class generic_stage_target(generic_target): def run(self): for x in self.settings["action_sequence"]: print "Running action sequence: "+x + sys.stdout.flush() try: apply(getattr(self,x)) except: -- 2.26.2