1 # Copyright 1999-2005 Gentoo Foundation
2 # Distributed under the terms of the GNU General Public License v2
3 # $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.51 2005/07/07 21:35:00 rocket Exp $
5 import sys,string,os,types,re,signal,traceback,md5,time
6 selinux_capable = False
7 #userpriv_capable = (os.getuid() == 0)
8 #fakeroot_capable = False
9 BASH_BINARY = "/bin/bash"
13 max_fd_limit=resource.getrlimit(RLIMIT_NOFILE)
17 # hokay, no resource module.
20 # pids this process knows of.
28 def cleanup(pids,block_exceptions=True):
29 """function to go through and reap the list of pids passed to it"""
35 os.kill(x,signal.SIGTERM)
36 if os.waitpid(x,os.WNOHANG)[1] == 0:
37 # feisty bugger, still alive.
38 os.kill(x,signal.SIGKILL)
44 if oe.errno not in (10,3):
51 try: spawned_pids.remove(x)
52 except IndexError: pass
56 # a function to turn a string of non-printable characters into a string of
59 hexStr = string.hexdigits
63 r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
67 # A function to calculate the md5 sum of a file
71 for line in f.readlines():
74 md5sum = hexify(m.digest())
75 print "MD5 (%s) = %s" % (file, md5sum)
79 def read_from_clst(file):
85 raise CatalystError, "Could not open file "+file
86 for line in myf.readlines():
87 line = string.replace(line, "\n", "") # drop newline
88 myline = myline + line
93 # these should never be touched
94 required_build_targets=["generic_target","generic_stage_target"]
96 # new build types should be added here
97 valid_build_targets=["stage1_target","stage2_target","stage3_target","stage4_target","grp_target",
98 "livecd_stage1_target","livecd_stage2_target","embedded_target",
99 "tinderbox_target","snapshot_target","netboot_target"]
101 required_config_file_values=["storedir","sharedir","distdir","portdir"]
102 valid_config_file_values=required_config_file_values[:]
103 valid_config_file_values.append("PKGCACHE")
104 valid_config_file_values.append("KERNCACHE")
105 valid_config_file_values.append("CCACHE")
106 valid_config_file_values.append("DISTCC")
107 valid_config_file_values.append("ENVSCRIPT")
108 valid_config_file_values.append("AUTORESUME")
109 valid_config_file_values.append("FETCH")
110 valid_config_file_values.append("CLEAR_AUTORESUME")
111 valid_config_file_values.append("options")
112 valid_config_file_values.append("DEBUG")
113 valid_config_file_values.append("VERBOSE")
114 valid_config_file_values.append("PURGE")
118 def list_bashify(mylist):
119 if type(mylist)==types.StringType:
123 for x in range(0,len(mypack)):
124 # surround args with quotes for passing to bash,
125 # allows things like "<" to remain intact
126 mypack[x]="'"+mypack[x]+"'"
127 mypack=string.join(mypack)
130 def list_to_string(mylist):
131 if type(mylist)==types.StringType:
135 for x in range(0,len(mypack)):
136 # surround args with quotes for passing to bash,
137 # allows things like "<" to remain intact
139 mypack=string.join(mypack)
142 class CatalystError(Exception):
143 def __init__(self, message):
145 (type,value)=sys.exc_info()[:2]
148 print traceback.print_exc(file=sys.stdout)
150 print "!!! catalyst: "+message
158 print "!!! catalyst: "+msg
161 def find_binary(myc):
162 """look through the environmental path for an executable file named whatever myc is"""
167 for x in p.split(":"):
168 #if it exists, and is executable
169 if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
170 return "%s/%s" % (x,myc)
174 def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
175 """spawn mycommand as an arguement to bash"""
178 opt_name=mycommand.split()[0]
179 if not env.has_key("BASH_ENV"):
180 env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
184 args.append(mycommand)
185 return spawn(args,env=env,opt_name=opt_name,**keywords)
187 #def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \
188 # collect_fds=[1],fd_pipes=None,**keywords):
189 def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
190 collect_fds=[1],fd_pipes=None,**keywords):
191 """call spawn, collecting the output to fd's specified in collect_fds list
192 emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
193 requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
194 'lets let log only stdin and let stderr slide by'.
196 emulate_gso was deprecated from the day it was added, so convert your code over.
197 spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
198 global selinux_capable
201 #if type(spawn_type) not in [types.FunctionType, types.MethodType]:
202 # s="spawn_type must be passed a function, not",type(spawn_type),spawn_type
209 for x in collect_fds:
211 keywords["returnpid"]=True
213 mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
215 if type(mypid) != types.ListType:
217 return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
220 mydata=fd.readlines()
223 mydata=string.join(mydata)
224 if len(mydata) and mydata[-1] == "\n":
226 retval=os.waitpid(mypid[0],0)[1]
229 return [retval,mydata]
230 retval=process_exit_code(retval)
231 return [retval, mydata]
234 # base spawn function
235 def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\
236 uid=None,gid=None,groups=None,umask=None,logfile=None,path_lookup=True,\
237 selinux_context=None, raise_signals=False, func_call=False):
238 """base fork/execve function.
239 mycommand is the desired command- if you need a command to execute in a bash/sandbox/fakeroot
240 environment, use the appropriate spawn call. This is a straight fork/exec code path.
241 Can either have a tuple, or a string passed in. If uid/gid/groups/umask specified, it changes
242 the forked process to said value. If path_lookup is on, a non-absolute command will be converted
243 to an absolute command, otherwise it returns None.
245 selinux_context is the desired context, dependant on selinux being available.
246 opt_name controls the name the processor goes by.
247 fd_pipes controls which file descriptor numbers are left open in the forked process- it's a dict of
248 current fd's raw fd #, desired #.
250 func_call is a boolean for specifying to execute a python function- use spawn_func instead.
251 raise_signals is questionable. Basically throw an exception if signal'd. No exception is thrown
254 logfile overloads the specified fd's to write to a tee process which logs to logfile
255 returnpid returns the relevant pids (a list, including the logging process if logfile is on).
257 non-returnpid calls to spawn will block till the process has exited, returning the exitcode/signal
258 raw_exit_code controls whether the actual waitpid result is returned, or intrepretted."""
263 if type(mycommand)==types.StringType:
264 mycommand=mycommand.split()
266 if not os.access(myc, os.X_OK):
269 myc = find_binary(myc)
275 mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2}))
276 retval=os.waitpid(mypid[-1],os.WNOHANG)[1]
281 return process_exit_code(retval)
290 opt_name = mycommand[0]
292 myargs.extend(mycommand[1:])
294 mypid.append(os.fork())
297 spawned_pids.extend(mypid)
303 # this may look ugly, but basically it moves file descriptors around to ensure no
304 # handles that are needed are accidentally closed during the final dup2 calls.
306 if type(fd_pipes)==types.DictType:
311 #build list of which fds will be where, and where they are at currently
314 src_fd.append(fd_pipes[x])
316 # run through said list dup'ing descriptors so that they won't be waxed
317 # by other dup calls.
318 for x in range(0,len(trg_fd)):
319 if trg_fd[x] == src_fd[x]:
321 if trg_fd[x] in src_fd[x+1:]:
322 new=os.dup2(trg_fd[x],max(src_fd) + 1)
326 src_fd[s.index(trg_fd[x])]=new
327 except SystemExit, e:
332 # transfer the fds to their final pre-exec position.
333 for x in range(0,len(trg_fd)):
334 if trg_fd[x] != src_fd[x]:
335 os.dup2(src_fd[x], trg_fd[x])
339 # wax all open descriptors that weren't requested be left open.
340 for x in range(0,max_fd_limit):
344 except SystemExit, e:
349 # note this order must be preserved- can't change gid/groups if you change uid first.
350 if selinux_capable and selinux_context:
352 selinux.setexec(selinux_context)
363 #print "execing", myc, myargs
365 # either use a passed in func for interpretting the results, or return if no exception.
366 # note the passed in list, and dict are expanded.
367 if len(mycommand) == 4:
368 os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
370 mycommand[0](*mycommand[1],**mycommand[2])
372 print "caught exception",e," in forked func",mycommand[0]
375 os.execvp(myc,myargs)
376 #os.execve(myc,myargs,env)
377 except SystemExit, e:
381 raise str(e)+":\n "+myc+" "+string.join(myargs)
382 print "func call failed"
384 # If the execve fails, we need to report it, and exit
385 # *carefully* --- report error here
388 return # should never get reached
390 # if we were logging, kill the pipes.
398 # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
399 # if the main pid (mycommand) returned badly.
401 retval=os.waitpid(mypid[-1],0)[1]
403 cleanup(mypid[0:-1],block_exceptions=False)
404 # at this point we've killed all other kid pids generated via this call.
408 return process_exit_code(retval,throw_signals=raise_signals)
424 #def spawn(mystring,debug=0,fd_pipes=None):
426 # apparently, os.system mucks up return values, so this code
429 # Taken from portage.py - thanks to carpaski@gentoo.org
431 # print "Running command \""+mystring+"\""
433 # mycommand = "/bin/bash"
435 # myargs=["bash","-x","-c",mystring]
437 # myargs=["bash","-c",mystring]
442 # os.dup2(fd_pipes[0], 0) # stdin -- (Read)/Write
443 # os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write)
444 # os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write)
446 # os.execvp(mycommand,myargs)
447 # except Exception, e:
448 # raise CatalystError,myexc
450 # If the execve fails, we need to report it, and exit
451 # *carefully* --- report error here
454 # return # should never get reached
456 # retval=os.waitpid(mypid,0)[1]
457 # if (retval & 0xff)==0:
458 # return (retval >> 8) # return exit code
460 # return ((retval & 0xff) << 8) # interrupted by signal
463 # os.kill(mypid,signal.SIGTERM)
464 # if os.waitpid(mypid,os.WNOHANG)[1] == 0:
465 # # feisty bugger, still alive.
466 # os.kill(mypid,signal.SIGKILL)
470 def cmd(mycmd,myexc=""):
473 retval=spawn_bash(mycmd)
475 raise CatalystError,myexc
479 def process_exit_code(retval,throw_signals=False):
480 """process a waitpid returned exit code, returning exit code if it exit'd, or the
481 signal if it died from signalling
482 if throw_signals is on, it raises a SystemExit if the process was signaled.
483 This is intended for usage with threads, although at the moment you can't signal individual
484 threads in python, only the master thread, so it's a questionable option."""
485 if (retval & 0xff)==0:
486 return retval >> 8 # return exit code
489 #use systemexit, since portage is stupid about exception catching.
491 return (retval & 0xff) << 8 # interrupted by signal
494 def file_locate(settings,filelist,expand=1):
495 #if expand=1, non-absolute paths will be accepted and
496 # expanded to os.getcwd()+"/"+localpath if file exists
497 for myfile in filelist:
498 if not settings.has_key(myfile):
499 #filenames such as cdtar are optional, so we don't assume the variable is defined.
501 if len(settings[myfile])==0:
502 raise CatalystError, "File variable \""+myfile+"\" has a length of zero (not specified.)"
503 if settings[myfile][0]=="/":
504 if not os.path.exists(settings[myfile]):
505 raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]
506 elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
507 settings[myfile]=os.getcwd()+"/"+settings[myfile]
509 raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]+" (2nd try)"
513 The spec file format is a very simple and easy-to-use format for storing data. Here's an example
523 This file would be interpreted as defining three items: item1, item2 and item3. item1 would contain
524 the string value "value1". Item2 would contain an ordered list [ "foo", "bar", "oni" ]. item3
525 would contain an ordered list as well: [ "meep", "bark", "gleep", "moop" ]. It's important to note
526 that the order of multiple-value items is preserved, but the order that the items themselves are
527 defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1"
528 "item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash).
531 def parse_spec(mylines):
534 colon=re.compile(":")
535 trailing_comment=re.compile("#.*\n")
536 newline=re.compile("\n")
537 leading_white_space=re.compile("^\s+")
538 white_space=re.compile("\s+")
539 while pos<len(mylines):
540 # Force the line to be clean
541 # Remove Comments ( anything following # )
542 mylines[pos]=trailing_comment.sub("",mylines[pos])
544 # Remove newline character \n
545 mylines[pos]=newline.sub("",mylines[pos])
547 # Remove leading white space
548 mylines[pos]=leading_white_space.sub("",mylines[pos])
550 # Skip any blank lines
551 if len(mylines[pos])<=1:
554 msearch=colon.search(mylines[pos])
556 # If semicolon found assume its a new key
557 # This may cause problems if : are used for key values but works for now
559 # Split on the first semicolon creating two strings in the array mobjs
560 mobjs = colon.split(mylines[pos],1)
561 # Start a new array using the first element of mobjs
564 # split on white space creating additional array elements
565 subarray=mobjs[1].split()
569 # Store as a string if only one element is found.
570 # this is to keep with original catalyst behavior
571 # eventually this may go away if catalyst just works
573 newarray.append(subarray[0])
575 newarray.append(mobjs[1].split())
577 # Else add on to the last key we were working on
579 mobjs = white_space.split(mylines[pos])
585 myspec[newarray[0]]=newarray[1]
587 myspec[newarray[0]]=newarray[1:]
589 for x in myspec.keys():
590 # Convert myspec[x] to an array of strings
592 if type(myspec[x])!=types.StringType:
594 if type(y)==types.ListType:
595 newarray.append(y[0])
596 if type(y)==types.StringType:
599 # Delete empty key pairs
600 if len(myspec[x])==0:
601 print "\n\tWARNING: No value set for key: "+x
602 print "\tdeleting key: "+x+"\n"
607 def parse_makeconf(mylines):
610 pat=re.compile("([a-zA-Z_]*)=(.*)")
611 while pos<len(mylines):
612 if len(mylines[pos])<=1:
616 if mylines[pos][0] in ["#"," ","\t"]:
617 #skip indented lines, comments
622 mobj=pat.match(myline)
625 clean_string = re.sub(r"\"",r"",mobj.group(2))
626 mymakeconf[mobj.group(1)]=clean_string
629 def read_spec(myspecfile):
631 myf=open(myspecfile,"r")
633 raise CatalystError, "Could not open spec file "+myspecfile
634 mylines=myf.readlines()
636 return parse_spec(mylines)
638 def read_makeconf(mymakeconffile):
639 if os.path.exists(mymakeconffile):
641 myf=open(mymakeconffile,"r")
642 mylines=myf.readlines()
644 return parse_makeconf(mylines)
646 raise CatalystError, "Could not open make.conf file "+myspecfile
651 def msg(mymsg,verblevel=1):
652 if verbosity>=verblevel:
655 def pathcompare(path1,path2):
656 # Change double slashes to slash
657 path1 = re.sub(r"//",r"/",path1)
658 path2 = re.sub(r"//",r"/",path2)
659 # Removing ending slash
660 path1 = re.sub("/$","",path1)
661 path2 = re.sub("/$","",path2)
668 "enhanced to handle bind mounts"
669 if os.path.ismount(path):
671 a=open("/proc/mounts","r")
672 mylines=a.readlines()
676 if pathcompare(path,mysplit[1]):
680 def arg_parse(cmdline):
681 #global required_config_file_values
684 foo=string.split(x,"=")
686 raise CatalystError, "Invalid arg syntax: "+x
689 mydict[foo[0]]=foo[1]
691 # if all is well, we should return (we should have bailed before here if not)
694 def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
695 "helper function to help targets parse additional arguments"
696 global valid_config_file_values
697 for x in addlargs.keys():
698 if x not in validspec and x not in valid_config_file_values:
699 raise CatalystError, "Argument \""+x+"\" not recognized."
701 myspec[x]=addlargs[x]
702 for x in requiredspec:
703 if not myspec.has_key(x):
704 raise CatalystError, "Required argument \""+x+"\" not specified."
706 def spec_dump(myspec):
707 for x in myspec.keys():
708 print x+": "+repr(myspec[x])
715 raise CatalystError, "Could not touch "+myfile+"."
717 def countdown(secs=5, doing="Starting"):
719 print ">>> Waiting",secs,"seconds before starting..."
720 print ">>> (Control-C to abort)...\n"+doing+" in: ",
724 sys.stdout.write(str(sec+1)+" ")