catalyst/support.py

   1
   2 import sys,string,os,types,re,signal,traceback,time
   3 #import md5,sha
   4 selinux_capable = False
   5 #userpriv_capable = (os.getuid() == 0)
   6 #fakeroot_capable = False
   7 BASH_BINARY             = "/bin/bash"
   8
   9 # set it to 0 for the soft limit, 1 for the hard limit
  10 DESIRED_RLIMIT = 0
  11 try:
  12         import resource
  13         max_fd_limit=resource.getrlimit(resource.RLIMIT_NOFILE)[DESIRED_RLIMIT]
  14 except SystemExit, e:
  15         raise
  16 except:
  17         # hokay, no resource module.
  18         max_fd_limit=256
  19
  20 # pids this process knows of.
  21 spawned_pids = []
  22
  23 try:
  24         import urllib
  25 except SystemExit, e:
  26         raise
  27
  28 def cleanup(pids,block_exceptions=True):
  29         """function to go through and reap the list of pids passed to it"""
  30         global spawned_pids
  31         if type(pids) == int:
  32                 pids = [pids]
  33         for x in pids:
  34                 try:
  35                         os.kill(x,signal.SIGTERM)
  36                         if os.waitpid(x,os.WNOHANG)[1] == 0:
  37                                 # feisty bugger, still alive.
  38                                 os.kill(x,signal.SIGKILL)
  39                                 os.waitpid(x,0)
  40
  41                 except OSError, oe:
  42                         if block_exceptions:
  43                                 pass
  44                         if oe.errno not in (10,3):
  45                                 raise oe
  46                 except SystemExit:
  47                         raise
  48                 except Exception:
  49                         if block_exceptions:
  50                                 pass
  51                 try:                    spawned_pids.remove(x)
  52                 except IndexError:      pass
  53
  54
  55
  56 # a function to turn a string of non-printable characters into a string of
  57 # hex characters
  58 def hexify(str):
  59         hexStr = string.hexdigits
  60         r = ''
  61         for ch in str:
  62                 i = ord(ch)
  63                 r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
  64         return r
  65 # hexify()
  66
  67
  68 def read_from_clst(file):
  69         line = ''
  70         myline = ''
  71         try:
  72                 myf=open(file,"r")
  73         except:
  74                 return -1
  75                 #raise CatalystError, "Could not open file "+file
  76         for line in myf.readlines():
  77             #line = string.replace(line, "\n", "") # drop newline
  78             myline = myline + line
  79         myf.close()
  80         return myline
  81 # read_from_clst
  82
  83 # these should never be touched
  84 required_build_targets=["generic_target","generic_stage_target"]
  85
  86 # new build types should be added here
  87 valid_build_targets=["stage1_target","stage2_target","stage3_target","stage4_target","grp_target",
  88                         "livecd_stage1_target","livecd_stage2_target","embedded_target",
  89                         "tinderbox_target","snapshot_target","netboot_target","netboot2_target"]
  90
  91 required_config_file_values=["storedir","sharedir","distdir","portdir"]
  92 valid_config_file_values=required_config_file_values[:]
  93 valid_config_file_values.append("PKGCACHE")
  94 valid_config_file_values.append("KERNCACHE")
  95 valid_config_file_values.append("CCACHE")
  96 valid_config_file_values.append("DISTCC")
  97 valid_config_file_values.append("ICECREAM")
  98 valid_config_file_values.append("ENVSCRIPT")
  99 valid_config_file_values.append("AUTORESUME")
 100 valid_config_file_values.append("FETCH")
 101 valid_config_file_values.append("CLEAR_AUTORESUME")
 102 valid_config_file_values.append("options")
 103 valid_config_file_values.append("DEBUG")
 104 valid_config_file_values.append("VERBOSE")
 105 valid_config_file_values.append("PURGE")
 106 valid_config_file_values.append("PURGEONLY")
 107 valid_config_file_values.append("SNAPCACHE")
 108 valid_config_file_values.append("snapshot_cache")
 109 valid_config_file_values.append("hash_function")
 110 valid_config_file_values.append("digests")
 111 valid_config_file_values.append("contents")
 112 valid_config_file_values.append("SEEDCACHE")
 113
 114 verbosity=1
 115
 116 def list_bashify(mylist):
 117         if type(mylist)==types.StringType:
 118                 mypack=[mylist]
 119         else:
 120                 mypack=mylist[:]
 121         for x in range(0,len(mypack)):
 122                 # surround args with quotes for passing to bash,
 123                 # allows things like "<" to remain intact
 124                 mypack[x]="'"+mypack[x]+"'"
 125         mypack=string.join(mypack)
 126         return mypack
 127
 128 def list_to_string(mylist):
 129         if type(mylist)==types.StringType:
 130                 mypack=[mylist]
 131         else:
 132                 mypack=mylist[:]
 133         for x in range(0,len(mypack)):
 134                 # surround args with quotes for passing to bash,
 135                 # allows things like "<" to remain intact
 136                 mypack[x]=mypack[x]
 137         mypack=string.join(mypack)
 138         return mypack
 139
 140 class CatalystError(Exception):
 141         def __init__(self, message):
 142                 if message:
 143                         (type,value)=sys.exc_info()[:2]
 144                         if value!=None:
 145                                 print
 146                                 print traceback.print_exc(file=sys.stdout)
 147                         print
 148                         print "!!! catalyst: "+message
 149                         print
 150
 151 class LockInUse(Exception):
 152         def __init__(self, message):
 153                 if message:
 154                         #(type,value)=sys.exc_info()[:2]
 155                         #if value!=None:
 156                             #print
 157                             #kprint traceback.print_exc(file=sys.stdout)
 158                         print
 159                         print "!!! catalyst lock file in use: "+message
 160                         print
 161
 162 def die(msg=None):
 163         warn(msg)
 164         sys.exit(1)
 165
 166 def warn(msg):
 167         print "!!! catalyst: "+msg
 168
 169 def find_binary(myc):
 170         """look through the environmental path for an executable file named whatever myc is"""
 171         # this sucks. badly.
 172         p=os.getenv("PATH")
 173         if p == None:
 174                 return None
 175         for x in p.split(":"):
 176                 #if it exists, and is executable
 177                 if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
 178                         return "%s/%s" % (x,myc)
 179         return None
 180
 181 def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
 182         """spawn mycommand as an arguement to bash"""
 183         args=[BASH_BINARY]
 184         if not opt_name:
 185             opt_name=mycommand.split()[0]
 186         if "BASH_ENV" not in env:
 187             env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
 188         if debug:
 189             args.append("-x")
 190         args.append("-c")
 191         args.append(mycommand)
 192         return spawn(args,env=env,opt_name=opt_name,**keywords)
 193
 194 #def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \
 195 #        collect_fds=[1],fd_pipes=None,**keywords):
 196
 197 def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
 198         collect_fds=[1],fd_pipes=None,**keywords):
 199         """call spawn, collecting the output to fd's specified in collect_fds list
 200         emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
 201         requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
 202         'lets let log only stdin and let stderr slide by'.
 203
 204         emulate_gso was deprecated from the day it was added, so convert your code over.
 205         spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
 206         global selinux_capable
 207         pr,pw=os.pipe()
 208
 209         #if type(spawn_type) not in [types.FunctionType, types.MethodType]:
 210         #        s="spawn_type must be passed a function, not",type(spawn_type),spawn_type
 211         #        raise Exception,s
 212
 213         if fd_pipes==None:
 214                 fd_pipes={}
 215                 fd_pipes[0] = 0
 216
 217         for x in collect_fds:
 218                 fd_pipes[x] = pw
 219         keywords["returnpid"]=True
 220
 221         mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
 222         os.close(pw)
 223         if type(mypid) != types.ListType:
 224                 os.close(pr)
 225                 return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
 226
 227         fd=os.fdopen(pr,"r")
 228         mydata=fd.readlines()
 229         fd.close()
 230         if emulate_gso:
 231                 mydata=string.join(mydata)
 232                 if len(mydata) and mydata[-1] == "\n":
 233                         mydata=mydata[:-1]
 234         retval=os.waitpid(mypid[0],0)[1]
 235         cleanup(mypid)
 236         if raw_exit_code:
 237                 return [retval,mydata]
 238         retval=process_exit_code(retval)
 239         return [retval, mydata]
 240
 241 # base spawn function
 242 def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\
 243          uid=None,gid=None,groups=None,umask=None,logfile=None,path_lookup=True,\
 244          selinux_context=None, raise_signals=False, func_call=False):
 245         """base fork/execve function.
 246         mycommand is the desired command- if you need a command to execute in a bash/sandbox/fakeroot
 247         environment, use the appropriate spawn call.  This is a straight fork/exec code path.
 248         Can either have a tuple, or a string passed in.  If uid/gid/groups/umask specified, it changes
 249         the forked process to said value.  If path_lookup is on, a non-absolute command will be converted
 250         to an absolute command, otherwise it returns None.
 251
 252         selinux_context is the desired context, dependant on selinux being available.
 253         opt_name controls the name the processor goes by.
 254         fd_pipes controls which file descriptor numbers are left open in the forked process- it's a dict of
 255         current fd's raw fd #, desired #.
 256
 257         func_call is a boolean for specifying to execute a python function- use spawn_func instead.
 258         raise_signals is questionable.  Basically throw an exception if signal'd.  No exception is thrown
 259         if raw_input is on.
 260
 261         logfile overloads the specified fd's to write to a tee process which logs to logfile
 262         returnpid returns the relevant pids (a list, including the logging process if logfile is on).
 263
 264         non-returnpid calls to spawn will block till the process has exited, returning the exitcode/signal
 265         raw_exit_code controls whether the actual waitpid result is returned, or intrepretted."""
 266
 267         myc=''
 268         if not func_call:
 269                 if type(mycommand)==types.StringType:
 270                         mycommand=mycommand.split()
 271                 myc = mycommand[0]
 272                 if not os.access(myc, os.X_OK):
 273                         if not path_lookup:
 274                                 return None
 275                         myc = find_binary(myc)
 276                         if myc == None:
 277                             return None
 278         mypid=[]
 279         if logfile:
 280                 pr,pw=os.pipe()
 281                 mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2}))
 282                 retval=os.waitpid(mypid[-1],os.WNOHANG)[1]
 283                 if retval != 0:
 284                         # he's dead jim.
 285                         if raw_exit_code:
 286                                 return retval
 287                         return process_exit_code(retval)
 288
 289                 if fd_pipes == None:
 290                         fd_pipes={}
 291                         fd_pipes[0] = 0
 292                 fd_pipes[1]=pw
 293                 fd_pipes[2]=pw
 294
 295         if not opt_name:
 296                 opt_name = mycommand[0]
 297         myargs=[opt_name]
 298         myargs.extend(mycommand[1:])
 299         global spawned_pids
 300         mypid.append(os.fork())
 301         if mypid[-1] != 0:
 302                 #log the bugger.
 303                 spawned_pids.extend(mypid)
 304
 305         if mypid[-1] == 0:
 306                 if func_call:
 307                         spawned_pids = []
 308
 309                 # this may look ugly, but basically it moves file descriptors around to ensure no
 310                 # handles that are needed are accidentally closed during the final dup2 calls.
 311                 trg_fd=[]
 312                 if type(fd_pipes)==types.DictType:
 313                         src_fd=[]
 314                         k=fd_pipes.keys()
 315                         k.sort()
 316
 317                         #build list of which fds will be where, and where they are at currently
 318                         for x in k:
 319                                 trg_fd.append(x)
 320                                 src_fd.append(fd_pipes[x])
 321
 322                         # run through said list dup'ing descriptors so that they won't be waxed
 323                         # by other dup calls.
 324                         for x in range(0,len(trg_fd)):
 325                                 if trg_fd[x] == src_fd[x]:
 326                                         continue
 327                                 if trg_fd[x] in src_fd[x+1:]:
 328                                         new=os.dup2(trg_fd[x],max(src_fd) + 1)
 329                                         os.close(trg_fd[x])
 330                                         try:
 331                                                 while True:
 332                                                         src_fd[s.index(trg_fd[x])]=new
 333                                         except SystemExit, e:
 334                                                 raise
 335                                         except:
 336                                                 pass
 337
 338                         # transfer the fds to their final pre-exec position.
 339                         for x in range(0,len(trg_fd)):
 340                                 if trg_fd[x] != src_fd[x]:
 341                                         os.dup2(src_fd[x], trg_fd[x])
 342                 else:
 343                         trg_fd=[0,1,2]
 344
 345                 # wax all open descriptors that weren't requested be left open.
 346                 for x in range(0,max_fd_limit):
 347                         if x not in trg_fd:
 348                                 try:
 349                                         os.close(x)
 350                                 except SystemExit, e:
 351                                         raise
 352                                 except:
 353                                         pass
 354
 355                 # note this order must be preserved- can't change gid/groups if you change uid first.
 356                 if selinux_capable and selinux_context:
 357                         import selinux
 358                         selinux.setexec(selinux_context)
 359                 if gid:
 360                         os.setgid(gid)
 361                 if groups:
 362                         os.setgroups(groups)
 363                 if uid:
 364                         os.setuid(uid)
 365                 if umask:
 366                         os.umask(umask)
 367                 else:
 368                         os.umask(022)
 369
 370                 try:
 371                         #print "execing", myc, myargs
 372                         if func_call:
 373                                 # either use a passed in func for interpretting the results, or return if no exception.
 374                                 # note the passed in list, and dict are expanded.
 375                                 if len(mycommand) == 4:
 376                                         os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
 377                                 try:
 378                                         mycommand[0](*mycommand[1],**mycommand[2])
 379                                 except Exception,e:
 380                                         print "caught exception",e," in forked func",mycommand[0]
 381                                 sys.exit(0)
 382
 383                         #os.execvp(myc,myargs)
 384                         os.execve(myc,myargs,env)
 385                 except SystemExit, e:
 386                         raise
 387                 except Exception, e:
 388                         if not func_call:
 389                                 raise str(e)+":\n   "+myc+" "+string.join(myargs)
 390                         print "func call failed"
 391
 392                 # If the execve fails, we need to report it, and exit
 393                 # *carefully* --- report error here
 394                 os._exit(1)
 395                 sys.exit(1)
 396                 return # should never get reached
 397
 398         # if we were logging, kill the pipes.
 399         if logfile:
 400                 os.close(pr)
 401                 os.close(pw)
 402
 403         if returnpid:
 404                 return mypid
 405
 406         # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
 407         # if the main pid (mycommand) returned badly.
 408         while len(mypid):
 409                 retval=os.waitpid(mypid[-1],0)[1]
 410                 if retval != 0:
 411                         cleanup(mypid[0:-1],block_exceptions=False)
 412                         # at this point we've killed all other kid pids generated via this call.
 413                         # return now.
 414                         if raw_exit_code:
 415                                 return retval
 416                         return process_exit_code(retval,throw_signals=raise_signals)
 417                 else:
 418                         mypid.pop(-1)
 419         cleanup(mypid)
 420         return 0
 421
 422 def cmd(mycmd,myexc="",env={}):
 423         try:
 424                 sys.stdout.flush()
 425                 retval=spawn_bash(mycmd,env)
 426                 if retval != 0:
 427                         raise CatalystError,myexc
 428         except:
 429                 raise
 430
 431 def process_exit_code(retval,throw_signals=False):
 432         """process a waitpid returned exit code, returning exit code if it exit'd, or the
 433         signal if it died from signalling
 434         if throw_signals is on, it raises a SystemExit if the process was signaled.
 435         This is intended for usage with threads, although at the moment you can't signal individual
 436         threads in python, only the master thread, so it's a questionable option."""
 437         if (retval & 0xff)==0:
 438                 return retval >> 8 # return exit code
 439         else:
 440                 if throw_signals:
 441                         #use systemexit, since portage is stupid about exception catching.
 442                         raise SystemExit()
 443                 return (retval & 0xff) << 8 # interrupted by signal
 444
 445 def file_locate(settings,filelist,expand=1):
 446         #if expand=1, non-absolute paths will be accepted and
 447         # expanded to os.getcwd()+"/"+localpath if file exists
 448         for myfile in filelist:
 449                 if myfile not in settings:
 450                         #filenames such as cdtar are optional, so we don't assume the variable is defined.
 451                         pass
 452                 else:
 453                     if len(settings[myfile])==0:
 454                             raise CatalystError, "File variable \""+myfile+"\" has a length of zero (not specified.)"
 455                     if settings[myfile][0]=="/":
 456                             if not os.path.exists(settings[myfile]):
 457                                     raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]
 458                     elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
 459                             settings[myfile]=os.getcwd()+"/"+settings[myfile]
 460                     else:
 461                             raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]+" (2nd try)"
 462 """
 463 Spec file format:
 464
 465 The spec file format is a very simple and easy-to-use format for storing data. Here's an example
 466 file:
 467
 468 item1: value1
 469 item2: foo bar oni
 470 item3:
 471         meep
 472         bark
 473         gleep moop
 474
 475 This file would be interpreted as defining three items: item1, item2 and item3. item1 would contain
 476 the string value "value1". Item2 would contain an ordered list [ "foo", "bar", "oni" ]. item3
 477 would contain an ordered list as well: [ "meep", "bark", "gleep", "moop" ]. It's important to note
 478 that the order of multiple-value items is preserved, but the order that the items themselves are
 479 defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1"
 480 "item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash).
 481 """
 482
 483 def parse_makeconf(mylines):
 484         mymakeconf={}
 485         pos=0
 486         pat=re.compile("([0-9a-zA-Z_]*)=(.*)")
 487         while pos<len(mylines):
 488                 if len(mylines[pos])<=1:
 489                         #skip blanks
 490                         pos += 1
 491                         continue
 492                 if mylines[pos][0] in ["#"," ","\t"]:
 493                         #skip indented lines, comments
 494                         pos += 1
 495                         continue
 496                 else:
 497                         myline=mylines[pos]
 498                         mobj=pat.match(myline)
 499                         pos += 1
 500                         if mobj.group(2):
 501                             clean_string = re.sub(r"\"",r"",mobj.group(2))
 502                             mymakeconf[mobj.group(1)]=clean_string
 503         return mymakeconf
 504
 505 def read_makeconf(mymakeconffile):
 506         if os.path.exists(mymakeconffile):
 507                 try:
 508                         try:
 509                                 import snakeoil.bash #import snakeoil.fileutils
 510                                 return snakeoil.bash.read_bash_dict(mymakeconffile, sourcing_command="source")
 511                         except ImportError:
 512                                 try:
 513                                         import portage.util
 514                                         return portage.util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
 515                                 except:
 516                                         try:
 517                                                 import portage_util
 518                                                 return portage_util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
 519                                         except ImportError:
 520                                                 myf=open(mymakeconffile,"r")
 521                                                 mylines=myf.readlines()
 522                                                 myf.close()
 523                                                 return parse_makeconf(mylines)
 524                 except:
 525                         raise CatalystError, "Could not parse make.conf file "+mymakeconffile
 526         else:
 527                 makeconf={}
 528                 return makeconf
 529
 530 def msg(mymsg,verblevel=1):
 531         if verbosity>=verblevel:
 532                 print mymsg
 533
 534 def pathcompare(path1,path2):
 535         # Change double slashes to slash
 536         path1 = re.sub(r"//",r"/",path1)
 537         path2 = re.sub(r"//",r"/",path2)
 538         # Removing ending slash
 539         path1 = re.sub("/$","",path1)
 540         path2 = re.sub("/$","",path2)
 541
 542         if path1 == path2:
 543                 return 1
 544         return 0
 545
 546 def ismount(path):
 547         "enhanced to handle bind mounts"
 548         if os.path.ismount(path):
 549                 return 1
 550         a=os.popen("mount")
 551         mylines=a.readlines()
 552         a.close()
 553         for line in mylines:
 554                 mysplit=line.split()
 555                 if pathcompare(path,mysplit[2]):
 556                         return 1
 557         return 0
 558
 559 def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 560         "helper function to help targets parse additional arguments"
 561         global valid_config_file_values
 562
 563         messages = []
 564         for x in addlargs.keys():
 565                 if x not in validspec and x not in valid_config_file_values and x not in requiredspec:
 566                         messages.append("Argument \""+x+"\" not recognized.")
 567                 else:
 568                         myspec[x]=addlargs[x]
 569
 570         for x in requiredspec:
 571                 if x not in myspec:
 572                         messages.append("Required argument \""+x+"\" not specified.")
 573
 574         if messages:
 575                 raise CatalystError, '\n\tAlso: '.join(messages)
 576
 577 def touch(myfile):
 578         try:
 579                 myf=open(myfile,"w")
 580                 myf.close()
 581         except IOError:
 582                 raise CatalystError, "Could not touch "+myfile+"."
 583
 584 def countdown(secs=5, doing="Starting"):
 585         if secs:
 586                 print ">>> Waiting",secs,"seconds before starting..."
 587                 print ">>> (Control-C to abort)...\n"+doing+" in: ",
 588                 ticks=range(secs)
 589                 ticks.reverse()
 590                 for sec in ticks:
 591                         sys.stdout.write(str(sec+1)+" ")
 592                         sys.stdout.flush()
 593                         time.sleep(1)
 594                 print
 595
 596 def normpath(mypath):
 597         TrailingSlash=False
 598         if mypath[-1] == "/":
 599             TrailingSlash=True
 600         newpath = os.path.normpath(mypath)
 601         if len(newpath) > 1:
 602                 if newpath[:2] == "//":
 603                         newpath = newpath[1:]
 604         if TrailingSlash:
 605             newpath=newpath+'/'
 606         return newpath