catalyst/support.py

   1
   2 import sys,string,os,types,re,signal,traceback,time
   3 #import md5,sha
   4 selinux_capable = False
   5 #userpriv_capable = (os.getuid() == 0)
   6 #fakeroot_capable = False
   7 BASH_BINARY             = "/bin/bash"
   8
   9 try:
  10         import resource
  11         max_fd_limit=resource.getrlimit(RLIMIT_NOFILE)
  12 except SystemExit, e:
  13         raise
  14 except:
  15         # hokay, no resource module.
  16         max_fd_limit=256
  17
  18 # pids this process knows of.
  19 spawned_pids = []
  20
  21 try:
  22         import urllib
  23 except SystemExit, e:
  24         raise
  25
  26 def cleanup(pids,block_exceptions=True):
  27         """function to go through and reap the list of pids passed to it"""
  28         global spawned_pids
  29         if type(pids) == int:
  30                 pids = [pids]
  31         for x in pids:
  32                 try:
  33                         os.kill(x,signal.SIGTERM)
  34                         if os.waitpid(x,os.WNOHANG)[1] == 0:
  35                                 # feisty bugger, still alive.
  36                                 os.kill(x,signal.SIGKILL)
  37                                 os.waitpid(x,0)
  38
  39                 except OSError, oe:
  40                         if block_exceptions:
  41                                 pass
  42                         if oe.errno not in (10,3):
  43                                 raise oe
  44                 except SystemExit:
  45                         raise
  46                 except Exception:
  47                         if block_exceptions:
  48                                 pass
  49                 try:                    spawned_pids.remove(x)
  50                 except IndexError:      pass
  51
  52
  53
  54 # a function to turn a string of non-printable characters into a string of
  55 # hex characters
  56 def hexify(str):
  57         hexStr = string.hexdigits
  58         r = ''
  59         for ch in str:
  60                 i = ord(ch)
  61                 r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
  62         return r
  63 # hexify()
  64
  65
  66 def read_from_clst(file):
  67         line = ''
  68         myline = ''
  69         try:
  70                 myf=open(file,"r")
  71         except:
  72                 return -1
  73                 #raise CatalystError, "Could not open file "+file
  74         for line in myf.readlines():
  75             #line = string.replace(line, "\n", "") # drop newline
  76             myline = myline + line
  77         myf.close()
  78         return myline
  79 # read_from_clst
  80
  81 # these should never be touched
  82 required_build_targets=["generic_target","generic_stage_target"]
  83
  84 # new build types should be added here
  85 valid_build_targets=["stage1_target","stage2_target","stage3_target","stage4_target","grp_target",
  86                         "livecd_stage1_target","livecd_stage2_target","embedded_target",
  87                         "tinderbox_target","snapshot_target","netboot_target","netboot2_target"]
  88
  89 required_config_file_values=["storedir","sharedir","distdir","portdir"]
  90 valid_config_file_values=required_config_file_values[:]
  91 valid_config_file_values.append("PKGCACHE")
  92 valid_config_file_values.append("KERNCACHE")
  93 valid_config_file_values.append("CCACHE")
  94 valid_config_file_values.append("DISTCC")
  95 valid_config_file_values.append("ICECREAM")
  96 valid_config_file_values.append("ENVSCRIPT")
  97 valid_config_file_values.append("AUTORESUME")
  98 valid_config_file_values.append("FETCH")
  99 valid_config_file_values.append("CLEAR_AUTORESUME")
 100 valid_config_file_values.append("options")
 101 valid_config_file_values.append("DEBUG")
 102 valid_config_file_values.append("VERBOSE")
 103 valid_config_file_values.append("PURGE")
 104 valid_config_file_values.append("PURGEONLY")
 105 valid_config_file_values.append("SNAPCACHE")
 106 valid_config_file_values.append("snapshot_cache")
 107 valid_config_file_values.append("hash_function")
 108 valid_config_file_values.append("digests")
 109 valid_config_file_values.append("contents")
 110 valid_config_file_values.append("SEEDCACHE")
 111
 112 verbosity=1
 113
 114 def list_bashify(mylist):
 115         if type(mylist)==types.StringType:
 116                 mypack=[mylist]
 117         else:
 118                 mypack=mylist[:]
 119         for x in range(0,len(mypack)):
 120                 # surround args with quotes for passing to bash,
 121                 # allows things like "<" to remain intact
 122                 mypack[x]="'"+mypack[x]+"'"
 123         mypack=string.join(mypack)
 124         return mypack
 125
 126 def list_to_string(mylist):
 127         if type(mylist)==types.StringType:
 128                 mypack=[mylist]
 129         else:
 130                 mypack=mylist[:]
 131         for x in range(0,len(mypack)):
 132                 # surround args with quotes for passing to bash,
 133                 # allows things like "<" to remain intact
 134                 mypack[x]=mypack[x]
 135         mypack=string.join(mypack)
 136         return mypack
 137
 138 class CatalystError(Exception):
 139         def __init__(self, message):
 140                 if message:
 141                         (type,value)=sys.exc_info()[:2]
 142                         if value!=None:
 143                                 print
 144                                 print traceback.print_exc(file=sys.stdout)
 145                         print
 146                         print "!!! catalyst: "+message
 147                         print
 148
 149 class LockInUse(Exception):
 150         def __init__(self, message):
 151                 if message:
 152                         #(type,value)=sys.exc_info()[:2]
 153                         #if value!=None:
 154                             #print
 155                             #kprint traceback.print_exc(file=sys.stdout)
 156                         print
 157                         print "!!! catalyst lock file in use: "+message
 158                         print
 159
 160 def die(msg=None):
 161         warn(msg)
 162         sys.exit(1)
 163
 164 def warn(msg):
 165         print "!!! catalyst: "+msg
 166
 167 def find_binary(myc):
 168         """look through the environmental path for an executable file named whatever myc is"""
 169         # this sucks. badly.
 170         p=os.getenv("PATH")
 171         if p == None:
 172                 return None
 173         for x in p.split(":"):
 174                 #if it exists, and is executable
 175                 if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
 176                         return "%s/%s" % (x,myc)
 177         return None
 178
 179 def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
 180         """spawn mycommand as an arguement to bash"""
 181         args=[BASH_BINARY]
 182         if not opt_name:
 183             opt_name=mycommand.split()[0]
 184         if "BASH_ENV" not in env:
 185             env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
 186         if debug:
 187             args.append("-x")
 188         args.append("-c")
 189         args.append(mycommand)
 190         return spawn(args,env=env,opt_name=opt_name,**keywords)
 191
 192 #def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \
 193 #        collect_fds=[1],fd_pipes=None,**keywords):
 194
 195 def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
 196         collect_fds=[1],fd_pipes=None,**keywords):
 197         """call spawn, collecting the output to fd's specified in collect_fds list
 198         emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
 199         requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
 200         'lets let log only stdin and let stderr slide by'.
 201
 202         emulate_gso was deprecated from the day it was added, so convert your code over.
 203         spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
 204         global selinux_capable
 205         pr,pw=os.pipe()
 206
 207         #if type(spawn_type) not in [types.FunctionType, types.MethodType]:
 208         #        s="spawn_type must be passed a function, not",type(spawn_type),spawn_type
 209         #        raise Exception,s
 210
 211         if fd_pipes==None:
 212                 fd_pipes={}
 213                 fd_pipes[0] = 0
 214
 215         for x in collect_fds:
 216                 fd_pipes[x] = pw
 217         keywords["returnpid"]=True
 218
 219         mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
 220         os.close(pw)
 221         if type(mypid) != types.ListType:
 222                 os.close(pr)
 223                 return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
 224
 225         fd=os.fdopen(pr,"r")
 226         mydata=fd.readlines()
 227         fd.close()
 228         if emulate_gso:
 229                 mydata=string.join(mydata)
 230                 if len(mydata) and mydata[-1] == "\n":
 231                         mydata=mydata[:-1]
 232         retval=os.waitpid(mypid[0],0)[1]
 233         cleanup(mypid)
 234         if raw_exit_code:
 235                 return [retval,mydata]
 236         retval=process_exit_code(retval)
 237         return [retval, mydata]
 238
 239 # base spawn function
 240 def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\
 241          uid=None,gid=None,groups=None,umask=None,logfile=None,path_lookup=True,\
 242          selinux_context=None, raise_signals=False, func_call=False):
 243         """base fork/execve function.
 244         mycommand is the desired command- if you need a command to execute in a bash/sandbox/fakeroot
 245         environment, use the appropriate spawn call.  This is a straight fork/exec code path.
 246         Can either have a tuple, or a string passed in.  If uid/gid/groups/umask specified, it changes
 247         the forked process to said value.  If path_lookup is on, a non-absolute command will be converted
 248         to an absolute command, otherwise it returns None.
 249
 250         selinux_context is the desired context, dependant on selinux being available.
 251         opt_name controls the name the processor goes by.
 252         fd_pipes controls which file descriptor numbers are left open in the forked process- it's a dict of
 253         current fd's raw fd #, desired #.
 254
 255         func_call is a boolean for specifying to execute a python function- use spawn_func instead.
 256         raise_signals is questionable.  Basically throw an exception if signal'd.  No exception is thrown
 257         if raw_input is on.
 258
 259         logfile overloads the specified fd's to write to a tee process which logs to logfile
 260         returnpid returns the relevant pids (a list, including the logging process if logfile is on).
 261
 262         non-returnpid calls to spawn will block till the process has exited, returning the exitcode/signal
 263         raw_exit_code controls whether the actual waitpid result is returned, or intrepretted."""
 264
 265         myc=''
 266         if not func_call:
 267                 if type(mycommand)==types.StringType:
 268                         mycommand=mycommand.split()
 269                 myc = mycommand[0]
 270                 if not os.access(myc, os.X_OK):
 271                         if not path_lookup:
 272                                 return None
 273                         myc = find_binary(myc)
 274                         if myc == None:
 275                             return None
 276         mypid=[]
 277         if logfile:
 278                 pr,pw=os.pipe()
 279                 mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2}))
 280                 retval=os.waitpid(mypid[-1],os.WNOHANG)[1]
 281                 if retval != 0:
 282                         # he's dead jim.
 283                         if raw_exit_code:
 284                                 return retval
 285                         return process_exit_code(retval)
 286
 287                 if fd_pipes == None:
 288                         fd_pipes={}
 289                         fd_pipes[0] = 0
 290                 fd_pipes[1]=pw
 291                 fd_pipes[2]=pw
 292
 293         if not opt_name:
 294                 opt_name = mycommand[0]
 295         myargs=[opt_name]
 296         myargs.extend(mycommand[1:])
 297         global spawned_pids
 298         mypid.append(os.fork())
 299         if mypid[-1] != 0:
 300                 #log the bugger.
 301                 spawned_pids.extend(mypid)
 302
 303         if mypid[-1] == 0:
 304                 if func_call:
 305                         spawned_pids = []
 306
 307                 # this may look ugly, but basically it moves file descriptors around to ensure no
 308                 # handles that are needed are accidentally closed during the final dup2 calls.
 309                 trg_fd=[]
 310                 if type(fd_pipes)==types.DictType:
 311                         src_fd=[]
 312                         k=fd_pipes.keys()
 313                         k.sort()
 314
 315                         #build list of which fds will be where, and where they are at currently
 316                         for x in k:
 317                                 trg_fd.append(x)
 318                                 src_fd.append(fd_pipes[x])
 319
 320                         # run through said list dup'ing descriptors so that they won't be waxed
 321                         # by other dup calls.
 322                         for x in range(0,len(trg_fd)):
 323                                 if trg_fd[x] == src_fd[x]:
 324                                         continue
 325                                 if trg_fd[x] in src_fd[x+1:]:
 326                                         new=os.dup2(trg_fd[x],max(src_fd) + 1)
 327                                         os.close(trg_fd[x])
 328                                         try:
 329                                                 while True:
 330                                                         src_fd[s.index(trg_fd[x])]=new
 331                                         except SystemExit, e:
 332                                                 raise
 333                                         except:
 334                                                 pass
 335
 336                         # transfer the fds to their final pre-exec position.
 337                         for x in range(0,len(trg_fd)):
 338                                 if trg_fd[x] != src_fd[x]:
 339                                         os.dup2(src_fd[x], trg_fd[x])
 340                 else:
 341                         trg_fd=[0,1,2]
 342
 343                 # wax all open descriptors that weren't requested be left open.
 344                 for x in range(0,max_fd_limit):
 345                         if x not in trg_fd:
 346                                 try:
 347                                         os.close(x)
 348                                 except SystemExit, e:
 349                                         raise
 350                                 except:
 351                                         pass
 352
 353                 # note this order must be preserved- can't change gid/groups if you change uid first.
 354                 if selinux_capable and selinux_context:
 355                         import selinux
 356                         selinux.setexec(selinux_context)
 357                 if gid:
 358                         os.setgid(gid)
 359                 if groups:
 360                         os.setgroups(groups)
 361                 if uid:
 362                         os.setuid(uid)
 363                 if umask:
 364                         os.umask(umask)
 365                 else:
 366                         os.umask(022)
 367
 368                 try:
 369                         #print "execing", myc, myargs
 370                         if func_call:
 371                                 # either use a passed in func for interpretting the results, or return if no exception.
 372                                 # note the passed in list, and dict are expanded.
 373                                 if len(mycommand) == 4:
 374                                         os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
 375                                 try:
 376                                         mycommand[0](*mycommand[1],**mycommand[2])
 377                                 except Exception,e:
 378                                         print "caught exception",e," in forked func",mycommand[0]
 379                                 sys.exit(0)
 380
 381                         #os.execvp(myc,myargs)
 382                         os.execve(myc,myargs,env)
 383                 except SystemExit, e:
 384                         raise
 385                 except Exception, e:
 386                         if not func_call:
 387                                 raise str(e)+":\n   "+myc+" "+string.join(myargs)
 388                         print "func call failed"
 389
 390                 # If the execve fails, we need to report it, and exit
 391                 # *carefully* --- report error here
 392                 os._exit(1)
 393                 sys.exit(1)
 394                 return # should never get reached
 395
 396         # if we were logging, kill the pipes.
 397         if logfile:
 398                 os.close(pr)
 399                 os.close(pw)
 400
 401         if returnpid:
 402                 return mypid
 403
 404         # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
 405         # if the main pid (mycommand) returned badly.
 406         while len(mypid):
 407                 retval=os.waitpid(mypid[-1],0)[1]
 408                 if retval != 0:
 409                         cleanup(mypid[0:-1],block_exceptions=False)
 410                         # at this point we've killed all other kid pids generated via this call.
 411                         # return now.
 412                         if raw_exit_code:
 413                                 return retval
 414                         return process_exit_code(retval,throw_signals=raise_signals)
 415                 else:
 416                         mypid.pop(-1)
 417         cleanup(mypid)
 418         return 0
 419
 420 def cmd(mycmd,myexc="",env={}):
 421         try:
 422                 sys.stdout.flush()
 423                 retval=spawn_bash(mycmd,env)
 424                 if retval != 0:
 425                         raise CatalystError,myexc
 426         except:
 427                 raise
 428
 429 def process_exit_code(retval,throw_signals=False):
 430         """process a waitpid returned exit code, returning exit code if it exit'd, or the
 431         signal if it died from signalling
 432         if throw_signals is on, it raises a SystemExit if the process was signaled.
 433         This is intended for usage with threads, although at the moment you can't signal individual
 434         threads in python, only the master thread, so it's a questionable option."""
 435         if (retval & 0xff)==0:
 436                 return retval >> 8 # return exit code
 437         else:
 438                 if throw_signals:
 439                         #use systemexit, since portage is stupid about exception catching.
 440                         raise SystemExit()
 441                 return (retval & 0xff) << 8 # interrupted by signal
 442
 443 def file_locate(settings,filelist,expand=1):
 444         #if expand=1, non-absolute paths will be accepted and
 445         # expanded to os.getcwd()+"/"+localpath if file exists
 446         for myfile in filelist:
 447                 if myfile not in settings:
 448                         #filenames such as cdtar are optional, so we don't assume the variable is defined.
 449                         pass
 450                 else:
 451                     if len(settings[myfile])==0:
 452                             raise CatalystError, "File variable \""+myfile+"\" has a length of zero (not specified.)"
 453                     if settings[myfile][0]=="/":
 454                             if not os.path.exists(settings[myfile]):
 455                                     raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]
 456                     elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
 457                             settings[myfile]=os.getcwd()+"/"+settings[myfile]
 458                     else:
 459                             raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]+" (2nd try)"
 460 """
 461 Spec file format:
 462
 463 The spec file format is a very simple and easy-to-use format for storing data. Here's an example
 464 file:
 465
 466 item1: value1
 467 item2: foo bar oni
 468 item3:
 469         meep
 470         bark
 471         gleep moop
 472
 473 This file would be interpreted as defining three items: item1, item2 and item3. item1 would contain
 474 the string value "value1". Item2 would contain an ordered list [ "foo", "bar", "oni" ]. item3
 475 would contain an ordered list as well: [ "meep", "bark", "gleep", "moop" ]. It's important to note
 476 that the order of multiple-value items is preserved, but the order that the items themselves are
 477 defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1"
 478 "item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash).
 479 """
 480
 481 def parse_makeconf(mylines):
 482         mymakeconf={}
 483         pos=0
 484         pat=re.compile("([0-9a-zA-Z_]*)=(.*)")
 485         while pos<len(mylines):
 486                 if len(mylines[pos])<=1:
 487                         #skip blanks
 488                         pos += 1
 489                         continue
 490                 if mylines[pos][0] in ["#"," ","\t"]:
 491                         #skip indented lines, comments
 492                         pos += 1
 493                         continue
 494                 else:
 495                         myline=mylines[pos]
 496                         mobj=pat.match(myline)
 497                         pos += 1
 498                         if mobj.group(2):
 499                             clean_string = re.sub(r"\"",r"",mobj.group(2))
 500                             mymakeconf[mobj.group(1)]=clean_string
 501         return mymakeconf
 502
 503 def read_makeconf(mymakeconffile):
 504         if os.path.exists(mymakeconffile):
 505                 try:
 506                         try:
 507                                 import snakeoil.bash #import snakeoil.fileutils
 508                                 return snakeoil.bash.read_bash_dict(mymakeconffile, sourcing_command="source")
 509                         except ImportError:
 510                                 try:
 511                                         import portage.util
 512                                         return portage.util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
 513                                 except:
 514                                         try:
 515                                                 import portage_util
 516                                                 return portage_util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
 517                                         except ImportError:
 518                                                 myf=open(mymakeconffile,"r")
 519                                                 mylines=myf.readlines()
 520                                                 myf.close()
 521                                                 return parse_makeconf(mylines)
 522                 except:
 523                         raise CatalystError, "Could not parse make.conf file "+mymakeconffile
 524         else:
 525                 makeconf={}
 526                 return makeconf
 527
 528 def msg(mymsg,verblevel=1):
 529         if verbosity>=verblevel:
 530                 print mymsg
 531
 532 def pathcompare(path1,path2):
 533         # Change double slashes to slash
 534         path1 = re.sub(r"//",r"/",path1)
 535         path2 = re.sub(r"//",r"/",path2)
 536         # Removing ending slash
 537         path1 = re.sub("/$","",path1)
 538         path2 = re.sub("/$","",path2)
 539
 540         if path1 == path2:
 541                 return 1
 542         return 0
 543
 544 def ismount(path):
 545         "enhanced to handle bind mounts"
 546         if os.path.ismount(path):
 547                 return 1
 548         a=os.popen("mount")
 549         mylines=a.readlines()
 550         a.close()
 551         for line in mylines:
 552                 mysplit=line.split()
 553                 if pathcompare(path,mysplit[2]):
 554                         return 1
 555         return 0
 556
 557 def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 558         "helper function to help targets parse additional arguments"
 559         global valid_config_file_values
 560
 561         messages = []
 562         for x in addlargs.keys():
 563                 if x not in validspec and x not in valid_config_file_values and x not in requiredspec:
 564                         messages.append("Argument \""+x+"\" not recognized.")
 565                 else:
 566                         myspec[x]=addlargs[x]
 567
 568         for x in requiredspec:
 569                 if x not in myspec:
 570                         messages.append("Required argument \""+x+"\" not specified.")
 571
 572         if messages:
 573                 raise CatalystError, '\n\tAlso: '.join(messages)
 574
 575 def touch(myfile):
 576         try:
 577                 myf=open(myfile,"w")
 578                 myf.close()
 579         except IOError:
 580                 raise CatalystError, "Could not touch "+myfile+"."
 581
 582 def countdown(secs=5, doing="Starting"):
 583         if secs:
 584                 print ">>> Waiting",secs,"seconds before starting..."
 585                 print ">>> (Control-C to abort)...\n"+doing+" in: ",
 586                 ticks=range(secs)
 587                 ticks.reverse()
 588                 for sec in ticks:
 589                         sys.stdout.write(str(sec+1)+" ")
 590                         sys.stdout.flush()
 591                         time.sleep(1)
 592                 print
 593
 594 def normpath(mypath):
 595         TrailingSlash=False
 596         if mypath[-1] == "/":
 597             TrailingSlash=True
 598         newpath = os.path.normpath(mypath)
 599         if len(newpath) > 1:
 600                 if newpath[:2] == "//":
 601                         newpath = newpath[1:]
 602         if TrailingSlash:
 603             newpath=newpath+'/'
 604         return newpath