modules/catalyst_support.py

   1
   2 import sys,string,os,types,re,signal,traceback,time
   3 #import md5,sha
   4 selinux_capable = False
   5 #userpriv_capable = (os.getuid() == 0)
   6 #fakeroot_capable = False
   7 BASH_BINARY             = "/bin/bash"
   8
   9 try:
  10         import resource
  11         max_fd_limit=resource.getrlimit(RLIMIT_NOFILE)
  12 except SystemExit, e:
  13         raise
  14 except:
  15         # hokay, no resource module.
  16         max_fd_limit=256
  17
  18 # pids this process knows of.
  19 spawned_pids = []
  20
  21 try:
  22         import urllib
  23 except SystemExit, e:
  24         raise
  25
  26 def cleanup(pids,block_exceptions=True):
  27         """function to go through and reap the list of pids passed to it"""
  28         global spawned_pids
  29         if type(pids) == int:
  30                 pids = [pids]
  31         for x in pids:
  32                 try:
  33                         os.kill(x,signal.SIGTERM)
  34                         if os.waitpid(x,os.WNOHANG)[1] == 0:
  35                                 # feisty bugger, still alive.
  36                                 os.kill(x,signal.SIGKILL)
  37                                 os.waitpid(x,0)
  38
  39                 except OSError, oe:
  40                         if block_exceptions:
  41                                 pass
  42                         if oe.errno not in (10,3):
  43                                 raise oe
  44                 except SystemExit:
  45                         raise
  46                 except Exception:
  47                         if block_exceptions:
  48                                 pass
  49                 try:                    spawned_pids.remove(x)
  50                 except IndexError:      pass
  51
  52
  53
  54 # a function to turn a string of non-printable characters into a string of
  55 # hex characters
  56 def hexify(str):
  57         hexStr = string.hexdigits
  58         r = ''
  59         for ch in str:
  60                 i = ord(ch)
  61                 r = r + hexStr[(i >> 4) & 0xF] + hexStr[i & 0xF]
  62         return r
  63 # hexify()
  64
  65 def generate_contents(file,contents_function="auto",verbose=False):
  66         try:
  67                 _ = contents_function
  68                 if _ == 'auto' and file.endswith('.iso'):
  69                         _ = 'isoinfo-l'
  70                 if (_ in ['tar-tv','auto']):
  71                         if file.endswith('.tgz') or file.endswith('.tar.gz'):
  72                                 _ = 'tar-tvz'
  73                         elif file.endswith('.tbz2') or file.endswith('.tar.bz2'):
  74                                 _ = 'tar-tvj'
  75                         elif file.endswith('.tar'):
  76                                 _ = 'tar-tv'
  77
  78                 if _ == 'auto':
  79                         warn('File %r has unknown type for automatic detection.' % (file, ))
  80                         return None
  81                 else:
  82                         contents_function = _
  83                         _ = contents_map[contents_function]
  84                         return _[0](file,_[1],verbose)
  85         except:
  86                 raise CatalystError,\
  87                         "Error generating contents, is appropriate utility (%s) installed on your system?" \
  88                         % (contents_function, )
  89
  90 def calc_contents(file,cmd,verbose):
  91         args={ 'file': file }
  92         cmd=cmd % dict(args)
  93         a=os.popen(cmd)
  94         mylines=a.readlines()
  95         a.close()
  96         result="".join(mylines)
  97         if verbose:
  98                 print result
  99         return result
 100
 101 # This has map must be defined after the function calc_content
 102 # It is possible to call different functions from this but they must be defined
 103 # before hash_map
 104 # Key,function,cmd
 105 contents_map={
 106         # 'find' is disabled because it requires the source path, which is not
 107         # always available
 108         #"find"         :[calc_contents,"find %(path)s"],
 109         "tar-tv":[calc_contents,"tar tvf %(file)s"],
 110         "tar-tvz":[calc_contents,"tar tvzf %(file)s"],
 111         "tar-tvj":[calc_contents,"tar tvjf %(file)s"],
 112         "isoinfo-l":[calc_contents,"isoinfo -l -i %(file)s"],
 113         # isoinfo-f should be a last resort only
 114         "isoinfo-f":[calc_contents,"isoinfo -f -i %(file)s"],
 115 }
 116
 117 def generate_hash(file,hash_function="crc32",verbose=False):
 118         try:
 119                 return hash_map[hash_function][0](file,hash_map[hash_function][1],hash_map[hash_function][2],\
 120                         hash_map[hash_function][3],verbose)
 121         except:
 122                 raise CatalystError,"Error generating hash, is appropriate utility installed on your system?"
 123
 124 def calc_hash(file,cmd,cmd_args,id_string="MD5",verbose=False):
 125         a=os.popen(cmd+" "+cmd_args+" "+file)
 126         mylines=a.readlines()
 127         a.close()
 128         mylines=mylines[0].split()
 129         result=mylines[0]
 130         if verbose:
 131                 print id_string+" (%s) = %s" % (file, result)
 132         return result
 133
 134 def calc_hash2(file,cmd,cmd_args,id_string="MD5",verbose=False):
 135         a=os.popen(cmd+" "+cmd_args+" "+file)
 136         header=a.readline()
 137         mylines=a.readline().split()
 138         hash=mylines[0]
 139         short_file=os.path.split(mylines[1])[1]
 140         a.close()
 141         result=header+hash+"  "+short_file+"\n"
 142         if verbose:
 143                 print header+" (%s) = %s" % (short_file, result)
 144         return result
 145
 146 # This has map must be defined after the function calc_hash
 147 # It is possible to call different functions from this but they must be defined
 148 # before hash_map
 149 # Key,function,cmd,cmd_args,Print string
 150 hash_map={
 151          "adler32":[calc_hash2,"shash","-a ADLER32","ADLER32"],\
 152          "crc32":[calc_hash2,"shash","-a CRC32","CRC32"],\
 153          "crc32b":[calc_hash2,"shash","-a CRC32B","CRC32B"],\
 154          "gost":[calc_hash2,"shash","-a GOST","GOST"],\
 155          "haval128":[calc_hash2,"shash","-a HAVAL128","HAVAL128"],\
 156          "haval160":[calc_hash2,"shash","-a HAVAL160","HAVAL160"],\
 157          "haval192":[calc_hash2,"shash","-a HAVAL192","HAVAL192"],\
 158          "haval224":[calc_hash2,"shash","-a HAVAL224","HAVAL224"],\
 159          "haval256":[calc_hash2,"shash","-a HAVAL256","HAVAL256"],\
 160          "md2":[calc_hash2,"shash","-a MD2","MD2"],\
 161          "md4":[calc_hash2,"shash","-a MD4","MD4"],\
 162          "md5":[calc_hash2,"shash","-a MD5","MD5"],\
 163          "ripemd128":[calc_hash2,"shash","-a RIPEMD128","RIPEMD128"],\
 164          "ripemd160":[calc_hash2,"shash","-a RIPEMD160","RIPEMD160"],\
 165          "ripemd256":[calc_hash2,"shash","-a RIPEMD256","RIPEMD256"],\
 166          "ripemd320":[calc_hash2,"shash","-a RIPEMD320","RIPEMD320"],\
 167          "sha1":[calc_hash2,"shash","-a SHA1","SHA1"],\
 168          "sha224":[calc_hash2,"shash","-a SHA224","SHA224"],\
 169          "sha256":[calc_hash2,"shash","-a SHA256","SHA256"],\
 170          "sha384":[calc_hash2,"shash","-a SHA384","SHA384"],\
 171          "sha512":[calc_hash2,"shash","-a SHA512","SHA512"],\
 172          "snefru128":[calc_hash2,"shash","-a SNEFRU128","SNEFRU128"],\
 173          "snefru256":[calc_hash2,"shash","-a SNEFRU256","SNEFRU256"],\
 174          "tiger":[calc_hash2,"shash","-a TIGER","TIGER"],\
 175          "tiger128":[calc_hash2,"shash","-a TIGER128","TIGER128"],\
 176          "tiger160":[calc_hash2,"shash","-a TIGER160","TIGER160"],\
 177          "whirlpool":[calc_hash2,"shash","-a WHIRLPOOL","WHIRLPOOL"],\
 178          }
 179
 180 def read_from_clst(file):
 181         line = ''
 182         myline = ''
 183         try:
 184                 myf=open(file,"r")
 185         except:
 186                 return -1
 187                 #raise CatalystError, "Could not open file "+file
 188         for line in myf.readlines():
 189             #line = string.replace(line, "\n", "") # drop newline
 190             myline = myline + line
 191         myf.close()
 192         return myline
 193 # read_from_clst
 194
 195 # these should never be touched
 196 required_build_targets=["generic_target","generic_stage_target"]
 197
 198 # new build types should be added here
 199 valid_build_targets=["stage1_target","stage2_target","stage3_target","stage4_target","grp_target",
 200                         "livecd_stage1_target","livecd_stage2_target","embedded_target",
 201                         "tinderbox_target","snapshot_target","netboot_target","netboot2_target"]
 202
 203 required_config_file_values=["storedir","sharedir","distdir","portdir"]
 204 valid_config_file_values=required_config_file_values[:]
 205 valid_config_file_values.append("PKGCACHE")
 206 valid_config_file_values.append("KERNCACHE")
 207 valid_config_file_values.append("CCACHE")
 208 valid_config_file_values.append("DISTCC")
 209 valid_config_file_values.append("ICECREAM")
 210 valid_config_file_values.append("ENVSCRIPT")
 211 valid_config_file_values.append("AUTORESUME")
 212 valid_config_file_values.append("FETCH")
 213 valid_config_file_values.append("CLEAR_AUTORESUME")
 214 valid_config_file_values.append("options")
 215 valid_config_file_values.append("DEBUG")
 216 valid_config_file_values.append("VERBOSE")
 217 valid_config_file_values.append("PURGE")
 218 valid_config_file_values.append("SNAPCACHE")
 219 valid_config_file_values.append("snapshot_cache")
 220 valid_config_file_values.append("hash_function")
 221 valid_config_file_values.append("digests")
 222 valid_config_file_values.append("contents")
 223 valid_config_file_values.append("SEEDCACHE")
 224
 225 verbosity=1
 226
 227 def list_bashify(mylist):
 228         if type(mylist)==types.StringType:
 229                 mypack=[mylist]
 230         else:
 231                 mypack=mylist[:]
 232         for x in range(0,len(mypack)):
 233                 # surround args with quotes for passing to bash,
 234                 # allows things like "<" to remain intact
 235                 mypack[x]="'"+mypack[x]+"'"
 236         mypack=string.join(mypack)
 237         return mypack
 238
 239 def list_to_string(mylist):
 240         if type(mylist)==types.StringType:
 241                 mypack=[mylist]
 242         else:
 243                 mypack=mylist[:]
 244         for x in range(0,len(mypack)):
 245                 # surround args with quotes for passing to bash,
 246                 # allows things like "<" to remain intact
 247                 mypack[x]=mypack[x]
 248         mypack=string.join(mypack)
 249         return mypack
 250
 251 class CatalystError(Exception):
 252         def __init__(self, message):
 253                 if message:
 254                         (type,value)=sys.exc_info()[:2]
 255                         if value!=None:
 256                                 print
 257                                 print traceback.print_exc(file=sys.stdout)
 258                         print
 259                         print "!!! catalyst: "+message
 260                         print
 261
 262 class LockInUse(Exception):
 263         def __init__(self, message):
 264                 if message:
 265                         #(type,value)=sys.exc_info()[:2]
 266                         #if value!=None:
 267                             #print
 268                             #kprint traceback.print_exc(file=sys.stdout)
 269                         print
 270                         print "!!! catalyst lock file in use: "+message
 271                         print
 272
 273 def die(msg=None):
 274         warn(msg)
 275         sys.exit(1)
 276
 277 def warn(msg):
 278         print "!!! catalyst: "+msg
 279
 280
 281 def find_binary(myc):
 282         """look through the environmental path for an executable file named whatever myc is"""
 283         # this sucks. badly.
 284         p=os.getenv("PATH")
 285         if p == None:
 286                 return None
 287         for x in p.split(":"):
 288                 #if it exists, and is executable
 289                 if os.path.exists("%s/%s" % (x,myc)) and os.stat("%s/%s" % (x,myc))[0] & 0x0248:
 290                         return "%s/%s" % (x,myc)
 291         return None
 292
 293
 294 def spawn_bash(mycommand,env={},debug=False,opt_name=None,**keywords):
 295         """spawn mycommand as an arguement to bash"""
 296         args=[BASH_BINARY]
 297         if not opt_name:
 298             opt_name=mycommand.split()[0]
 299         if not env.has_key("BASH_ENV"):
 300             env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
 301         if debug:
 302             args.append("-x")
 303         args.append("-c")
 304         args.append(mycommand)
 305         return spawn(args,env=env,opt_name=opt_name,**keywords)
 306
 307 #def spawn_get_output(mycommand,spawn_type=spawn,raw_exit_code=False,emulate_gso=True, \
 308 #        collect_fds=[1],fd_pipes=None,**keywords):
 309 def spawn_get_output(mycommand,raw_exit_code=False,emulate_gso=True, \
 310         collect_fds=[1],fd_pipes=None,**keywords):
 311         """call spawn, collecting the output to fd's specified in collect_fds list
 312         emulate_gso is a compatability hack to emulate commands.getstatusoutput's return, minus the
 313         requirement it always be a bash call (spawn_type controls the actual spawn call), and minus the
 314         'lets let log only stdin and let stderr slide by'.
 315
 316         emulate_gso was deprecated from the day it was added, so convert your code over.
 317         spawn_type is the passed in function to call- typically spawn_bash, spawn, spawn_sandbox, or spawn_fakeroot"""
 318         global selinux_capable
 319         pr,pw=os.pipe()
 320
 321         #if type(spawn_type) not in [types.FunctionType, types.MethodType]:
 322         #        s="spawn_type must be passed a function, not",type(spawn_type),spawn_type
 323         #        raise Exception,s
 324
 325         if fd_pipes==None:
 326                 fd_pipes={}
 327                 fd_pipes[0] = 0
 328
 329         for x in collect_fds:
 330                 fd_pipes[x] = pw
 331         keywords["returnpid"]=True
 332
 333         mypid=spawn_bash(mycommand,fd_pipes=fd_pipes,**keywords)
 334         os.close(pw)
 335         if type(mypid) != types.ListType:
 336                 os.close(pr)
 337                 return [mypid, "%s: No such file or directory" % mycommand.split()[0]]
 338
 339         fd=os.fdopen(pr,"r")
 340         mydata=fd.readlines()
 341         fd.close()
 342         if emulate_gso:
 343                 mydata=string.join(mydata)
 344                 if len(mydata) and mydata[-1] == "\n":
 345                         mydata=mydata[:-1]
 346         retval=os.waitpid(mypid[0],0)[1]
 347         cleanup(mypid)
 348         if raw_exit_code:
 349                 return [retval,mydata]
 350         retval=process_exit_code(retval)
 351         return [retval, mydata]
 352
 353
 354 # base spawn function
 355 def spawn(mycommand,env={},raw_exit_code=False,opt_name=None,fd_pipes=None,returnpid=False,\
 356          uid=None,gid=None,groups=None,umask=None,logfile=None,path_lookup=True,\
 357          selinux_context=None, raise_signals=False, func_call=False):
 358         """base fork/execve function.
 359         mycommand is the desired command- if you need a command to execute in a bash/sandbox/fakeroot
 360         environment, use the appropriate spawn call.  This is a straight fork/exec code path.
 361         Can either have a tuple, or a string passed in.  If uid/gid/groups/umask specified, it changes
 362         the forked process to said value.  If path_lookup is on, a non-absolute command will be converted
 363         to an absolute command, otherwise it returns None.
 364
 365         selinux_context is the desired context, dependant on selinux being available.
 366         opt_name controls the name the processor goes by.
 367         fd_pipes controls which file descriptor numbers are left open in the forked process- it's a dict of
 368         current fd's raw fd #, desired #.
 369
 370         func_call is a boolean for specifying to execute a python function- use spawn_func instead.
 371         raise_signals is questionable.  Basically throw an exception if signal'd.  No exception is thrown
 372         if raw_input is on.
 373
 374         logfile overloads the specified fd's to write to a tee process which logs to logfile
 375         returnpid returns the relevant pids (a list, including the logging process if logfile is on).
 376
 377         non-returnpid calls to spawn will block till the process has exited, returning the exitcode/signal
 378         raw_exit_code controls whether the actual waitpid result is returned, or intrepretted."""
 379
 380
 381         myc=''
 382         if not func_call:
 383                 if type(mycommand)==types.StringType:
 384                         mycommand=mycommand.split()
 385                 myc = mycommand[0]
 386                 if not os.access(myc, os.X_OK):
 387                         if not path_lookup:
 388                                 return None
 389                         myc = find_binary(myc)
 390                         if myc == None:
 391                             return None
 392         mypid=[]
 393         if logfile:
 394                 pr,pw=os.pipe()
 395                 mypid.extend(spawn(('tee','-i','-a',logfile),returnpid=True,fd_pipes={0:pr,1:1,2:2}))
 396                 retval=os.waitpid(mypid[-1],os.WNOHANG)[1]
 397                 if retval != 0:
 398                         # he's dead jim.
 399                         if raw_exit_code:
 400                                 return retval
 401                         return process_exit_code(retval)
 402
 403                 if fd_pipes == None:
 404                         fd_pipes={}
 405                         fd_pipes[0] = 0
 406                 fd_pipes[1]=pw
 407                 fd_pipes[2]=pw
 408
 409         if not opt_name:
 410                 opt_name = mycommand[0]
 411         myargs=[opt_name]
 412         myargs.extend(mycommand[1:])
 413         global spawned_pids
 414         mypid.append(os.fork())
 415         if mypid[-1] != 0:
 416                 #log the bugger.
 417                 spawned_pids.extend(mypid)
 418
 419         if mypid[-1] == 0:
 420                 if func_call:
 421                         spawned_pids = []
 422
 423                 # this may look ugly, but basically it moves file descriptors around to ensure no
 424                 # handles that are needed are accidentally closed during the final dup2 calls.
 425                 trg_fd=[]
 426                 if type(fd_pipes)==types.DictType:
 427                         src_fd=[]
 428                         k=fd_pipes.keys()
 429                         k.sort()
 430
 431                         #build list of which fds will be where, and where they are at currently
 432                         for x in k:
 433                                 trg_fd.append(x)
 434                                 src_fd.append(fd_pipes[x])
 435
 436                         # run through said list dup'ing descriptors so that they won't be waxed
 437                         # by other dup calls.
 438                         for x in range(0,len(trg_fd)):
 439                                 if trg_fd[x] == src_fd[x]:
 440                                         continue
 441                                 if trg_fd[x] in src_fd[x+1:]:
 442                                         new=os.dup2(trg_fd[x],max(src_fd) + 1)
 443                                         os.close(trg_fd[x])
 444                                         try:
 445                                                 while True:
 446                                                         src_fd[s.index(trg_fd[x])]=new
 447                                         except SystemExit, e:
 448                                                 raise
 449                                         except:
 450                                                 pass
 451
 452                         # transfer the fds to their final pre-exec position.
 453                         for x in range(0,len(trg_fd)):
 454                                 if trg_fd[x] != src_fd[x]:
 455                                         os.dup2(src_fd[x], trg_fd[x])
 456                 else:
 457                         trg_fd=[0,1,2]
 458
 459                 # wax all open descriptors that weren't requested be left open.
 460                 for x in range(0,max_fd_limit):
 461                         if x not in trg_fd:
 462                                 try:
 463                                         os.close(x)
 464                                 except SystemExit, e:
 465                                         raise
 466                                 except:
 467                                         pass
 468
 469                 # note this order must be preserved- can't change gid/groups if you change uid first.
 470                 if selinux_capable and selinux_context:
 471                         import selinux
 472                         selinux.setexec(selinux_context)
 473                 if gid:
 474                         os.setgid(gid)
 475                 if groups:
 476                         os.setgroups(groups)
 477                 if uid:
 478                         os.setuid(uid)
 479                 if umask:
 480                         os.umask(umask)
 481
 482                 try:
 483                         #print "execing", myc, myargs
 484                         if func_call:
 485                                 # either use a passed in func for interpretting the results, or return if no exception.
 486                                 # note the passed in list, and dict are expanded.
 487                                 if len(mycommand) == 4:
 488                                         os._exit(mycommand[3](mycommand[0](*mycommand[1],**mycommand[2])))
 489                                 try:
 490                                         mycommand[0](*mycommand[1],**mycommand[2])
 491                                 except Exception,e:
 492                                         print "caught exception",e," in forked func",mycommand[0]
 493                                 sys.exit(0)
 494
 495                         #os.execvp(myc,myargs)
 496                         os.execve(myc,myargs,env)
 497                 except SystemExit, e:
 498                         raise
 499                 except Exception, e:
 500                         if not func_call:
 501                                 raise str(e)+":\n   "+myc+" "+string.join(myargs)
 502                         print "func call failed"
 503
 504                 # If the execve fails, we need to report it, and exit
 505                 # *carefully* --- report error here
 506                 os._exit(1)
 507                 sys.exit(1)
 508                 return # should never get reached
 509
 510         # if we were logging, kill the pipes.
 511         if logfile:
 512                 os.close(pr)
 513                 os.close(pw)
 514
 515         if returnpid:
 516                 return mypid
 517
 518         # loop through pids (typically one, unless logging), either waiting on their death, or waxing them
 519         # if the main pid (mycommand) returned badly.
 520         while len(mypid):
 521                 retval=os.waitpid(mypid[-1],0)[1]
 522                 if retval != 0:
 523                         cleanup(mypid[0:-1],block_exceptions=False)
 524                         # at this point we've killed all other kid pids generated via this call.
 525                         # return now.
 526                         if raw_exit_code:
 527                                 return retval
 528                         return process_exit_code(retval,throw_signals=raise_signals)
 529                 else:
 530                         mypid.pop(-1)
 531         cleanup(mypid)
 532         return 0
 533
 534 def cmd(mycmd,myexc="",env={}):
 535         try:
 536                 sys.stdout.flush()
 537                 retval=spawn_bash(mycmd,env)
 538                 if retval != 0:
 539                         raise CatalystError,myexc
 540         except:
 541                 raise
 542
 543 def process_exit_code(retval,throw_signals=False):
 544         """process a waitpid returned exit code, returning exit code if it exit'd, or the
 545         signal if it died from signalling
 546         if throw_signals is on, it raises a SystemExit if the process was signaled.
 547         This is intended for usage with threads, although at the moment you can't signal individual
 548         threads in python, only the master thread, so it's a questionable option."""
 549         if (retval & 0xff)==0:
 550                 return retval >> 8 # return exit code
 551         else:
 552                 if throw_signals:
 553                         #use systemexit, since portage is stupid about exception catching.
 554                         raise SystemExit()
 555                 return (retval & 0xff) << 8 # interrupted by signal
 556
 557
 558 def file_locate(settings,filelist,expand=1):
 559         #if expand=1, non-absolute paths will be accepted and
 560         # expanded to os.getcwd()+"/"+localpath if file exists
 561         for myfile in filelist:
 562                 if not settings.has_key(myfile):
 563                         #filenames such as cdtar are optional, so we don't assume the variable is defined.
 564                         pass
 565                 else:
 566                     if len(settings[myfile])==0:
 567                             raise CatalystError, "File variable \""+myfile+"\" has a length of zero (not specified.)"
 568                     if settings[myfile][0]=="/":
 569                             if not os.path.exists(settings[myfile]):
 570                                     raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]
 571                     elif expand and os.path.exists(os.getcwd()+"/"+settings[myfile]):
 572                             settings[myfile]=os.getcwd()+"/"+settings[myfile]
 573                     else:
 574                             raise CatalystError, "Cannot locate specified "+myfile+": "+settings[myfile]+" (2nd try)"
 575 """
 576 Spec file format:
 577
 578 The spec file format is a very simple and easy-to-use format for storing data. Here's an example
 579 file:
 580
 581 item1: value1
 582 item2: foo bar oni
 583 item3:
 584         meep
 585         bark
 586         gleep moop
 587
 588 This file would be interpreted as defining three items: item1, item2 and item3. item1 would contain
 589 the string value "value1". Item2 would contain an ordered list [ "foo", "bar", "oni" ]. item3
 590 would contain an ordered list as well: [ "meep", "bark", "gleep", "moop" ]. It's important to note
 591 that the order of multiple-value items is preserved, but the order that the items themselves are
 592 defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1"
 593 "item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash).
 594 """
 595
 596 def parse_spec(mylines):
 597         myspec = {}
 598         cur_array = []
 599         trailing_comment=re.compile("#.*$")
 600         white_space=re.compile("\s+")
 601         while len(mylines):
 602                 myline = mylines.pop(0).strip()
 603
 604                 # Force the line to be clean
 605                 # Remove Comments ( anything following # )
 606                 myline = trailing_comment.sub("", myline)
 607
 608                 # Skip any blank lines
 609                 if not myline: continue
 610
 611                 # Look for colon
 612                 msearch = myline.find(':')
 613
 614                 # If semicolon found assume its a new key
 615                 # This may cause problems if : are used for key values but works for now
 616                 if msearch != -1:
 617                         # Split on the first semicolon creating two strings in the array mobjs
 618                         mobjs = myline.split(':', 1)
 619                         mobjs[1] = mobjs[1].strip()
 620
 621                         # Check that this key doesn't exist already in the spec
 622                         if myspec.has_key(mobjs[0]):
 623                                 raise Exception("You have a duplicate key (" + mobjs[0] + ") in your spec. Please fix it")
 624
 625                         # Start a new array using the first element of mobjs
 626                         cur_array = [mobjs[0]]
 627                         if mobjs[1]:
 628                                 # split on white space creating additional array elements
 629                                 subarray = white_space.split(mobjs[1])
 630                                 if subarray:
 631                                         if len(subarray)==1:
 632                                                 # Store as a string if only one element is found.
 633                                                 # this is to keep with original catalyst behavior
 634                                                 # eventually this may go away if catalyst just works
 635                                                 # with arrays.
 636                                                 cur_array.append(subarray[0])
 637                                         else:
 638                                                 cur_array += subarray
 639
 640                 # Else add on to the last key we were working on
 641                 else:
 642                         mobjs = white_space.split(myline)
 643                         cur_array += mobjs
 644
 645                 # XXX: Do we really still need this "single value is a string" behavior?
 646                 if len(cur_array) == 2:
 647                         myspec[cur_array[0]] = cur_array[1]
 648                 else:
 649                         myspec[cur_array[0]] = cur_array[1:]
 650
 651         for x in myspec.keys():
 652                 # Delete empty key pairs
 653                 if not myspec[x]:
 654                         print "\n\tWARNING: No value set for key " + x + "...deleting"
 655                         del myspec[x]
 656         return myspec
 657
 658 def parse_makeconf(mylines):
 659         mymakeconf={}
 660         pos=0
 661         pat=re.compile("([0-9a-zA-Z_]*)=(.*)")
 662         while pos<len(mylines):
 663                 if len(mylines[pos])<=1:
 664                         #skip blanks
 665                         pos += 1
 666                         continue
 667                 if mylines[pos][0] in ["#"," ","\t"]:
 668                         #skip indented lines, comments
 669                         pos += 1
 670                         continue
 671                 else:
 672                         myline=mylines[pos]
 673                         mobj=pat.match(myline)
 674                         pos += 1
 675                         if mobj.group(2):
 676                             clean_string = re.sub(r"\"",r"",mobj.group(2))
 677                             mymakeconf[mobj.group(1)]=clean_string
 678         return mymakeconf
 679
 680 def read_spec(myspecfile):
 681         try:
 682                 myf=open(myspecfile,"r")
 683         except:
 684                 raise CatalystError, "Could not open spec file "+myspecfile
 685         mylines=myf.readlines()
 686         myf.close()
 687         return parse_spec(mylines)
 688
 689 def read_makeconf(mymakeconffile):
 690         if os.path.exists(mymakeconffile):
 691                 try:
 692                         try:
 693                                 import snakeoil.fileutils
 694                                 return snakeoil.fileutils.read_bash_dict(mymakeconffile, sourcing_command="source")
 695                         except ImportError:
 696                                 try:
 697                                         import portage_util
 698                                         return portage_util.getconfig(mymakeconffile, tolerant=1, allow_sourcing=True)
 699                                 except ImportError:
 700                                         myf=open(mymakeconffile,"r")
 701                                         mylines=myf.readlines()
 702                                         myf.close()
 703                                         return parse_makeconf(mylines)
 704                 except:
 705                         raise CatalystError, "Could not parse make.conf file "+mymakeconffile
 706         else:
 707                 makeconf={}
 708                 return makeconf
 709
 710 def msg(mymsg,verblevel=1):
 711         if verbosity>=verblevel:
 712                 print mymsg
 713
 714 def pathcompare(path1,path2):
 715         # Change double slashes to slash
 716         path1 = re.sub(r"//",r"/",path1)
 717         path2 = re.sub(r"//",r"/",path2)
 718         # Removing ending slash
 719         path1 = re.sub("/$","",path1)
 720         path2 = re.sub("/$","",path2)
 721
 722         if path1 == path2:
 723                 return 1
 724         return 0
 725
 726 def ismount(path):
 727         "enhanced to handle bind mounts"
 728         if os.path.ismount(path):
 729                 return 1
 730         a=os.popen("mount")
 731         mylines=a.readlines()
 732         a.close()
 733         for line in mylines:
 734                 mysplit=line.split()
 735                 if pathcompare(path,mysplit[2]):
 736                         return 1
 737         return 0
 738
 739 def arg_parse(cmdline):
 740         #global required_config_file_values
 741         mydict={}
 742         for x in cmdline:
 743                 foo=string.split(x,"=",1)
 744                 if len(foo)!=2:
 745                         raise CatalystError, "Invalid arg syntax: "+x
 746
 747                 else:
 748                         mydict[foo[0]]=foo[1]
 749
 750         # if all is well, we should return (we should have bailed before here if not)
 751         return mydict
 752
 753 def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 754         "helper function to help targets parse additional arguments"
 755         global valid_config_file_values
 756
 757         for x in addlargs.keys():
 758                 if x not in validspec and x not in valid_config_file_values and x not in requiredspec:
 759                         raise CatalystError, "Argument \""+x+"\" not recognized."
 760                 else:
 761                         myspec[x]=addlargs[x]
 762
 763         for x in requiredspec:
 764                 if not myspec.has_key(x):
 765                         raise CatalystError, "Required argument \""+x+"\" not specified."
 766
 767 def spec_dump(myspec):
 768         for x in myspec.keys():
 769                 print x+": "+repr(myspec[x])
 770
 771 def touch(myfile):
 772         try:
 773                 myf=open(myfile,"w")
 774                 myf.close()
 775         except IOError:
 776                 raise CatalystError, "Could not touch "+myfile+"."
 777
 778 def countdown(secs=5, doing="Starting"):
 779         if secs:
 780                 print ">>> Waiting",secs,"seconds before starting..."
 781                 print ">>> (Control-C to abort)...\n"+doing+" in: ",
 782                 ticks=range(secs)
 783                 ticks.reverse()
 784                 for sec in ticks:
 785                         sys.stdout.write(str(sec+1)+" ")
 786                         sys.stdout.flush()
 787                         time.sleep(1)
 788                 print
 789
 790 def normpath(mypath):
 791         TrailingSlash=False
 792         if mypath[-1] == "/":
 793             TrailingSlash=True
 794         newpath = os.path.normpath(mypath)
 795         if len(newpath) > 1:
 796                 if newpath[:2] == "//":
 797                         newpath = newpath[1:]
 798         if TrailingSlash:
 799             newpath=newpath+'/'
 800         return newpath
 801