From: Daniel Robbins Date: Sat, 8 Nov 2003 18:19:17 +0000 (+0000) Subject: spec file reading and parsing functions added X-Git-Tag: CATALYST_1_0_1~174 X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=4cbbd192977366eba8ae92bb96774ea4d6121d0a;p=catalyst.git spec file reading and parsing functions added git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@66 d1e1f19c-881f-0410-ab34-b69fee027534 --- diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py index 195f6b81..3e2089fc 100644 --- a/modules/catalyst_support.py +++ b/modules/catalyst_support.py @@ -26,6 +26,89 @@ def cmd(mycmd,myexc=""): if retval != 0: raise CatalystError,myexc +""" +Spec file format: + +The spec file format is a very simple and easy-to-use format for storing data. Here's an example +file: + +item1: value1 +item2: foo bar oni +item3: + meep + bark + gleep moop + +This file would be interpreted as defining three items: item1, item2 and item3. item1 would contain +the string value "value1". Item2 would contain an ordered list [ "foo", "bar", "oni" ]. item3 +would contain an ordered list as well: [ "meep", "bark", "gleep", "moop" ]. It's important to note +that the order of multiple-value items is preserved, but the order that the items themselves are +defined are not preserved. In other words, "foo", "bar", "oni" ordering is preserved but "item1" +"item2" "item3" ordering is not, as the item strings are stored in a dictionary (hash). +""" + +def parse_spec(mylines): + myspec={} + pos=0 + while pos foo:"bar" + myspec[myline[0]]=myline[1] + pos += 1 + elif len(myline)>2: + #foo: bar oni --> foo: [ "bar", "oni" ] + myspec[myline[0]]=myline[1:] + pos += 1 + else: + #foo: + # bar + # oni meep + # --> foo: [ "bar", "oni", "meep" ] + accum=[] + pos += 1 + while (pos=verblevel: print mymsg diff --git a/modules/catalyst_util.py b/modules/catalyst_util.py deleted file mode 100755 index d04df45d..00000000 --- a/modules/catalyst_util.py +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/python -# Copyright 2003 Gentoo Technologies, Inc.; http://www.gentoo.org -# Released under the GNU General Public License version 2 - -#TODO: add snapshotting of portage trees, which will not be handled by spec files. -#Then, add spec file support and env var export support, then mount/umount support... -#then we should be getting close to completion and usability. - -import sys,os,string,stat - -subarches=["amd64", "hppa", "hppa1.1", "hppa2.0", "x86", "i386", "i486", "i586", "i686", -"athlon", "athlon-xp", "athlon-mp", "pentium-mmx", "pentium3", "pentium4", "ppc", "g3", -"g4", "sparc", "sparc64", "mips", "alpha", "ev4", "ev5", "ev56", "pca56", "ev6", "ev67" ] - -#this target stuff is not completed, but I've added it as a general template. -""" -setting name default/example defined how? -===================================================================================================================== - -globals: - -storedir* /var/tmp/catalyst default (where to store all the stuff, ie snapshots, etc.) -sharedir* /usr/share/catalyst default (where the dir that holds targets is) - -global external paths: - -distdir /usr/portage/distfiles default (/usr/portage/distfiles) -portdir* /usr/portage default (not set inside chroot) - -locals from spec: - -version_stamp 20031016 user (from spec) -target stage3 user (from spec) -subarch pentium4 user (from spec) -rel_type default user (from spec) (was BUILDTYPE) -rel_version 1.4 user (from spec) (was MAINVERSION) -snapshot 20031016 user (from spec) -source_subpath default-x86-1.4/stage2-pentium4-20031016 user (from spec) - - -target_subpath default-x86-1.4/stage3-pentium4-20031016 - rel_type+"-"+mainarch+"-"+rel_version+"/"+target+"-"+subarch+"-"+version_stamp - -snapshot_path /var/tmp/catalyst/snapshots/portage-20031016.tar.bz2 - storedir+"/snapshots/portage"+snapshot+".tar.bz2" - -target_path /var/tmp/catalyst/builds/default-x86-1.4/stage3-pentium4-20031016.tar.bz2 - storedir+"/builds/"+target_subpath+".tar.bz2" - -source_path /var/tmp/catalyst/builds/default-x86-1.4/stage2-pentium4-20031016.tar.bz2 - storedir+"/builds/"+source+".tar.bz2" - -chroot_path /var/tmp/catalyst/tmp/default-x86-1.4/stage3-pentium4-20031016 - storedir+"/tmp/"+target_subpath - -pkgcache_path /var/tmp/catalyst/packages/default-x86-1.4/stage3-pentium4-20031016 - storedir+"/packages/"+target_subpath - -locals, auto-generated: - -mainarch x86 auto -cflags -O2 auto, or user -cxxflags -O2 auto, or user -hostuse mmx sse auto, or user -chost i686-pc-linux-gnu auto, or user -makeopts -j2 auto (but overridable from catalyst.conf) -chroot chroot auto (linux32 chroot or chroot) - - - -Config file sources: - 1. defaults can come from /etc/catalyst.conf (since it's run as root, might as well put it in /etc - (these defaults can be things like pkgdir, distdir, but not rel_version or rel_type - which we specifically want in the spec file so that we have a complete description there.) - This file can also tell catalyst whether to use ccache or not. - 2. spec file - spec can override any of the "auto" variables. we need to remember cxxflags too. -""" -class generic_target: - def __init__(self,myset): - self.settings=myset - self.envmap={"CFLAGS":"cflags" } - def path(self): - #temp file paths: - #/var/tmp/catalyst/default-x86-1.4/stage1-pentium4-20030911/work - #/var/tmp/catalyst/default-x86-1.4/stage1-pentium4-20030911/packages (if there is a package cache) - - #final paths: - #snapshots/portage-20030911.tar.bz2 - #builds/default-x86-1.4/stage1-pentium4-20030911.tar.bz2 - #builds/default-x86-1.4/stage2-pentium4-20030911.tar.bz2 - #builds/default-x86-1.4/stage3-pentium4-20030911.tar.bz2 - #builds/default-x86-1.4/grp-pentium4-20030911/cd1/packages/All - #builds/default-x86-1.4/grp-pentium4-20030911/cd2/packages/All - #builds/default-x86-1.4/livecd-20030911.tar.bz2 - #builds/buildtype-mainarch-mainversion/buildtype-subarch-version_stamp - #now where to put the work files. - return "stages/stage1-"+self.settings["subarch"]+"-"+self.settings["buildno"]+".tar.bz2" - def read_spec_file(self,myfile): - #settings from this file: - #pkgdir, distdir, subarch, mainversion, version_stamp, snapshot, source tarball - #default default user user user user user - #read in a spec file, grab settings we need. - pass - def execute_script(self,myscript,myargs): - #export env vars here - return os.system(myscript+" "+string.join(myargs," ")) - pass - def export_variables(self): - #export environment variables - #export: - # CFLAGS, HOSTUSE, CHOST, MAINARCH, MAINVERSION, BUILDTYPE, MAKEOPTS, BASEDIR, CHROOTDIR, FEATURES - # auto auto auto auto spec spec auto default auto default - pass - def build(self): - #do the actual stage1 building - return execute_script("targets/stage1/build.sh") - def unpack(self): - #unpack stages - pass - def mount_all(self): - #mount mount points; let's handle this from python - pass - def umount_all(self): - #umount mount points; let's handle this from python - pass - def mount_safety_check(self,mypath): - #check and verify that none of our paths in mypath are mounted. We don't want to clean up with things still - #mounted, and this allows us to check. returns 1 on ok, 0 on "something is still mounted" case. - paths=["usr/portage/packages","/usr/portage/distfiles", "/var/tmp/distfiles", "/proc", "/root/.ccache", "/dev"] - if not os.path.exists(mypath): - return 1 - mypstat=os.stat(mypath)[ST_DEV] - for x in paths: - if not os.path.exists(x): - continue - teststat=os.stat(x)[ST_DEV] - if teststat!=mypstat: - #something is still mounted - return 0 - return 1 - def prep(self): - #prepare stage for packing up - pass - def clean(self): - #clean up temporary build directory - pass - def pack(self): - #tar up anything like a stage and put in right place - pass - def run(self): - self.mount_safety_check() - self.unpack() - self.setup() - self.mount_all() - retval=self.build() #check for failure - self.umount_all() - self.mount_safety_check() - self.prep() - self.pack() - self.clean() - -class stage2(generic_target): - #subclass of generic_target - def path(self): - return "stages/stage2-"+self.settings["subarch"]+"-"+self.settings["buildno"]+".tar.bz2" - -class stage3(generic_target): - def path(self): - return "stages/stage3-"+self.settings["subarch"]+"-"+self.settings["buildno"]+".tar.bz2" - -class grp(generic_target): - def path(self): - return "stages/stage3-"+self.settings["subarch"]+"-"+self.settings["buildno"]+".tar.bz2" - -class livecd(generic_target): - def path(self): - return "stages/stage3-"+self.settings["subarch"]+"-"+self.settings["buildno"]+".tar.bz2" - -def verify_os(myset): - if sys.platform=="linux2": - myset["os_userland"]="GNU" - else: - raise OSError, "Platform "+sys.platform+" not recognized." - -def job_defaults(mainarch): - return "to be completed" - -def verify_subarch(myset,subarch): - global subarches - if subarch not in subarches: - raise ValueError, "Sub-architecture "+mysubarch+" not recognized." - myset["subarch"]=subarch - - if subarch == "athlon-mp": - subarch="athlon-xp" - - results=None - - if subarch == "ia64": - results=["ia64","-O2","ia64-unknown-linux-gnu",[]] - elif subarch == "amd64": - results=["amd64","-O2 -fPIC","x86_64-pc-linux-gnu",[]] - elif subarch in ["hppa","hppa1.1","hppa2.0"]: - results=["hppa","-O2","hppa-unknown-linux-gnu",[]] - if subarch == "hppa2.0": - results[1] += " -march=2.0" - else: - results[1] += " -march=1.1" - elif subarch in ["x86","i386","i486","i586","i686","athlon","athlon-xp","athlon-mp","pentium-mmx","pentium3","pentium4"]: - #With recent gcc compilers (gcc-3.1+,) gcc produces generally slower code with -O3 as compared to -O2 - #So we are tweaking things here. - uf=[] - if subarch == "x86": - cf="-O2 -mcpu=i686 -fomit-frame-pointer" - elif subarch == "athlon-xp": - #we've intentionally lowered optimizations here to address compile bugs. It is very likely safe to - #go up to -O3 without any additional -f goodies (like unroll-loops and prefetch-loop-arrays) tacked - #on. Note that we don't add any extra -f goodies if we are athlon* below. - cf="-O2 -march=athlon-xp -fomit-frame-pointer" - else: - cf="-O2 -march="+subarch+" -fomit-frame-pointer" - if subarch[0:6] == "athlon": - uf.append("3dnow") - if subarch in [ "athlon","athlon-xp","athlon-mp","pentium-mmx","pentium3","pentium4" ]: - uf.append("mmx") - if subarch in [ "pentium3", "pentium4" ]: - uf.append("sse") - if subarch in ["pentium3", "pentium4", "athlon", "athlon-xp", "athlon-mp", "i686"]: - cf += " -finline-functions -finline-limit=800" - if subarch in ["i386","i486","i586"]: - results=["x86",cf,subarch+"-pc-linux-gnu",[]] - elif subarch == "x86": - results=["x86",cf,"i486-pc-linux-gnu",[]] - elif subarch == "pentium-mmx": - results=["x86",cf,"i586-pc-linux-gnu",uf] - else: - results=["x86",cf,"i686-pc-linux-gnu",uf] - - elif subarch in ["ppc","g3","g4"]: - if subarch == "ppc": - results=["ppc","-O2 -fsigned-char",[]] - elif subarch == "g3": - results=["ppc","-O2 -mcpu=750 -mpowerpc-gfxopt -fsigned-char"] - elif subarch == "g4": - results=["ppc","-O2 -mcpu=7400 -maltivec -mabi=altivec -mpowerpc-gfxopt -fsigned-char"] - results.extend(["powerpc-unknown-linux-gnu",[]]) - elif subarch in ["sparc","sparc64"]: - if subarch == "sparc": - results=["sparc","-O2"] - else: - results=["sparc64","-O3 -mcpu=ultrasparc -mtune=ultrasparc"] - results.extend(["sparc-unknown-linux-gnu",[]]) - elif subarch == "mips": - results=["mips","-O2","mips-unknown-linux-gnu",[]] - elif subarch in ["alpha","ev4","ev5","ev56","pca56","ev6","ev67"]: - if subarch == "alpha": - results=["alpha","-O3 -mcpu=ev5","alpha-unknown-linux-gnu",[]] - else: - results=["alpha","-O3 -mcpu="+subarch,"alpha"+subarch+"-linux-gnu",[]] - if results==None: - raise ValueError, "Invalid subarch value passed to compile_defaults (you should not see this)" - #the main architecture we're building for: (string) - myset["mainarch"]=results[0] - #the CFLAGS we should use on this specific architecture (ie pentium4): (string) - myset["cflags"]=results[1] - #the CHOST setting (ie i686-pc-linux-gnu): (string) - myset["chost"]=results[2] - #any USE variables that should be enabled on this platform (mmx, sse, 3dnow) (list) - myset["hostuse"]=results[3] - -def die(msg=None): - if msg: - print "catalyst: "+msg - sys.exit(1) - -def warn(msg): - print "catalyst: "+msg - -modes=["snap","enter","umount","livecd","stage"] -modesdesc={ "snap":"Create a snapshot of the Portage tree for building", - "enter":"Enter the specified build chroot (interactive)", - "umount":"Unmount the specified build chroot mount points", - "livecd":"Build the specified LiveCD runtime image", - "stage":"Build the specified stage tarball or package set", -} - -def do_snapshot(portdir,snap_temp_dir,snapdir,snapversion): - print "Creating Portage tree snapshot "+snapversion+" from "+portdir+"..." - mytmp=snap_temp_dir+"/snap-"+snapversion - if os.path.exists(mytmp): - retval=os.system("rm -rf "+mytmp) - if retval != 0: - die("Could not remove existing directory: "+mytmp) - os.makedirs(mytmp) - retval=os.system("rsync -a --exclude /packages/ --exclude /distfiles/ --exclude CVS/ "+portdir+"/ "+mytmp+"/portage/") - if retval != 0: - die("snapshot failure.") - print "Compressing Portage snapshot tarball..." - retval=os.system("( cd "+mytmp+"; tar cjf "+snapdir+"/portage-"+snapversion+".tar.bz2 portage )") - if retval != 0: - die("snapshot tarball creation failure.") - print "Cleaning up temporary snapshot directory..." - #Be a good citizen and clean up after ourselves - retval=os.system("rm -rf "+mytmp) - if retval != 0: - die("Unable to clean up directory: "+mytmp) - -def usage(): - print "catalyst: Gentoo Linux stage/LiveCD/GRP building tool" - print - for x in modes: - print x+":",modesdesc[x] - die() - -def read_settings(myset,myfn): - """Grab local settings from a specified file myfn, dump values we are interested in to settings dictionary myset""" - if not os.path.exists(myfn): - raise OSError, "Cannot find settings file "+myfn - valdict={} - #this next line might throw an exception; it runs the file myfn as python code - #and dumps all variable definitions in the valdict dictionary. - try: - execfile(myfn,valdict,valdict) - except: - raise ValueError, "Error parsing settings file: "+myfn - #now we look inside the valdict dictionary and grab the settings we're interested - #in. - for x in ["buildtype","portdir","distdir","ccache"]: - if valdict.has_key(x): - myset[x]=valdict[x] - -def global_settings_init(myset): - #now, we read in global configuration settings from /etc/catalyst.conf - if os.path.exists("/etc/catalyst.conf"): - read_settings(myset,"/etc/catalyst.conf") - #set reasonable defaults if none were provided in /etc/catalyst.conf - mydefaults={"portdir":"/usr/portage","storedir":"/var/tmp/catalyst","sharedir":"/usr/share/catalyst","distdir":"/usr/share/distfiles"} - for x in mydefaults.keys(): - if not myset.has_key(x): - myset[x]=mydefaults[x] - if not myset.has_key("snapdir"): - myset["snapdir"]=myset["storedir"]+"/snapshots" - if not myset.has_key("cat_tmpdir"): - myset["cat_tmpdir"]=myset["storedir"]+"/tmp" - -def init_writable_dirs(myset): - #create the initial main directories that we need to write to. - for x in ["storedir","snapdir","cat_tmpdir"]: - if not os.path.exists(myset[x]): - os.makedirs(myset[x]) - -def dump_settings(myset): - for x in myset.keys(): - print x+":",myset[x] - print "Done!" - -def mainloop(): - global subarches - #argument processing - - if len(sys.argv)==1 or sys.argv[1] in ["-h","--help"]: - usage() - elif os.getuid()!=0: - #non-root callers can still get -h and --help to work. - die("This script requires root privileges to operate.") - elif sys.argv[1] in modes: - #set up internal configuration settings dictionary - #myset=settings() - myset={} - verify_os(myset) - global_settings_init(myset) - init_writable_dirs(myset) - #dump_settings(myset) - if sys.argv[1]=="snap": - if len(sys.argv)!=3: - die("invalid number of arguments for snapshot.") - do_snapshot(myset["portdir"],myset["cat_tmpdir"],myset["snapdir"],sys.argv[2]) - #do snapshot here - sys.exit(0) - sys.exit(0) - else: - usage() - -if __name__ == "__main__": - mainloop() - - -