From 1394254dcfd80cee6286aa674d98db809a5d0b21 Mon Sep 17 00:00:00 2001 From: "John P. Davis" Date: Wed, 16 Jun 2004 18:34:23 +0000 Subject: [PATCH] beta code to wrap commands git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@399 d1e1f19c-881f-0410-ab34-b69fee027534 --- ChangeLog | 14 +++- TODO | 9 +-- modules/catalyst_support.py | 58 +++++++++++++-- modules/generic_stage_target.py | 121 ++++++++++++++++++-------------- 4 files changed, 136 insertions(+), 66 deletions(-) diff --git a/ChangeLog b/ChangeLog index e6db652f..a3ed84b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,18 @@ # ChangeLog for gentoo/src/catalyst # Copyright 2002-2004 Gentoo Technologies, Inc.; Distributed under the GPL v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.64 2004/06/13 15:48:51 zhen Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.65 2004/06/16 18:34:23 zhen Exp $ + + 16 Jun 2004; John Davis TODO, modules/catalyst_support.py, + modules/generic_stage_target.py: + revamped the cmd() structure so that it could properly return error codes. + Please note that this might break catalyst until there is some further + testing. SO DO NOT USE IT FOR BUILDING ANYTHING IMPORTANT (yet). The benefit + of me doing this is that SIGINT (ctrl-c) makes catalyst die nice and proper + now. Additionally, catalyst will stop when there is an error with an ebuild + ... it didn't do this before, it just plowed along and packed things up. + + Much thanks to for (writing) pointing me to the spawn() code in + portage.py and then answering my noob questions. 13 Jun 2004; John Davis livecd/runscript/x86-archscript.sh, livecd/runscript-support/kmerge.sh, diff --git a/TODO b/TODO index ee2bb115..e4ca747c 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,8 @@ # Copyright 1999-2004 Gentoo Technologies, Inc. # Distributed under the terms of the GNU General Public License v2 -# $Header: /var/cvsroot/gentoo/src/catalyst/Attic/TODO,v 1.6 2004/06/15 05:41:31 zhen Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/Attic/TODO,v 1.7 2004/06/16 18:34:23 zhen Exp $ + +catalyst man page resuming of builds (sucks having to unpack and repack seeds every time) @@ -9,8 +11,3 @@ finish catalyst howto re-do catalyst main script to be more robust Fix the hardcoded arch information in modules/generic_stage_target.py - -Add support for ~x86, etc builds - -Make sure to turn off grsec chroot restrictions before starting the build -(hint: echo 0 > /proc/sys/kernel/grsecurity/chroot_*) diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py index 6dd31a6a..f78079e3 100644 --- a/modules/catalyst_support.py +++ b/modules/catalyst_support.py @@ -1,6 +1,6 @@ # Distributed under the GNU General Public License version 2 # Copyright 2003-2004 Gentoo Technologies, Inc. -# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.22 2004/06/08 04:07:34 zhen Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.23 2004/06/16 18:34:23 zhen Exp $ import sys,string,os,types @@ -45,13 +45,52 @@ def die(msg=None): sys.exit(1) def warn(msg): - print "catalyst: "+msg + print "!!! catalyst: "+msg + +def spawn(mystring,debug=0,fd_pipes=None): + """ + apparently, os.system mucks up return values, so this code + should fix that. + + Taken from portage.py - thanks to carpaski@gentoo.org + """ + print "Running command \""+mystring+"\"" + myargs=[] + mycommand = "/bin/bash" + if debug: + myargs=["bash","-x","-c",mystring] + else: + myargs=["bash","-c",mystring] + + mypid=os.fork() + if mypid==0: + if fd_pipes: + os.dup2(fd_pipes[0], 0) # stdin -- (Read)/Write + os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write) + os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write) + try: + os.execvp(mycommand,myargs) + except Exception, e: + raise CatalystError,myexc + + # If the execve fails, we need to report it, and exit + # *carefully* --- report error here + os._exit(1) + sys.exit(1) + return # should never get reached + retval=os.waitpid(mypid,0)[1] + if (retval & 0xff)==0: + return (retval >> 8) # return exit code + else: + return ((retval & 0xff) << 8) # interrupted by signal def cmd(mycmd,myexc=""): - print "Running command \""+mycmd+"\"" - retval=os.system(mycmd) - if retval != 0: - raise CatalystError,myexc + try: + retval=spawn(mycmd) + if retval != 0: + raise CatalystError,myexc + except KeyboardInterrupt: + raise CatalystError,"Build aborting due to user intervention" def file_locate(settings,filelist,expand=1): #if expand=1, non-absolute paths will be accepted and @@ -201,3 +240,10 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec): def spec_dump(myspec): for x in myspec.keys(): print x+": "+repr(myspec[x]) + +def touch(myfile): + try: + myf=open(myfile,"w") + myf.close() + except IOError: + raise CatalystError, "Could not touch "+myfile+"." diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py index 4502941c..7f4cc449 100644 --- a/modules/generic_stage_target.py +++ b/modules/generic_stage_target.py @@ -1,6 +1,6 @@ # Distributed under the GNU General Public License version 2 # Copyright 2003-2004 Gentoo Technologies, Inc. -# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.2 2004/06/04 14:03:46 zhen Exp $ +# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.3 2004/06/16 18:34:23 zhen Exp $ """ This class does all of the chroot setup, copying of files, etc. It is @@ -121,7 +121,7 @@ class generic_stage_target(generic_target): if self.settings["target"]=="grp": self.mounts.append("/tmp/grp") self.mountmap["/tmp/grp"]=self.settings["target_path"] - + def mount_safety_check(self): mypath=self.settings["chroot_path"] #check and verify that none of our paths in mypath are mounted. We don't want to clean up with things still @@ -147,20 +147,33 @@ class generic_stage_target(generic_target): def dir_setup(self): print "Setting up directories..." self.mount_safety_check() - cmd("rm -rf "+self.settings["chroot_path"],"Could not remove existing directory: "+self.settings["chroot_path"]) - os.makedirs(self.settings["chroot_path"]) + if not os.path.exists(self.settings["chroot_path"]+"/tmp/unpacked"): + cmd("rm -rf "+self.settings["chroot_path"],"Could not remove existing directory: "+self.settings["chroot_path"]) + else: + print "Directories previously setup, resuming..." + + if not os.path.exists(self.settings["chroot_path"]): + os.makedirs(self.settings["chroot_path"]) if self.settings.has_key("PKGCACHE"): if not os.path.exists(self.settings["pkgcache_path"]): os.makedirs(self.settings["pkgcache_path"]) - + def unpack_and_bind(self): - print "Unpacking stage tarball..." - cmd("tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"],"Error unpacking tarball") - if os.path.exists(self.settings["chroot_path"]+"/usr/portage"): - print "Cleaning up existing portage tree snapshot..." - cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage","Error removing existing snapshot directory.") - print "Unpacking portage tree snapshot..." - cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr","Error unpacking snapshot") + if not os.path.exists(self.settings["chroot_path"]+"/tmp/unpacked"): + print "Unpacking stage tarball..." + cmd("tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"],"Error unpacking tarball") + if os.path.exists(self.settings["chroot_path"]+"/usr/portage"): + print "Cleaning up existing portage tree snapshot..." + cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage","Error removing existing snapshot directory.") + print "Unpacking portage tree snapshot..." + cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr","Error unpacking snapshot") + print "Configuring profile link..." + cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile","Error zapping profile link") + cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+" "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link") + touch(self.settings["chroot_path"]+"/tmp/unpacked") + else: + print "CHROOT previously unpacked and bind mounted, resuming..." + for x in self.mounts: if not os.path.exists(self.settings["chroot_path"]+x): os.makedirs(self.settings["chroot_path"]+x) @@ -171,10 +184,7 @@ class generic_stage_target(generic_target): if retval!=0: self.unbind() raise CatalystError,"Couldn't bind mount "+src - print "Configuring profile link..." - cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile","Error zapping profile link") - cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+" "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link") - + def unbind(self): ouch=0 mypath=self.settings["chroot_path"] @@ -198,45 +208,49 @@ class generic_stage_target(generic_target): raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety." def chroot_setup(self): - cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc","Could not copy resolv.conf into place.") - if self.settings.has_key("ENVSCRIPT"): - if not os.path.exists(self.settings["ENVSCRIPT"]): - raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"] - cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript","Could not copy envscript into place.") - cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf") + if not os.path.exists(self.settings["chroot_path"]+"/tmp/chroot_setup"): + cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc","Could not copy resolv.conf into place.") + if self.settings.has_key("ENVSCRIPT"): + if not os.path.exists(self.settings["ENVSCRIPT"]): + raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"] + cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript","Could not copy envscript into place.") + cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf") - myf=open(self.settings["chroot_path"]+"/etc/make.conf","w") - myf.write("# These settings were set by the catalyst build script that automatically built this stage\n") - myf.write("# Please consult /etc/make.conf.example for a more detailed example\n") - myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n') - myf.write('CHOST="'+self.settings["CHOST"]+'"\n') - myusevars=[] - if self.settings.has_key("HOSTUSE"): - myusevars.extend(self.settings["HOSTUSE"]) - if self.settings["target"]=="grp": - myusevars.append("bindist") - myusevars.extend(self.settings["grp/use"]) - myf.write('USE="'+string.join(myusevars)+'"\n') - elif self.settings["target"]=="tinderbox": - myusevars.extend(self.settings["tinderbox/use"]) - myf.write('USE="'+string.join(myusevars)+'"\n') - elif self.settings["target"]=="livecd-stage1": - myusevars.extend(self.settings["livecd/use"]) - myf.write('USE="'+string.join(myusevars)+'"\n') - elif self.settings["target"]=="embedded": - myusevars.extend(self.settings["embedded/use"]) - myf.write('USE="'+string.join(myusevars)+'"\n') - if self.settings.has_key("CXXFLAGS"): - myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n') - else: - myf.write('CXXFLAGS="${CFLAGS}"\n') - myf.close() - - #create entry in /etc/passwd for distcc user - if self.settings.has_key("DISTCC"): - myf=open(self.settings["chroot_path"]+"/etc/passwd","a") - myf.write("distcc:x:7980:2:distccd:/dev/null:/bin/false\n") + myf=open(self.settings["chroot_path"]+"/etc/make.conf","w") + myf.write("# These settings were set by the catalyst build script that automatically built this stage\n") + myf.write("# Please consult /etc/make.conf.example for a more detailed example\n") + myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n') + myf.write('CHOST="'+self.settings["CHOST"]+'"\n') + myusevars=[] + if self.settings.has_key("HOSTUSE"): + myusevars.extend(self.settings["HOSTUSE"]) + if self.settings["target"]=="grp": + myusevars.append("bindist") + myusevars.extend(self.settings["grp/use"]) + myf.write('USE="'+string.join(myusevars)+'"\n') + elif self.settings["target"]=="tinderbox": + myusevars.extend(self.settings["tinderbox/use"]) + myf.write('USE="'+string.join(myusevars)+'"\n') + elif self.settings["target"]=="livecd-stage1": + myusevars.extend(self.settings["livecd/use"]) + myf.write('USE="'+string.join(myusevars)+'"\n') + elif self.settings["target"]=="embedded": + myusevars.extend(self.settings["embedded/use"]) + myf.write('USE="'+string.join(myusevars)+'"\n') + if self.settings.has_key("CXXFLAGS"): + myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n') + else: + myf.write('CXXFLAGS="${CFLAGS}"\n') myf.close() + + #create entry in /etc/passwd for distcc user + if self.settings.has_key("DISTCC"): + myf=open(self.settings["chroot_path"]+"/etc/passwd","a") + myf.write("distcc:x:7980:2:distccd:/dev/null:/bin/false\n") + myf.close() + touch(self.settings["chroot_path"]+"/tmp/chroot_setup") + else: + print "CHROOT previously setup, resuming..." def clean(self): destpath=self.settings["chroot_path"] @@ -316,6 +330,7 @@ class generic_stage_target(generic_target): def run_local(self): try: cmd("/bin/bash "+self.settings["sharedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+".sh run","build script failed") + except CatalystError: self.unbind() raise CatalystError,"Stage build aborting due to error." -- 2.26.2