beta code to wrap commands

git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@399 d1e1f19c-881f-0410-ab34-b69fee027534

diff --git a/ChangeLog b/ChangeLog
index e6db652fc398a50ae9a9c402719199711e19b772..a3ed84b4416b05ef343d7739f10dc007dd5d0576 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,18 @@
 # ChangeLog for gentoo/src/catalyst 
 # Copyright 2002-2004 Gentoo Technologies, Inc.; Distributed under the GPL v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.64 2004/06/13 15:48:51 zhen Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/ChangeLog,v 1.65 2004/06/16 18:34:23 zhen Exp $
+
+  16 Jun 2004; John Davis <zhen@gentoo.org> TODO, modules/catalyst_support.py,
+  modules/generic_stage_target.py:
+  revamped the cmd() structure so that it could properly return error codes.
+  Please note that this might break catalyst until there is some further
+  testing. SO DO NOT USE IT FOR BUILDING ANYTHING IMPORTANT (yet). The benefit
+  of me doing this is that SIGINT (ctrl-c) makes catalyst die nice and proper
+  now. Additionally, catalyst will stop when there is an error with an ebuild
+  ... it didn't do this before, it just plowed along and packed things up.
+  
+  Much thanks to <carpaski@gentoo.org> for (writing) pointing me to the spawn() code in
+  portage.py and then answering my noob questions.
 
   13 Jun 2004; John Davis <zhen@gentoo.org>
   livecd/runscript/x86-archscript.sh, livecd/runscript-support/kmerge.sh,

diff --git a/TODO b/TODO
index ee2bb115033430c612a2c7f62c78f3c33c6cec12..e4ca747c42d58522f76525b32e882e4fca380ba6 100644 (file)
--- a/TODO
+++ b/TODO
@@ -1,6 +1,8 @@
 # Copyright 1999-2004 Gentoo Technologies, Inc.
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/Attic/TODO,v 1.6 2004/06/15 05:41:31 zhen Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/Attic/TODO,v 1.7 2004/06/16 18:34:23 zhen Exp $
+
+catalyst man page
 
 resuming of builds (sucks having to unpack and repack seeds every time)
 
@@ -9,8 +11,3 @@ finish catalyst howto
 re-do catalyst main script to be more robust
 
 Fix the hardcoded arch information in modules/generic_stage_target.py
-
-Add support for ~x86, etc builds
-
-Make sure to turn off grsec chroot restrictions before starting the build
-(hint: echo 0 > /proc/sys/kernel/grsecurity/chroot_*)

diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py
index 6dd31a6a23eb7b380232eea0e1126beb9a81b28f..f78079e340db8afd01381bc9719875ac249adbf3 100644 (file)
--- a/modules/catalyst_support.py
+++ b/modules/catalyst_support.py
@@ -1,6 +1,6 @@
 # Distributed under the GNU General Public License version 2
 # Copyright 2003-2004 Gentoo Technologies, Inc.
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.22 2004/06/08 04:07:34 zhen Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/catalyst_support.py,v 1.23 2004/06/16 18:34:23 zhen Exp $
 
 import sys,string,os,types
 
@@ -45,13 +45,52 @@ def die(msg=None):
        sys.exit(1)
 
 def warn(msg):
-       print "catalyst: "+msg
+       print "!!! catalyst: "+msg
+
+def spawn(mystring,debug=0,fd_pipes=None):
+       """
+       apparently, os.system mucks up return values, so this code
+       should fix that.
+
+       Taken from portage.py - thanks to carpaski@gentoo.org
+       """
+       print "Running command \""+mystring+"\""
+       myargs=[]
+       mycommand = "/bin/bash"
+       if debug:
+               myargs=["bash","-x","-c",mystring]
+       else:
+               myargs=["bash","-c",mystring]
+       
+       mypid=os.fork()
+       if mypid==0:
+               if fd_pipes:
+                       os.dup2(fd_pipes[0], 0) # stdin  -- (Read)/Write
+                       os.dup2(fd_pipes[1], 1) # stdout -- Read/(Write)
+                       os.dup2(fd_pipes[2], 2) # stderr -- Read/(Write)
+               try:
+                       os.execvp(mycommand,myargs)
+               except Exception, e:
+                       raise CatalystError,myexc
+               
+               # If the execve fails, we need to report it, and exit
+               # *carefully* --- report error here
+               os._exit(1)
+               sys.exit(1)
+               return # should never get reached
+       retval=os.waitpid(mypid,0)[1]
+       if (retval & 0xff)==0:
+               return (retval >> 8) # return exit code
+       else:
+               return ((retval & 0xff) << 8) # interrupted by signal
 
 def cmd(mycmd,myexc=""):
-       print "Running command \""+mycmd+"\""
-       retval=os.system(mycmd)
-       if retval != 0:
-               raise CatalystError,myexc
+       try:
+               retval=spawn(mycmd)
+               if retval != 0:
+                       raise CatalystError,myexc
+       except KeyboardInterrupt:
+               raise CatalystError,"Build aborting due to user intervention"
 
 def file_locate(settings,filelist,expand=1):
        #if expand=1, non-absolute paths will be accepted and
@@ -201,3 +240,10 @@ def addl_arg_parse(myspec,addlargs,requiredspec,validspec):
 def spec_dump(myspec):
        for x in myspec.keys():
                print x+": "+repr(myspec[x])
+
+def touch(myfile):
+       try:
+               myf=open(myfile,"w")
+               myf.close()
+       except IOError:
+               raise CatalystError, "Could not touch "+myfile+"."

diff --git a/modules/generic_stage_target.py b/modules/generic_stage_target.py
index 4502941ce44826d7239a2edd6cc61a34b4d98e63..7f4cc4498f9efa47d2cb8d52229c8ce08226977d 100644 (file)
--- a/modules/generic_stage_target.py
+++ b/modules/generic_stage_target.py
@@ -1,6 +1,6 @@
 # Distributed under the GNU General Public License version 2
 # Copyright 2003-2004 Gentoo Technologies, Inc.
-# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.2 2004/06/04 14:03:46 zhen Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/modules/generic_stage_target.py,v 1.3 2004/06/16 18:34:23 zhen Exp $
 
 """
 This class does all of the chroot setup, copying of files, etc. It is
@@ -121,7 +121,7 @@ class generic_stage_target(generic_target):
                if self.settings["target"]=="grp":
                        self.mounts.append("/tmp/grp")
                        self.mountmap["/tmp/grp"]=self.settings["target_path"]
-       
+               
        def mount_safety_check(self):
                mypath=self.settings["chroot_path"]
                #check and verify that none of our paths in mypath are mounted. We don't want to clean up with things still
@@ -147,20 +147,33 @@ class generic_stage_target(generic_target):
        def dir_setup(self):
                print "Setting up directories..."
                self.mount_safety_check()
-               cmd("rm -rf "+self.settings["chroot_path"],"Could not remove existing directory: "+self.settings["chroot_path"])
-               os.makedirs(self.settings["chroot_path"])
+               if not os.path.exists(self.settings["chroot_path"]+"/tmp/unpacked"):
+                       cmd("rm -rf "+self.settings["chroot_path"],"Could not remove existing directory: "+self.settings["chroot_path"])
+               else:
+                       print "Directories previously setup, resuming..."
+                       
+               if not os.path.exists(self.settings["chroot_path"]):
+                       os.makedirs(self.settings["chroot_path"])
                if self.settings.has_key("PKGCACHE"):   
                        if not os.path.exists(self.settings["pkgcache_path"]):
                                os.makedirs(self.settings["pkgcache_path"])
-               
+                                       
        def unpack_and_bind(self):
-               print "Unpacking stage tarball..."
-               cmd("tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"],"Error unpacking tarball")
-               if os.path.exists(self.settings["chroot_path"]+"/usr/portage"):
-                       print "Cleaning up existing portage tree snapshot..."
-                       cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage","Error removing existing snapshot directory.")
-               print "Unpacking portage tree snapshot..."
-               cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr","Error unpacking snapshot")
+               if not os.path.exists(self.settings["chroot_path"]+"/tmp/unpacked"):
+                       print "Unpacking stage tarball..."
+                       cmd("tar xjpf "+self.settings["source_path"]+" -C "+self.settings["chroot_path"],"Error unpacking tarball")
+                       if os.path.exists(self.settings["chroot_path"]+"/usr/portage"):
+                               print "Cleaning up existing portage tree snapshot..."
+                               cmd("rm -rf "+self.settings["chroot_path"]+"/usr/portage","Error removing existing snapshot directory.")
+                       print "Unpacking portage tree snapshot..."
+                       cmd("tar xjpf "+self.settings["snapshot_path"]+" -C "+self.settings["chroot_path"]+"/usr","Error unpacking snapshot")
+                       print "Configuring profile link..."
+                       cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile","Error zapping profile link")
+                       cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+" "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link")
+                       touch(self.settings["chroot_path"]+"/tmp/unpacked")
+               else:
+                       print "CHROOT previously unpacked and bind mounted, resuming..."
+               
                for x in self.mounts: 
                        if not os.path.exists(self.settings["chroot_path"]+x):
                                os.makedirs(self.settings["chroot_path"]+x)
@@ -171,10 +184,7 @@ class generic_stage_target(generic_target):
                        if retval!=0:
                                self.unbind()
                                raise CatalystError,"Couldn't bind mount "+src
-               print "Configuring profile link..."
-               cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.profile","Error zapping profile link")
-               cmd("ln -sf ../usr/portage/profiles/"+self.settings["target_profile"]+" "+self.settings["chroot_path"]+"/etc/make.profile","Error creating profile link")
-
+               
        def unbind(self):
                ouch=0
                mypath=self.settings["chroot_path"]
@@ -198,45 +208,49 @@ class generic_stage_target(generic_target):
                        raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety."
 
        def chroot_setup(self):
-               cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc","Could not copy resolv.conf into place.")
-               if self.settings.has_key("ENVSCRIPT"):
-                       if not os.path.exists(self.settings["ENVSCRIPT"]):
-                               raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"]
-                       cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript","Could not copy envscript into place.")
-               cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf")
+               if not os.path.exists(self.settings["chroot_path"]+"/tmp/chroot_setup"):
+                       cmd("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc","Could not copy resolv.conf into place.")
+                       if self.settings.has_key("ENVSCRIPT"):
+                               if not os.path.exists(self.settings["ENVSCRIPT"]):
+                                       raise CatalystError, "Can't find envscript "+self.settings["ENVSCRIPT"]
+                               cmd("cp "+self.settings["ENVSCRIPT"]+" "+self.settings["chroot_path"]+"/tmp/envscript","Could not copy envscript into place.")
+                       cmd("rm -f "+self.settings["chroot_path"]+"/etc/make.conf")
 
-               myf=open(self.settings["chroot_path"]+"/etc/make.conf","w")
-               myf.write("# These settings were set by the catalyst build script that automatically built this stage\n")
-               myf.write("# Please consult /etc/make.conf.example for a more detailed example\n")
-               myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n')
-               myf.write('CHOST="'+self.settings["CHOST"]+'"\n')
-               myusevars=[]
-               if self.settings.has_key("HOSTUSE"):
-                       myusevars.extend(self.settings["HOSTUSE"])
-               if self.settings["target"]=="grp":
-                       myusevars.append("bindist")
-                       myusevars.extend(self.settings["grp/use"])
-                       myf.write('USE="'+string.join(myusevars)+'"\n')
-               elif self.settings["target"]=="tinderbox":
-                       myusevars.extend(self.settings["tinderbox/use"])
-                       myf.write('USE="'+string.join(myusevars)+'"\n')
-               elif self.settings["target"]=="livecd-stage1":
-                       myusevars.extend(self.settings["livecd/use"])
-                       myf.write('USE="'+string.join(myusevars)+'"\n')
-               elif self.settings["target"]=="embedded":
-                       myusevars.extend(self.settings["embedded/use"])
-                       myf.write('USE="'+string.join(myusevars)+'"\n')
-               if self.settings.has_key("CXXFLAGS"):
-                       myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n')
-               else:
-                       myf.write('CXXFLAGS="${CFLAGS}"\n')
-               myf.close()
-
-               #create entry in /etc/passwd for distcc user
-               if self.settings.has_key("DISTCC"): 
-                       myf=open(self.settings["chroot_path"]+"/etc/passwd","a")
-                       myf.write("distcc:x:7980:2:distccd:/dev/null:/bin/false\n")
+                       myf=open(self.settings["chroot_path"]+"/etc/make.conf","w")
+                       myf.write("# These settings were set by the catalyst build script that automatically built this stage\n")
+                       myf.write("# Please consult /etc/make.conf.example for a more detailed example\n")
+                       myf.write('CFLAGS="'+self.settings["CFLAGS"]+'"\n')
+                       myf.write('CHOST="'+self.settings["CHOST"]+'"\n')
+                       myusevars=[]
+                       if self.settings.has_key("HOSTUSE"):
+                               myusevars.extend(self.settings["HOSTUSE"])
+                       if self.settings["target"]=="grp":
+                               myusevars.append("bindist")
+                               myusevars.extend(self.settings["grp/use"])
+                               myf.write('USE="'+string.join(myusevars)+'"\n')
+                       elif self.settings["target"]=="tinderbox":
+                               myusevars.extend(self.settings["tinderbox/use"])
+                               myf.write('USE="'+string.join(myusevars)+'"\n')
+                       elif self.settings["target"]=="livecd-stage1":
+                               myusevars.extend(self.settings["livecd/use"])
+                               myf.write('USE="'+string.join(myusevars)+'"\n')
+                       elif self.settings["target"]=="embedded":
+                               myusevars.extend(self.settings["embedded/use"])
+                               myf.write('USE="'+string.join(myusevars)+'"\n')
+                       if self.settings.has_key("CXXFLAGS"):
+                               myf.write('CXXFLAGS="'+self.settings["CXXFLAGS"]+'"\n')
+                       else:
+                               myf.write('CXXFLAGS="${CFLAGS}"\n')
                        myf.close()
+
+                       #create entry in /etc/passwd for distcc user
+                       if self.settings.has_key("DISTCC"): 
+                               myf=open(self.settings["chroot_path"]+"/etc/passwd","a")
+                               myf.write("distcc:x:7980:2:distccd:/dev/null:/bin/false\n")
+                               myf.close()
+                       touch(self.settings["chroot_path"]+"/tmp/chroot_setup")
+               else:
+                       print "CHROOT previously setup, resuming..."
                
        def clean(self):
                destpath=self.settings["chroot_path"]
@@ -316,6 +330,7 @@ class generic_stage_target(generic_target):
        def run_local(self):
                try:
                        cmd("/bin/bash "+self.settings["sharedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+".sh run","build script failed")
+
                except CatalystError:
                        self.unbind()
                        raise CatalystError,"Stage build aborting due to error."