From: Daniel Robbins <drobbins@gentoo.org>
Date: Tue, 28 Oct 2003 22:09:23 +0000 (+0000)
Subject: lots of improvements
X-Git-Tag: CATALYST_1_0_1~209
X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=25b002630ea90de91fbb861ea7c299cc8cc27339;p=catalyst.git

lots of improvements


git-svn-id: svn+ssh://svn.gentoo.org/var/svnroot/catalyst/trunk@31 d1e1f19c-881f-0410-ab34-b69fee027534
---

diff --git a/catalyst b/catalyst
index b00f1c96..a209ffe2 100755
--- a/catalyst
+++ b/catalyst
@@ -13,50 +13,6 @@ def usage():
 if len(sys.argv)==1 or sys.argv[1] in ["-h","--help"]:
 	usage()
 	sys.exit(1)
-"""
-Overview of catalyst operation
-==============================
-
-* The program starts, and the local machine type is detected. 
-
-* Based on this information, catalyst determines what kind of machine types
-  it can build for (amd64 and ia64 can build for x86 as well, for example.)
-  The appropriate arch plugins are loaded, which contain builder classes
-  for each supported sub-arch.
-
-* Command-line arguments are parsed. If specified, a spec file is read.
-
-* These build variables are stored in an internal "spec" object, which will
-  be a standard python dictionary. This spec dictionary contains all relevant
-  build-related information.
-
-* The spec object is passed to the appropriate target constructor.
-  The target constructor updates the spec object to contain data relevant
-  to the particular target (stage1, stage3, grp, etc.)
-
-*** PROTOTYPE CODE UP TO HERE IS COMPLETE (drobbins, 26 Oct '03)
-
-* The full data of the spec object is written to disc, so there is a complete
-  record of all variables that will be used to build what we're building.  This
-  will allow for another person to re-use this information to replicate our
-  work (it should be possible to distribute a spec file along with a portage
-  snapshot and a starter tarball, and our build can be replicated exactly on
-  any machine.) The spec object contains data like CFLAGS, CHOST, subarch,
-  mainarch, the profile used to build, and for GRP and LiveCDs the complete
-  package build list. This is important to allow work to be replicated. It's
-  possible that the stage1/2/3.sh and other scripts should be distributed as
-  well, to allow proper replication of work.
-
-* The build process begins by calling the appropriate method of the builder
-  instance. This includes cleanup, setup of chroot, entering the chroot,
-  running the appropriate bash build script, checking for error conditions,
-  and finishing up.
-  
-* The catalyst process is now complete :)
-"""
-
-#map current machine information from uname() to the mainarch we are running
-#under
 
 import targets
 targetmap={}
@@ -73,20 +29,26 @@ myspec={}
 myspec["storedir"]="/var/tmp/catalyst"
 myspec["sharedir"]="/usr/share/catalyst"
 #these would come from there too?:
-myspec["distdir"]="/usr/portage/distfiles"
+myspec["distdir"]="/mnt/misc/distfiles"
 myspec["portdir"]="/usr/portage"
 
-#all the main program needs to do is figure out the target. The rest of the args go to the
-#target constructor
-
-#call target constructor, pass our "myspec" settings as well as cmdline arguments for parsing
-
-addlargs={}
-arg_parse(myspec,addlargs)
-if not targetmap.has_key(myspec["target"]):
-	raise CatalystError,"Target \""+myspec["target"]+"\" not available."
-mytarget=targetmap[myspec["target"]](myspec,addlargs)
-mytarget.run()
-
+#determine target, call target constructor and hand it the rest of the arguments
+
+try:
+	addlargs={}
+	arg_parse(myspec,addlargs)
+	if not targetmap.has_key(myspec["target"]):
+		raise CatalystError,"Target \""+myspec["target"]+"\" not available."
+	mytarget=targetmap[myspec["target"]](myspec,addlargs)
+	
+	#let's display our spec information in all its glory
+	print
+	spec_dump(myspec)
+	print
+	
+	mytarget.run()
+except CatalystError:
+	sys.exit(1)
+# Examples:
 # ./catalyst subarch=pentium4 version_stamp=20031016 target=stage3 rel_type=default rel_version=1.4 snapshot=20031016 source_subpath=default-x86-1.4/stage2-pentium4-20031016
-
+# ./catalyst target=snapshot version_stamp=20031028
diff --git a/modules/catalyst_support.py b/modules/catalyst_support.py
index afc559d2..ad171679 100644
--- a/modules/catalyst_support.py
+++ b/modules/catalyst_support.py
@@ -1,4 +1,4 @@
-import sys,string
+import sys,string,os
 
 config_file_values=["storedir","sharedir","distdir","portdir"]
 
@@ -6,14 +6,26 @@ class CatalystError(Exception):
 	def __init__(self, message):
 		if message:
 			print "catalyst: "+message
-		sys.exit(1)
-
 def die(msg=None):
-	raise CatalystError, msg
+	warn(msg)
+	sys.exit(1)
 
 def warn(msg):
 	print "catalyst: "+msg
 
+def ismount(path):
+	"enhanced to handle bind mounts"
+	if os.path.ismount(path):
+		return 1
+	a=open("/proc/mounts","r")
+	mylines=a.readlines()
+	a.close()
+	for line in mylines:
+		mysplit=line.split()
+		if path == mysplit[1]:
+			return 1
+	return 0
+
 def arg_parse(mydict,remaining={}):
 	global config_file_values
 	for x in sys.argv[1:]:
diff --git a/modules/targets.py b/modules/targets.py
index 934622af..9a86501e 100644
--- a/modules/targets.py
+++ b/modules/targets.py
@@ -1,4 +1,4 @@
-import os,stat,string,imp
+import os,stat,string,imp,types
 from catalyst_support import *
 
 class generic_target:
@@ -16,11 +16,6 @@ class generic_stage_target(generic_target):
 		self.valid_values=self.required_values
 		generic_target.__init__(self,addlargs,myspec)
 
-		#ensure we have all required user-supplied settings before proceeding
-		for x in requiredspec:
-			if not self.settings.has_key(x):
-				raise CatalystError, "Required value \""+x+"\" not specified."
-		
 		# map the mainarch we are running under to the mainarches we support for
 		# building stages and LiveCDs. (for example, on amd64, we can build stages for
 		# x86 or amd64.
@@ -74,15 +69,12 @@ class generic_stage_target(generic_target):
 		paths=["/usr/portage/packages","/usr/portage/distfiles", "/var/tmp/distfiles", "/proc", "/root/.ccache", "/dev"]
 		if not os.path.exists(mypath):
 			return 
-		mypstat=os.stat(mypath)[stat.ST_DEV]
 		for x in paths:
 			if not os.path.exists(mypath+x):
 				continue
-			teststat=os.stat(mypath+x)[stat.ST_DEV]
-			if teststat!=mypstat:
+			if ismount(mypath+x):
 				#something is still mounted
 				raise CatalystError, x+" is still mounted; aborting."
-		return 1
 		
 	def dir_setup(self):
 		self.mount_safety_check()
@@ -111,24 +103,56 @@ class generic_stage_target(generic_target):
 				raise CatalystError,"Couldn't bind mount "+x[0]
 
 	def unbind(self):
+		ouch=0
+		mypath=self.settings["chroot_path"]
 		for x in ["/usr/portage/distfiles","/proc","/dev"]:
-			retval=os.system("umount "+self.settings["chroot_path"]+x)
+			if not os.path.exists(mypath+x):
+				continue
+			if not ismount(mypath+x):
+				#it's not mounted, continue
+				continue
+			retval=os.system("umount "+mypath+x)
 			if retval!=0:
-				warning("Couldn't umount bind mount: "+self.settings["chroot_path"]+x)
-
-	def setup(self):
-		#setup will leave everything in unbound state if there is a failure
-		self.dir_setup()
+				ouch=1
+				warn("Couldn't umount bind mount: "+mypath+x)
+				#keep trying to umount the others, to minimize damage if developer makes a mistake
+		if ouch:
+			#if any bind mounts really failed, then we need to raise this to potentially prevent
+			#an upcoming bash stage cleanup script from wiping our bind mounts.
+			raise CatalystError,"Couldn't umount one or more bind-mounts; aborting for safety."
+
+	def chroot_setup(self):
 		self.unpack_and_bind()
+		retval=os.system("cp /etc/resolv.conf "+self.settings["chroot_path"]+"/etc")
+		if retval!=0:
+			raise CatalystError,"Could not copy resolv.conf into place."
 
+	def clean(self):
+		"do not call without first unbinding"
+		retval=os.system("rm "+self.settings["chroot_path"]+"/etc/resolv.conf")
+		if retval!=0:
+			raise CatalystError,"Could not clean up resolv.conf."
+		retval=os.system(self.settings["storedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+".sh clean")
+		if retval!=0:
+			raise CatalystError,"clean script failed."
+		
 	def run(self):
-		self.setup()
+		self.dir_setup()
+		self.chroot_setup()
+		#modify the current environment. This is an ugly hack that should be fixed. We need this
+		#to use the os.system() call since we can't specify our own environ:
+		for x in self.settings.keys():
+			if type(self.settings[x])==types.StringType:
+				os.environ[x]=self.settings[x]
 		try:
-			pass
+			retval=os.system(self.settings["storedir"]+"/targets/"+self.settings["target"]+"/"+self.settings["target"]+".sh run")
+			if retval!=0:
+				raise CatalystError,"build script failed."
 		finally:
-			#always unbind
 			self.unbind()
-	
+		self.clean()
+
+			
 class snapshot_target(generic_target):
 	def __init__(self,myspec,addlargs):
 		self.valid_values=["version_stamp","target"]
diff --git a/targets/stage2/stage2.sh b/targets/stage2/stage2.sh
index 4e5d90d0..fd3e0272 100755
--- a/targets/stage2/stage2.sh
+++ b/targets/stage2/stage2.sh
@@ -1,8 +1,10 @@
 # Copyright 1999-2003 Gentoo Technologies, Inc.
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/targets/stage2/Attic/stage2.sh,v 1.1 2003/10/15 05:23:14 zhen Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/targets/stage2/Attic/stage2.sh,v 1.2 2003/10/28 22:09:23 drobbins Exp $
 
-$CHROOT . /bin/bash << EOF
+case $1 in
+run)
+	$CHROOT . /bin/bash << EOF
 	env-update
 	source /etc/profile
 	mkdir -p /usr/portage/packages/All
@@ -14,6 +16,22 @@ $CHROOT . /bin/bash << EOF
 
 	/usr/portage/scripts/bootstrap.sh || exit 1
 EOF
-
-[ $? -ne 0 ] && die "Stage 2 build failure"
-
+	[ $? -ne 0 ] && exit 1 
+	;;
+clean)
+	# we need to have catalyst un-bind-mount things before
+	# we clean up.
+	$CHROOT . /bin/bash << EOF
+	if [ ${CCACHE} -eq 1 ]
+	then
+		emerge -C ccache
+	fi
+	rm -rf /usr/portage
+	rm -rf /tmp/*
+EOF
+	[ $? -ne 0 ] && exit 1 
+	;;
+*)
+	exit 1
+	;;
+esac
diff --git a/targets/stage3/stage3.sh b/targets/stage3/stage3.sh
index 864d0630..d913fb57 100755
--- a/targets/stage3/stage3.sh
+++ b/targets/stage3/stage3.sh
@@ -1,21 +1,40 @@
 # Copyright 1999-2003 Gentoo Technologies, Inc.
 # Distributed under the terms of the GNU General Public License v2
-# $Header: /var/cvsroot/gentoo/src/catalyst/targets/stage3/Attic/stage3.sh,v 1.1 2003/10/15 05:33:03 zhen Exp $
+# $Header: /var/cvsroot/gentoo/src/catalyst/targets/stage3/Attic/stage3.sh,v 1.2 2003/10/28 22:09:23 drobbins Exp $
 
-$CHROOT . /bin/bash << EOF
+case $1 in
+enter)
+	$CHROOT $chroot_path
+	;;
+run)
+	$CHROOT $chroot_path /bin/bash << EOF
 	env-update
 	source /etc/profile
-	if [ ${CCACHE} -eq 1 ]
+	if [ -n "${CCACHE}" ]
 	then
 		emerge --oneshot --nodeps --usepkg --buildpkg ccache || exit 1
 	fi
-	if [ ${REL_TYPE} = "hardened" ]
+	if [ ${rel_type} = "hardened" ]
 	then
 		emerge --oneshot --nodeps hardened-gcc || exit 1
 	fi
 	export CONFIG_PROTECT="-*"
 	emerge system --usepkg --buildpkg || exit 1
 EOF
-
-[ $? -ne 0 ] && die "Stage 3 build failure"
-
+	[ $? -ne 0 ] && exit 1
+	;;
+clean)
+	$CHROOT $chroot_path /bin/bash << EOF
+	env-update
+	source /etc/profile
+	if [ -n "${CCACHE}" ]
+	then
+		emerge -C ccache || exit 1
+	fi
+EOF
+	[ $? -ne 0 ] && exit 1
+	;;
+*)
+	exit 1
+	;;
+esac