From: Brian Dolbec Date: Fri, 31 May 2013 16:11:30 +0000 (-0700) Subject: Initial creation of fileops.py X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=ac04141af92422fdb2e99673da14a11db6a16056;p=catalyst.git Initial creation of fileops.py Migrate to using ensure_dirs() --- diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py index 8b25ef88..9ce1dfdb 100644 --- a/catalyst/base/clearbase.py +++ b/catalyst/base/clearbase.py @@ -5,7 +5,7 @@ from stat import ST_UID, ST_GID, ST_MODE from catalyst.support import cmd, countdown - +from catalyst.fileops import ensure_dirs class ClearBase(object): """ @@ -35,7 +35,7 @@ class ClearBase(object): +myemp) #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, 0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) @@ -54,7 +54,7 @@ class ClearBase(object): if os.uname()[0] == "FreeBSD": os.system("chflags -R noschg "+myemp) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) @@ -73,7 +73,7 @@ class ClearBase(object): mystat=os.stat(myemp) #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) @@ -92,7 +92,7 @@ class ClearBase(object): mystat=os.stat(myemp) #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index 0e717bf5..f7d92fff 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -15,7 +15,7 @@ from catalyst.base.clearbase import ClearBase from catalyst.base.genbase import GenBase from catalyst.defaults import target_mounts from catalyst.lock import LockDir - +from catalyst.fileops import ensure_dirs, pjoin class StageBase(TargetBase, ClearBase, GenBase): """ @@ -330,8 +330,7 @@ class StageBase(TargetBase, ClearBase, GenBase): # +self.settings["target_path"],env=self.env) touch(self.settings["autoresume_path"]+"setup_target_path") - if not os.path.exists(self.settings["storedir"]+"/builds/"): - os.makedirs(self.settings["storedir"]+"/builds/") + ensure_dirs(self.settings["storedir"] + "/builds/") def set_fsscript(self): if self.settings["spec_prefix"]+"/fsscript" in self.settings: @@ -475,8 +474,7 @@ class StageBase(TargetBase, ClearBase, GenBase): self.settings["version_stamp"]+"/") if "autoresume" in self.settings["options"]: print "The autoresume path is " + self.settings["autoresume_path"] - if not os.path.exists(self.settings["autoresume_path"]): - os.makedirs(self.settings["autoresume_path"],0755) + ensure_dirs(self.settings["autoresume_path"], mode=0755) def set_controller_file(self): self.settings["controller_file"]=normpath(self.settings["shdir"] + "/" + @@ -765,19 +763,15 @@ class StageBase(TargetBase, ClearBase, GenBase): print "unpack() clear_chroot()" self.clear_chroot() - if not os.path.exists(self.settings["chroot_path"]): - os.makedirs(self.settings["chroot_path"]) + ensure_dirs(self.settings["chroot_path"]) - if not os.path.exists(self.settings["chroot_path"]+"/tmp"): - os.makedirs(self.settings["chroot_path"]+"/tmp",1777) + ensure_dirs(self.settings["chroot_path"]+"/tmp",mode=1777) if "pkgcache" in self.settings["options"]: - if not os.path.exists(self.settings["pkgcache_path"]): - os.makedirs(self.settings["pkgcache_path"],0755) + ensure_dirs(self.settings["pkgcache_path"],mode=0755) if "kerncache" in self.settings["options"]: - if not os.path.exists(self.settings["kerncache_path"]): - os.makedirs(self.settings["kerncache_path"],0755) + ensure_dirs(self.settings["kerncache_path"],mode=0755) print display_msg cmd(unpack_cmd,error_msg,env=self.env) @@ -845,8 +839,7 @@ class StageBase(TargetBase, ClearBase, GenBase): print cleanup_msg cleanup_cmd="rm -rf "+destdir cmd(cleanup_cmd,cleanup_errmsg,env=self.env) - if not os.path.exists(destdir): - os.makedirs(destdir,0755) + ensure_dirs(destdir,mode=0755) print "Unpacking portage tree (This can take a long time) ..." cmd(unpack_cmd,unpack_errmsg,env=self.env) @@ -932,12 +925,11 @@ class StageBase(TargetBase, ClearBase, GenBase): for x in self.mounts: #print "bind(); x =", x target = normpath(self.settings["chroot_path"] + self.target_mounts[x]) - if not os.path.exists(target): - os.makedirs(target, 0755) + ensure_dirs(target, mode=0755) if not os.path.exists(self.mountmap[x]): if not self.mountmap[x] == "tmpfs": - os.makedirs(self.mountmap[x],0755) + ensure_dirs(self.mountmap[x], mode=0755) src=self.mountmap[x] #print "bind(); src =", src @@ -1195,7 +1187,7 @@ class StageBase(TargetBase, ClearBase, GenBase): """ mystat=os.stat(myemp) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) touch(self.settings["autoresume_path"]+"empty") @@ -1248,8 +1240,7 @@ class StageBase(TargetBase, ClearBase, GenBase): mypath=string.join(mypath[:-1],"/") """ Now make sure path exists """ - if not os.path.exists(mypath): - os.makedirs(mypath) + ensure_dirs(mypath) print "Creating stage tarball..." diff --git a/catalyst/fileops.py b/catalyst/fileops.py new file mode 100644 index 00000000..009bf1cd --- /dev/null +++ b/catalyst/fileops.py @@ -0,0 +1,45 @@ +#!/usr/bin/python + +# Maintained in full by: +# Catalyst Team +# Release Engineering Team + +'''fileops.py + +Performs file operations such as pack/unpack, +ensuring directories exist,... imports snakeoils osutils +functions for use throughout catalyst. +''' + +from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs, normpath, + pjoin, listdir_files) +from catalyst.support import CatalystError + + +def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True, + failback=None, fatal=False): + '''Wrapper to snakeoil.osutil's ensure_dirs() + This additionally allows for failures to run + cleanup or other code and/or raise fatal errors. + + :param path: directory to ensure exists on disk + :param gid: a valid GID to set any created directories to + :param uid: a valid UID to set any created directories to + :param mode: permissions to set any created directories to + :param minimal: boolean controlling whether or not the specified mode + must be enforced, or is the minimal permissions necessary. For example, + if mode=0755, minimal=True, and a directory exists with mode 0707, + this will restore the missing group perms resulting in 757. + :param failback: function to run in the event of a failed attemp + to create the directory. + :return: True if the directory could be created/ensured to have those + permissions, False if not. + ''' + succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True) + if not succeeded: + if failback: + failback() + if fatal: + raise CatalystError( + "Failed to create directory: %s" % path, print_traceback=True) + return succeeded diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py index 93d30fcc..a17f9bd9 100644 --- a/catalyst/targets/grp.py +++ b/catalyst/targets/grp.py @@ -10,7 +10,7 @@ import glob from catalyst.support import (CatalystError, normpath, touch, cmd, list_bashify) - +from catalyst.fileops import ensure_dirs from catalyst.base.stagebase import StageBase @@ -51,8 +51,7 @@ class grp(StageBase): #if os.path.isdir(self.settings["target_path"]): #cmd("rm -rf "+self.settings["target_path"], #"Could not remove existing directory: "+self.settings["target_path"],env=self.env) - if not os.path.exists(self.settings["target_path"]): - os.makedirs(self.settings["target_path"]) + ensure_dirs(self.settings["target_path"]) touch(self.settings["autoresume_path"]+"setup_target_path") diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py index 3b19c3f9..668960bb 100644 --- a/catalyst/targets/livecd_stage1.py +++ b/catalyst/targets/livecd_stage1.py @@ -10,6 +10,7 @@ import string from catalyst.support import (normpath, touch, cmd) +from catalyst.fileops import ensure_dirs from catalyst.base.stagebase import StageBase @@ -43,8 +44,7 @@ class livecd_stage1(StageBase): "Could not remove existing directory: "+self.settings["target_path"],env=self.env) touch(self.settings["autoresume_path"]+"setup_target_path") - if not os.path.exists(self.settings["target_path"]): - os.makedirs(self.settings["target_path"]) + ensure_dirs(self.settings["target_path"]) def set_spec_prefix(self): self.settings["spec_prefix"]="livecd" diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py index 37cdd40a..7de16d1f 100644 --- a/catalyst/targets/livecd_stage2.py +++ b/catalyst/targets/livecd_stage2.py @@ -8,6 +8,7 @@ import os from catalyst.support import (normpath, file_locate, CatalystError, cmd, read_from_clst, touch) +from catalyst.fileops import ensure_dirs from catalyst.base.stagebase import StageBase @@ -63,8 +64,7 @@ class livecd_stage2(StageBase): cmd("rm -rf "+self.settings["target_path"], "Could not remove existing directory: "+self.settings["target_path"],env=self.env) touch(self.settings["autoresume_path"]+"setup_target_path") - if not os.path.exists(self.settings["target_path"]): - os.makedirs(self.settings["target_path"]) + ensure_dirs(self.settings["target_path"]) def run_local(self): # what modules do we want to blacklist? @@ -114,15 +114,10 @@ class livecd_stage2(StageBase): self.clear_chroot() #self.dir_setup() - if not os.path.exists(self.settings["chroot_path"]): - os.makedirs(self.settings["chroot_path"]) - - if not os.path.exists(self.settings["chroot_path"]+"/tmp"): - os.makedirs(self.settings["chroot_path"]+"/tmp",1777) + ensure_dirs(self.settings["chroot_path"]+"/tmp", mode=1777) if "pkgcache" in self.settings["options"]: - if not os.path.exists(self.settings["pkgcache_path"]): - os.makedirs(self.settings["pkgcache_path"],0755) + ensure_dirs(self.settings["pkgcache_path"], mode=0755) if not display_msg: raise CatalystError("Could not find appropriate source.\n" diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py index f9b2a7b3..dbf608d8 100644 --- a/catalyst/targets/netboot2.py +++ b/catalyst/targets/netboot2.py @@ -11,6 +11,7 @@ from stat import ST_UID, ST_GID, ST_MODE from catalyst.support import (CatalystError, normpath, touch, cmd, list_bashify) +from catalyst.fileops import ensure_dirs from catalyst.base.stagebase import StageBase @@ -63,8 +64,7 @@ class netboot2(StageBase): "Could not remove existing file: "+self.settings["target_path"],env=self.env) touch(self.settings["autoresume_path"]+"setup_target_path") - if not os.path.exists(self.settings["storedir"]+"/builds/"): - os.makedirs(self.settings["storedir"]+"/builds/") + ensure_dirs(self.settings["storedir"]+"/builds/") def copy_files_to_image(self): # copies specific files from the buildroot to merge_path @@ -159,7 +159,7 @@ class netboot2(StageBase): # the proper perms and ownership mystat=os.stat(myemp) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) touch(self.settings["autoresume_path"]+"empty") diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py index aa0a251b..9a80e71f 100644 --- a/catalyst/targets/snapshot.py +++ b/catalyst/targets/snapshot.py @@ -10,7 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE from catalyst.support import normpath, cmd from catalyst.base.targetbase import TargetBase from catalyst.base.genbase import GenBase - +from catalyst.fileops import ensure_dirs class snapshot(TargetBase, GenBase): """ Builder class for snapshots. @@ -31,8 +31,7 @@ class snapshot(TargetBase, GenBase): def setup(self): x=normpath(self.settings["storedir"]+"/snapshots") - if not os.path.exists(x): - os.makedirs(x) + ensure_dirs(x) def mount_safety_check(self): pass @@ -50,8 +49,7 @@ class snapshot(TargetBase, GenBase): " from "+self.settings["portdir"]+"..." mytmp=self.settings["tmp_path"] - if not os.path.exists(mytmp): - os.makedirs(mytmp) + ensure_dirs(mytmp) cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " + \ "--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* "+\ @@ -88,6 +86,6 @@ class snapshot(TargetBase, GenBase): if os.uname()[0] == "FreeBSD": os.system("chflags -R noschg "+myemp) shutil.rmtree(myemp) - os.makedirs(myemp,0755) + ensure_dirs(myemp, mode=0755) os.chown(myemp,mystat[ST_UID],mystat[ST_GID]) os.chmod(myemp,mystat[ST_MODE]) diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py index e9369290..fc730b20 100644 --- a/catalyst/targets/stage1.py +++ b/catalyst/targets/stage1.py @@ -8,6 +8,7 @@ import os from catalyst.support import normpath, list_to_string +from catalyst.fileops import ensure_dirs from catalyst.base.stagebase import StageBase @@ -66,10 +67,8 @@ class stage1(StageBase): # baselayout no longer creates the .keep files in proc and dev for FreeBSD as it # would create them too late...we need them earlier before bind mounting filesystems # since proc and dev are not writeable, so...create them here - if not os.path.exists(self.settings["stage_path"]+"/proc"): - os.makedirs(self.settings["stage_path"]+"/proc") - if not os.path.exists(self.settings["stage_path"]+"/dev"): - os.makedirs(self.settings["stage_path"]+"/dev") + ensure_dirs(self.settings["stage_path"]+"/proc") + ensure_dirs(self.settings["stage_path"]+"/dev") if not os.path.isfile(self.settings["stage_path"]+"/proc/.keep"): try: proc_keepfile = open(self.settings["stage_path"]+"/proc/.keep","w") @@ -89,8 +88,7 @@ class stage1(StageBase): def set_mounts(self): # stage_path/proc probably doesn't exist yet, so create it - if not os.path.exists(self.settings["stage_path"]+"/proc"): - os.makedirs(self.settings["stage_path"]+"/proc") + ensure_dirs(self.settings["stage_path"]+"/proc") # alter the mount mappings to bind mount proc onto it self.mounts.append("stage1root/proc")