Initial creation of fileops.py

Migrate to using ensure_dirs()

diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index 8b25ef88b4ab718c097f909e686dd9dd31663ce4..9ce1dfdbb3b3ee643865caf5bc761347be34ad1a 100644 (file)
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -5,7 +5,7 @@ from stat import ST_UID, ST_GID, ST_MODE
 
 
 from catalyst.support import cmd, countdown
-
+from catalyst.fileops import ensure_dirs
 
 class ClearBase(object):
        """
@@ -35,7 +35,7 @@ class ClearBase(object):
                                                +myemp)
                                #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
                                shutil.rmtree(myemp)
-                               os.makedirs(myemp,0755)
+                               ensure_dirs(myemp, 0755)
                                os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                                os.chmod(myemp,mystat[ST_MODE])
 
@@ -54,7 +54,7 @@ class ClearBase(object):
                        if os.uname()[0] == "FreeBSD":
                                os.system("chflags -R noschg "+myemp)
                        shutil.rmtree(myemp)
-                       os.makedirs(myemp,0755)
+                       ensure_dirs(myemp, mode=0755)
                        os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                        os.chmod(myemp,mystat[ST_MODE])
 
@@ -73,7 +73,7 @@ class ClearBase(object):
                                mystat=os.stat(myemp)
                                #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
                                shutil.rmtree(myemp)
-                               os.makedirs(myemp,0755)
+                               ensure_dirs(myemp, mode=0755)
                                os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                                os.chmod(myemp,mystat[ST_MODE])
 
@@ -92,7 +92,7 @@ class ClearBase(object):
                                mystat=os.stat(myemp)
                                #cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
                                shutil.rmtree(myemp)
-                               os.makedirs(myemp,0755)
+                               ensure_dirs(myemp, mode=0755)
                                os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                                os.chmod(myemp,mystat[ST_MODE])
 

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 0e717bf57b3f51c1178a30ed55573eee97444bf4..f7d92fffa20244f60890300dc6e1b4776df859ce 100644 (file)
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -15,7 +15,7 @@ from catalyst.base.clearbase import ClearBase
 from catalyst.base.genbase import GenBase
 from catalyst.defaults import target_mounts
 from catalyst.lock import LockDir
-
+from catalyst.fileops import ensure_dirs, pjoin
 
 class StageBase(TargetBase, ClearBase, GenBase):
        """
@@ -330,8 +330,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
 #                                      +self.settings["target_path"],env=self.env)
                        touch(self.settings["autoresume_path"]+"setup_target_path")
 
-                       if not os.path.exists(self.settings["storedir"]+"/builds/"):
-                               os.makedirs(self.settings["storedir"]+"/builds/")
+                       ensure_dirs(self.settings["storedir"] + "/builds/")
 
        def set_fsscript(self):
                if self.settings["spec_prefix"]+"/fsscript" in self.settings:
@@ -475,8 +474,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
                        self.settings["version_stamp"]+"/")
                if "autoresume" in self.settings["options"]:
                        print "The autoresume path is " + self.settings["autoresume_path"]
-               if not os.path.exists(self.settings["autoresume_path"]):
-                       os.makedirs(self.settings["autoresume_path"],0755)
+               ensure_dirs(self.settings["autoresume_path"], mode=0755)
 
        def set_controller_file(self):
                self.settings["controller_file"]=normpath(self.settings["shdir"] + "/" +
@@ -765,19 +763,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
                                print "unpack()  clear_chroot()"
                                self.clear_chroot()
 
-                       if not os.path.exists(self.settings["chroot_path"]):
-                               os.makedirs(self.settings["chroot_path"])
+                       ensure_dirs(self.settings["chroot_path"])
 
-                       if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
-                               os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
+                       ensure_dirs(self.settings["chroot_path"]+"/tmp",mode=1777)
 
                        if "pkgcache" in self.settings["options"]:
-                               if not os.path.exists(self.settings["pkgcache_path"]):
-                                       os.makedirs(self.settings["pkgcache_path"],0755)
+                               ensure_dirs(self.settings["pkgcache_path"],mode=0755)
 
                        if "kerncache" in self.settings["options"]:
-                               if not os.path.exists(self.settings["kerncache_path"]):
-                                       os.makedirs(self.settings["kerncache_path"],0755)
+                               ensure_dirs(self.settings["kerncache_path"],mode=0755)
 
                        print display_msg
                        cmd(unpack_cmd,error_msg,env=self.env)
@@ -845,8 +839,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
                                print cleanup_msg
                                cleanup_cmd="rm -rf "+destdir
                                cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
-                       if not os.path.exists(destdir):
-                               os.makedirs(destdir,0755)
+                       ensure_dirs(destdir,mode=0755)
 
                        print "Unpacking portage tree (This can take a long time) ..."
                        cmd(unpack_cmd,unpack_errmsg,env=self.env)
@@ -932,12 +925,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
                for x in self.mounts:
                        #print "bind(); x =", x
                        target = normpath(self.settings["chroot_path"] + self.target_mounts[x])
-                       if not os.path.exists(target):
-                               os.makedirs(target, 0755)
+                       ensure_dirs(target, mode=0755)
 
                        if not os.path.exists(self.mountmap[x]):
                                if not self.mountmap[x] == "tmpfs":
-                                       os.makedirs(self.mountmap[x],0755)
+                                       ensure_dirs(self.mountmap[x], mode=0755)
 
                        src=self.mountmap[x]
                        #print "bind(); src =", src
@@ -1195,7 +1187,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
                                        """
                                        mystat=os.stat(myemp)
                                        shutil.rmtree(myemp)
-                                       os.makedirs(myemp,0755)
+                                       ensure_dirs(myemp, mode=0755)
                                        os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                                        os.chmod(myemp,mystat[ST_MODE])
                        touch(self.settings["autoresume_path"]+"empty")
@@ -1248,8 +1240,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
                        mypath=string.join(mypath[:-1],"/")
 
                        """ Now make sure path exists """
-                       if not os.path.exists(mypath):
-                               os.makedirs(mypath)
+                       ensure_dirs(mypath)
 
                        print "Creating stage tarball..."
 

diff --git a/catalyst/fileops.py b/catalyst/fileops.py
new file mode 100644 (file)
index 0000000..009bf1c
--- /dev/null
+++ b/catalyst/fileops.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python
+
+# Maintained in full by:
+# Catalyst Team <catalyst@gentoo.org>
+# Release Engineering Team <releng@gentoo.org>
+
+'''fileops.py
+
+Performs file operations such as pack/unpack,
+ensuring directories exist,... imports snakeoils osutils
+functions for use throughout catalyst.
+'''
+
+from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs, normpath,
+       pjoin, listdir_files)
+from catalyst.support import CatalystError
+
+
+def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
+               failback=None, fatal=False):
+       '''Wrapper to snakeoil.osutil's ensure_dirs()
+       This additionally allows for failures to run
+       cleanup or other code and/or raise fatal errors.
+
+       :param path: directory to ensure exists on disk
+       :param gid: a valid GID to set any created directories to
+       :param uid: a valid UID to set any created directories to
+       :param mode: permissions to set any created directories to
+       :param minimal: boolean controlling whether or not the specified mode
+               must be enforced, or is the minimal permissions necessary.  For example,
+               if mode=0755, minimal=True, and a directory exists with mode 0707,
+               this will restore the missing group perms resulting in 757.
+       :param failback: function to run in the event of a failed attemp
+               to create the directory.
+       :return: True if the directory could be created/ensured to have those
+               permissions, False if not.
+       '''
+       succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True)
+       if not succeeded:
+               if failback:
+                       failback()
+               if fatal:
+                       raise CatalystError(
+                               "Failed to create directory: %s" % path, print_traceback=True)
+       return succeeded

diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py
index 93d30fccb64306d3eae13d74c26ec51c74ad2cfb..a17f9bd929263ccb58fe99b42663cf3410759a53 100644 (file)
--- a/catalyst/targets/grp.py
+++ b/catalyst/targets/grp.py
@@ -10,7 +10,7 @@ import glob
 
 from catalyst.support import (CatalystError, normpath,
        touch, cmd, list_bashify)
-
+from catalyst.fileops import ensure_dirs
 from catalyst.base.stagebase import StageBase
 
 
@@ -51,8 +51,7 @@ class grp(StageBase):
                        #if os.path.isdir(self.settings["target_path"]):
                                #cmd("rm -rf "+self.settings["target_path"],
                                #"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
-                       if not os.path.exists(self.settings["target_path"]):
-                               os.makedirs(self.settings["target_path"])
+                       ensure_dirs(self.settings["target_path"])
 
                        touch(self.settings["autoresume_path"]+"setup_target_path")
 

diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py
index 3b19c3f93b5db353d977d4a638ef1da502bb52ca..668960bbd14eb22ff0cbe5f51921280f1e699b73 100644 (file)
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -10,6 +10,7 @@ import string
 
 from catalyst.support import (normpath,
        touch, cmd)
+from catalyst.fileops import ensure_dirs
 
 from catalyst.base.stagebase import StageBase
 
@@ -43,8 +44,7 @@ class livecd_stage1(StageBase):
                                        "Could not remove existing directory: "+self.settings["target_path"],env=self.env)
                                touch(self.settings["autoresume_path"]+"setup_target_path")
 
-                       if not os.path.exists(self.settings["target_path"]):
-                               os.makedirs(self.settings["target_path"])
+                       ensure_dirs(self.settings["target_path"])
 
        def set_spec_prefix(self):
                self.settings["spec_prefix"]="livecd"

diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py
index 37cdd40aa0e7aac79dddaae3a67fbc14bce71e5b..7de16d1f15dbd830849cda4b0e1c8b2b22683930 100644 (file)
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -8,6 +8,7 @@ import os
 
 from catalyst.support import (normpath, file_locate, CatalystError, cmd,
        read_from_clst, touch)
+from catalyst.fileops import ensure_dirs
 from catalyst.base.stagebase import StageBase
 
 
@@ -63,8 +64,7 @@ class livecd_stage2(StageBase):
                                cmd("rm -rf "+self.settings["target_path"],
                                "Could not remove existing directory: "+self.settings["target_path"],env=self.env)
                                touch(self.settings["autoresume_path"]+"setup_target_path")
-                       if not os.path.exists(self.settings["target_path"]):
-                               os.makedirs(self.settings["target_path"])
+                       ensure_dirs(self.settings["target_path"])
 
        def run_local(self):
                # what modules do we want to blacklist?
@@ -114,15 +114,10 @@ class livecd_stage2(StageBase):
                                self.clear_chroot()
                                #self.dir_setup()
 
-                       if not os.path.exists(self.settings["chroot_path"]):
-                               os.makedirs(self.settings["chroot_path"])
-
-                       if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
-                               os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
+                       ensure_dirs(self.settings["chroot_path"]+"/tmp", mode=1777)
 
                        if "pkgcache" in self.settings["options"]:
-                               if not os.path.exists(self.settings["pkgcache_path"]):
-                                       os.makedirs(self.settings["pkgcache_path"],0755)
+                               ensure_dirs(self.settings["pkgcache_path"], mode=0755)
 
                        if not display_msg:
                                raise CatalystError("Could not find appropriate source.\n"

diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py
index f9b2a7b38b09882abf62e0d143c033fe13b629d0..dbf608d8eb3d018703f841dbe18b4c535f85195d 100644 (file)
--- a/catalyst/targets/netboot2.py
+++ b/catalyst/targets/netboot2.py
@@ -11,6 +11,7 @@ from stat import ST_UID, ST_GID, ST_MODE
 
 from catalyst.support import (CatalystError, normpath,
        touch, cmd, list_bashify)
+from catalyst.fileops import ensure_dirs
 
 from catalyst.base.stagebase import StageBase
 
@@ -63,8 +64,7 @@ class netboot2(StageBase):
                                        "Could not remove existing file: "+self.settings["target_path"],env=self.env)
                                touch(self.settings["autoresume_path"]+"setup_target_path")
 
-               if not os.path.exists(self.settings["storedir"]+"/builds/"):
-                       os.makedirs(self.settings["storedir"]+"/builds/")
+               ensure_dirs(self.settings["storedir"]+"/builds/")
 
        def copy_files_to_image(self):
                # copies specific files from the buildroot to merge_path
@@ -159,7 +159,7 @@ class netboot2(StageBase):
                                        # the proper perms and ownership
                                        mystat=os.stat(myemp)
                                        shutil.rmtree(myemp)
-                                       os.makedirs(myemp,0755)
+                                       ensure_dirs(myemp, mode=0755)
                                        os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                                        os.chmod(myemp,mystat[ST_MODE])
                touch(self.settings["autoresume_path"]+"empty")

diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index aa0a251b8d1151037ee883664b33c28d7f204723..9a80e71fe1b0ed8d0fa5b94a2cea5f5cf6c147f3 100644 (file)
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -10,7 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
 from catalyst.support import normpath, cmd
 from catalyst.base.targetbase import TargetBase
 from catalyst.base.genbase import GenBase
-
+from catalyst.fileops import ensure_dirs
 class snapshot(TargetBase, GenBase):
        """
        Builder class for snapshots.
@@ -31,8 +31,7 @@ class snapshot(TargetBase, GenBase):
 
        def setup(self):
                x=normpath(self.settings["storedir"]+"/snapshots")
-               if not os.path.exists(x):
-                       os.makedirs(x)
+               ensure_dirs(x)
 
        def mount_safety_check(self):
                pass
@@ -50,8 +49,7 @@ class snapshot(TargetBase, GenBase):
                        " from "+self.settings["portdir"]+"..."
 
                mytmp=self.settings["tmp_path"]
-               if not os.path.exists(mytmp):
-                       os.makedirs(mytmp)
+               ensure_dirs(mytmp)
 
                cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " + \
                        "--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* "+\
@@ -88,6 +86,6 @@ class snapshot(TargetBase, GenBase):
                        if os.uname()[0] == "FreeBSD":
                                os.system("chflags -R noschg "+myemp)
                        shutil.rmtree(myemp)
-                       os.makedirs(myemp,0755)
+                       ensure_dirs(myemp, mode=0755)
                        os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
                        os.chmod(myemp,mystat[ST_MODE])

diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index e9369290d0b0d6411be0cd065ac169522aa1392f..fc730b20106fc814f6498c0d1ba9f138f0d8410c 100644 (file)
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -8,6 +8,7 @@ import os
 
 
 from catalyst.support import normpath, list_to_string
+from catalyst.fileops import ensure_dirs
 from catalyst.base.stagebase import StageBase
 
 
@@ -66,10 +67,8 @@ class stage1(StageBase):
                        # baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
                        # would create them too late...we need them earlier before bind mounting filesystems
                        # since proc and dev are not writeable, so...create them here
-                       if not os.path.exists(self.settings["stage_path"]+"/proc"):
-                               os.makedirs(self.settings["stage_path"]+"/proc")
-                       if not os.path.exists(self.settings["stage_path"]+"/dev"):
-                               os.makedirs(self.settings["stage_path"]+"/dev")
+                       ensure_dirs(self.settings["stage_path"]+"/proc")
+                       ensure_dirs(self.settings["stage_path"]+"/dev")
                        if not os.path.isfile(self.settings["stage_path"]+"/proc/.keep"):
                                try:
                                        proc_keepfile = open(self.settings["stage_path"]+"/proc/.keep","w")
@@ -89,8 +88,7 @@ class stage1(StageBase):
 
        def set_mounts(self):
                # stage_path/proc probably doesn't exist yet, so create it
-               if not os.path.exists(self.settings["stage_path"]+"/proc"):
-                       os.makedirs(self.settings["stage_path"]+"/proc")
+               ensure_dirs(self.settings["stage_path"]+"/proc")
 
                # alter the mount mappings to bind mount proc onto it
                self.mounts.append("stage1root/proc")