--- /dev/null
+#!/usr/bin/python
+#
+# rmerge2 is a robust version of "emerge -e" which supports
+# resumption of builds. It was inspired by the original rmerge
+# bash script by Dan Doel <dolio@zoomtown.com> which did basically
+# the same thing (but not quite).
+#
+# Copyright (C) 2002 Bardur Arantsson <bardur@imada.sdu.dk>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# See http://www.gnu.org/copyleft/gpl.html for further information.
+#
+
+VERSION = "0.9.7"
+
+# system configuration (there should be no need to change these)
+STATEDIR="/var/lib/rmerge2"
+EMERGE="/usr/bin/emerge"
+
+# default option values (DO NOT CHANGE UNLESS YOU KNOW WHAT YOU ARE DOING)
+opt_empty = 0
+opt_pretend = 0
+opt_skip = 0
+opt_resume = 1
+opt_force = 0
+opt_quiet = 0
+opt_statefile = None
+
+# This is an option which is purely for testing.
+# It causes merge_pkg to not do anything dangerous,
+# and in addition randomly fail emerge runs. Useful
+# for testing if resumption of builds works properly.
+opt_test = 0
+
+#
+# Start of the actual code
+#
+import getopt
+import os
+import md5
+import re
+import whrandom
+from popen2 import *
+import cPickle
+pickle = cPickle
+import portage
+
+# read portage configuration to avoid relying on
+# the environment
+portage_config = portage.config()
+
+# fatal error
+def fatal(message):
+ os.sys.stderr.write("rmerge2: %s\n" % (message,))
+ os.sys.stderr.flush()
+ os.sys.exit(1)
+
+class MergeError(Exception):
+ pass
+
+class State:
+ """The state object represents the state of the
+ compilation. It is pickled/unpickled to preserve
+ state information across invocations."""
+
+ def __init__(self, todo, _failed = [], skipped = []):
+ """Create an initially empty state."""
+ self.todo = todo
+ self._failed = _failed
+ self.skipped = skipped
+
+ def skip(self, n):
+ """Skip n packages."""
+ self.skipped.extend(self.todo[:n])
+ self.todo = self.todo[n:]
+
+ def any_left(self):
+ """Are there any packages left to do?"""
+ return len(self.todo) != 0
+
+ def get_next(self):
+ """Return the next package from the to do list."""
+ return self.todo[0]
+
+ def get_failed(self):
+ """Return the list of failed packages."""
+ return self._failed
+
+ def get_todo(self):
+ """Return the list of packages left to do."""
+ return self.todo
+
+ def done(self):
+ """Mark the first package as done."""
+ del self.todo[0]
+
+ def failed(self):
+ """The current package failed (completely)."""
+ self._failed.append(self.todo[0])
+ del self.todo[0]
+
+ def dump(self, fname):
+ """Write state to file named fname in a crashproof way."""
+ f = open("%s.tmp" % fname,"w")
+ try:
+ pickle.dump(self, f)
+ os.rename("%s.tmp" % fname, fname)
+ except:
+ os.unlink("%s.tmp" % fname) # don't need anymore
+ raise
+
+def load_state(fname):
+ return pickle.load(open(fname,"r"))
+
+def split_pkgname(pkgname):
+ """Chops off the version number and returns a tuple
+ of (basename, version) for the given package name."""
+ m = re.search("(.*?)(-[.0-9a-zA-Z]+(-r[0-9]+)?)$",pkgname)
+ if m:
+ pkg_ver = m.group(2)[1:]
+ pkgname = m.group(1)
+ else:
+ pkg_ver = None
+
+ return (pkgname,pkg_ver)
+
+def get_state_filename(pkgnames, create=1):
+ """Compute the status directory name for the
+ given list of packages."""
+ if opt_statefile:
+ return opt_statefile
+ else:
+ # Generate a state file name if none is given
+ pkgnames = pkgnames[:] # sorting is destructive
+ pkgnames.sort() # we want to ignore ordering
+
+ # hash each of the package names
+ h = md5.new()
+ for pkgname in pkgnames:
+ h.update(pkgname.strip())
+
+ return os.path.join(STATEDIR,h.hexdigest())
+
+def get_prereqs(pkgnames):
+ """Compute prerequisite packages for the given packages and
+ return the list. The list contains full package names (category,
+ package name and version number). If opt_empty is NOT set,
+ only packages which were originally listed are returned. However,
+ this is still useful for determining the proper ordering of those
+ packages."""
+ if not opt_empty:
+ # build a dictionary of package names for quick lookup.
+ # this is used to determine if we should merge a given
+ # package. this is a very crude way of doing this, but
+ # portage doesn't allow us to find out (inexpensively)
+ # whether a given package is part of another which is
+ # really what we need if pkgnames contains virtual
+ # packages. Oh, well.
+ pkgnames_dict = {}
+ for pkgname in pkgnames:
+ pkgnames_dict[pkgname] = None # value doesn't matter
+
+ # If any input packages contain version numbers, we should
+ # repesct them. Do this by adding a '=' in front so that
+ # emerge knows that we want a specific version
+ qpkgnames = pkgnames[:] # copy to avoid overwriting
+ for i in xrange(0,len(pkgnames)):
+ pkgname = pkgnames[i]
+ if split_pkgname(pkgname)[1]!=None: # fixed version?
+ qpkgnames[i] = ("=%s" % pkgname)
+
+ # Evil hack to retrieve list of package names (and versions)
+ r = re.compile("\\[ebuild......\\]\\s+([^\\s]+)")
+
+ # Use emerge to get the canonical package list.
+ # NOTE: If "-e" is not present, emerge will not
+ # compute interdependencies between packages properly
+ emerge = Popen3([EMERGE,"-p","-e"] + qpkgnames)
+ prereqs = [] # current list of prerequisite packages
+ lines = emerge.fromchild.readlines()
+ for line in lines:
+ m = r.match(line)
+ if m:
+ if not opt_empty: # only want requested packages to be merged?
+ # full package name(s)
+ fpkgname = m.group(1)
+ fpkgname_nocat = fpkgname.split("/")[1] # without category
+ # get base name of package
+ pkgname = split_pkgname(fpkgname)[0]
+ pkgname_nocat = pkgname.split("/")[1] # without category
+ if not fpkgname in pkgnames_dict and \
+ not fpkgname_nocat in pkgnames_dict and \
+ not pkgname in pkgnames_dict and \
+ not pkgname_nocat in pkgnames_dict:
+ continue # move on to next package
+ # append package to list of prereqs
+ prereqs.append(m.group(1))
+
+ # status?
+ status = emerge.wait()
+ if not (os.WIFEXITED(status) and os.WEXITSTATUS(status)==0):
+ os.sys.stderr.write("emerge output:\n\n")
+ os.sys.stderr.writelines(lines)
+ fatal("emerge returned bad exit status. cannot continue.")
+
+ return prereqs
+
+def merge_pkg(pkgname):
+ """This function merges a single package. The package name
+ must be given as a full package name, i.e. with a category
+ and version number."""
+ if opt_test:
+ if whrandom.choice([0,1,1]): # "fail" random packages
+ pass # fake successful merge
+ else:
+ raise MergeError("Fake merge error")
+ else:
+ print "Merging %s..." % pkgname
+ os.sys.stdout.flush()
+ # This is a bit more complicated than using Popen,
+ # but we really don't have any choice if we want
+ # to "forward" the output
+ pid = os.fork()
+ if pid==0: # child?
+ if opt_quiet:
+ try: # pipe to the great void
+ devnull = open("/dev/null","w")
+ os.dup2(devnull.fileno(),0)
+ os.dup2(devnull.fileno(),1)
+ devnull.close()
+ except os.error,e:
+ pass # not that important
+ os.execv(EMERGE,[EMERGE,"--oneshot","=%s" % pkgname])
+ fatal("execv() failed: cannot continue")
+
+ # wait for emerge to finish
+ try:
+ status = os.waitpid(pid,0)[1]
+ except os.error,e:
+ status = 0 # child finished already?
+ # abnormal exit?
+ if not (os.WIFEXITED(status) and os.WEXITSTATUS(status)==0):
+ raise MergeError("Failed to merge %s..." % pkgname)
+
+def merge_pkgs(pkgnames):
+ """Merge given packages using emerge. The input list of package names
+ contains package names *without* version numbers. This function
+ maintains state between each merge to avoid having to remerge
+ previously merged packages."""
+
+ # state file
+ state_filename = get_state_filename(pkgnames)
+
+ # start environment (note that the current environment
+ # overrides Portage variables as usual).
+ start_env = { "CFLAGS" : os.environ.get("CFLAGS",
+ portage_config["CFLAGS"]),
+ "CXXFLAGS" : os.environ.get("CXXFLAGS",
+ portage_config["CXXFLAGS"]) }
+
+ # environment to use when a package fails to merge
+ BCFLAGS = os.environ.get("BCFLAGS",None)
+ if BCFLAGS==None and portage_config.has_key("BCFLAGS"):
+ BCFLAGS=portage_config["BCFLAGS"]
+
+ BCXXFLAGS = os.environ.get("BCXXFLAGS",None)
+ if BCXXFLAGS==None and portage_config.has_key("BCXXFLAGS"):
+ BCXXFLAGS=portage_config["BCXXFLAGS"]
+
+ # backup environment to use when emerge fails
+ if BCFLAGS==None and BCXXFLAGS==None:
+ backup_env = None
+ elif BCFLAGS!=None and BCXXFLAGS!=None:
+ backup_env = { "CFLAGS": BCFLAGS, "CXXFLAGS": BCXXFLAGS }
+ else:
+ raise MergeError("Must either set both BCFLAGS and BCXXFLAGS or neither.")
+
+ # if we want to continue and a saved state exists
+ if opt_resume and os.path.exists(state_filename):
+ state = load_state(state_filename)
+ else:
+ state = State(get_prereqs(pkgnames))
+
+ # skip opt_skip packages
+ state.skip(opt_skip)
+
+ # if we're just pretending, display
+ # a list of packages that would be merged
+ if opt_pretend:
+ print "I would merge the following packages (in order):\n\n"
+ print "\n".join(state.get_todo())
+ else:
+ # merge each package in turn
+ while state.any_left():
+ # get the name of the first package left to do
+ pkgname = state.get_next()
+ tries=0 # how many times have we tried this package?
+ while (tries <= 1):
+ try:
+ # merge and restore original flags
+ try:
+ merge_pkg(pkgname)
+ finally:
+ os.environ.update(start_env)
+ # update the list of packages left to do.
+ state.done()
+ break # stop trying this package
+ except MergeError,e:
+ # Don't retry unless there is a backup_env
+ if (tries == 0) and backup_env:
+ # Let's retry with different flags
+ tries += 1
+ os.environ.update(backup_env)
+ else:
+ if opt_force:
+ state.failed() # register failure and continue
+ tries += 1
+ else:
+ raise # give up
+ # dump state each time through the loop
+ state.dump(state_filename)
+
+ # list the packages that failed to merge
+ if len(state.get_failed())>0:
+ print
+ print "The following packages failed to merge:"
+ print
+ print "\n".join(state.get_failed())
+
+#
+# Main program
+#
+try:
+ # parse options
+ opts,args = getopt.getopt(os.sys.argv[1:],"epsfqV",
+ ["emptytree","pretend","skip=","scratch",
+ "force","quiet","help","statefile=","version"])
+ for (opt,val) in opts:
+ if opt in ["-e","--emptytree"]:
+ opt_empty = 1
+ elif opt in ["-p","--pretend"]:
+ opt_pretend = 1
+ elif opt in ["--skip"]:
+ opt_skip = int(val)
+ elif opt in ["-s","--scratch"]:
+ opt_resume = 0
+ elif opt in ["-f","--force"]:
+ opt_force = 1
+ elif opt in ["-q","--quiet"]:
+ opt_quiet = 1
+ elif opt in ["-h","--help"]:
+ raise getopt.error("",None)
+ elif opt in ["-V","--version"]:
+ print "rmerge2 %s" % (VERSION,)
+ os.sys.exit(0)
+ elif opt in ["--statefile"]:
+ opt_statefile = val
+
+ # require explicit package names
+ if len(args)==0:
+ raise getopt.error("Please specify a package name.","")
+
+ # check that CFLAGS and CXXFLAGS are set
+ if not os.environ.has_key("CFLAGS"):
+ fatal("must set CFLAGS in /etc/make.conf or environment")
+ if not os.environ.has_key("CXXFLAGS"):
+ fatal("must set CXXFLAGS in /etc/make.conf or environment")
+
+ # anything other than a "pretend" run requires
+ # root privileges.
+ if (not opt_pretend) and (os.getuid() != 0):
+ fatal("root privileges required")
+
+ # merge packages
+ merge_pkgs(args)
+
+except getopt.error,e:
+ # any reason for exception or do we just want
+ # to display usage?
+ if e:
+ print "rmerge2: %s" % (e,)
+ print
+ # print the standard usage
+ print """Usage:
+
+ rmerge2 [options] package1 ...
+
+Synopsis:
+
+ Robustly merges all the given packages using emerge.
+
+Options:
+
+-h, --help
+ Prints this message.
+
+-e, --emptytree
+ Pretend that no packages (besides glibc) are installed.
+ This will cause all the packages the given package(s)
+ depend on to be remerged. Useful for rebuilding the
+ entire distribution from scratch.
+
+-p, --pretend
+ Do not merge any packages, just display which packages
+ would be merged.
+
+--skip=N
+ Skip the given number of packages. Useful if you are
+ doing a manual merge of a lot of packages and just want
+ to skip a package that is giving you trouble.
+
+-s, --scratch
+ Start a new merge of the given packages from scratch.
+ Forces rmerge2 to ignore any previous progress information.
+ Useful if you want to be absolutely certain that
+ ALL packages are remerged, even after having merged some
+ of them previously. NOTE: This only applies to the given
+ set of packages, you could (in theory at least) have
+ other package batches which could be continued independently
+ of this run.
+
+-f, --force
+ Keep running even after packages fail to merge. The
+ list of packages which failed to merge is displayed
+ at the end of a complete run. Very useful for a
+ completely automated rebuild.
+
+ NOTE: If a package fails to merge with the normal
+ CFLAGS/CXXFLAGS the rmerge2 script will retry merging
+ using BCFLAGS/BCXXFLAGS from your environment or porage
+ configuration files.
+
+-q, --quiet
+ Do not display output from emerge runs. This makes the
+ run almost completely noise-free, but errors from portage
+ will NOT be displayed. Also, watching these compiles can
+ be VERY tedious if the individual packages take a long time
+ to merge.
+
+--statefile
+ Save the program state in this file instead of the standard
+ location (which is determined from package names and versions
+ given on the command line).
+
+-V, --version
+ Show program version.
+
+Examples:
+
+ Remerge all installed packages (except glibc):
+
+ rmerge2 -f -s -e world
+
+ Remerge given packages in the appropriate order
+ (as determined by their dependency lists):
+
+ rmerge2 package1 package2 ...
+
+ Note that this does not work properly on "virtual"
+ packages (e.g. "kde") because of the way portage
+ currently works.
+
+ Remerge given package(s) and all the packages it
+ depend on:
+
+ rmerge2 -e package1 [package2 ...]
+"""
projects / gentoo.git / commitdiff