bin/quickpkg

   1 #!/usr/bin/python -bb
   2 # Copyright 1999-2014 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4
   5 from __future__ import print_function
   6
   7 import errno
   8 import math
   9 import signal
  10 import sys
  11 import tarfile
  12
  13 from os import path as osp
  14 pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")
  15 sys.path.insert(0, pym_path)
  16 import portage
  17 portage._internal_caller = True
  18 from portage import os
  19 from portage import xpak
  20 from portage.dbapi.dep_expand import dep_expand
  21 from portage.dep import Atom, use_reduce
  22 from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData,
  23         InvalidDependString, PackageSetNotFound, PermissionDenied)
  24 from portage.util import ConfigProtect, ensure_dirs, shlex_split
  25 from portage.dbapi.vartree import dblink, tar_contents
  26 from portage.checksum import perform_md5
  27 from portage._sets import load_default_config, SETPREFIX
  28 from portage.util._argparse import ArgumentParser
  29
  30 def quickpkg_atom(options, infos, arg, eout):
  31         settings = portage.settings
  32         root = portage.settings['ROOT']
  33         eroot = portage.settings['EROOT']
  34         trees = portage.db[eroot]
  35         vartree = trees["vartree"]
  36         vardb = vartree.dbapi
  37         bintree = trees["bintree"]
  38
  39         include_config = options.include_config == "y"
  40         include_unmodified_config = options.include_unmodified_config == "y"
  41         fix_metadata_keys = ["PF", "CATEGORY"]
  42
  43         try:
  44                 atom = dep_expand(arg, mydb=vardb, settings=vartree.settings)
  45         except AmbiguousPackageName as e:
  46                 # Multiple matches thrown from cpv_expand
  47                 eout.eerror("Please use a more specific atom: %s" % \
  48                         " ".join(e.args[0]))
  49                 del e
  50                 infos["missing"].append(arg)
  51                 return
  52         except (InvalidAtom, InvalidData):
  53                 eout.eerror("Invalid atom: %s" % (arg,))
  54                 infos["missing"].append(arg)
  55                 return
  56         if atom[:1] == '=' and arg[:1] != '=':
  57                 # dep_expand() allows missing '=' but it's really invalid
  58                 eout.eerror("Invalid atom: %s" % (arg,))
  59                 infos["missing"].append(arg)
  60                 return
  61
  62         matches = vardb.match(atom)
  63         pkgs_for_arg = 0
  64         for cpv in matches:
  65                 excluded_config_files = []
  66                 bintree.prevent_collision(cpv)
  67                 dblnk = vardb._dblink(cpv)
  68                 have_lock = False
  69
  70                 if "__PORTAGE_INHERIT_VARDB_LOCK" not in settings:
  71                         try:
  72                                 dblnk.lockdb()
  73                                 have_lock = True
  74                         except PermissionDenied:
  75                                 pass
  76
  77                 try:
  78                         if not dblnk.exists():
  79                                 # unmerged by a concurrent process
  80                                 continue
  81                         iuse, use, restrict = vardb.aux_get(cpv,
  82                                 ["IUSE","USE","RESTRICT"])
  83                         iuse = [ x.lstrip("+-") for x in iuse.split() ]
  84                         use = use.split()
  85                         try:
  86                                 restrict = use_reduce(restrict, uselist=use, flat=True)
  87                         except InvalidDependString as e:
  88                                 eout.eerror("Invalid RESTRICT metadata " + \
  89                                         "for '%s': %s; skipping" % (cpv, str(e)))
  90                                 del e
  91                                 continue
  92                         if "bindist" in iuse and "bindist" not in use:
  93                                 eout.ewarn("%s: package was emerged with USE=-bindist!" % cpv)
  94                                 eout.ewarn("%s: it might not be legal to redistribute this." % cpv)
  95                         elif "bindist" in restrict:
  96                                 eout.ewarn("%s: package has RESTRICT=bindist!" % cpv)
  97                                 eout.ewarn("%s: it might not be legal to redistribute this." % cpv)
  98                         eout.ebegin("Building package for %s" % cpv)
  99                         pkgs_for_arg += 1
 100                         contents = dblnk.getcontents()
 101                         protect = None
 102                         if not include_config:
 103                                 confprot = ConfigProtect(eroot,
 104                                         shlex_split(settings.get("CONFIG_PROTECT", "")),
 105                                         shlex_split(settings.get("CONFIG_PROTECT_MASK", "")))
 106                                 def protect(filename):
 107                                         if not confprot.isprotected(filename):
 108                                                 return False
 109                                         if include_unmodified_config:
 110                                                 file_data = contents[filename]
 111                                                 if file_data[0] == "obj":
 112                                                         orig_md5 = file_data[2].lower()
 113                                                         cur_md5 = perform_md5(filename, calc_prelink=1)
 114                                                         if orig_md5 == cur_md5:
 115                                                                 return False
 116                                         excluded_config_files.append(filename)
 117                                         return True
 118                         existing_metadata = dict(zip(fix_metadata_keys,
 119                                 vardb.aux_get(cpv, fix_metadata_keys)))
 120                         category, pf = portage.catsplit(cpv)
 121                         required_metadata = {}
 122                         required_metadata["CATEGORY"] = category
 123                         required_metadata["PF"] = pf
 124                         update_metadata = {}
 125                         for k, v in required_metadata.items():
 126                                 if v != existing_metadata[k]:
 127                                         update_metadata[k] = v
 128                         if update_metadata:
 129                                 vardb.aux_update(cpv, update_metadata)
 130                         xpdata = xpak.xpak(dblnk.dbdir)
 131                         binpkg_tmpfile = os.path.join(bintree.pkgdir,
 132                                 cpv + ".tbz2." + str(os.getpid()))
 133                         ensure_dirs(os.path.dirname(binpkg_tmpfile))
 134                         tar = tarfile.open(binpkg_tmpfile, "w:bz2")
 135                         tar_contents(contents, root, tar, protect=protect)
 136                         tar.close()
 137                         xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata)
 138                 finally:
 139                         if have_lock:
 140                                 dblnk.unlockdb()
 141                 bintree.inject(cpv, filename=binpkg_tmpfile)
 142                 binpkg_path = bintree.getname(cpv)
 143                 try:
 144                         s = os.stat(binpkg_path)
 145                 except OSError as e:
 146                         # Sanity check, shouldn't happen normally.
 147                         eout.eend(1)
 148                         eout.eerror(str(e))
 149                         del e
 150                         eout.eerror("Failed to create package: '%s'" % binpkg_path)
 151                 else:
 152                         eout.eend(0)
 153                         infos["successes"].append((cpv, s.st_size))
 154                         infos["config_files_excluded"] += len(excluded_config_files)
 155                         for filename in excluded_config_files:
 156                                 eout.ewarn("Excluded config: '%s'" % filename)
 157         if not pkgs_for_arg:
 158                 eout.eerror("Could not find anything " + \
 159                         "to match '%s'; skipping" % arg)
 160                 infos["missing"].append(arg)
 161
 162 def quickpkg_set(options, infos, arg, eout):
 163         eroot = portage.settings['EROOT']
 164         trees = portage.db[eroot]
 165         vartree = trees["vartree"]
 166
 167         settings = vartree.settings
 168         settings._init_dirs()
 169         setconfig = load_default_config(settings, trees)
 170         sets = setconfig.getSets()
 171
 172         set = arg[1:]
 173         if not set in sets:
 174                 eout.eerror("Package set not found: '%s'; skipping" % (arg,))
 175                 infos["missing"].append(arg)
 176                 return
 177
 178         try:
 179                 atoms = setconfig.getSetAtoms(set)
 180         except PackageSetNotFound as e:
 181                 eout.eerror("Failed to process package set '%s' because " % set +
 182                         "it contains the non-existent package set '%s'; skipping" % e)
 183                 infos["missing"].append(arg)
 184                 return
 185
 186         for atom in atoms:
 187                 quickpkg_atom(options, infos, atom, eout)
 188
 189
 190 def quickpkg_extended_atom(options, infos, atom, eout):
 191         eroot = portage.settings['EROOT']
 192         trees = portage.db[eroot]
 193         vartree = trees["vartree"]
 194         vardb = vartree.dbapi
 195
 196         require_metadata = atom.slot or atom.repo
 197         atoms = []
 198         for cpv in vardb.cpv_all():
 199                 cpv_atom = Atom("=%s" % cpv)
 200
 201                 if atom == "*/*":
 202                         atoms.append(cpv_atom)
 203                         continue
 204
 205                 if not portage.match_from_list(atom, [cpv]):
 206                         continue
 207
 208                 if require_metadata:
 209                         try:
 210                                 cpv = vardb._pkg_str(cpv, atom.repo)
 211                         except (KeyError, InvalidData):
 212                                 continue
 213                         if not portage.match_from_list(atom, [cpv]):
 214                                 continue
 215
 216                 atoms.append(cpv_atom)
 217
 218         for atom in atoms:
 219                 quickpkg_atom(options, infos, atom, eout)
 220
 221
 222 def quickpkg_main(options, args, eout):
 223         eroot = portage.settings['EROOT']
 224         trees = portage.db[eroot]
 225         bintree = trees["bintree"]
 226
 227         try:
 228                 ensure_dirs(bintree.pkgdir)
 229         except portage.exception.PortageException:
 230                 pass
 231         if not os.access(bintree.pkgdir, os.W_OK):
 232                 eout.eerror("No write access to '%s'" % bintree.pkgdir)
 233                 return errno.EACCES
 234
 235         infos = {}
 236         infos["successes"] = []
 237         infos["missing"] = []
 238         infos["config_files_excluded"] = 0
 239         for arg in args:
 240                 if arg[0] == SETPREFIX:
 241                         quickpkg_set(options, infos, arg, eout)
 242                         continue
 243                 try:
 244                         atom = Atom(arg, allow_wildcard=True, allow_repo=True)
 245                 except (InvalidAtom, InvalidData):
 246                         # maybe it's valid but missing category (requires dep_expand)
 247                         quickpkg_atom(options, infos, arg, eout)
 248                 else:
 249                         if atom.extended_syntax:
 250                                 quickpkg_extended_atom(options, infos, atom, eout)
 251                         else:
 252                                 quickpkg_atom(options, infos, atom, eout)
 253
 254         if not infos["successes"]:
 255                 eout.eerror("No packages found")
 256                 return 1
 257         print()
 258         eout.einfo("Packages now in '%s':" % bintree.pkgdir)
 259         units = {10:'K', 20:'M', 30:'G', 40:'T',
 260                 50:'P', 60:'E', 70:'Z', 80:'Y'}
 261         for cpv, size in infos["successes"]:
 262                 if not size:
 263                         # avoid OverflowError in math.log()
 264                         size_str = "0"
 265                 else:
 266                         power_of_2 = math.log(size, 2)
 267                         power_of_2 = 10*int(power_of_2/10)
 268                         unit = units.get(power_of_2)
 269                         if unit:
 270                                 size = float(size)/(2**power_of_2)
 271                                 size_str = "%.1f" % size
 272                                 if len(size_str) > 4:
 273                                         # emulate `du -h`, don't show too many sig figs
 274                                         size_str = str(int(size))
 275                                 size_str += unit
 276                         else:
 277                                 size_str = str(size)
 278                 eout.einfo("%s: %s" % (cpv, size_str))
 279         if infos["config_files_excluded"]:
 280                 print()
 281                 eout.ewarn("Excluded config files: %d" % infos["config_files_excluded"])
 282                 eout.ewarn("See --help if you would like to include config files.")
 283         if infos["missing"]:
 284                 print()
 285                 eout.ewarn("The following packages could not be found:")
 286                 eout.ewarn(" ".join(infos["missing"]))
 287                 return 2
 288         return os.EX_OK
 289
 290 if __name__ == "__main__":
 291         usage = "quickpkg [options] <list of package atoms or package sets>"
 292         parser = ArgumentParser(usage=usage)
 293         parser.add_argument("--umask",
 294                 default="0077",
 295                 help="umask used during package creation (default is 0077)")
 296         parser.add_argument("--ignore-default-opts",
 297                 action="store_true",
 298                 help="do not use the QUICKPKG_DEFAULT_OPTS environment variable")
 299         parser.add_argument("--include-config",
 300                 choices=["y","n"],
 301                 default="n",
 302                 metavar="<y|n>",
 303                 help="include all files protected by CONFIG_PROTECT (as a security precaution, default is 'n')")
 304         parser.add_argument("--include-unmodified-config",
 305                 choices=["y","n"],
 306                 default="n",
 307                 metavar="<y|n>",
 308                 help="include files protected by CONFIG_PROTECT that have not been modified since installation (as a security precaution, default is 'n')")
 309         options, args = parser.parse_known_args(sys.argv[1:])
 310         if not options.ignore_default_opts:
 311                 default_opts = shlex_split(
 312                         portage.settings.get("QUICKPKG_DEFAULT_OPTS", ""))
 313                 options, args = parser.parse_known_args(default_opts + sys.argv[1:])
 314         if not args:
 315                 parser.error("no packages atoms given")
 316         try:
 317                 umask = int(options.umask, 8)
 318         except ValueError:
 319                 parser.error("invalid umask: %s" % options.umask)
 320         # We need to ensure a sane umask for the packages that will be created.
 321         old_umask = os.umask(umask)
 322         eout = portage.output.EOutput()
 323         def sigwinch_handler(signum, frame):
 324                 lines, eout.term_columns =  portage.output.get_term_size()
 325         signal.signal(signal.SIGWINCH, sigwinch_handler)
 326         try:
 327                 retval = quickpkg_main(options, args, eout)
 328         finally:
 329                 os.umask(old_umask)
 330                 signal.signal(signal.SIGWINCH, signal.SIG_DFL)
 331         sys.exit(retval)