bin/emaint

   1 #!/usr/bin/python -O
   2 # vim: noet :
   3
   4 from __future__ import print_function
   5
   6 import re
   7 import signal
   8 import sys
   9 import textwrap
  10 import time
  11 from optparse import OptionParser, OptionValueError
  12
  13 try:
  14         import portage
  15 except ImportError:
  16         from os import path as osp
  17         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  18         import portage
  19
  20 from portage import os
  21
  22 class WorldHandler(object):
  23
  24         short_desc = "Fix problems in the world file"
  25
  26         def name():
  27                 return "world"
  28         name = staticmethod(name)
  29
  30         def __init__(self):
  31                 self.invalid = []
  32                 self.not_installed = []
  33                 self.invalid_category = []
  34                 self.okay = []
  35                 from portage.sets import load_default_config
  36                 setconfig = load_default_config(portage.settings,
  37                         portage.db[portage.settings["ROOT"]])
  38                 self._sets = setconfig.getSets()
  39
  40         def _check_world(self, onProgress):
  41                 categories = set(portage.settings.categories)
  42                 myroot = portage.settings["ROOT"]
  43                 self.world_file = os.path.join(myroot, portage.const.WORLD_FILE)
  44                 self.found = os.access(self.world_file, os.R_OK)
  45                 vardb = portage.db[myroot]["vartree"].dbapi
  46
  47                 from portage.sets import SETPREFIX
  48                 sets = self._sets
  49                 world_atoms = list(sets["world"])
  50                 maxval = len(world_atoms)
  51                 if onProgress:
  52                         onProgress(maxval, 0)
  53                 for i, atom in enumerate(world_atoms):
  54                         if not portage.isvalidatom(atom):
  55                                 if atom.startswith(SETPREFIX):
  56                                         s = atom[len(SETPREFIX):]
  57                                         if s in sets:
  58                                                 self.okay.append(atom)
  59                                         else:
  60                                                 self.not_installed.append(atom)
  61                                 else:
  62                                         self.invalid.append(atom)
  63                                 if onProgress:
  64                                         onProgress(maxval, i+1)
  65                                 continue
  66                         cp = portage.dep_getkey(atom)
  67                         okay = True
  68                         if not vardb.match(atom):
  69                                 self.not_installed.append(atom)
  70                                 okay = False
  71                         if portage.catsplit(cp)[0] not in categories:
  72                                 self.invalid_category.append(atom)
  73                                 okay = False
  74                         if okay:
  75                                 self.okay.append(atom)
  76                         if onProgress:
  77                                 onProgress(maxval, i+1)
  78
  79         def check(self, onProgress=None):
  80                 self._check_world(onProgress)
  81                 errors = []
  82                 if self.found:
  83                         errors += map(lambda x: "'%s' is not a valid atom" % x, self.invalid)
  84                         errors += map(lambda x: "'%s' is not installed" % x, self.not_installed)
  85                         errors += map(lambda x: "'%s' has a category that is not listed in /etc/portage/categories" % x, self.invalid_category)
  86                 else:
  87                         errors.append(self.world_file + " could not be opened for reading")
  88                 return errors
  89
  90         def fix(self, onProgress=None):
  91                 world_set = self._sets["world"]
  92                 world_set.lock()
  93                 try:
  94                         world_set.load() # maybe it's changed on disk
  95                         before = set(world_set)
  96                         self._check_world(onProgress)
  97                         after = set(self.okay)
  98                         errors = []
  99                         if before != after:
 100                                 try:
 101                                         world_set.replace(self.okay)
 102                                 except portage.exception.PortageException:
 103                                         errors.append("%s could not be opened for writing" % \
 104                                                 self.world_file)
 105                         return errors
 106                 finally:
 107                         world_set.unlock()
 108
 109 class BinhostHandler(object):
 110
 111         short_desc = "Generate a metadata index for binary packages"
 112
 113         def name():
 114                 return "binhost"
 115         name = staticmethod(name)
 116
 117         def __init__(self):
 118                 myroot = portage.settings["ROOT"]
 119                 self._bintree = portage.db[myroot]["bintree"]
 120                 self._bintree.populate()
 121                 self._pkgindex_file = self._bintree._pkgindex_file
 122                 self._pkgindex = self._bintree._load_pkgindex()
 123
 124         def check(self, onProgress=None):
 125                 missing = []
 126                 cpv_all = self._bintree.dbapi.cpv_all()
 127                 cpv_all.sort()
 128                 maxval = len(cpv_all)
 129                 if onProgress:
 130                         onProgress(maxval, 0)
 131                 pkgindex = self._pkgindex
 132                 missing = []
 133                 metadata = {}
 134                 for d in pkgindex.packages:
 135                         metadata[d["CPV"]] = d
 136                 for i, cpv in enumerate(cpv_all):
 137                         d = metadata.get(cpv)
 138                         if not d or "MD5" not in d:
 139                                 missing.append(cpv)
 140                         if onProgress:
 141                                 onProgress(maxval, i+1)
 142                 errors = ["'%s' is not in Packages" % cpv for cpv in missing]
 143                 stale = set(metadata).difference(cpv_all)
 144                 for cpv in stale:
 145                         errors.append("'%s' is not in the repository" % cpv)
 146                 return errors
 147
 148         def fix(self, onProgress=None):
 149                 bintree = self._bintree
 150                 cpv_all = self._bintree.dbapi.cpv_all()
 151                 cpv_all.sort()
 152                 missing = []
 153                 maxval = 0
 154                 if onProgress:
 155                         onProgress(maxval, 0)
 156                 pkgindex = self._pkgindex
 157                 missing = []
 158                 metadata = {}
 159                 for d in pkgindex.packages:
 160                         metadata[d["CPV"]] = d
 161
 162                 for i, cpv in enumerate(cpv_all):
 163                         d = metadata.get(cpv)
 164                         if not d or "MD5" not in d:
 165                                 missing.append(cpv)
 166
 167                 stale = set(metadata).difference(cpv_all)
 168                 if missing or stale:
 169                         from portage import locks
 170                         pkgindex_lock = locks.lockfile(
 171                                 self._pkgindex_file, wantnewlockfile=1)
 172                         try:
 173                                 # Repopulate with lock held.
 174                                 bintree._populate()
 175                                 cpv_all = self._bintree.dbapi.cpv_all()
 176                                 cpv_all.sort()
 177
 178                                 pkgindex = bintree._load_pkgindex()
 179                                 self._pkgindex = pkgindex
 180
 181                                 metadata = {}
 182                                 for d in pkgindex.packages:
 183                                         metadata[d["CPV"]] = d
 184
 185                                 # Recount missing packages, with lock held.
 186                                 del missing[:]
 187                                 for i, cpv in enumerate(cpv_all):
 188                                         d = metadata.get(cpv)
 189                                         if not d or "MD5" not in d:
 190                                                 missing.append(cpv)
 191
 192                                 maxval = len(missing)
 193                                 for i, cpv in enumerate(missing):
 194                                         try:
 195                                                 metadata[cpv] = bintree._pkgindex_entry(cpv)
 196                                         except portage.exception.InvalidDependString:
 197                                                 writemsg("!!! Invalid binary package: '%s'\n" % \
 198                                                         bintree.getname(cpv), noiselevel=-1)
 199
 200                                         if onProgress:
 201                                                 onProgress(maxval, i+1)
 202
 203                                 for cpv in set(metadata).difference(
 204                                         self._bintree.dbapi.cpv_all()):
 205                                         del metadata[cpv]
 206
 207                                 # We've updated the pkgindex, so set it to
 208                                 # repopulate when necessary.
 209                                 bintree.populated = False
 210
 211                                 del pkgindex.packages[:]
 212                                 pkgindex.packages.extend(metadata.values())
 213                                 from portage.util import atomic_ofstream
 214                                 f = atomic_ofstream(self._pkgindex_file)
 215                                 try:
 216                                         self._pkgindex.write(f)
 217                                 finally:
 218                                         f.close()
 219                         finally:
 220                                 locks.unlockfile(pkgindex_lock)
 221
 222                 if onProgress:
 223                         if maxval == 0:
 224                                 maxval = 1
 225                         onProgress(maxval, maxval)
 226                 return None
 227
 228 class MoveHandler(object):
 229
 230         def __init__(self, tree):
 231                 self._tree = tree
 232                 self._portdir = tree.settings["PORTDIR"]
 233                 self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"]
 234
 235         def _grab_global_updates(self, portdir):
 236                 from portage.update import grab_updates, parse_updates
 237                 updpath = os.path.join(portdir, "profiles", "updates")
 238                 try:
 239                         rawupdates = grab_updates(updpath)
 240                 except portage.exception.DirectoryNotFound:
 241                         rawupdates = []
 242                 upd_commands = []
 243                 errors = []
 244                 for mykey, mystat, mycontent in rawupdates:
 245                         commands, errors = parse_updates(mycontent)
 246                         upd_commands.extend(commands)
 247                         errors.extend(errors)
 248                 return upd_commands, errors
 249
 250         def check(self, onProgress=None):
 251                 updates, errors = self._grab_global_updates(self._portdir)
 252                 # Matching packages and moving them is relatively fast, so the
 253                 # progress bar is updated in indeterminate mode.
 254                 match = self._tree.dbapi.match
 255                 aux_get = self._tree.dbapi.aux_get
 256                 if onProgress:
 257                         onProgress(0, 0)
 258                 for i, update_cmd in enumerate(updates):
 259                         if update_cmd[0] == "move":
 260                                 origcp, newcp = update_cmd[1:]
 261                                 for cpv in match(origcp):
 262                                         errors.append("'%s' moved to '%s'" % (cpv, newcp))
 263                         elif update_cmd[0] == "slotmove":
 264                                 pkg, origslot, newslot = update_cmd[1:]
 265                                 for cpv in match(pkg):
 266                                         slot = aux_get(cpv, ["SLOT"])[0]
 267                                         if slot == origslot:
 268                                                 errors.append("'%s' slot moved from '%s' to '%s'" % \
 269                                                         (cpv, origslot, newslot))
 270                         if onProgress:
 271                                 onProgress(0, 0)
 272
 273                 # Searching for updates in all the metadata is relatively slow, so this
 274                 # is where the progress bar comes out of indeterminate mode.
 275                 cpv_all = self._tree.dbapi.cpv_all()
 276                 cpv_all.sort()
 277                 maxval = len(cpv_all)
 278                 aux_update = self._tree.dbapi.aux_update
 279                 update_keys = self._update_keys
 280                 from portage.update import update_dbentries
 281                 if onProgress:
 282                         onProgress(maxval, 0)
 283                 for i, cpv in enumerate(cpv_all):
 284                         metadata = dict(zip(update_keys, aux_get(cpv, update_keys)))
 285                         metadata_updates = update_dbentries(updates, metadata)
 286                         if metadata_updates:
 287                                 errors.append("'%s' has outdated metadata" % cpv)
 288                         if onProgress:
 289                                 onProgress(maxval, i+1)
 290                 return errors
 291
 292         def fix(self, onProgress=None):
 293                 updates, errors = self._grab_global_updates(self._portdir)
 294                 # Matching packages and moving them is relatively fast, so the
 295                 # progress bar is updated in indeterminate mode.
 296                 move = self._tree.dbapi.move_ent
 297                 slotmove = self._tree.dbapi.move_slot_ent
 298                 if onProgress:
 299                         onProgress(0, 0)
 300                 for i, update_cmd in enumerate(updates):
 301                         if update_cmd[0] == "move":
 302                                 move(update_cmd)
 303                         elif update_cmd[0] == "slotmove":
 304                                 slotmove(update_cmd)
 305                         if onProgress:
 306                                 onProgress(0, 0)
 307
 308                 # Searching for updates in all the metadata is relatively slow, so this
 309                 # is where the progress bar comes out of indeterminate mode.
 310                 self._tree.dbapi.update_ents(updates, onProgress=onProgress)
 311                 return errors
 312
 313 class MoveInstalled(MoveHandler):
 314
 315         short_desc = "Perform package move updates for installed packages"
 316
 317         def name():
 318                 return "moveinst"
 319         name = staticmethod(name)
 320         def __init__(self):
 321                 myroot = portage.settings["ROOT"]
 322                 MoveHandler.__init__(self, portage.db[myroot]["vartree"])
 323
 324 class MoveBinary(MoveHandler):
 325
 326         short_desc = "Perform package move updates for binary packages"
 327
 328         def name():
 329                 return "movebin"
 330         name = staticmethod(name)
 331         def __init__(self):
 332                 myroot = portage.settings["ROOT"]
 333                 MoveHandler.__init__(self, portage.db[myroot]["bintree"])
 334
 335 class VdbKeyHandler(object):
 336         def name():
 337                 return "vdbkeys"
 338         name = staticmethod(name)
 339
 340         def __init__(self):
 341                 self.list = portage.db["/"]["vartree"].dbapi.cpv_all()
 342                 self.missing = []
 343                 self.keys = ["HOMEPAGE", "SRC_URI", "KEYWORDS", "DESCRIPTION"]
 344
 345                 for p in self.list:
 346                         mydir = os.path.join(os.sep, portage.settings["ROOT"], portage.const.VDB_PATH, p)+os.sep
 347                         ismissing = True
 348                         for k in self.keys:
 349                                 if os.path.exists(mydir+k):
 350                                         ismissing = False
 351                                         break
 352                         if ismissing:
 353                                 self.missing.append(p)
 354
 355         def check(self):
 356                 return ["%s has missing keys" % x for x in self.missing]
 357
 358         def fix(self):
 359
 360                 errors = []
 361
 362                 for p in self.missing:
 363                         mydir = os.path.join(os.sep, portage.settings["ROOT"], portage.const.VDB_PATH, p)+os.sep
 364                         if not os.access(mydir+"environment.bz2", os.R_OK):
 365                                 errors.append("Can't access %s" % (mydir+"environment.bz2"))
 366                         elif not os.access(mydir, os.W_OK):
 367                                 errors.append("Can't create files in %s" % mydir)
 368                         else:
 369                                 env = os.popen("bzip2 -dcq "+mydir+"environment.bz2", "r")
 370                                 envlines = env.read().split("\n")
 371                                 env.close()
 372                                 for k in self.keys:
 373                                         s = [l for l in envlines if l.startswith(k+"=")]
 374                                         if len(s) > 1:
 375                                                 errors.append("multiple matches for %s found in %senvironment.bz2" % (k, mydir))
 376                                         elif len(s) == 0:
 377                                                 s = ""
 378                                         else:
 379                                                 s = s[0].split("=",1)[1]
 380                                                 s = s.lstrip("$").strip("\'\"")
 381                                                 s = re.sub("(\\\\[nrt])+", " ", s)
 382                                                 s = " ".join(s.split()).strip()
 383                                                 if s != "":
 384                                                         try:
 385                                                                 keyfile = open(mydir+os.sep+k, "w")
 386                                                                 keyfile.write(s+"\n")
 387                                                                 keyfile.close()
 388                                                         except (IOError, OSError) as e:
 389                                                                 errors.append("Could not write %s, reason was: %s" % (mydir+k, e))
 390
 391                 return errors
 392
 393 class ProgressHandler(object):
 394         def __init__(self):
 395                 self.curval = 0
 396                 self.maxval = 0
 397                 self.last_update = 0
 398                 self.min_display_latency = 0.2
 399
 400         def onProgress(self, maxval, curval):
 401                 self.maxval = maxval
 402                 self.curval = curval
 403                 cur_time = time.time()
 404                 if cur_time - self.last_update >= self.min_display_latency:
 405                         self.last_update = cur_time
 406                         self.display()
 407
 408         def display(self):
 409                 raise NotImplementedError(self)
 410
 411 class CleanResume(object):
 412
 413         short_desc = "Discard emerge --resume merge lists"
 414
 415         def name():
 416                 return "cleanresume"
 417         name = staticmethod(name)
 418
 419         def check(self, onProgress=None):
 420                 messages = []
 421                 mtimedb = portage.mtimedb
 422                 resume_keys = ("resume", "resume_backup")
 423                 maxval = len(resume_keys)
 424                 if onProgress:
 425                         onProgress(maxval, 0)
 426                 for i, k in enumerate(resume_keys):
 427                         try:
 428                                 d = mtimedb.get(k)
 429                                 if d is None:
 430                                         continue
 431                                 if not isinstance(d, dict):
 432                                         messages.append("unrecognized resume list: '%s'" % k)
 433                                         continue
 434                                 mergelist = d.get("mergelist")
 435                                 if mergelist is None or not hasattr(mergelist, "__len__"):
 436                                         messages.append("unrecognized resume list: '%s'" % k)
 437                                         continue
 438                                 messages.append("resume list '%s' contains %d packages" % \
 439                                         (k, len(mergelist)))
 440                         finally:
 441                                 if onProgress:
 442                                         onProgress(maxval, i+1)
 443                 return messages
 444
 445         def fix(self, onProgress=None):
 446                 delete_count = 0
 447                 mtimedb = portage.mtimedb
 448                 resume_keys = ("resume", "resume_backup")
 449                 maxval = len(resume_keys)
 450                 if onProgress:
 451                         onProgress(maxval, 0)
 452                 for i, k in enumerate(resume_keys):
 453                         try:
 454                                 if mtimedb.pop(k, None) is not None:
 455                                         delete_count += 1
 456                         finally:
 457                                 if onProgress:
 458                                         onProgress(maxval, i+1)
 459                 if delete_count:
 460                         mtimedb.commit()
 461
 462 def emaint_main(myargv):
 463
 464         # Similar to emerge, emaint needs a default umask so that created
 465         # files (such as the world file) have sane permissions.
 466         os.umask(0o22)
 467
 468         # TODO: Create a system that allows external modules to be added without
 469         #       the need for hard coding.
 470         modules = {
 471                 "world" : WorldHandler,
 472                 "binhost":BinhostHandler,
 473                 "moveinst":MoveInstalled,
 474                 "movebin":MoveBinary,
 475                 "cleanresume":CleanResume
 476         }
 477
 478         module_names = list(modules.keys())
 479         module_names.sort()
 480         module_names.insert(0, "all")
 481
 482         def exclusive(option, *args, **kw):
 483                 var = kw.get("var", None)
 484                 if var is None:
 485                         raise ValueError("var not specified to exclusive()")
 486                 if getattr(parser, var, ""):
 487                         raise OptionValueError("%s and %s are exclusive options" % (getattr(parser, var), option))
 488                 setattr(parser, var, str(option))
 489
 490
 491         usage = "usage: emaint [options] COMMAND"
 492
 493         desc = "The emaint program provides an interface to system health " + \
 494                 "checks and maintenance. See the emaint(1) man page " + \
 495                 "for additional information about the following commands:"
 496
 497         usage += "\n\n"
 498         for line in textwrap.wrap(desc, 65):
 499                 usage += "%s\n" % line
 500         usage += "\n"
 501         usage += "  %s" % "all".ljust(15) + \
 502                 "Perform all supported commands\n"
 503         for m in module_names[1:]:
 504                 usage += "  %s%s\n" % (m.ljust(15), modules[m].short_desc)
 505
 506         parser = OptionParser(usage=usage, version=portage.VERSION)
 507         parser.add_option("-c", "--check", help="check for problems",
 508                 action="callback", callback=exclusive, callback_kwargs={"var":"action"})
 509         parser.add_option("-f", "--fix", help="attempt to fix problems",
 510                 action="callback", callback=exclusive, callback_kwargs={"var":"action"})
 511         parser.action = None
 512
 513
 514         (options, args) = parser.parse_args(args=myargv)
 515         if len(args) != 1:
 516                 parser.error("Incorrect number of arguments")
 517         if args[0] not in module_names:
 518                 parser.error("%s target is not a known target" % args[0])
 519
 520         if parser.action:
 521                 action = parser.action
 522         else:
 523                 print("Defaulting to --check")
 524                 action = "-c/--check"
 525
 526         if args[0] == "all":
 527                 tasks = modules.values()
 528         else:
 529                 tasks = [modules[args[0]]]
 530
 531
 532         if action == "-c/--check":
 533                 status = "Checking %s for problems"
 534                 func = "check"
 535         else:
 536                 status = "Attempting to fix %s"
 537                 func = "fix"
 538
 539         isatty = sys.stdout.isatty()
 540         for task in tasks:
 541                 print(status % task.name())
 542                 inst = task()
 543                 onProgress = None
 544                 if isatty:
 545                         progressBar = portage.output.TermProgressBar()
 546                         progressHandler = ProgressHandler()
 547                         onProgress = progressHandler.onProgress
 548                         def display():
 549                                 progressBar.set(progressHandler.curval, progressHandler.maxval)
 550                         progressHandler.display = display
 551                         def sigwinch_handler(signum, frame):
 552                                 lines, progressBar.term_columns = \
 553                                         portage.output.get_term_size()
 554                         signal.signal(signal.SIGWINCH, sigwinch_handler)
 555                 result = getattr(inst, func)(onProgress=onProgress)
 556                 if isatty:
 557                         # make sure the final progress is displayed
 558                         progressHandler.display()
 559                         print()
 560                         signal.signal(signal.SIGWINCH, signal.SIG_DFL)
 561                 if result:
 562                         print()
 563                         print("\n".join(result))
 564                         print("\n")
 565
 566         print("Finished")
 567
 568 if __name__ == "__main__":
 569         emaint_main(sys.argv[1:])