bin/emaint

   1 #!/usr/bin/python -O
   2 # vim: noet :
   3
   4 from __future__ import print_function
   5
   6 import errno
   7 import re
   8 import signal
   9 import stat
  10 import sys
  11 import textwrap
  12 import time
  13 from optparse import OptionParser, OptionValueError
  14
  15 try:
  16         import portage
  17 except ImportError:
  18         from os import path as osp
  19         sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym"))
  20         import portage
  21
  22 from portage import os
  23 from portage.util import writemsg
  24
  25 if sys.hexversion >= 0x3000000:
  26         long = int
  27
  28 class WorldHandler(object):
  29
  30         short_desc = "Fix problems in the world file"
  31
  32         def name():
  33                 return "world"
  34         name = staticmethod(name)
  35
  36         def __init__(self):
  37                 self.invalid = []
  38                 self.not_installed = []
  39                 self.invalid_category = []
  40                 self.okay = []
  41                 from portage._sets import load_default_config
  42                 setconfig = load_default_config(portage.settings,
  43                         portage.db[portage.settings['EROOT']])
  44                 self._sets = setconfig.getSets()
  45
  46         def _check_world(self, onProgress):
  47                 categories = set(portage.settings.categories)
  48                 eroot = portage.settings['EROOT']
  49                 self.world_file = os.path.join(eroot, portage.const.WORLD_FILE)
  50                 self.found = os.access(self.world_file, os.R_OK)
  51                 vardb = portage.db[eroot]["vartree"].dbapi
  52
  53                 from portage._sets import SETPREFIX
  54                 sets = self._sets
  55                 world_atoms = list(sets["selected"])
  56                 maxval = len(world_atoms)
  57                 if onProgress:
  58                         onProgress(maxval, 0)
  59                 for i, atom in enumerate(world_atoms):
  60                         if not isinstance(atom, portage.dep.Atom):
  61                                 if atom.startswith(SETPREFIX):
  62                                         s = atom[len(SETPREFIX):]
  63                                         if s in sets:
  64                                                 self.okay.append(atom)
  65                                         else:
  66                                                 self.not_installed.append(atom)
  67                                 else:
  68                                         self.invalid.append(atom)
  69                                 if onProgress:
  70                                         onProgress(maxval, i+1)
  71                                 continue
  72                         okay = True
  73                         if not vardb.match(atom):
  74                                 self.not_installed.append(atom)
  75                                 okay = False
  76                         if portage.catsplit(atom.cp)[0] not in categories:
  77                                 self.invalid_category.append(atom)
  78                                 okay = False
  79                         if okay:
  80                                 self.okay.append(atom)
  81                         if onProgress:
  82                                 onProgress(maxval, i+1)
  83
  84         def check(self, onProgress=None):
  85                 self._check_world(onProgress)
  86                 errors = []
  87                 if self.found:
  88                         errors += ["'%s' is not a valid atom" % x for x in self.invalid]
  89                         errors += ["'%s' is not installed" % x for x in self.not_installed]
  90                         errors += ["'%s' has a category that is not listed in /etc/portage/categories" % x for x in self.invalid_category]
  91                 else:
  92                         errors.append(self.world_file + " could not be opened for reading")
  93                 return errors
  94
  95         def fix(self, onProgress=None):
  96                 world_set = self._sets["selected"]
  97                 world_set.lock()
  98                 try:
  99                         world_set.load() # maybe it's changed on disk
 100                         before = set(world_set)
 101                         self._check_world(onProgress)
 102                         after = set(self.okay)
 103                         errors = []
 104                         if before != after:
 105                                 try:
 106                                         world_set.replace(self.okay)
 107                                 except portage.exception.PortageException:
 108                                         errors.append("%s could not be opened for writing" % \
 109                                                 self.world_file)
 110                         return errors
 111                 finally:
 112                         world_set.unlock()
 113
 114 class BinhostHandler(object):
 115
 116         short_desc = "Generate a metadata index for binary packages"
 117
 118         def name():
 119                 return "binhost"
 120         name = staticmethod(name)
 121
 122         def __init__(self):
 123                 eroot = portage.settings['EROOT']
 124                 self._bintree = portage.db[eroot]["bintree"]
 125                 self._bintree.populate()
 126                 self._pkgindex_file = self._bintree._pkgindex_file
 127                 self._pkgindex = self._bintree._load_pkgindex()
 128
 129         def _need_update(self, cpv, data):
 130
 131                 if "MD5" not in data:
 132                         return True
 133
 134                 size = data.get("SIZE")
 135                 if size is None:
 136                         return True
 137
 138                 mtime = data.get("MTIME")
 139                 if mtime is None:
 140                         return True
 141
 142                 pkg_path = self._bintree.getname(cpv)
 143                 try:
 144                         s = os.lstat(pkg_path)
 145                 except OSError as e:
 146                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 147                                 raise
 148                         # We can't update the index for this one because
 149                         # it disappeared.
 150                         return False
 151
 152                 try:
 153                         if long(mtime) != s[stat.ST_MTIME]:
 154                                 return True
 155                         if long(size) != long(s.st_size):
 156                                 return True
 157                 except ValueError:
 158                         return True
 159
 160                 return False
 161
 162         def check(self, onProgress=None):
 163                 missing = []
 164                 cpv_all = self._bintree.dbapi.cpv_all()
 165                 cpv_all.sort()
 166                 maxval = len(cpv_all)
 167                 if onProgress:
 168                         onProgress(maxval, 0)
 169                 pkgindex = self._pkgindex
 170                 missing = []
 171                 metadata = {}
 172                 for d in pkgindex.packages:
 173                         metadata[d["CPV"]] = d
 174                 for i, cpv in enumerate(cpv_all):
 175                         d = metadata.get(cpv)
 176                         if not d or self._need_update(cpv, d):
 177                                 missing.append(cpv)
 178                         if onProgress:
 179                                 onProgress(maxval, i+1)
 180                 errors = ["'%s' is not in Packages" % cpv for cpv in missing]
 181                 stale = set(metadata).difference(cpv_all)
 182                 for cpv in stale:
 183                         errors.append("'%s' is not in the repository" % cpv)
 184                 return errors
 185
 186         def fix(self, onProgress=None):
 187                 bintree = self._bintree
 188                 cpv_all = self._bintree.dbapi.cpv_all()
 189                 cpv_all.sort()
 190                 missing = []
 191                 maxval = 0
 192                 if onProgress:
 193                         onProgress(maxval, 0)
 194                 pkgindex = self._pkgindex
 195                 missing = []
 196                 metadata = {}
 197                 for d in pkgindex.packages:
 198                         metadata[d["CPV"]] = d
 199
 200                 for i, cpv in enumerate(cpv_all):
 201                         d = metadata.get(cpv)
 202                         if not d or self._need_update(cpv, d):
 203                                 missing.append(cpv)
 204
 205                 stale = set(metadata).difference(cpv_all)
 206                 if missing or stale:
 207                         from portage import locks
 208                         pkgindex_lock = locks.lockfile(
 209                                 self._pkgindex_file, wantnewlockfile=1)
 210                         try:
 211                                 # Repopulate with lock held.
 212                                 bintree._populate()
 213                                 cpv_all = self._bintree.dbapi.cpv_all()
 214                                 cpv_all.sort()
 215
 216                                 pkgindex = bintree._load_pkgindex()
 217                                 self._pkgindex = pkgindex
 218
 219                                 metadata = {}
 220                                 for d in pkgindex.packages:
 221                                         metadata[d["CPV"]] = d
 222
 223                                 # Recount missing packages, with lock held.
 224                                 del missing[:]
 225                                 for i, cpv in enumerate(cpv_all):
 226                                         d = metadata.get(cpv)
 227                                         if not d or self._need_update(cpv, d):
 228                                                 missing.append(cpv)
 229
 230                                 maxval = len(missing)
 231                                 for i, cpv in enumerate(missing):
 232                                         try:
 233                                                 metadata[cpv] = bintree._pkgindex_entry(cpv)
 234                                         except portage.exception.InvalidDependString:
 235                                                 writemsg("!!! Invalid binary package: '%s'\n" % \
 236                                                         bintree.getname(cpv), noiselevel=-1)
 237
 238                                         if onProgress:
 239                                                 onProgress(maxval, i+1)
 240
 241                                 for cpv in set(metadata).difference(
 242                                         self._bintree.dbapi.cpv_all()):
 243                                         del metadata[cpv]
 244
 245                                 # We've updated the pkgindex, so set it to
 246                                 # repopulate when necessary.
 247                                 bintree.populated = False
 248
 249                                 del pkgindex.packages[:]
 250                                 pkgindex.packages.extend(metadata.values())
 251                                 from portage.util import atomic_ofstream
 252                                 f = atomic_ofstream(self._pkgindex_file)
 253                                 try:
 254                                         self._pkgindex.write(f)
 255                                 finally:
 256                                         f.close()
 257                         finally:
 258                                 locks.unlockfile(pkgindex_lock)
 259
 260                 if onProgress:
 261                         if maxval == 0:
 262                                 maxval = 1
 263                         onProgress(maxval, maxval)
 264                 return None
 265
 266 class MoveHandler(object):
 267
 268         def __init__(self, tree, porttree):
 269                 self._tree = tree
 270                 self._portdb = porttree.dbapi
 271                 self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"]
 272                 self._master_repo = \
 273                         self._portdb.getRepositoryName(self._portdb.porttree_root)
 274
 275         def _grab_global_updates(self):
 276                 from portage.update import grab_updates, parse_updates
 277                 retupdates = {}
 278                 errors = []
 279
 280                 for repo_name in self._portdb.getRepositories():
 281                         repo = self._portdb.getRepositoryPath(repo_name)
 282                         updpath = os.path.join(repo, "profiles", "updates")
 283                         if not os.path.isdir(updpath):
 284                                 continue
 285
 286                         try:
 287                                 rawupdates = grab_updates(updpath)
 288                         except portage.exception.DirectoryNotFound:
 289                                 rawupdates = []
 290                         upd_commands = []
 291                         for mykey, mystat, mycontent in rawupdates:
 292                                 commands, errors = parse_updates(mycontent)
 293                                 upd_commands.extend(commands)
 294                                 errors.extend(errors)
 295                         retupdates[repo_name] = upd_commands
 296
 297                 if self._master_repo in retupdates:
 298                         retupdates['DEFAULT'] = retupdates[self._master_repo]
 299
 300                 return retupdates, errors
 301
 302         def check(self, onProgress=None):
 303                 allupdates, errors = self._grab_global_updates()
 304                 # Matching packages and moving them is relatively fast, so the
 305                 # progress bar is updated in indeterminate mode.
 306                 match = self._tree.dbapi.match
 307                 aux_get = self._tree.dbapi.aux_get
 308                 if onProgress:
 309                         onProgress(0, 0)
 310                 for repo, updates in allupdates.items():
 311                         if repo == 'DEFAULT':
 312                                 continue
 313                         if not updates:
 314                                 continue
 315
 316                         def repo_match(repository):
 317                                 return repository == repo or \
 318                                         (repo == self._master_repo and \
 319                                         repository not in allupdates)
 320
 321                         for i, update_cmd in enumerate(updates):
 322                                 if update_cmd[0] == "move":
 323                                         origcp, newcp = update_cmd[1:]
 324                                         for cpv in match(origcp):
 325                                                 if repo_match(aux_get(cpv, ["repository"])[0]):
 326                                                         errors.append("'%s' moved to '%s'" % (cpv, newcp))
 327                                 elif update_cmd[0] == "slotmove":
 328                                         pkg, origslot, newslot = update_cmd[1:]
 329                                         for cpv in match(pkg):
 330                                                 slot, prepo = aux_get(cpv, ["SLOT", "repository"])
 331                                                 if slot == origslot and repo_match(prepo):
 332                                                         errors.append("'%s' slot moved from '%s' to '%s'" % \
 333                                                                 (cpv, origslot, newslot))
 334                                 if onProgress:
 335                                         onProgress(0, 0)
 336
 337                 # Searching for updates in all the metadata is relatively slow, so this
 338                 # is where the progress bar comes out of indeterminate mode.
 339                 cpv_all = self._tree.dbapi.cpv_all()
 340                 cpv_all.sort()
 341                 maxval = len(cpv_all)
 342                 aux_update = self._tree.dbapi.aux_update
 343                 meta_keys = self._update_keys + ['repository']
 344                 from portage.update import update_dbentries
 345                 if onProgress:
 346                         onProgress(maxval, 0)
 347                 for i, cpv in enumerate(cpv_all):
 348                         metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys)))
 349                         repository = metadata.pop('repository')
 350                         try:
 351                                 updates = allupdates[repository]
 352                         except KeyError:
 353                                 try:
 354                                         updates = allupdates['DEFAULT']
 355                                 except KeyError:
 356                                         continue
 357                         if not updates:
 358                                 continue
 359                         metadata_updates = update_dbentries(updates, metadata)
 360                         if metadata_updates:
 361                                 errors.append("'%s' has outdated metadata" % cpv)
 362                         if onProgress:
 363                                 onProgress(maxval, i+1)
 364                 return errors
 365
 366         def fix(self, onProgress=None):
 367                 allupdates, errors = self._grab_global_updates()
 368                 # Matching packages and moving them is relatively fast, so the
 369                 # progress bar is updated in indeterminate mode.
 370                 move = self._tree.dbapi.move_ent
 371                 slotmove = self._tree.dbapi.move_slot_ent
 372                 if onProgress:
 373                         onProgress(0, 0)
 374                 for repo, updates in allupdates.items():
 375                         if repo == 'DEFAULT':
 376                                 continue
 377                         if not updates:
 378                                 continue
 379
 380                         def repo_match(repository):
 381                                 return repository == repo or \
 382                                         (repo == self._master_repo and \
 383                                         repository not in allupdates)
 384
 385                         for i, update_cmd in enumerate(updates):
 386                                 if update_cmd[0] == "move":
 387                                         move(update_cmd, repo_match=repo_match)
 388                                 elif update_cmd[0] == "slotmove":
 389                                         slotmove(update_cmd, repo_match=repo_match)
 390                                 if onProgress:
 391                                         onProgress(0, 0)
 392
 393                 # Searching for updates in all the metadata is relatively slow, so this
 394                 # is where the progress bar comes out of indeterminate mode.
 395                 self._tree.dbapi.update_ents(allupdates, onProgress=onProgress)
 396                 return errors
 397
 398 class MoveInstalled(MoveHandler):
 399
 400         short_desc = "Perform package move updates for installed packages"
 401
 402         def name():
 403                 return "moveinst"
 404         name = staticmethod(name)
 405         def __init__(self):
 406                 eroot = portage.settings['EROOT']
 407                 MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"])
 408
 409 class MoveBinary(MoveHandler):
 410
 411         short_desc = "Perform package move updates for binary packages"
 412
 413         def name():
 414                 return "movebin"
 415         name = staticmethod(name)
 416         def __init__(self):
 417                 eroot = portage.settings['EROOT']
 418                 MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree'])
 419
 420 class VdbKeyHandler(object):
 421         def name():
 422                 return "vdbkeys"
 423         name = staticmethod(name)
 424
 425         def __init__(self):
 426                 self.list = portage.db[portage.settings["EROOT"]]["vartree"].dbapi.cpv_all()
 427                 self.missing = []
 428                 self.keys = ["HOMEPAGE", "SRC_URI", "KEYWORDS", "DESCRIPTION"]
 429
 430                 for p in self.list:
 431                         mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep
 432                         ismissing = True
 433                         for k in self.keys:
 434                                 if os.path.exists(mydir+k):
 435                                         ismissing = False
 436                                         break
 437                         if ismissing:
 438                                 self.missing.append(p)
 439
 440         def check(self):
 441                 return ["%s has missing keys" % x for x in self.missing]
 442
 443         def fix(self):
 444
 445                 errors = []
 446
 447                 for p in self.missing:
 448                         mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep
 449                         if not os.access(mydir+"environment.bz2", os.R_OK):
 450                                 errors.append("Can't access %s" % (mydir+"environment.bz2"))
 451                         elif not os.access(mydir, os.W_OK):
 452                                 errors.append("Can't create files in %s" % mydir)
 453                         else:
 454                                 env = os.popen("bzip2 -dcq "+mydir+"environment.bz2", "r")
 455                                 envlines = env.read().split("\n")
 456                                 env.close()
 457                                 for k in self.keys:
 458                                         s = [l for l in envlines if l.startswith(k+"=")]
 459                                         if len(s) > 1:
 460                                                 errors.append("multiple matches for %s found in %senvironment.bz2" % (k, mydir))
 461                                         elif len(s) == 0:
 462                                                 s = ""
 463                                         else:
 464                                                 s = s[0].split("=",1)[1]
 465                                                 s = s.lstrip("$").strip("\'\"")
 466                                                 s = re.sub("(\\\\[nrt])+", " ", s)
 467                                                 s = " ".join(s.split()).strip()
 468                                                 if s != "":
 469                                                         try:
 470                                                                 keyfile = open(mydir+os.sep+k, "w")
 471                                                                 keyfile.write(s+"\n")
 472                                                                 keyfile.close()
 473                                                         except (IOError, OSError) as e:
 474                                                                 errors.append("Could not write %s, reason was: %s" % (mydir+k, e))
 475
 476                 return errors
 477
 478 class ProgressHandler(object):
 479         def __init__(self):
 480                 self.curval = 0
 481                 self.maxval = 0
 482                 self.last_update = 0
 483                 self.min_display_latency = 0.2
 484
 485         def onProgress(self, maxval, curval):
 486                 self.maxval = maxval
 487                 self.curval = curval
 488                 cur_time = time.time()
 489                 if cur_time - self.last_update >= self.min_display_latency:
 490                         self.last_update = cur_time
 491                         self.display()
 492
 493         def display(self):
 494                 raise NotImplementedError(self)
 495
 496 class CleanResume(object):
 497
 498         short_desc = "Discard emerge --resume merge lists"
 499
 500         def name():
 501                 return "cleanresume"
 502         name = staticmethod(name)
 503
 504         def check(self, onProgress=None):
 505                 messages = []
 506                 mtimedb = portage.mtimedb
 507                 resume_keys = ("resume", "resume_backup")
 508                 maxval = len(resume_keys)
 509                 if onProgress:
 510                         onProgress(maxval, 0)
 511                 for i, k in enumerate(resume_keys):
 512                         try:
 513                                 d = mtimedb.get(k)
 514                                 if d is None:
 515                                         continue
 516                                 if not isinstance(d, dict):
 517                                         messages.append("unrecognized resume list: '%s'" % k)
 518                                         continue
 519                                 mergelist = d.get("mergelist")
 520                                 if mergelist is None or not hasattr(mergelist, "__len__"):
 521                                         messages.append("unrecognized resume list: '%s'" % k)
 522                                         continue
 523                                 messages.append("resume list '%s' contains %d packages" % \
 524                                         (k, len(mergelist)))
 525                         finally:
 526                                 if onProgress:
 527                                         onProgress(maxval, i+1)
 528                 return messages
 529
 530         def fix(self, onProgress=None):
 531                 delete_count = 0
 532                 mtimedb = portage.mtimedb
 533                 resume_keys = ("resume", "resume_backup")
 534                 maxval = len(resume_keys)
 535                 if onProgress:
 536                         onProgress(maxval, 0)
 537                 for i, k in enumerate(resume_keys):
 538                         try:
 539                                 if mtimedb.pop(k, None) is not None:
 540                                         delete_count += 1
 541                         finally:
 542                                 if onProgress:
 543                                         onProgress(maxval, i+1)
 544                 if delete_count:
 545                         mtimedb.commit()
 546
 547 def emaint_main(myargv):
 548
 549         # Similar to emerge, emaint needs a default umask so that created
 550         # files (such as the world file) have sane permissions.
 551         os.umask(0o22)
 552
 553         # TODO: Create a system that allows external modules to be added without
 554         #       the need for hard coding.
 555         modules = {
 556                 "world" : WorldHandler,
 557                 "binhost":BinhostHandler,
 558                 "moveinst":MoveInstalled,
 559                 "movebin":MoveBinary,
 560                 "cleanresume":CleanResume
 561         }
 562
 563         module_names = list(modules)
 564         module_names.sort()
 565         module_names.insert(0, "all")
 566
 567         def exclusive(option, *args, **kw):
 568                 var = kw.get("var", None)
 569                 if var is None:
 570                         raise ValueError("var not specified to exclusive()")
 571                 if getattr(parser, var, ""):
 572                         raise OptionValueError("%s and %s are exclusive options" % (getattr(parser, var), option))
 573                 setattr(parser, var, str(option))
 574
 575
 576         usage = "usage: emaint [options] COMMAND"
 577
 578         desc = "The emaint program provides an interface to system health " + \
 579                 "checks and maintenance. See the emaint(1) man page " + \
 580                 "for additional information about the following commands:"
 581
 582         usage += "\n\n"
 583         for line in textwrap.wrap(desc, 65):
 584                 usage += "%s\n" % line
 585         usage += "\n"
 586         usage += "  %s" % "all".ljust(15) + \
 587                 "Perform all supported commands\n"
 588         for m in module_names[1:]:
 589                 usage += "  %s%s\n" % (m.ljust(15), modules[m].short_desc)
 590
 591         parser = OptionParser(usage=usage, version=portage.VERSION)
 592         parser.add_option("-c", "--check", help="check for problems",
 593                 action="callback", callback=exclusive, callback_kwargs={"var":"action"})
 594         parser.add_option("-f", "--fix", help="attempt to fix problems",
 595                 action="callback", callback=exclusive, callback_kwargs={"var":"action"})
 596         parser.action = None
 597
 598
 599         (options, args) = parser.parse_args(args=myargv)
 600         if len(args) != 1:
 601                 parser.error("Incorrect number of arguments")
 602         if args[0] not in module_names:
 603                 parser.error("%s target is not a known target" % args[0])
 604
 605         if parser.action:
 606                 action = parser.action
 607         else:
 608                 print("Defaulting to --check")
 609                 action = "-c/--check"
 610
 611         if args[0] == "all":
 612                 tasks = modules.values()
 613         else:
 614                 tasks = [modules[args[0]]]
 615
 616
 617         if action == "-c/--check":
 618                 status = "Checking %s for problems"
 619                 func = "check"
 620         else:
 621                 status = "Attempting to fix %s"
 622                 func = "fix"
 623
 624         isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty()
 625         for task in tasks:
 626                 print(status % task.name())
 627                 inst = task()
 628                 onProgress = None
 629                 if isatty:
 630                         progressBar = portage.output.TermProgressBar()
 631                         progressHandler = ProgressHandler()
 632                         onProgress = progressHandler.onProgress
 633                         def display():
 634                                 progressBar.set(progressHandler.curval, progressHandler.maxval)
 635                         progressHandler.display = display
 636                         def sigwinch_handler(signum, frame):
 637                                 lines, progressBar.term_columns = \
 638                                         portage.output.get_term_size()
 639                         signal.signal(signal.SIGWINCH, sigwinch_handler)
 640                 result = getattr(inst, func)(onProgress=onProgress)
 641                 if isatty:
 642                         # make sure the final progress is displayed
 643                         progressHandler.display()
 644                         print()
 645                         signal.signal(signal.SIGWINCH, signal.SIG_DFL)
 646                 if result:
 647                         print()
 648                         print("\n".join(result))
 649                         print("\n")
 650
 651         print("Finished")
 652
 653 if __name__ == "__main__":
 654         emaint_main(sys.argv[1:])