pym/portage/manifest.py

   1 # Copyright 1999-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 import errno
   7 import io
   8 import re
   9 import sys
  10 import warnings
  11
  12 import portage
  13 portage.proxy.lazyimport.lazyimport(globals(),
  14         'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
  15                 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes',
  16         'portage.repository.config:_find_invalid_path_char',
  17         'portage.util:write_atomic',
  18 )
  19
  20 from portage import os
  21 from portage import _encodings
  22 from portage import _unicode_decode
  23 from portage import _unicode_encode
  24 from portage.exception import DigestException, FileNotFound, \
  25         InvalidDataType, MissingParameter, PermissionDenied, \
  26         PortageException, PortagePackageException
  27 from portage.const import (MANIFEST1_HASH_FUNCTIONS, MANIFEST2_HASH_DEFAULTS,
  28         MANIFEST2_HASH_FUNCTIONS, MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH,
  29         VCS_DIRS)
  30 from portage.localization import _
  31
  32 _manifest_re = re.compile(
  33         r'^(' + '|'.join(MANIFEST2_IDENTIFIERS) + r') (.*)( \d+( \S+ \S+)+)$',
  34         re.UNICODE)
  35
  36 if sys.hexversion >= 0x3000000:
  37         _unicode = str
  38         basestring = str
  39 else:
  40         _unicode = unicode
  41
  42 class FileNotInManifestException(PortageException):
  43         pass
  44
  45 def manifest2AuxfileFilter(filename):
  46         filename = filename.strip(os.sep)
  47         mysplit = filename.split(os.path.sep)
  48         if "CVS" in mysplit:
  49                 return False
  50         for x in mysplit:
  51                 if x[:1] == '.':
  52                         return False
  53         return not filename[:7] == 'digest-'
  54
  55 def manifest2MiscfileFilter(filename):
  56         filename = filename.strip(os.sep)
  57         return not (filename in VCS_DIRS + ("files", "Manifest") or filename.endswith(".ebuild"))
  58
  59 def guessManifestFileType(filename):
  60         """ Perform a best effort guess of which type the given filename is, avoid using this if possible """
  61         if filename.startswith("files" + os.sep + "digest-"):
  62                 return None
  63         if filename.startswith("files" + os.sep):
  64                 return "AUX"
  65         elif filename.endswith(".ebuild"):
  66                 return "EBUILD"
  67         elif filename in ["ChangeLog", "metadata.xml"]:
  68                 return "MISC"
  69         else:
  70                 return "DIST"
  71
  72 def guessThinManifestFileType(filename):
  73         type = guessManifestFileType(filename)
  74         if type != "DIST":
  75                 return None
  76         return "DIST"
  77
  78 def parseManifest2(line):
  79         if not isinstance(line, basestring):
  80                 line = ' '.join(line)
  81         myentry = None
  82         match = _manifest_re.match(line)
  83         if match is not None:
  84                 tokens = match.group(3).split()
  85                 hashes = dict(zip(tokens[1::2], tokens[2::2]))
  86                 hashes["size"] = int(tokens[0])
  87                 myentry = Manifest2Entry(type=match.group(1),
  88                         name=match.group(2), hashes=hashes)
  89         return myentry
  90
  91 class ManifestEntry(object):
  92         __slots__ = ("type", "name", "hashes")
  93         def __init__(self, **kwargs):
  94                 for k, v in kwargs.items():
  95                         setattr(self, k, v)
  96
  97 class Manifest2Entry(ManifestEntry):
  98         def __str__(self):
  99                 myline = " ".join([self.type, self.name, str(self.hashes["size"])])
 100                 myhashkeys = list(self.hashes)
 101                 myhashkeys.remove("size")
 102                 myhashkeys.sort()
 103                 for h in myhashkeys:
 104                         myline += " " + h + " " + str(self.hashes[h])
 105                 return myline
 106
 107         def __eq__(self, other):
 108                 if not isinstance(other, Manifest2Entry) or \
 109                         self.type != other.type or \
 110                         self.name != other.name or \
 111                         self.hashes != other.hashes:
 112                         return False
 113                 return True
 114
 115         def __ne__(self, other):
 116                 return not self.__eq__(other)
 117
 118         if sys.hexversion < 0x3000000:
 119
 120                 __unicode__ = __str__
 121
 122                 def __str__(self):
 123                         return _unicode_encode(self.__unicode__(),
 124                                 encoding=_encodings['repo.content'], errors='strict')
 125
 126 class Manifest(object):
 127         parsers = (parseManifest2,)
 128         def __init__(self, pkgdir, distdir=None, fetchlist_dict=None,
 129                 manifest1_compat=DeprecationWarning, from_scratch=False, thin=False,
 130                 allow_missing=False, allow_create=True, hashes=None,
 131                 find_invalid_path_char=None):
 132                 """ Create new Manifest instance for package in pkgdir.
 133                     Do not parse Manifest file if from_scratch == True (only for internal use)
 134                         The fetchlist_dict parameter is required only for generation of
 135                         a Manifest (not needed for parsing and checking sums).
 136                         If thin is specified, then the manifest carries only info for
 137                         distfiles."""
 138
 139                 if manifest1_compat is not DeprecationWarning:
 140                         warnings.warn("The manifest1_compat parameter of the "
 141                                 "portage.manifest.Manifest constructor is deprecated.",
 142                                 DeprecationWarning, stacklevel=2)
 143
 144                 if find_invalid_path_char is None:
 145                         find_invalid_path_char = _find_invalid_path_char
 146                 self._find_invalid_path_char = find_invalid_path_char
 147                 self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
 148                 self.fhashdict = {}
 149                 self.hashes = set()
 150
 151                 if hashes is None:
 152                         hashes = MANIFEST2_HASH_DEFAULTS
 153
 154                 self.hashes.update(hashes.intersection(MANIFEST2_HASH_FUNCTIONS))
 155                 self.hashes.difference_update(hashname for hashname in \
 156                         list(self.hashes) if hashname not in hashfunc_map)
 157                 self.hashes.add("size")
 158                 self.hashes.add(MANIFEST2_REQUIRED_HASH)
 159                 for t in MANIFEST2_IDENTIFIERS:
 160                         self.fhashdict[t] = {}
 161                 if not from_scratch:
 162                         self._read()
 163                 if fetchlist_dict != None:
 164                         self.fetchlist_dict = fetchlist_dict
 165                 else:
 166                         self.fetchlist_dict = {}
 167                 self.distdir = distdir
 168                 self.thin = thin
 169                 if thin:
 170                         self.guessType = guessThinManifestFileType
 171                 else:
 172                         self.guessType = guessManifestFileType
 173                 self.allow_missing = allow_missing
 174                 self.allow_create = allow_create
 175
 176         def getFullname(self):
 177                 """ Returns the absolute path to the Manifest file for this instance """
 178                 return os.path.join(self.pkgdir, "Manifest")
 179
 180         def getDigests(self):
 181                 """ Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue} """
 182                 rval = {}
 183                 for t in MANIFEST2_IDENTIFIERS:
 184                         rval.update(self.fhashdict[t])
 185                 return rval
 186
 187         def getTypeDigests(self, ftype):
 188                 """ Similar to getDigests(), but restricted to files of the given type. """
 189                 return self.fhashdict[ftype]
 190
 191         def _readManifest(self, file_path, myhashdict=None, **kwargs):
 192                 """Parse a manifest.  If myhashdict is given then data will be added too it.
 193                    Otherwise, a new dict will be created and returned."""
 194                 try:
 195                         with io.open(_unicode_encode(file_path,
 196                                 encoding=_encodings['fs'], errors='strict'), mode='r',
 197                                 encoding=_encodings['repo.content'], errors='replace') as f:
 198                                 if myhashdict is None:
 199                                         myhashdict = {}
 200                                 self._parseDigests(f, myhashdict=myhashdict, **kwargs)
 201                         return myhashdict
 202                 except (OSError, IOError) as e:
 203                         if e.errno == errno.ENOENT:
 204                                 raise FileNotFound(file_path)
 205                         else:
 206                                 raise
 207
 208         def _read(self):
 209                 """ Parse Manifest file for this instance """
 210                 try:
 211                         self._readManifest(self.getFullname(), myhashdict=self.fhashdict)
 212                 except FileNotFound:
 213                         pass
 214
 215         def _parseManifestLines(self, mylines):
 216                 """Parse manifest lines and return a list of manifest entries."""
 217                 for myline in mylines:
 218                         myentry = None
 219                         for parser in self.parsers:
 220                                 myentry = parser(myline)
 221                                 if myentry is not None:
 222                                         yield myentry
 223                                         break # go to the next line
 224
 225         def _parseDigests(self, mylines, myhashdict=None, mytype=None):
 226                 """Parse manifest entries and store the data in myhashdict.  If mytype
 227                 is specified, it will override the type for all parsed entries."""
 228                 if myhashdict is None:
 229                         myhashdict = {}
 230                 for myentry in self._parseManifestLines(mylines):
 231                         if mytype is None:
 232                                 myentry_type = myentry.type
 233                         else:
 234                                 myentry_type = mytype
 235                         myhashdict.setdefault(myentry_type, {})
 236                         myhashdict[myentry_type].setdefault(myentry.name, {})
 237                         myhashdict[myentry_type][myentry.name].update(myentry.hashes)
 238                 return myhashdict
 239
 240         def _getDigestData(self, distlist):
 241                 """create a hash dict for a specific list of files"""
 242                 myhashdict = {}
 243                 for myname in distlist:
 244                         for mytype in self.fhashdict:
 245                                 if myname in self.fhashdict[mytype]:
 246                                         myhashdict.setdefault(mytype, {})
 247                                         myhashdict[mytype].setdefault(myname, {})
 248                                         myhashdict[mytype][myname].update(self.fhashdict[mytype][myname])
 249                 return myhashdict
 250
 251         def _createManifestEntries(self):
 252                 valid_hashes = set(MANIFEST2_HASH_FUNCTIONS)
 253                 valid_hashes.add('size')
 254                 mytypes = list(self.fhashdict)
 255                 mytypes.sort()
 256                 for t in mytypes:
 257                         myfiles = list(self.fhashdict[t])
 258                         myfiles.sort()
 259                         for f in myfiles:
 260                                 myentry = Manifest2Entry(
 261                                         type=t, name=f, hashes=self.fhashdict[t][f].copy())
 262                                 for h in list(myentry.hashes):
 263                                         if h not in valid_hashes:
 264                                                 del myentry.hashes[h]
 265                                 yield myentry
 266
 267         def checkIntegrity(self):
 268                 for t in self.fhashdict:
 269                         for f in self.fhashdict[t]:
 270                                 if MANIFEST2_REQUIRED_HASH not in self.fhashdict[t][f]:
 271                                         raise MissingParameter(_("Missing %s checksum: %s %s") %
 272                                                 (MANIFEST2_REQUIRED_HASH, t, f))
 273
 274         def write(self, sign=False, force=False):
 275                 """ Write Manifest instance to disk, optionally signing it. Returns
 276                 True if the Manifest is actually written, and False if the write
 277                 is skipped due to existing Manifest being identical."""
 278                 rval = False
 279                 if not self.allow_create:
 280                         return rval
 281                 self.checkIntegrity()
 282                 try:
 283                         myentries = list(self._createManifestEntries())
 284                         update_manifest = True
 285                         if myentries and not force:
 286                                 try:
 287                                         f = io.open(_unicode_encode(self.getFullname(),
 288                                                 encoding=_encodings['fs'], errors='strict'),
 289                                                 mode='r', encoding=_encodings['repo.content'],
 290                                                 errors='replace')
 291                                         oldentries = list(self._parseManifestLines(f))
 292                                         f.close()
 293                                         if len(oldentries) == len(myentries):
 294                                                 update_manifest = False
 295                                                 for i in range(len(oldentries)):
 296                                                         if oldentries[i] != myentries[i]:
 297                                                                 update_manifest = True
 298                                                                 break
 299                                 except (IOError, OSError) as e:
 300                                         if e.errno == errno.ENOENT:
 301                                                 pass
 302                                         else:
 303                                                 raise
 304
 305                         if update_manifest:
 306                                 if myentries or not (self.thin or self.allow_missing):
 307                                         # If myentries is empty, don't write an empty manifest
 308                                         # when thin or allow_missing is enabled. Except for
 309                                         # thin manifests with no DIST entries, myentries is
 310                                         # non-empty for all currently known use cases.
 311                                         write_atomic(self.getFullname(), "".join("%s\n" %
 312                                                 _unicode(myentry) for myentry in myentries))
 313                                         rval = True
 314                                 else:
 315                                         # With thin manifest, there's no need to have
 316                                         # a Manifest file if there are no DIST entries.
 317                                         try:
 318                                                 os.unlink(self.getFullname())
 319                                         except OSError as e:
 320                                                 if e.errno != errno.ENOENT:
 321                                                         raise
 322                                         rval = True
 323
 324                         if sign:
 325                                 self.sign()
 326                 except (IOError, OSError) as e:
 327                         if e.errno == errno.EACCES:
 328                                 raise PermissionDenied(str(e))
 329                         raise
 330                 return rval
 331
 332         def sign(self):
 333                 """ Sign the Manifest """
 334                 raise NotImplementedError()
 335
 336         def validateSignature(self):
 337                 """ Validate signature on Manifest """
 338                 raise NotImplementedError()
 339
 340         def addFile(self, ftype, fname, hashdict=None, ignoreMissing=False):
 341                 """ Add entry to Manifest optionally using hashdict to avoid recalculation of hashes """
 342                 if ftype == "AUX" and not fname.startswith("files/"):
 343                         fname = os.path.join("files", fname)
 344                 if not os.path.exists(self.pkgdir+fname) and not ignoreMissing:
 345                         raise FileNotFound(fname)
 346                 if not ftype in MANIFEST2_IDENTIFIERS:
 347                         raise InvalidDataType(ftype)
 348                 if ftype == "AUX" and fname.startswith("files"):
 349                         fname = fname[6:]
 350                 self.fhashdict[ftype][fname] = {}
 351                 if hashdict != None:
 352                         self.fhashdict[ftype][fname].update(hashdict)
 353                 if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]:
 354                         self.updateFileHashes(ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing)
 355
 356         def removeFile(self, ftype, fname):
 357                 """ Remove given entry from Manifest """
 358                 del self.fhashdict[ftype][fname]
 359
 360         def hasFile(self, ftype, fname):
 361                 """ Return whether the Manifest contains an entry for the given type,filename pair """
 362                 return (fname in self.fhashdict[ftype])
 363
 364         def findFile(self, fname):
 365                 """ Return entrytype of the given file if present in Manifest or None if not present """
 366                 for t in MANIFEST2_IDENTIFIERS:
 367                         if fname in self.fhashdict[t]:
 368                                 return t
 369                 return None
 370
 371         def create(self, checkExisting=False, assumeDistHashesSometimes=False,
 372                 assumeDistHashesAlways=False, requiredDistfiles=[]):
 373                 """ Recreate this Manifest from scratch.  This will not use any
 374                 existing checksums unless assumeDistHashesSometimes or
 375                 assumeDistHashesAlways is true (assumeDistHashesSometimes will only
 376                 cause DIST checksums to be reused if the file doesn't exist in
 377                 DISTDIR).  The requiredDistfiles parameter specifies a list of
 378                 distfiles to raise a FileNotFound exception for (if no file or existing
 379                 checksums are available), and defaults to all distfiles when not
 380                 specified."""
 381                 if not self.allow_create:
 382                         return
 383                 if checkExisting:
 384                         self.checkAllHashes()
 385                 if assumeDistHashesSometimes or assumeDistHashesAlways:
 386                         distfilehashes = self.fhashdict["DIST"]
 387                 else:
 388                         distfilehashes = {}
 389                 self.__init__(self.pkgdir, distdir=self.distdir,
 390                         fetchlist_dict=self.fetchlist_dict, from_scratch=True,
 391                         thin=self.thin, allow_missing=self.allow_missing,
 392                         allow_create=self.allow_create, hashes=self.hashes,
 393                         find_invalid_path_char=self._find_invalid_path_char)
 394                 pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
 395                 cat = self._pkgdir_category()
 396
 397                 pkgdir = self.pkgdir
 398                 if self.thin:
 399                         cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir)
 400                 else:
 401                         cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir)
 402
 403                 distlist = set()
 404                 for cpv in cpvlist:
 405                         distlist.update(self._getCpvDistfiles(cpv))
 406
 407                 if requiredDistfiles is None:
 408                         # This allows us to force removal of stale digests for the
 409                         # ebuild --force digest option (no distfiles are required).
 410                         requiredDistfiles = set()
 411                 elif len(requiredDistfiles) == 0:
 412                         # repoman passes in an empty list, which implies that all distfiles
 413                         # are required.
 414                         requiredDistfiles = distlist.copy()
 415                 required_hash_types = set()
 416                 required_hash_types.add("size")
 417                 required_hash_types.add(MANIFEST2_REQUIRED_HASH)
 418                 for f in distlist:
 419                         fname = os.path.join(self.distdir, f)
 420                         mystat = None
 421                         try:
 422                                 mystat = os.stat(fname)
 423                         except OSError:
 424                                 pass
 425                         if f in distfilehashes and \
 426                                 not required_hash_types.difference(distfilehashes[f]) and \
 427                                 ((assumeDistHashesSometimes and mystat is None) or \
 428                                 (assumeDistHashesAlways and mystat is None) or \
 429                                 (assumeDistHashesAlways and mystat is not None and \
 430                                 set(distfilehashes[f]) == set(self.hashes) and \
 431                                 distfilehashes[f]["size"] == mystat.st_size)):
 432                                 self.fhashdict["DIST"][f] = distfilehashes[f]
 433                         else:
 434                                 try:
 435                                         self.fhashdict["DIST"][f] = perform_multiple_checksums(fname, self.hashes)
 436                                 except FileNotFound:
 437                                         if f in requiredDistfiles:
 438                                                 raise
 439
 440         def _is_cpv(self, cat, pn, filename):
 441                 if not filename.endswith(".ebuild"):
 442                         return None
 443                 pf = filename[:-7]
 444                 ps = portage.versions._pkgsplit(pf)
 445                 cpv = "%s/%s" % (cat, pf)
 446                 if not ps:
 447                         raise PortagePackageException(
 448                                 _("Invalid package name: '%s'") % cpv)
 449                 if ps[0] != pn:
 450                         raise PortagePackageException(
 451                                 _("Package name does not "
 452                                 "match directory name: '%s'") % cpv)
 453                 return cpv
 454
 455         def _update_thin_pkgdir(self, cat, pn, pkgdir):
 456                 for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
 457                         break
 458                 cpvlist = []
 459                 for f in pkgdir_files:
 460                         try:
 461                                 f = _unicode_decode(f,
 462                                         encoding=_encodings['fs'], errors='strict')
 463                         except UnicodeDecodeError:
 464                                 continue
 465                         if f[:1] == '.':
 466                                 continue
 467                         pf = self._is_cpv(cat, pn, f)
 468                         if pf is not None:
 469                                 cpvlist.append(pf)
 470                 return cpvlist
 471
 472         def _update_thick_pkgdir(self, cat, pn, pkgdir):
 473                 cpvlist = []
 474                 for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
 475                         break
 476                 for f in pkgdir_files:
 477                         try:
 478                                 f = _unicode_decode(f,
 479                                         encoding=_encodings['fs'], errors='strict')
 480                         except UnicodeDecodeError:
 481                                 continue
 482                         if f[:1] == ".":
 483                                 continue
 484                         pf = self._is_cpv(cat, pn, f)
 485                         if pf is not None:
 486                                 mytype = "EBUILD"
 487                                 cpvlist.append(pf)
 488                         elif self._find_invalid_path_char(f) == -1 and \
 489                                 manifest2MiscfileFilter(f):
 490                                 mytype = "MISC"
 491                         else:
 492                                 continue
 493                         self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
 494                 recursive_files = []
 495
 496                 pkgdir = self.pkgdir
 497                 cut_len = len(os.path.join(pkgdir, "files") + os.sep)
 498                 for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
 499                         for f in files:
 500                                 try:
 501                                         f = _unicode_decode(f,
 502                                                 encoding=_encodings['fs'], errors='strict')
 503                                 except UnicodeDecodeError:
 504                                         continue
 505                                 full_path = os.path.join(parentdir, f)
 506                                 recursive_files.append(full_path[cut_len:])
 507                 for f in recursive_files:
 508                         if self._find_invalid_path_char(f) != -1 or \
 509                                 not manifest2AuxfileFilter(f):
 510                                 continue
 511                         self.fhashdict["AUX"][f] = perform_multiple_checksums(
 512                                 os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
 513                 return cpvlist
 514
 515         def _pkgdir_category(self):
 516                 return self.pkgdir.rstrip(os.sep).split(os.sep)[-2]
 517
 518         def _getAbsname(self, ftype, fname):
 519                 if ftype == "DIST":
 520                         absname = os.path.join(self.distdir, fname)
 521                 elif ftype == "AUX":
 522                         absname = os.path.join(self.pkgdir, "files", fname)
 523                 else:
 524                         absname = os.path.join(self.pkgdir, fname)
 525                 return absname
 526
 527         def checkAllHashes(self, ignoreMissingFiles=False):
 528                 for t in MANIFEST2_IDENTIFIERS:
 529                         self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles)
 530
 531         def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None):
 532                 for f in self.fhashdict[idtype]:
 533                         self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles,
 534                                 hash_filter=hash_filter)
 535
 536         def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None):
 537                 digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname])
 538                 if hash_filter is not None:
 539                         digests = _apply_hash_filter(digests, hash_filter)
 540                 try:
 541                         ok, reason = verify_all(self._getAbsname(ftype, fname), digests)
 542                         if not ok:
 543                                 raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason)))
 544                         return ok, reason
 545                 except FileNotFound as e:
 546                         if not ignoreMissing:
 547                                 raise
 548                         return False, _("File Not Found: '%s'") % str(e)
 549
 550         def checkCpvHashes(self, cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=False):
 551                 """ check the hashes for all files associated to the given cpv, include all
 552                 AUX files and optionally all MISC files. """
 553                 if not onlyDistfiles:
 554                         self.checkTypeHashes("AUX", ignoreMissingFiles=False)
 555                         if checkMiscfiles:
 556                                 self.checkTypeHashes("MISC", ignoreMissingFiles=False)
 557                         ebuildname = "%s.ebuild" % self._catsplit(cpv)[1]
 558                         self.checkFileHashes("EBUILD", ebuildname, ignoreMissing=False)
 559                 if checkDistfiles or onlyDistfiles:
 560                         for f in self._getCpvDistfiles(cpv):
 561                                 self.checkFileHashes("DIST", f, ignoreMissing=False)
 562
 563         def _getCpvDistfiles(self, cpv):
 564                 """ Get a list of all DIST files associated to the given cpv """
 565                 return self.fetchlist_dict[cpv]
 566
 567         def getDistfilesSize(self, fetchlist):
 568                 total_bytes = 0
 569                 for f in fetchlist:
 570                         total_bytes += int(self.fhashdict["DIST"][f]["size"])
 571                 return total_bytes
 572
 573         def updateFileHashes(self, ftype, fname, checkExisting=True, ignoreMissing=True, reuseExisting=False):
 574                 """ Regenerate hashes for the given file """
 575                 if checkExisting:
 576                         self.checkFileHashes(ftype, fname, ignoreMissing=ignoreMissing)
 577                 if not ignoreMissing and fname not in self.fhashdict[ftype]:
 578                         raise FileNotInManifestException(fname)
 579                 if fname not in self.fhashdict[ftype]:
 580                         self.fhashdict[ftype][fname] = {}
 581                 myhashkeys = list(self.hashes)
 582                 if reuseExisting:
 583                         for k in [h for h in self.fhashdict[ftype][fname] if h in myhashkeys]:
 584                                 myhashkeys.remove(k)
 585                 myhashes = perform_multiple_checksums(self._getAbsname(ftype, fname), myhashkeys)
 586                 self.fhashdict[ftype][fname].update(myhashes)
 587
 588         def updateTypeHashes(self, idtype, checkExisting=False, ignoreMissingFiles=True):
 589                 """ Regenerate all hashes for all files of the given type """
 590                 for fname in self.fhashdict[idtype]:
 591                         self.updateFileHashes(idtype, fname, checkExisting)
 592
 593         def updateAllHashes(self, checkExisting=False, ignoreMissingFiles=True):
 594                 """ Regenerate all hashes for all files in this Manifest. """
 595                 for idtype in MANIFEST2_IDENTIFIERS:
 596                         self.updateTypeHashes(idtype, checkExisting=checkExisting,
 597                                 ignoreMissingFiles=ignoreMissingFiles)
 598
 599         def updateCpvHashes(self, cpv, ignoreMissingFiles=True):
 600                 """ Regenerate all hashes associated to the given cpv (includes all AUX and MISC
 601                 files)."""
 602                 self.updateTypeHashes("AUX", ignoreMissingFiles=ignoreMissingFiles)
 603                 self.updateTypeHashes("MISC", ignoreMissingFiles=ignoreMissingFiles)
 604                 ebuildname = "%s.ebuild" % self._catsplit(cpv)[1]
 605                 self.updateFileHashes("EBUILD", ebuildname, ignoreMissingFiles=ignoreMissingFiles)
 606                 for f in self._getCpvDistfiles(cpv):
 607                         self.updateFileHashes("DIST", f, ignoreMissingFiles=ignoreMissingFiles)
 608
 609         def updateHashesGuessType(self, fname, *args, **kwargs):
 610                 """ Regenerate hashes for the given file (guesses the type and then
 611                 calls updateFileHashes)."""
 612                 mytype = self.guessType(fname)
 613                 if mytype == "AUX":
 614                         fname = fname[len("files" + os.sep):]
 615                 elif mytype is None:
 616                         return
 617                 myrealtype = self.findFile(fname)
 618                 if myrealtype is not None:
 619                         mytype = myrealtype
 620                 return self.updateFileHashes(mytype, fname, *args, **kwargs)
 621
 622         def getFileData(self, ftype, fname, key):
 623                 """ Return the value of a specific (type,filename,key) triple, mainly useful
 624                 to get the size for distfiles."""
 625                 return self.fhashdict[ftype][fname][key]
 626
 627         def getVersions(self):
 628                 """ Returns a list of manifest versions present in the manifest file. """
 629                 rVal = []
 630                 mfname = self.getFullname()
 631                 if not os.path.exists(mfname):
 632                         return rVal
 633                 myfile = io.open(_unicode_encode(mfname,
 634                         encoding=_encodings['fs'], errors='strict'),
 635                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 636                 lines = myfile.readlines()
 637                 myfile.close()
 638                 for l in lines:
 639                         mysplit = l.split()
 640                         if len(mysplit) == 4 and mysplit[0] in MANIFEST1_HASH_FUNCTIONS \
 641                                 and 1 not in rVal:
 642                                 rVal.append(1)
 643                         elif len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS \
 644                                 and ((len(mysplit) - 3) % 2) == 0 and not 2 in rVal:
 645                                 rVal.append(2)
 646                 return rVal
 647
 648         def _catsplit(self, pkg_key):
 649                 """Split a category and package, returning a list of [cat, pkg].
 650                 This is compatible with portage.catsplit()"""
 651                 return pkg_key.split("/", 1)