pym/portage/manifest.py

   1 # Copyright 1999-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 import errno
   7 import io
   8 import re
   9 import sys
  10 import warnings
  11
  12 import portage
  13 portage.proxy.lazyimport.lazyimport(globals(),
  14         'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
  15                 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes',
  16         'portage.repository.config:_find_invalid_path_char',
  17         'portage.util:write_atomic',
  18 )
  19
  20 from portage import os
  21 from portage import _encodings
  22 from portage import _unicode_decode
  23 from portage import _unicode_encode
  24 from portage.exception import DigestException, FileNotFound, \
  25         InvalidDataType, MissingParameter, PermissionDenied, \
  26         PortageException, PortagePackageException
  27 from portage.const import (MANIFEST1_HASH_FUNCTIONS, MANIFEST2_HASH_DEFAULTS,
  28         MANIFEST2_HASH_FUNCTIONS, MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH)
  29 from portage.localization import _
  30
  31 _manifest_re = re.compile(
  32         r'^(' + '|'.join(MANIFEST2_IDENTIFIERS) + r') (.*)( \d+( \S+ \S+)+)$',
  33         re.UNICODE)
  34
  35 if sys.hexversion >= 0x3000000:
  36         _unicode = str
  37         basestring = str
  38 else:
  39         _unicode = unicode
  40
  41 class FileNotInManifestException(PortageException):
  42         pass
  43
  44 def manifest2AuxfileFilter(filename):
  45         filename = filename.strip(os.sep)
  46         mysplit = filename.split(os.path.sep)
  47         if "CVS" in mysplit:
  48                 return False
  49         for x in mysplit:
  50                 if x[:1] == '.':
  51                         return False
  52         return not filename[:7] == 'digest-'
  53
  54 def manifest2MiscfileFilter(filename):
  55         return not (filename == "Manifest" or filename.endswith(".ebuild"))
  56
  57 def guessManifestFileType(filename):
  58         """ Perform a best effort guess of which type the given filename is, avoid using this if possible """
  59         if filename.startswith("files" + os.sep + "digest-"):
  60                 return None
  61         if filename.startswith("files" + os.sep):
  62                 return "AUX"
  63         elif filename.endswith(".ebuild"):
  64                 return "EBUILD"
  65         elif filename in ["ChangeLog", "metadata.xml"]:
  66                 return "MISC"
  67         else:
  68                 return "DIST"
  69
  70 def guessThinManifestFileType(filename):
  71         type = guessManifestFileType(filename)
  72         if type != "DIST":
  73                 return None
  74         return "DIST"
  75
  76 def parseManifest2(line):
  77         if not isinstance(line, basestring):
  78                 line = ' '.join(line)
  79         myentry = None
  80         match = _manifest_re.match(line)
  81         if match is not None:
  82                 tokens = match.group(3).split()
  83                 hashes = dict(zip(tokens[1::2], tokens[2::2]))
  84                 hashes["size"] = int(tokens[0])
  85                 myentry = Manifest2Entry(type=match.group(1),
  86                         name=match.group(2), hashes=hashes)
  87         return myentry
  88
  89 class ManifestEntry(object):
  90         __slots__ = ("type", "name", "hashes")
  91         def __init__(self, **kwargs):
  92                 for k, v in kwargs.items():
  93                         setattr(self, k, v)
  94
  95 class Manifest2Entry(ManifestEntry):
  96         def __str__(self):
  97                 myline = " ".join([self.type, self.name, str(self.hashes["size"])])
  98                 myhashkeys = list(self.hashes)
  99                 myhashkeys.remove("size")
 100                 myhashkeys.sort()
 101                 for h in myhashkeys:
 102                         myline += " " + h + " " + str(self.hashes[h])
 103                 return myline
 104
 105         def __eq__(self, other):
 106                 if not isinstance(other, Manifest2Entry) or \
 107                         self.type != other.type or \
 108                         self.name != other.name or \
 109                         self.hashes != other.hashes:
 110                         return False
 111                 return True
 112
 113         def __ne__(self, other):
 114                 return not self.__eq__(other)
 115
 116         if sys.hexversion < 0x3000000:
 117
 118                 __unicode__ = __str__
 119
 120                 def __str__(self):
 121                         return _unicode_encode(self.__unicode__(),
 122                                 encoding=_encodings['repo.content'], errors='strict')
 123
 124 class Manifest(object):
 125         parsers = (parseManifest2,)
 126         def __init__(self, pkgdir, distdir=None, fetchlist_dict=None,
 127                 manifest1_compat=DeprecationWarning, from_scratch=False, thin=False,
 128                 allow_missing=False, allow_create=True, hashes=None,
 129                 find_invalid_path_char=None):
 130                 """ Create new Manifest instance for package in pkgdir.
 131                     Do not parse Manifest file if from_scratch == True (only for internal use)
 132                         The fetchlist_dict parameter is required only for generation of
 133                         a Manifest (not needed for parsing and checking sums).
 134                         If thin is specified, then the manifest carries only info for
 135                         distfiles."""
 136
 137                 if manifest1_compat is not DeprecationWarning:
 138                         warnings.warn("The manifest1_compat parameter of the "
 139                                 "portage.manifest.Manifest constructor is deprecated.",
 140                                 DeprecationWarning, stacklevel=2)
 141
 142                 if find_invalid_path_char is None:
 143                         find_invalid_path_char = _find_invalid_path_char
 144                 self._find_invalid_path_char = find_invalid_path_char
 145                 self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
 146                 self.fhashdict = {}
 147                 self.hashes = set()
 148
 149                 if hashes is None:
 150                         hashes = MANIFEST2_HASH_DEFAULTS
 151
 152                 self.hashes.update(hashes.intersection(MANIFEST2_HASH_FUNCTIONS))
 153                 self.hashes.difference_update(hashname for hashname in \
 154                         list(self.hashes) if hashname not in hashfunc_map)
 155                 self.hashes.add("size")
 156                 self.hashes.add(MANIFEST2_REQUIRED_HASH)
 157                 for t in MANIFEST2_IDENTIFIERS:
 158                         self.fhashdict[t] = {}
 159                 if not from_scratch:
 160                         self._read()
 161                 if fetchlist_dict != None:
 162                         self.fetchlist_dict = fetchlist_dict
 163                 else:
 164                         self.fetchlist_dict = {}
 165                 self.distdir = distdir
 166                 self.thin = thin
 167                 if thin:
 168                         self.guessType = guessThinManifestFileType
 169                 else:
 170                         self.guessType = guessManifestFileType
 171                 self.allow_missing = allow_missing
 172                 self.allow_create = allow_create
 173
 174         def getFullname(self):
 175                 """ Returns the absolute path to the Manifest file for this instance """
 176                 return os.path.join(self.pkgdir, "Manifest")
 177
 178         def getDigests(self):
 179                 """ Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue} """
 180                 rval = {}
 181                 for t in MANIFEST2_IDENTIFIERS:
 182                         rval.update(self.fhashdict[t])
 183                 return rval
 184
 185         def getTypeDigests(self, ftype):
 186                 """ Similar to getDigests(), but restricted to files of the given type. """
 187                 return self.fhashdict[ftype]
 188
 189         def _readManifest(self, file_path, myhashdict=None, **kwargs):
 190                 """Parse a manifest.  If myhashdict is given then data will be added too it.
 191                    Otherwise, a new dict will be created and returned."""
 192                 try:
 193                         with io.open(_unicode_encode(file_path,
 194                                 encoding=_encodings['fs'], errors='strict'), mode='r',
 195                                 encoding=_encodings['repo.content'], errors='replace') as f:
 196                                 if myhashdict is None:
 197                                         myhashdict = {}
 198                                 self._parseDigests(f, myhashdict=myhashdict, **kwargs)
 199                         return myhashdict
 200                 except (OSError, IOError) as e:
 201                         if e.errno == errno.ENOENT:
 202                                 raise FileNotFound(file_path)
 203                         else:
 204                                 raise
 205
 206         def _read(self):
 207                 """ Parse Manifest file for this instance """
 208                 try:
 209                         self._readManifest(self.getFullname(), myhashdict=self.fhashdict)
 210                 except FileNotFound:
 211                         pass
 212
 213         def _parseManifestLines(self, mylines):
 214                 """Parse manifest lines and return a list of manifest entries."""
 215                 for myline in mylines:
 216                         myentry = None
 217                         for parser in self.parsers:
 218                                 myentry = parser(myline)
 219                                 if myentry is not None:
 220                                         yield myentry
 221                                         break # go to the next line
 222
 223         def _parseDigests(self, mylines, myhashdict=None, mytype=None):
 224                 """Parse manifest entries and store the data in myhashdict.  If mytype
 225                 is specified, it will override the type for all parsed entries."""
 226                 if myhashdict is None:
 227                         myhashdict = {}
 228                 for myentry in self._parseManifestLines(mylines):
 229                         if mytype is None:
 230                                 myentry_type = myentry.type
 231                         else:
 232                                 myentry_type = mytype
 233                         myhashdict.setdefault(myentry_type, {})
 234                         myhashdict[myentry_type].setdefault(myentry.name, {})
 235                         myhashdict[myentry_type][myentry.name].update(myentry.hashes)
 236                 return myhashdict
 237
 238         def _getDigestData(self, distlist):
 239                 """create a hash dict for a specific list of files"""
 240                 myhashdict = {}
 241                 for myname in distlist:
 242                         for mytype in self.fhashdict:
 243                                 if myname in self.fhashdict[mytype]:
 244                                         myhashdict.setdefault(mytype, {})
 245                                         myhashdict[mytype].setdefault(myname, {})
 246                                         myhashdict[mytype][myname].update(self.fhashdict[mytype][myname])
 247                 return myhashdict
 248
 249         def _createManifestEntries(self):
 250                 valid_hashes = set(MANIFEST2_HASH_FUNCTIONS)
 251                 valid_hashes.add('size')
 252                 mytypes = list(self.fhashdict)
 253                 mytypes.sort()
 254                 for t in mytypes:
 255                         myfiles = list(self.fhashdict[t])
 256                         myfiles.sort()
 257                         for f in myfiles:
 258                                 myentry = Manifest2Entry(
 259                                         type=t, name=f, hashes=self.fhashdict[t][f].copy())
 260                                 for h in list(myentry.hashes):
 261                                         if h not in valid_hashes:
 262                                                 del myentry.hashes[h]
 263                                 yield myentry
 264
 265         def checkIntegrity(self):
 266                 for t in self.fhashdict:
 267                         for f in self.fhashdict[t]:
 268                                 if MANIFEST2_REQUIRED_HASH not in self.fhashdict[t][f]:
 269                                         raise MissingParameter(_("Missing %s checksum: %s %s") %
 270                                                 (MANIFEST2_REQUIRED_HASH, t, f))
 271
 272         def write(self, sign=False, force=False):
 273                 """ Write Manifest instance to disk, optionally signing it. Returns
 274                 True if the Manifest is actually written, and False if the write
 275                 is skipped due to existing Manifest being identical."""
 276                 rval = False
 277                 if not self.allow_create:
 278                         return rval
 279                 self.checkIntegrity()
 280                 try:
 281                         myentries = list(self._createManifestEntries())
 282                         update_manifest = True
 283                         if myentries and not force:
 284                                 try:
 285                                         f = io.open(_unicode_encode(self.getFullname(),
 286                                                 encoding=_encodings['fs'], errors='strict'),
 287                                                 mode='r', encoding=_encodings['repo.content'],
 288                                                 errors='replace')
 289                                         oldentries = list(self._parseManifestLines(f))
 290                                         f.close()
 291                                         if len(oldentries) == len(myentries):
 292                                                 update_manifest = False
 293                                                 for i in range(len(oldentries)):
 294                                                         if oldentries[i] != myentries[i]:
 295                                                                 update_manifest = True
 296                                                                 break
 297                                 except (IOError, OSError) as e:
 298                                         if e.errno == errno.ENOENT:
 299                                                 pass
 300                                         else:
 301                                                 raise
 302
 303                         if update_manifest:
 304                                 if myentries or not (self.thin or self.allow_missing):
 305                                         # If myentries is empty, don't write an empty manifest
 306                                         # when thin or allow_missing is enabled. Except for
 307                                         # thin manifests with no DIST entries, myentries is
 308                                         # non-empty for all currently known use cases.
 309                                         write_atomic(self.getFullname(), "".join("%s\n" %
 310                                                 _unicode(myentry) for myentry in myentries))
 311                                         rval = True
 312                                 else:
 313                                         # With thin manifest, there's no need to have
 314                                         # a Manifest file if there are no DIST entries.
 315                                         try:
 316                                                 os.unlink(self.getFullname())
 317                                         except OSError as e:
 318                                                 if e.errno != errno.ENOENT:
 319                                                         raise
 320                                         rval = True
 321
 322                         if sign:
 323                                 self.sign()
 324                 except (IOError, OSError) as e:
 325                         if e.errno == errno.EACCES:
 326                                 raise PermissionDenied(str(e))
 327                         raise
 328                 return rval
 329
 330         def sign(self):
 331                 """ Sign the Manifest """
 332                 raise NotImplementedError()
 333
 334         def validateSignature(self):
 335                 """ Validate signature on Manifest """
 336                 raise NotImplementedError()
 337
 338         def addFile(self, ftype, fname, hashdict=None, ignoreMissing=False):
 339                 """ Add entry to Manifest optionally using hashdict to avoid recalculation of hashes """
 340                 if ftype == "AUX" and not fname.startswith("files/"):
 341                         fname = os.path.join("files", fname)
 342                 if not os.path.exists(self.pkgdir+fname) and not ignoreMissing:
 343                         raise FileNotFound(fname)
 344                 if not ftype in MANIFEST2_IDENTIFIERS:
 345                         raise InvalidDataType(ftype)
 346                 if ftype == "AUX" and fname.startswith("files"):
 347                         fname = fname[6:]
 348                 self.fhashdict[ftype][fname] = {}
 349                 if hashdict != None:
 350                         self.fhashdict[ftype][fname].update(hashdict)
 351                 if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]:
 352                         self.updateFileHashes(ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing)
 353
 354         def removeFile(self, ftype, fname):
 355                 """ Remove given entry from Manifest """
 356                 del self.fhashdict[ftype][fname]
 357
 358         def hasFile(self, ftype, fname):
 359                 """ Return whether the Manifest contains an entry for the given type,filename pair """
 360                 return (fname in self.fhashdict[ftype])
 361
 362         def findFile(self, fname):
 363                 """ Return entrytype of the given file if present in Manifest or None if not present """
 364                 for t in MANIFEST2_IDENTIFIERS:
 365                         if fname in self.fhashdict[t]:
 366                                 return t
 367                 return None
 368
 369         def create(self, checkExisting=False, assumeDistHashesSometimes=False,
 370                 assumeDistHashesAlways=False, requiredDistfiles=[]):
 371                 """ Recreate this Manifest from scratch.  This will not use any
 372                 existing checksums unless assumeDistHashesSometimes or
 373                 assumeDistHashesAlways is true (assumeDistHashesSometimes will only
 374                 cause DIST checksums to be reused if the file doesn't exist in
 375                 DISTDIR).  The requiredDistfiles parameter specifies a list of
 376                 distfiles to raise a FileNotFound exception for (if no file or existing
 377                 checksums are available), and defaults to all distfiles when not
 378                 specified."""
 379                 if not self.allow_create:
 380                         return
 381                 if checkExisting:
 382                         self.checkAllHashes()
 383                 if assumeDistHashesSometimes or assumeDistHashesAlways:
 384                         distfilehashes = self.fhashdict["DIST"]
 385                 else:
 386                         distfilehashes = {}
 387                 self.__init__(self.pkgdir, distdir=self.distdir,
 388                         fetchlist_dict=self.fetchlist_dict, from_scratch=True,
 389                         thin=self.thin, allow_missing=self.allow_missing,
 390                         allow_create=self.allow_create, hashes=self.hashes,
 391                         find_invalid_path_char=self._find_invalid_path_char)
 392                 pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
 393                 cat = self._pkgdir_category()
 394
 395                 pkgdir = self.pkgdir
 396                 if self.thin:
 397                         cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir)
 398                 else:
 399                         cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir)
 400
 401                 distlist = set()
 402                 for cpv in cpvlist:
 403                         distlist.update(self._getCpvDistfiles(cpv))
 404
 405                 if requiredDistfiles is None:
 406                         # This allows us to force removal of stale digests for the
 407                         # ebuild --force digest option (no distfiles are required).
 408                         requiredDistfiles = set()
 409                 elif len(requiredDistfiles) == 0:
 410                         # repoman passes in an empty list, which implies that all distfiles
 411                         # are required.
 412                         requiredDistfiles = distlist.copy()
 413                 required_hash_types = set()
 414                 required_hash_types.add("size")
 415                 required_hash_types.add(MANIFEST2_REQUIRED_HASH)
 416                 for f in distlist:
 417                         fname = os.path.join(self.distdir, f)
 418                         mystat = None
 419                         try:
 420                                 mystat = os.stat(fname)
 421                         except OSError:
 422                                 pass
 423                         if f in distfilehashes and \
 424                                 not required_hash_types.difference(distfilehashes[f]) and \
 425                                 ((assumeDistHashesSometimes and mystat is None) or \
 426                                 (assumeDistHashesAlways and mystat is None) or \
 427                                 (assumeDistHashesAlways and mystat is not None and \
 428                                 set(distfilehashes[f]) == set(self.hashes) and \
 429                                 distfilehashes[f]["size"] == mystat.st_size)):
 430                                 self.fhashdict["DIST"][f] = distfilehashes[f]
 431                         else:
 432                                 try:
 433                                         self.fhashdict["DIST"][f] = perform_multiple_checksums(fname, self.hashes)
 434                                 except FileNotFound:
 435                                         if f in requiredDistfiles:
 436                                                 raise
 437
 438         def _is_cpv(self, cat, pn, filename):
 439                 if not filename.endswith(".ebuild"):
 440                         return None
 441                 pf = filename[:-7]
 442                 ps = portage.versions._pkgsplit(pf)
 443                 cpv = "%s/%s" % (cat, pf)
 444                 if not ps:
 445                         raise PortagePackageException(
 446                                 _("Invalid package name: '%s'") % cpv)
 447                 if ps[0] != pn:
 448                         raise PortagePackageException(
 449                                 _("Package name does not "
 450                                 "match directory name: '%s'") % cpv)
 451                 return cpv
 452
 453         def _update_thin_pkgdir(self, cat, pn, pkgdir):
 454                 for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
 455                         break
 456                 cpvlist = []
 457                 for f in pkgdir_files:
 458                         try:
 459                                 f = _unicode_decode(f,
 460                                         encoding=_encodings['fs'], errors='strict')
 461                         except UnicodeDecodeError:
 462                                 continue
 463                         if f[:1] == '.':
 464                                 continue
 465                         pf = self._is_cpv(cat, pn, f)
 466                         if pf is not None:
 467                                 cpvlist.append(pf)
 468                 return cpvlist
 469
 470         def _update_thick_pkgdir(self, cat, pn, pkgdir):
 471                 cpvlist = []
 472                 for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
 473                         break
 474                 for f in pkgdir_files:
 475                         try:
 476                                 f = _unicode_decode(f,
 477                                         encoding=_encodings['fs'], errors='strict')
 478                         except UnicodeDecodeError:
 479                                 continue
 480                         if f[:1] == ".":
 481                                 continue
 482                         pf = self._is_cpv(cat, pn, f)
 483                         if pf is not None:
 484                                 mytype = "EBUILD"
 485                                 cpvlist.append(pf)
 486                         elif self._find_invalid_path_char(f) == -1 and \
 487                                 manifest2MiscfileFilter(f):
 488                                 mytype = "MISC"
 489                         else:
 490                                 continue
 491                         self.fhashdict[mytype][f] = perform_multiple_checksums(self.pkgdir+f, self.hashes)
 492                 recursive_files = []
 493
 494                 pkgdir = self.pkgdir
 495                 cut_len = len(os.path.join(pkgdir, "files") + os.sep)
 496                 for parentdir, dirs, files in os.walk(os.path.join(pkgdir, "files")):
 497                         for f in files:
 498                                 try:
 499                                         f = _unicode_decode(f,
 500                                                 encoding=_encodings['fs'], errors='strict')
 501                                 except UnicodeDecodeError:
 502                                         continue
 503                                 full_path = os.path.join(parentdir, f)
 504                                 recursive_files.append(full_path[cut_len:])
 505                 for f in recursive_files:
 506                         if self._find_invalid_path_char(f) != -1 or \
 507                                 not manifest2AuxfileFilter(f):
 508                                 continue
 509                         self.fhashdict["AUX"][f] = perform_multiple_checksums(
 510                                 os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes)
 511                 return cpvlist
 512
 513         def _pkgdir_category(self):
 514                 return self.pkgdir.rstrip(os.sep).split(os.sep)[-2]
 515
 516         def _getAbsname(self, ftype, fname):
 517                 if ftype == "DIST":
 518                         absname = os.path.join(self.distdir, fname)
 519                 elif ftype == "AUX":
 520                         absname = os.path.join(self.pkgdir, "files", fname)
 521                 else:
 522                         absname = os.path.join(self.pkgdir, fname)
 523                 return absname
 524
 525         def checkAllHashes(self, ignoreMissingFiles=False):
 526                 for t in MANIFEST2_IDENTIFIERS:
 527                         self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles)
 528
 529         def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None):
 530                 for f in self.fhashdict[idtype]:
 531                         self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles,
 532                                 hash_filter=hash_filter)
 533
 534         def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None):
 535                 digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname])
 536                 if hash_filter is not None:
 537                         digests = _apply_hash_filter(digests, hash_filter)
 538                 try:
 539                         ok, reason = verify_all(self._getAbsname(ftype, fname), digests)
 540                         if not ok:
 541                                 raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason)))
 542                         return ok, reason
 543                 except FileNotFound as e:
 544                         if not ignoreMissing:
 545                                 raise
 546                         return False, _("File Not Found: '%s'") % str(e)
 547
 548         def checkCpvHashes(self, cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=False):
 549                 """ check the hashes for all files associated to the given cpv, include all
 550                 AUX files and optionally all MISC files. """
 551                 if not onlyDistfiles:
 552                         self.checkTypeHashes("AUX", ignoreMissingFiles=False)
 553                         if checkMiscfiles:
 554                                 self.checkTypeHashes("MISC", ignoreMissingFiles=False)
 555                         ebuildname = "%s.ebuild" % self._catsplit(cpv)[1]
 556                         self.checkFileHashes("EBUILD", ebuildname, ignoreMissing=False)
 557                 if checkDistfiles or onlyDistfiles:
 558                         for f in self._getCpvDistfiles(cpv):
 559                                 self.checkFileHashes("DIST", f, ignoreMissing=False)
 560
 561         def _getCpvDistfiles(self, cpv):
 562                 """ Get a list of all DIST files associated to the given cpv """
 563                 return self.fetchlist_dict[cpv]
 564
 565         def getDistfilesSize(self, fetchlist):
 566                 total_bytes = 0
 567                 for f in fetchlist:
 568                         total_bytes += int(self.fhashdict["DIST"][f]["size"])
 569                 return total_bytes
 570
 571         def updateFileHashes(self, ftype, fname, checkExisting=True, ignoreMissing=True, reuseExisting=False):
 572                 """ Regenerate hashes for the given file """
 573                 if checkExisting:
 574                         self.checkFileHashes(ftype, fname, ignoreMissing=ignoreMissing)
 575                 if not ignoreMissing and fname not in self.fhashdict[ftype]:
 576                         raise FileNotInManifestException(fname)
 577                 if fname not in self.fhashdict[ftype]:
 578                         self.fhashdict[ftype][fname] = {}
 579                 myhashkeys = list(self.hashes)
 580                 if reuseExisting:
 581                         for k in [h for h in self.fhashdict[ftype][fname] if h in myhashkeys]:
 582                                 myhashkeys.remove(k)
 583                 myhashes = perform_multiple_checksums(self._getAbsname(ftype, fname), myhashkeys)
 584                 self.fhashdict[ftype][fname].update(myhashes)
 585
 586         def updateTypeHashes(self, idtype, checkExisting=False, ignoreMissingFiles=True):
 587                 """ Regenerate all hashes for all files of the given type """
 588                 for fname in self.fhashdict[idtype]:
 589                         self.updateFileHashes(idtype, fname, checkExisting)
 590
 591         def updateAllHashes(self, checkExisting=False, ignoreMissingFiles=True):
 592                 """ Regenerate all hashes for all files in this Manifest. """
 593                 for idtype in MANIFEST2_IDENTIFIERS:
 594                         self.updateTypeHashes(idtype, checkExisting=checkExisting,
 595                                 ignoreMissingFiles=ignoreMissingFiles)
 596
 597         def updateCpvHashes(self, cpv, ignoreMissingFiles=True):
 598                 """ Regenerate all hashes associated to the given cpv (includes all AUX and MISC
 599                 files)."""
 600                 self.updateTypeHashes("AUX", ignoreMissingFiles=ignoreMissingFiles)
 601                 self.updateTypeHashes("MISC", ignoreMissingFiles=ignoreMissingFiles)
 602                 ebuildname = "%s.ebuild" % self._catsplit(cpv)[1]
 603                 self.updateFileHashes("EBUILD", ebuildname, ignoreMissingFiles=ignoreMissingFiles)
 604                 for f in self._getCpvDistfiles(cpv):
 605                         self.updateFileHashes("DIST", f, ignoreMissingFiles=ignoreMissingFiles)
 606
 607         def updateHashesGuessType(self, fname, *args, **kwargs):
 608                 """ Regenerate hashes for the given file (guesses the type and then
 609                 calls updateFileHashes)."""
 610                 mytype = self.guessType(fname)
 611                 if mytype == "AUX":
 612                         fname = fname[len("files" + os.sep):]
 613                 elif mytype is None:
 614                         return
 615                 myrealtype = self.findFile(fname)
 616                 if myrealtype is not None:
 617                         mytype = myrealtype
 618                 return self.updateFileHashes(mytype, fname, *args, **kwargs)
 619
 620         def getFileData(self, ftype, fname, key):
 621                 """ Return the value of a specific (type,filename,key) triple, mainly useful
 622                 to get the size for distfiles."""
 623                 return self.fhashdict[ftype][fname][key]
 624
 625         def getVersions(self):
 626                 """ Returns a list of manifest versions present in the manifest file. """
 627                 rVal = []
 628                 mfname = self.getFullname()
 629                 if not os.path.exists(mfname):
 630                         return rVal
 631                 myfile = io.open(_unicode_encode(mfname,
 632                         encoding=_encodings['fs'], errors='strict'),
 633                         mode='r', encoding=_encodings['repo.content'], errors='replace')
 634                 lines = myfile.readlines()
 635                 myfile.close()
 636                 for l in lines:
 637                         mysplit = l.split()
 638                         if len(mysplit) == 4 and mysplit[0] in MANIFEST1_HASH_FUNCTIONS \
 639                                 and 1 not in rVal:
 640                                 rVal.append(1)
 641                         elif len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS \
 642                                 and ((len(mysplit) - 3) % 2) == 0 and not 2 in rVal:
 643                                 rVal.append(2)
 644                 return rVal
 645
 646         def _catsplit(self, pkg_key):
 647                 """Split a category and package, returning a list of [cat, pkg].
 648                 This is compatible with portage.catsplit()"""
 649                 return pkg_key.split("/", 1)