pym/portage/dbapi/bintree.py

   1 # Copyright 1998-2012 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 __all__ = ["bindbapi", "binarytree"]
   5
   6 import portage
   7 portage.proxy.lazyimport.lazyimport(globals(),
   8         'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all',
   9         'portage.dbapi.dep_expand:dep_expand',
  10         'portage.dep:dep_getkey,isjustname,match_from_list',
  11         'portage.output:EOutput,colorize',
  12         'portage.locks:lockfile,unlockfile',
  13         'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms',
  14         'portage.package.ebuild.fetch:_check_distfile',
  15         'portage.update:update_dbentries',
  16         'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
  17                 'writemsg,writemsg_stdout',
  18         'portage.util.listdir:listdir',
  19         'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
  20 )
  21
  22 from portage.cache.mappings import slot_dict_class
  23 from portage.const import CACHE_PATH
  24 from portage.dbapi.virtual import fakedbapi
  25 from portage.dep import Atom, use_reduce, paren_enclose
  26 from portage.exception import AlarmSignal, InvalidPackageName, \
  27         PermissionDenied, PortageException
  28 from portage.localization import _
  29 from portage.util import urlopen
  30 from portage import _movefile
  31 from portage import os
  32 from portage import _encodings
  33 from portage import _unicode_decode
  34 from portage import _unicode_encode
  35
  36 import codecs
  37 import errno
  38 import io
  39 import re
  40 import stat
  41 import subprocess
  42 import sys
  43 import tempfile
  44 import textwrap
  45 import warnings
  46 from itertools import chain
  47 try:
  48         from urllib.parse import urlparse
  49 except ImportError:
  50         from urlparse import urlparse
  51
  52 if sys.hexversion >= 0x3000000:
  53         basestring = str
  54         long = int
  55
  56 class bindbapi(fakedbapi):
  57         _known_keys = frozenset(list(fakedbapi._known_keys) + \
  58                 ["CHOST", "repository", "USE"])
  59         def __init__(self, mybintree=None, **kwargs):
  60                 fakedbapi.__init__(self, **kwargs)
  61                 self.bintree = mybintree
  62                 self.move_ent = mybintree.move_ent
  63                 self.cpvdict={}
  64                 self.cpdict={}
  65                 # Selectively cache metadata in order to optimize dep matching.
  66                 self._aux_cache_keys = set(
  67                         ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
  68                         "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE",
  69                         "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES",
  70                         ])
  71                 self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
  72                 self._aux_cache = {}
  73
  74         def match(self, *pargs, **kwargs):
  75                 if self.bintree and not self.bintree.populated:
  76                         self.bintree.populate()
  77                 return fakedbapi.match(self, *pargs, **kwargs)
  78
  79         def cpv_exists(self, cpv, myrepo=None):
  80                 if self.bintree and not self.bintree.populated:
  81                         self.bintree.populate()
  82                 return fakedbapi.cpv_exists(self, cpv)
  83
  84         def cpv_inject(self, cpv, **kwargs):
  85                 self._aux_cache.pop(cpv, None)
  86                 fakedbapi.cpv_inject(self, cpv, **kwargs)
  87
  88         def cpv_remove(self, cpv):
  89                 self._aux_cache.pop(cpv, None)
  90                 fakedbapi.cpv_remove(self, cpv)
  91
  92         def aux_get(self, mycpv, wants, myrepo=None):
  93                 if self.bintree and not self.bintree.populated:
  94                         self.bintree.populate()
  95                 cache_me = False
  96                 if not self._known_keys.intersection(
  97                         wants).difference(self._aux_cache_keys):
  98                         aux_cache = self._aux_cache.get(mycpv)
  99                         if aux_cache is not None:
 100                                 return [aux_cache.get(x, "") for x in wants]
 101                         cache_me = True
 102                 mysplit = mycpv.split("/")
 103                 mylist = []
 104                 tbz2name = mysplit[1]+".tbz2"
 105                 if not self.bintree._remotepkgs or \
 106                         not self.bintree.isremote(mycpv):
 107                         tbz2_path = self.bintree.getname(mycpv)
 108                         if not os.path.exists(tbz2_path):
 109                                 raise KeyError(mycpv)
 110                         metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data()
 111                         def getitem(k):
 112                                 v = metadata_bytes.get(_unicode_encode(k,
 113                                         encoding=_encodings['repo.content'],
 114                                         errors='backslashreplace'))
 115                                 if v is not None:
 116                                         v = _unicode_decode(v,
 117                                                 encoding=_encodings['repo.content'], errors='replace')
 118                                 return v
 119                 else:
 120                         getitem = self.bintree._remotepkgs[mycpv].get
 121                 mydata = {}
 122                 mykeys = wants
 123                 if cache_me:
 124                         mykeys = self._aux_cache_keys.union(wants)
 125                 for x in mykeys:
 126                         myval = getitem(x)
 127                         # myval is None if the key doesn't exist
 128                         # or the tbz2 is corrupt.
 129                         if myval:
 130                                 mydata[x] = " ".join(myval.split())
 131
 132                 if not mydata.setdefault('EAPI', _unicode_decode('0')):
 133                         mydata['EAPI'] = _unicode_decode('0')
 134
 135                 if cache_me:
 136                         aux_cache = self._aux_cache_slot_dict()
 137                         for x in self._aux_cache_keys:
 138                                 aux_cache[x] = mydata.get(x, _unicode_decode(''))
 139                         self._aux_cache[mycpv] = aux_cache
 140                 return [mydata.get(x, _unicode_decode('')) for x in wants]
 141
 142         def aux_update(self, cpv, values):
 143                 if not self.bintree.populated:
 144                         self.bintree.populate()
 145                 tbz2path = self.bintree.getname(cpv)
 146                 if not os.path.exists(tbz2path):
 147                         raise KeyError(cpv)
 148                 mytbz2 = portage.xpak.tbz2(tbz2path)
 149                 mydata = mytbz2.get_data()
 150
 151                 for k, v in values.items():
 152                         k = _unicode_encode(k,
 153                                 encoding=_encodings['repo.content'], errors='backslashreplace')
 154                         v = _unicode_encode(v,
 155                                 encoding=_encodings['repo.content'], errors='backslashreplace')
 156                         mydata[k] = v
 157
 158                 for k, v in list(mydata.items()):
 159                         if not v:
 160                                 del mydata[k]
 161                 mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 162                 # inject will clear stale caches via cpv_inject.
 163                 self.bintree.inject(cpv)
 164
 165         def cp_list(self, *pargs, **kwargs):
 166                 if not self.bintree.populated:
 167                         self.bintree.populate()
 168                 return fakedbapi.cp_list(self, *pargs, **kwargs)
 169
 170         def cp_all(self):
 171                 if not self.bintree.populated:
 172                         self.bintree.populate()
 173                 return fakedbapi.cp_all(self)
 174
 175         def cpv_all(self):
 176                 if not self.bintree.populated:
 177                         self.bintree.populate()
 178                 return fakedbapi.cpv_all(self)
 179
 180         def getfetchsizes(self, pkg):
 181                 """
 182                 This will raise MissingSignature if SIZE signature is not available,
 183                 or InvalidSignature if SIZE signature is invalid.
 184                 """
 185
 186                 if not self.bintree.populated:
 187                         self.bintree.populate()
 188
 189                 pkg = getattr(pkg, 'cpv', pkg)
 190
 191                 filesdict = {}
 192                 if not self.bintree.isremote(pkg):
 193                         pass
 194                 else:
 195                         metadata = self.bintree._remotepkgs[pkg]
 196                         try:
 197                                 size = int(metadata["SIZE"])
 198                         except KeyError:
 199                                 raise portage.exception.MissingSignature("SIZE")
 200                         except ValueError:
 201                                 raise portage.exception.InvalidSignature(
 202                                         "SIZE: %s" % metadata["SIZE"])
 203                         else:
 204                                 filesdict[os.path.basename(self.bintree.getname(pkg))] = size
 205
 206                 return filesdict
 207
 208 def _pkgindex_cpv_map_latest_build(pkgindex):
 209         """
 210         Given a PackageIndex instance, create a dict of cpv -> metadata map.
 211         If multiple packages have identical CPV values, prefer the package
 212         with latest BUILD_TIME value.
 213         @param pkgindex: A PackageIndex instance.
 214         @type pkgindex: PackageIndex
 215         @rtype: dict
 216         @return: a dict containing entry for the give cpv.
 217         """
 218         cpv_map = {}
 219
 220         for d in pkgindex.packages:
 221                 cpv = d["CPV"]
 222
 223                 btime = d.get('BUILD_TIME', '')
 224                 try:
 225                         btime = int(btime)
 226                 except ValueError:
 227                         btime = None
 228
 229                 other_d = cpv_map.get(cpv)
 230                 if other_d is not None:
 231                         other_btime = other_d.get('BUILD_TIME', '')
 232                         try:
 233                                 other_btime = int(other_btime)
 234                         except ValueError:
 235                                 other_btime = None
 236                         if other_btime and (not btime or other_btime > btime):
 237                                 continue
 238
 239                 cpv_map[_pkg_str(cpv)] = d
 240
 241         return cpv_map
 242
 243 class binarytree(object):
 244         "this tree scans for a list of all packages available in PKGDIR"
 245         def __init__(self, _unused=None, pkgdir=None,
 246                 virtual=DeprecationWarning, settings=None):
 247
 248                 if pkgdir is None:
 249                         raise TypeError("pkgdir parameter is required")
 250
 251                 if settings is None:
 252                         raise TypeError("settings parameter is required")
 253
 254                 if _unused is not None and _unused != settings['ROOT']:
 255                         warnings.warn("The root parameter of the "
 256                                 "portage.dbapi.bintree.binarytree"
 257                                 " constructor is now unused. Use "
 258                                 "settings['ROOT'] instead.",
 259                                 DeprecationWarning, stacklevel=2)
 260
 261                 if virtual is not DeprecationWarning:
 262                         warnings.warn("The 'virtual' parameter of the "
 263                                 "portage.dbapi.bintree.binarytree"
 264                                 " constructor is unused",
 265                                 DeprecationWarning, stacklevel=2)
 266
 267                 if True:
 268                         self.pkgdir = normalize_path(pkgdir)
 269                         self.dbapi = bindbapi(self, settings=settings)
 270                         self.update_ents = self.dbapi.update_ents
 271                         self.move_slot_ent = self.dbapi.move_slot_ent
 272                         self.populated = 0
 273                         self.tree = {}
 274                         self._remote_has_index = False
 275                         self._remotepkgs = None # remote metadata indexed by cpv
 276                         self.invalids = []
 277                         self.settings = settings
 278                         self._pkg_paths = {}
 279                         self._pkgindex_uri = {}
 280                         self._populating = False
 281                         self._all_directory = os.path.isdir(
 282                                 os.path.join(self.pkgdir, "All"))
 283                         self._pkgindex_version = 0
 284                         self._pkgindex_hashes = ["MD5","SHA1"]
 285                         self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
 286                         self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
 287                         self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"])
 288                         self._pkgindex_aux_keys = \
 289                                 ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
 290                                 "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
 291                                 "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
 292                                 "BASE_URI"]
 293                         self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 294                         self._pkgindex_use_evaluated_keys = \
 295                                 ("LICENSE", "RDEPEND", "DEPEND",
 296                                 "PDEPEND", "PROPERTIES", "PROVIDE")
 297                         self._pkgindex_header_keys = set([
 298                                 "ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 299                                 "ACCEPT_PROPERTIES", "CBUILD",
 300                                 "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
 301                                 "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"])
 302                         self._pkgindex_default_pkg_data = {
 303                                 "BUILD_TIME"         : "",
 304                                 "DEPEND"  : "",
 305                                 "EAPI"    : "0",
 306                                 "IUSE"    : "",
 307                                 "KEYWORDS": "",
 308                                 "LICENSE" : "",
 309                                 "PATH"    : "",
 310                                 "PDEPEND" : "",
 311                                 "PROPERTIES" : "",
 312                                 "PROVIDE" : "",
 313                                 "RDEPEND" : "",
 314                                 "RESTRICT": "",
 315                                 "SLOT"    : "0",
 316                                 "USE"     : "",
 317                                 "DEFINED_PHASES" : "",
 318                         }
 319                         self._pkgindex_inherited_keys = ["CHOST", "repository"]
 320
 321                         # Populate the header with appropriate defaults.
 322                         self._pkgindex_default_header_data = {
 323                                 "CHOST"        : self.settings.get("CHOST", ""),
 324                                 "repository"   : "",
 325                         }
 326
 327                         # It is especially important to populate keys like
 328                         # "repository" that save space when entries can
 329                         # inherit them from the header. If an existing
 330                         # pkgindex header already defines these keys, then
 331                         # they will appropriately override our defaults.
 332                         main_repo = self.settings.repositories.mainRepo()
 333                         if main_repo is not None and not main_repo.missing_repo_name:
 334                                 self._pkgindex_default_header_data["repository"] = \
 335                                         main_repo.name
 336
 337                         self._pkgindex_translated_keys = (
 338                                 ("DESCRIPTION"   ,   "DESC"),
 339                                 ("repository"    ,   "REPO"),
 340                         )
 341
 342                         self._pkgindex_allowed_pkg_keys = set(chain(
 343                                 self._pkgindex_keys,
 344                                 self._pkgindex_aux_keys,
 345                                 self._pkgindex_hashes,
 346                                 self._pkgindex_default_pkg_data,
 347                                 self._pkgindex_inherited_keys,
 348                                 chain(*self._pkgindex_translated_keys)
 349                         ))
 350
 351         @property
 352         def root(self):
 353                 warnings.warn("The root attribute of "
 354                         "portage.dbapi.bintree.binarytree"
 355                         " is deprecated. Use "
 356                         "settings['ROOT'] instead.",
 357                         DeprecationWarning, stacklevel=3)
 358                 return self.settings['ROOT']
 359
 360         def move_ent(self, mylist, repo_match=None):
 361                 if not self.populated:
 362                         self.populate()
 363                 origcp = mylist[1]
 364                 newcp = mylist[2]
 365                 # sanity check
 366                 for atom in (origcp, newcp):
 367                         if not isjustname(atom):
 368                                 raise InvalidPackageName(str(atom))
 369                 mynewcat = catsplit(newcp)[0]
 370                 origmatches=self.dbapi.cp_list(origcp)
 371                 moves = 0
 372                 if not origmatches:
 373                         return moves
 374                 for mycpv in origmatches:
 375                         mycpv_cp = portage.cpv_getkey(mycpv)
 376                         if mycpv_cp != origcp:
 377                                 # Ignore PROVIDE virtual match.
 378                                 continue
 379                         if repo_match is not None \
 380                                 and not repo_match(self.dbapi.aux_get(mycpv,
 381                                         ['repository'])[0]):
 382                                 continue
 383                         mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1)
 384                         myoldpkg = catsplit(mycpv)[1]
 385                         mynewpkg = catsplit(mynewcpv)[1]
 386
 387                         if (mynewpkg != myoldpkg) and os.path.exists(self.getname(mynewcpv)):
 388                                 writemsg(_("!!! Cannot update binary: Destination exists.\n"),
 389                                         noiselevel=-1)
 390                                 writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", noiselevel=-1)
 391                                 continue
 392
 393                         tbz2path = self.getname(mycpv)
 394                         if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK):
 395                                 writemsg(_("!!! Cannot update readonly binary: %s\n") % mycpv,
 396                                         noiselevel=-1)
 397                                 continue
 398
 399                         moves += 1
 400                         mytbz2 = portage.xpak.tbz2(tbz2path)
 401                         mydata = mytbz2.get_data()
 402                         updated_items = update_dbentries([mylist], mydata)
 403                         mydata.update(updated_items)
 404                         mydata[b'PF'] = \
 405                                 _unicode_encode(mynewpkg + "\n",
 406                                 encoding=_encodings['repo.content'])
 407                         mydata[b'CATEGORY'] = \
 408                                 _unicode_encode(mynewcat + "\n",
 409                                 encoding=_encodings['repo.content'])
 410                         if mynewpkg != myoldpkg:
 411                                 ebuild_data = mydata.pop(_unicode_encode(myoldpkg + '.ebuild',
 412                                         encoding=_encodings['repo.content']), None)
 413                                 if ebuild_data is not None:
 414                                         mydata[_unicode_encode(mynewpkg + '.ebuild',
 415                                                 encoding=_encodings['repo.content'])] = ebuild_data
 416
 417                         mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 418
 419                         self.dbapi.cpv_remove(mycpv)
 420                         del self._pkg_paths[mycpv]
 421                         new_path = self.getname(mynewcpv)
 422                         self._pkg_paths[mynewcpv] = os.path.join(
 423                                 *new_path.split(os.path.sep)[-2:])
 424                         if new_path != mytbz2:
 425                                 self._ensure_dir(os.path.dirname(new_path))
 426                                 _movefile(tbz2path, new_path, mysettings=self.settings)
 427                                 self._remove_symlink(mycpv)
 428                                 if new_path.split(os.path.sep)[-2] == "All":
 429                                         self._create_symlink(mynewcpv)
 430                         self.inject(mynewcpv)
 431
 432                 return moves
 433
 434         def _remove_symlink(self, cpv):
 435                 """Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove
 436                 the ${PKGDIR}/${CATEGORY} directory if empty.  The file will not be
 437                 removed if os.path.islink() returns False."""
 438                 mycat, mypkg = catsplit(cpv)
 439                 mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
 440                 if os.path.islink(mylink):
 441                         """Only remove it if it's really a link so that this method never
 442                         removes a real package that was placed here to avoid a collision."""
 443                         os.unlink(mylink)
 444                 try:
 445                         os.rmdir(os.path.join(self.pkgdir, mycat))
 446                 except OSError as e:
 447                         if e.errno not in (errno.ENOENT,
 448                                 errno.ENOTEMPTY, errno.EEXIST):
 449                                 raise
 450                         del e
 451
 452         def _create_symlink(self, cpv):
 453                 """Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and
 454                 ${PKGDIR}/${CATEGORY} directory, if necessary).  Any file that may
 455                 exist in the location of the symlink will first be removed."""
 456                 mycat, mypkg = catsplit(cpv)
 457                 full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
 458                 self._ensure_dir(os.path.dirname(full_path))
 459                 try:
 460                         os.unlink(full_path)
 461                 except OSError as e:
 462                         if e.errno != errno.ENOENT:
 463                                 raise
 464                         del e
 465                 os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path)
 466
 467         def prevent_collision(self, cpv):
 468                 """Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to
 469                 use for a given cpv.  If a collision will occur with an existing
 470                 package from another category, the existing package will be bumped to
 471                 ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist."""
 472                 if not self._all_directory:
 473                         return
 474
 475                 # Copy group permissions for new directories that
 476                 # may have been created.
 477                 for path in ("All", catsplit(cpv)[0]):
 478                         path = os.path.join(self.pkgdir, path)
 479                         self._ensure_dir(path)
 480                         if not os.access(path, os.W_OK):
 481                                 raise PermissionDenied("access('%s', W_OK)" % path)
 482
 483                 full_path = self.getname(cpv)
 484                 if "All" == full_path.split(os.path.sep)[-2]:
 485                         return
 486                 """Move a colliding package if it exists.  Code below this point only
 487                 executes in rare cases."""
 488                 mycat, mypkg = catsplit(cpv)
 489                 myfile = mypkg + ".tbz2"
 490                 mypath = os.path.join("All", myfile)
 491                 dest_path = os.path.join(self.pkgdir, mypath)
 492
 493                 try:
 494                         st = os.lstat(dest_path)
 495                 except OSError:
 496                         st = None
 497                 else:
 498                         if stat.S_ISLNK(st.st_mode):
 499                                 st = None
 500                                 try:
 501                                         os.unlink(dest_path)
 502                                 except OSError:
 503                                         if os.path.exists(dest_path):
 504                                                 raise
 505
 506                 if st is not None:
 507                         # For invalid packages, other_cat could be None.
 508                         other_cat = portage.xpak.tbz2(dest_path).getfile(b"CATEGORY")
 509                         if other_cat:
 510                                 other_cat = _unicode_decode(other_cat,
 511                                         encoding=_encodings['repo.content'], errors='replace')
 512                                 other_cat = other_cat.strip()
 513                                 other_cpv = other_cat + "/" + mypkg
 514                                 self._move_from_all(other_cpv)
 515                                 self.inject(other_cpv)
 516                 self._move_to_all(cpv)
 517
 518         def _ensure_dir(self, path):
 519                 """
 520                 Create the specified directory. Also, copy gid and group mode
 521                 bits from self.pkgdir if possible.
 522                 @param cat_dir: Absolute path of the directory to be created.
 523                 @type cat_dir: String
 524                 """
 525                 try:
 526                         pkgdir_st = os.stat(self.pkgdir)
 527                 except OSError:
 528                         ensure_dirs(path)
 529                         return
 530                 pkgdir_gid = pkgdir_st.st_gid
 531                 pkgdir_grp_mode = 0o2070 & pkgdir_st.st_mode
 532                 try:
 533                         ensure_dirs(path, gid=pkgdir_gid, mode=pkgdir_grp_mode, mask=0)
 534                 except PortageException:
 535                         if not os.path.isdir(path):
 536                                 raise
 537
 538         def _move_to_all(self, cpv):
 539                 """If the file exists, move it.  Whether or not it exists, update state
 540                 for future getname() calls."""
 541                 mycat, mypkg = catsplit(cpv)
 542                 myfile = mypkg + ".tbz2"
 543                 self._pkg_paths[cpv] = os.path.join("All", myfile)
 544                 src_path = os.path.join(self.pkgdir, mycat, myfile)
 545                 try:
 546                         mystat = os.lstat(src_path)
 547                 except OSError as e:
 548                         mystat = None
 549                 if mystat and stat.S_ISREG(mystat.st_mode):
 550                         self._ensure_dir(os.path.join(self.pkgdir, "All"))
 551                         dest_path = os.path.join(self.pkgdir, "All", myfile)
 552                         _movefile(src_path, dest_path, mysettings=self.settings)
 553                         self._create_symlink(cpv)
 554                         self.inject(cpv)
 555
 556         def _move_from_all(self, cpv):
 557                 """Move a package from ${PKGDIR}/All/${PF}.tbz2 to
 558                 ${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls."""
 559                 self._remove_symlink(cpv)
 560                 mycat, mypkg = catsplit(cpv)
 561                 myfile = mypkg + ".tbz2"
 562                 mypath = os.path.join(mycat, myfile)
 563                 dest_path = os.path.join(self.pkgdir, mypath)
 564                 self._ensure_dir(os.path.dirname(dest_path))
 565                 src_path = os.path.join(self.pkgdir, "All", myfile)
 566                 _movefile(src_path, dest_path, mysettings=self.settings)
 567                 self._pkg_paths[cpv] = mypath
 568
 569         def populate(self, getbinpkgs=0):
 570                 "populates the binarytree"
 571
 572                 if self._populating:
 573                         return
 574
 575                 pkgindex_lock = None
 576                 try:
 577                         if os.access(self.pkgdir, os.W_OK):
 578                                 pkgindex_lock = lockfile(self._pkgindex_file,
 579                                         wantnewlockfile=1)
 580                         self._populating = True
 581                         self._populate(getbinpkgs)
 582                 finally:
 583                         if pkgindex_lock:
 584                                 unlockfile(pkgindex_lock)
 585                         self._populating = False
 586
 587         def _populate(self, getbinpkgs=0):
 588                 if (not os.path.isdir(self.pkgdir) and not getbinpkgs):
 589                         return 0
 590
 591                 # Clear all caches in case populate is called multiple times
 592                 # as may be the case when _global_updates calls populate()
 593                 # prior to performing package moves since it only wants to
 594                 # operate on local packages (getbinpkgs=0).
 595                 self._remotepkgs = None
 596                 self.dbapi._clear_cache()
 597                 self.dbapi._aux_cache.clear()
 598                 if True:
 599                         pkg_paths = {}
 600                         self._pkg_paths = pkg_paths
 601                         dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True)
 602                         if "All" in dirs:
 603                                 dirs.remove("All")
 604                         dirs.sort()
 605                         dirs.insert(0, "All")
 606                         pkgindex = self._load_pkgindex()
 607                         pf_index = None
 608                         if not self._pkgindex_version_supported(pkgindex):
 609                                 pkgindex = self._new_pkgindex()
 610                         header = pkgindex.header
 611                         metadata = {}
 612                         for d in pkgindex.packages:
 613                                 metadata[d["CPV"]] = d
 614                         update_pkgindex = False
 615                         for mydir in dirs:
 616                                 for myfile in listdir(os.path.join(self.pkgdir, mydir)):
 617                                         if not myfile.endswith(".tbz2"):
 618                                                 continue
 619                                         mypath = os.path.join(mydir, myfile)
 620                                         full_path = os.path.join(self.pkgdir, mypath)
 621                                         s = os.lstat(full_path)
 622                                         if stat.S_ISLNK(s.st_mode):
 623                                                 continue
 624
 625                                         # Validate data from the package index and try to avoid
 626                                         # reading the xpak if possible.
 627                                         if mydir != "All":
 628                                                 possibilities = None
 629                                                 d = metadata.get(mydir+"/"+myfile[:-5])
 630                                                 if d:
 631                                                         possibilities = [d]
 632                                         else:
 633                                                 if pf_index is None:
 634                                                         pf_index = {}
 635                                                         for mycpv in metadata:
 636                                                                 mycat, mypf = catsplit(mycpv)
 637                                                                 pf_index.setdefault(
 638                                                                         mypf, []).append(metadata[mycpv])
 639                                                 possibilities = pf_index.get(myfile[:-5])
 640                                         if possibilities:
 641                                                 match = None
 642                                                 for d in possibilities:
 643                                                         try:
 644                                                                 if long(d["MTIME"]) != s[stat.ST_MTIME]:
 645                                                                         continue
 646                                                         except (KeyError, ValueError):
 647                                                                 continue
 648                                                         try:
 649                                                                 if long(d["SIZE"]) != long(s.st_size):
 650                                                                         continue
 651                                                         except (KeyError, ValueError):
 652                                                                 continue
 653                                                         if not self._pkgindex_keys.difference(d):
 654                                                                 match = d
 655                                                                 break
 656                                                 if match:
 657                                                         mycpv = match["CPV"]
 658                                                         if mycpv in pkg_paths:
 659                                                                 # discard duplicates (All/ is preferred)
 660                                                                 continue
 661                                                         mycpv = _pkg_str(mycpv)
 662                                                         pkg_paths[mycpv] = mypath
 663                                                         # update the path if the package has been moved
 664                                                         oldpath = d.get("PATH")
 665                                                         if oldpath and oldpath != mypath:
 666                                                                 update_pkgindex = True
 667                                                         if mypath != mycpv + ".tbz2":
 668                                                                 d["PATH"] = mypath
 669                                                                 if not oldpath:
 670                                                                         update_pkgindex = True
 671                                                         else:
 672                                                                 d.pop("PATH", None)
 673                                                                 if oldpath:
 674                                                                         update_pkgindex = True
 675                                                         self.dbapi.cpv_inject(mycpv)
 676                                                         if not self.dbapi._aux_cache_keys.difference(d):
 677                                                                 aux_cache = self.dbapi._aux_cache_slot_dict()
 678                                                                 for k in self.dbapi._aux_cache_keys:
 679                                                                         aux_cache[k] = d[k]
 680                                                                 self.dbapi._aux_cache[mycpv] = aux_cache
 681                                                         continue
 682                                         if not os.access(full_path, os.R_OK):
 683                                                 writemsg(_("!!! Permission denied to read " \
 684                                                         "binary package: '%s'\n") % full_path,
 685                                                         noiselevel=-1)
 686                                                 self.invalids.append(myfile[:-5])
 687                                                 continue
 688                                         metadata_bytes = portage.xpak.tbz2(full_path).get_data()
 689                                         mycat = _unicode_decode(metadata_bytes.get(b"CATEGORY", ""),
 690                                                 encoding=_encodings['repo.content'], errors='replace')
 691                                         mypf = _unicode_decode(metadata_bytes.get(b"PF", ""),
 692                                                 encoding=_encodings['repo.content'], errors='replace')
 693                                         slot = _unicode_decode(metadata_bytes.get(b"SLOT", ""),
 694                                                 encoding=_encodings['repo.content'], errors='replace')
 695                                         mypkg = myfile[:-5]
 696                                         if not mycat or not mypf or not slot:
 697                                                 #old-style or corrupt package
 698                                                 writemsg(_("\n!!! Invalid binary package: '%s'\n") % full_path,
 699                                                         noiselevel=-1)
 700                                                 missing_keys = []
 701                                                 if not mycat:
 702                                                         missing_keys.append("CATEGORY")
 703                                                 if not mypf:
 704                                                         missing_keys.append("PF")
 705                                                 if not slot:
 706                                                         missing_keys.append("SLOT")
 707                                                 msg = []
 708                                                 if missing_keys:
 709                                                         missing_keys.sort()
 710                                                         msg.append(_("Missing metadata key(s): %s.") % \
 711                                                                 ", ".join(missing_keys))
 712                                                 msg.append(_(" This binary package is not " \
 713                                                         "recoverable and should be deleted."))
 714                                                 for line in textwrap.wrap("".join(msg), 72):
 715                                                         writemsg("!!! %s\n" % line, noiselevel=-1)
 716                                                 self.invalids.append(mypkg)
 717                                                 continue
 718                                         mycat = mycat.strip()
 719                                         slot = slot.strip()
 720                                         if mycat != mydir and mydir != "All":
 721                                                 continue
 722                                         if mypkg != mypf.strip():
 723                                                 continue
 724                                         mycpv = mycat + "/" + mypkg
 725                                         if mycpv in pkg_paths:
 726                                                 # All is first, so it's preferred.
 727                                                 continue
 728                                         if not self.dbapi._category_re.match(mycat):
 729                                                 writemsg(_("!!! Binary package has an " \
 730                                                         "unrecognized category: '%s'\n") % full_path,
 731                                                         noiselevel=-1)
 732                                                 writemsg(_("!!! '%s' has a category that is not" \
 733                                                         " listed in %setc/portage/categories\n") % \
 734                                                         (mycpv, self.settings["PORTAGE_CONFIGROOT"]),
 735                                                         noiselevel=-1)
 736                                                 continue
 737                                         mycpv = _pkg_str(mycpv)
 738                                         pkg_paths[mycpv] = mypath
 739                                         self.dbapi.cpv_inject(mycpv)
 740                                         update_pkgindex = True
 741                                         d = metadata.get(mycpv, {})
 742                                         if d:
 743                                                 try:
 744                                                         if long(d["MTIME"]) != s[stat.ST_MTIME]:
 745                                                                 d.clear()
 746                                                 except (KeyError, ValueError):
 747                                                         d.clear()
 748                                         if d:
 749                                                 try:
 750                                                         if long(d["SIZE"]) != long(s.st_size):
 751                                                                 d.clear()
 752                                                 except (KeyError, ValueError):
 753                                                         d.clear()
 754
 755                                         d["CPV"] = mycpv
 756                                         d["SLOT"] = slot
 757                                         d["MTIME"] = str(s[stat.ST_MTIME])
 758                                         d["SIZE"] = str(s.st_size)
 759
 760                                         d.update(zip(self._pkgindex_aux_keys,
 761                                                 self.dbapi.aux_get(mycpv, self._pkgindex_aux_keys)))
 762                                         try:
 763                                                 self._eval_use_flags(mycpv, d)
 764                                         except portage.exception.InvalidDependString:
 765                                                 writemsg(_("!!! Invalid binary package: '%s'\n") % \
 766                                                         self.getname(mycpv), noiselevel=-1)
 767                                                 self.dbapi.cpv_remove(mycpv)
 768                                                 del pkg_paths[mycpv]
 769
 770                                         # record location if it's non-default
 771                                         if mypath != mycpv + ".tbz2":
 772                                                 d["PATH"] = mypath
 773                                         else:
 774                                                 d.pop("PATH", None)
 775                                         metadata[mycpv] = d
 776                                         if not self.dbapi._aux_cache_keys.difference(d):
 777                                                 aux_cache = self.dbapi._aux_cache_slot_dict()
 778                                                 for k in self.dbapi._aux_cache_keys:
 779                                                         aux_cache[k] = d[k]
 780                                                 self.dbapi._aux_cache[mycpv] = aux_cache
 781
 782                         for cpv in list(metadata):
 783                                 if cpv not in pkg_paths:
 784                                         del metadata[cpv]
 785
 786                         # Do not bother to write the Packages index if $PKGDIR/All/ exists
 787                         # since it will provide no benefit due to the need to read CATEGORY
 788                         # from xpak.
 789                         if update_pkgindex and os.access(self.pkgdir, os.W_OK):
 790                                 del pkgindex.packages[:]
 791                                 pkgindex.packages.extend(iter(metadata.values()))
 792                                 self._update_pkgindex_header(pkgindex.header)
 793                                 f = atomic_ofstream(self._pkgindex_file)
 794                                 pkgindex.write(f)
 795                                 f.close()
 796
 797                 if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
 798                         writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
 799                                 noiselevel=-1)
 800
 801                 if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
 802                         self.populated=1
 803                         return
 804                 self._remotepkgs = {}
 805                 for base_url in self.settings["PORTAGE_BINHOST"].split():
 806                         parsed_url = urlparse(base_url)
 807                         host = parsed_url.netloc
 808                         port = parsed_url.port
 809                         user = None
 810                         passwd = None
 811                         user_passwd = ""
 812                         if "@" in host:
 813                                 user, host = host.split("@", 1)
 814                                 user_passwd = user + "@"
 815                                 if ":" in user:
 816                                         user, passwd = user.split(":", 1)
 817                         port_args = []
 818                         if port is not None:
 819                                 port_str = ":%s" % (port,)
 820                                 if host.endswith(port_str):
 821                                         host = host[:-len(port_str)]
 822                         pkgindex_file = os.path.join(self.settings["EROOT"], CACHE_PATH, "binhost",
 823                                 host, parsed_url.path.lstrip("/"), "Packages")
 824                         pkgindex = self._new_pkgindex()
 825                         try:
 826                                 f = io.open(_unicode_encode(pkgindex_file,
 827                                         encoding=_encodings['fs'], errors='strict'),
 828                                         mode='r', encoding=_encodings['repo.content'],
 829                                         errors='replace')
 830                                 try:
 831                                         pkgindex.read(f)
 832                                 finally:
 833                                         f.close()
 834                         except EnvironmentError as e:
 835                                 if e.errno != errno.ENOENT:
 836                                         raise
 837                         local_timestamp = pkgindex.header.get("TIMESTAMP", None)
 838                         rmt_idx = self._new_pkgindex()
 839                         proc = None
 840                         tmp_filename = None
 841                         try:
 842                                 # urlparse.urljoin() only works correctly with recognized
 843                                 # protocols and requires the base url to have a trailing
 844                                 # slash, so join manually...
 845                                 url = base_url.rstrip("/") + "/Packages"
 846                                 try:
 847                                         f = urlopen(url)
 848                                 except IOError:
 849                                         path = parsed_url.path.rstrip("/") + "/Packages"
 850
 851                                         if parsed_url.scheme == 'sftp':
 852                                                 # The sftp command complains about 'Illegal seek' if
 853                                                 # we try to make it write to /dev/stdout, so use a
 854                                                 # temp file instead.
 855                                                 fd, tmp_filename = tempfile.mkstemp()
 856                                                 os.close(fd)
 857                                                 if port is not None:
 858                                                         port_args = ['-P', "%s" % (port,)]
 859                                                 proc = subprocess.Popen(['sftp'] + port_args + \
 860                                                         [user_passwd + host + ":" + path, tmp_filename])
 861                                                 if proc.wait() != os.EX_OK:
 862                                                         raise
 863                                                 f = open(tmp_filename, 'rb')
 864                                         elif parsed_url.scheme == 'ssh':
 865                                                 if port is not None:
 866                                                         port_args = ['-p', "%s" % (port,)]
 867                                                 proc = subprocess.Popen(['ssh'] + port_args + \
 868                                                         [user_passwd + host, '--', 'cat', path],
 869                                                         stdout=subprocess.PIPE)
 870                                                 f = proc.stdout
 871                                         else:
 872                                                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
 873                                                 fcmd = self.settings.get(setting)
 874                                                 if not fcmd:
 875                                                         raise
 876                                                 fd, tmp_filename = tempfile.mkstemp()
 877                                                 tmp_dirname, tmp_basename = os.path.split(tmp_filename)
 878                                                 os.close(fd)
 879                                                 success = portage.getbinpkg.file_get(url,
 880                                                      tmp_dirname, fcmd=fcmd, filename=tmp_basename)
 881                                                 if not success:
 882                                                         raise EnvironmentError("%s failed" % (setting,))
 883                                                 f = open(tmp_filename, 'rb')
 884
 885                                 f_dec = codecs.iterdecode(f,
 886                                         _encodings['repo.content'], errors='replace')
 887                                 try:
 888                                         rmt_idx.readHeader(f_dec)
 889                                         remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
 890                                         if not remote_timestamp:
 891                                                 # no timestamp in the header, something's wrong
 892                                                 pkgindex = None
 893                                                 writemsg(_("\n\n!!! Binhost package index " \
 894                                                 " has no TIMESTAMP field.\n"), noiselevel=-1)
 895                                         else:
 896                                                 if not self._pkgindex_version_supported(rmt_idx):
 897                                                         writemsg(_("\n\n!!! Binhost package index version" \
 898                                                         " is not supported: '%s'\n") % \
 899                                                         rmt_idx.header.get("VERSION"), noiselevel=-1)
 900                                                         pkgindex = None
 901                                                 elif local_timestamp != remote_timestamp:
 902                                                         rmt_idx.readBody(f_dec)
 903                                                         pkgindex = rmt_idx
 904                                 finally:
 905                                         # Timeout after 5 seconds, in case close() blocks
 906                                         # indefinitely (see bug #350139).
 907                                         try:
 908                                                 try:
 909                                                         AlarmSignal.register(5)
 910                                                         f.close()
 911                                                 finally:
 912                                                         AlarmSignal.unregister()
 913                                         except AlarmSignal:
 914                                                 writemsg("\n\n!!! %s\n" % \
 915                                                         _("Timed out while closing connection to binhost"),
 916                                                         noiselevel=-1)
 917                         except EnvironmentError as e:
 918                                 writemsg(_("\n\n!!! Error fetching binhost package" \
 919                                         " info from '%s'\n") % base_url)
 920                                 writemsg("!!! %s\n\n" % str(e))
 921                                 del e
 922                                 pkgindex = None
 923                         if proc is not None:
 924                                 if proc.poll() is None:
 925                                         proc.kill()
 926                                         proc.wait()
 927                                 proc = None
 928                         if tmp_filename is not None:
 929                                 try:
 930                                         os.unlink(tmp_filename)
 931                                 except OSError:
 932                                         pass
 933                         if pkgindex is rmt_idx:
 934                                 pkgindex.modified = False # don't update the header
 935                                 try:
 936                                         ensure_dirs(os.path.dirname(pkgindex_file))
 937                                         f = atomic_ofstream(pkgindex_file)
 938                                         pkgindex.write(f)
 939                                         f.close()
 940                                 except (IOError, PortageException):
 941                                         if os.access(os.path.dirname(pkgindex_file), os.W_OK):
 942                                                 raise
 943                                         # The current user doesn't have permission to cache the
 944                                         # file, but that's alright.
 945                         if pkgindex:
 946                                 # Organize remote package list as a cpv -> metadata map.
 947                                 remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
 948                                 remote_base_uri = pkgindex.header.get("URI", base_url)
 949                                 for cpv, remote_metadata in remotepkgs.items():
 950                                         remote_metadata["BASE_URI"] = remote_base_uri
 951                                         self._pkgindex_uri[cpv] = url
 952                                 self._remotepkgs.update(remotepkgs)
 953                                 self._remote_has_index = True
 954                                 for cpv in remotepkgs:
 955                                         self.dbapi.cpv_inject(cpv)
 956                                 if True:
 957                                         # Remote package instances override local package
 958                                         # if they are not identical.
 959                                         hash_names = ["SIZE"] + self._pkgindex_hashes
 960                                         for cpv, local_metadata in metadata.items():
 961                                                 remote_metadata = self._remotepkgs.get(cpv)
 962                                                 if remote_metadata is None:
 963                                                         continue
 964                                                 # Use digests to compare identity.
 965                                                 identical = True
 966                                                 for hash_name in hash_names:
 967                                                         local_value = local_metadata.get(hash_name)
 968                                                         if local_value is None:
 969                                                                 continue
 970                                                         remote_value = remote_metadata.get(hash_name)
 971                                                         if remote_value is None:
 972                                                                 continue
 973                                                         if local_value != remote_value:
 974                                                                 identical = False
 975                                                                 break
 976                                                 if identical:
 977                                                         del self._remotepkgs[cpv]
 978                                                 else:
 979                                                         # Override the local package in the aux_get cache.
 980                                                         self.dbapi._aux_cache[cpv] = remote_metadata
 981                                 else:
 982                                         # Local package instances override remote instances.
 983                                         for cpv in metadata:
 984                                                 self._remotepkgs.pop(cpv, None)
 985                                 continue
 986                         try:
 987                                 chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"])
 988                                 if chunk_size < 8:
 989                                         chunk_size = 8
 990                         except (ValueError, KeyError):
 991                                 chunk_size = 3000
 992                         writemsg_stdout("\n")
 993                         writemsg_stdout(
 994                                 colorize("GOOD", _("Fetching bininfo from ")) + \
 995                                 re.sub(r'//(.+):.+@(.+)/', r'//\1:*password*@\2/', base_url) + "\n")
 996                         remotepkgs = portage.getbinpkg.dir_get_metadata(
 997                                 base_url, chunk_size=chunk_size)
 998
 999                         for mypkg, remote_metadata in remotepkgs.items():
1000                                 mycat = remote_metadata.get("CATEGORY")
1001                                 if mycat is None:
1002                                         #old-style or corrupt package
1003                                         writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg,
1004                                                 noiselevel=-1)
1005                                         continue
1006                                 mycat = mycat.strip()
1007                                 fullpkg = mycat+"/"+mypkg[:-5]
1008
1009                                 if fullpkg in metadata:
1010                                         # When using this old protocol, comparison with the remote
1011                                         # package isn't supported, so the local package is always
1012                                         # preferred even if getbinpkgsonly is enabled.
1013                                         continue
1014
1015                                 if not self.dbapi._category_re.match(mycat):
1016                                         writemsg(_("!!! Remote binary package has an " \
1017                                                 "unrecognized category: '%s'\n") % fullpkg,
1018                                                 noiselevel=-1)
1019                                         writemsg(_("!!! '%s' has a category that is not" \
1020                                                 " listed in %setc/portage/categories\n") % \
1021                                                 (fullpkg, self.settings["PORTAGE_CONFIGROOT"]),
1022                                                 noiselevel=-1)
1023                                         continue
1024                                 mykey = portage.cpv_getkey(fullpkg)
1025                                 try:
1026                                         # invalid tbz2's can hurt things.
1027                                         self.dbapi.cpv_inject(fullpkg)
1028                                         for k, v in remote_metadata.items():
1029                                                 remote_metadata[k] = v.strip()
1030                                         remote_metadata["BASE_URI"] = base_url
1031
1032                                         # Eliminate metadata values with names that digestCheck
1033                                         # uses, since they are not valid when using the old
1034                                         # protocol. Typically this is needed for SIZE metadata
1035                                         # which corresponds to the size of the unpacked files
1036                                         # rather than the binpkg file size, triggering digest
1037                                         # verification failures as reported in bug #303211.
1038                                         remote_metadata.pop('SIZE', None)
1039                                         for k in portage.checksum.hashfunc_map:
1040                                                 remote_metadata.pop(k, None)
1041
1042                                         self._remotepkgs[fullpkg] = remote_metadata
1043                                 except SystemExit as e:
1044                                         raise
1045                                 except:
1046                                         writemsg(_("!!! Failed to inject remote binary package: %s\n") % fullpkg,
1047                                                 noiselevel=-1)
1048                                         continue
1049                 self.populated=1
1050
1051         def inject(self, cpv, filename=None):
1052                 """Add a freshly built package to the database.  This updates
1053                 $PKGDIR/Packages with the new package metadata (including MD5).
1054                 @param cpv: The cpv of the new package to inject
1055                 @type cpv: string
1056                 @param filename: File path of the package to inject, or None if it's
1057                         already in the location returned by getname()
1058                 @type filename: string
1059                 @rtype: None
1060                 """
1061                 mycat, mypkg = catsplit(cpv)
1062                 if not self.populated:
1063                         self.populate()
1064                 if filename is None:
1065                         full_path = self.getname(cpv)
1066                 else:
1067                         full_path = filename
1068                 try:
1069                         s = os.stat(full_path)
1070                 except OSError as e:
1071                         if e.errno != errno.ENOENT:
1072                                 raise
1073                         del e
1074                         writemsg(_("!!! Binary package does not exist: '%s'\n") % full_path,
1075                                 noiselevel=-1)
1076                         return
1077                 mytbz2 = portage.xpak.tbz2(full_path)
1078                 slot = mytbz2.getfile("SLOT")
1079                 if slot is None:
1080                         writemsg(_("!!! Invalid binary package: '%s'\n") % full_path,
1081                                 noiselevel=-1)
1082                         return
1083                 slot = slot.strip()
1084                 self.dbapi.cpv_inject(cpv)
1085
1086                 # Reread the Packages index (in case it's been changed by another
1087                 # process) and then updated it, all while holding a lock.
1088                 pkgindex_lock = None
1089                 created_symlink = False
1090                 try:
1091                         pkgindex_lock = lockfile(self._pkgindex_file,
1092                                 wantnewlockfile=1)
1093                         if filename is not None:
1094                                 new_filename = self.getname(cpv)
1095                                 try:
1096                                         samefile = os.path.samefile(filename, new_filename)
1097                                 except OSError:
1098                                         samefile = False
1099                                 if not samefile:
1100                                         self._ensure_dir(os.path.dirname(new_filename))
1101                                         _movefile(filename, new_filename, mysettings=self.settings)
1102                         if self._all_directory and \
1103                                 self.getname(cpv).split(os.path.sep)[-2] == "All":
1104                                 self._create_symlink(cpv)
1105                                 created_symlink = True
1106                         pkgindex = self._load_pkgindex()
1107
1108                         if not self._pkgindex_version_supported(pkgindex):
1109                                 pkgindex = self._new_pkgindex()
1110
1111                         # Discard remote metadata to ensure that _pkgindex_entry
1112                         # gets the local metadata. This also updates state for future
1113                         # isremote calls.
1114                         if self._remotepkgs is not None:
1115                                 self._remotepkgs.pop(cpv, None)
1116
1117                         # Discard cached metadata to ensure that _pkgindex_entry
1118                         # doesn't return stale metadata.
1119                         self.dbapi._aux_cache.pop(cpv, None)
1120
1121                         try:
1122                                 d = self._pkgindex_entry(cpv)
1123                         except portage.exception.InvalidDependString:
1124                                 writemsg(_("!!! Invalid binary package: '%s'\n") % \
1125                                         self.getname(cpv), noiselevel=-1)
1126                                 self.dbapi.cpv_remove(cpv)
1127                                 del self._pkg_paths[cpv]
1128                                 return
1129
1130                         # If found, remove package(s) with duplicate path.
1131                         path = d.get("PATH", "")
1132                         for i in range(len(pkgindex.packages) - 1, -1, -1):
1133                                 d2 = pkgindex.packages[i]
1134                                 if path and path == d2.get("PATH"):
1135                                         # Handle path collisions in $PKGDIR/All
1136                                         # when CPV is not identical.
1137                                         del pkgindex.packages[i]
1138                                 elif cpv == d2.get("CPV"):
1139                                         if path == d2.get("PATH", ""):
1140                                                 del pkgindex.packages[i]
1141                                         elif created_symlink and not d2.get("PATH", ""):
1142                                                 # Delete entry for the package that was just
1143                                                 # overwritten by a symlink to this package.
1144                                                 del pkgindex.packages[i]
1145
1146                         pkgindex.packages.append(d)
1147
1148                         self._update_pkgindex_header(pkgindex.header)
1149                         f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
1150                         pkgindex.write(f)
1151                         f.close()
1152                 finally:
1153                         if pkgindex_lock:
1154                                 unlockfile(pkgindex_lock)
1155
1156         def _pkgindex_entry(self, cpv):
1157                 """
1158                 Performs checksums and evaluates USE flag conditionals.
1159                 Raises InvalidDependString if necessary.
1160                 @rtype: dict
1161                 @return: a dict containing entry for the give cpv.
1162                 """
1163
1164                 pkg_path = self.getname(cpv)
1165
1166                 d = dict(zip(self._pkgindex_aux_keys,
1167                         self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
1168
1169                 d.update(perform_multiple_checksums(
1170                         pkg_path, hashes=self._pkgindex_hashes))
1171
1172                 d["CPV"] = cpv
1173                 st = os.stat(pkg_path)
1174                 d["MTIME"] = str(st[stat.ST_MTIME])
1175                 d["SIZE"] = str(st.st_size)
1176
1177                 rel_path = self._pkg_paths[cpv]
1178                 # record location if it's non-default
1179                 if rel_path != cpv + ".tbz2":
1180                         d["PATH"] = rel_path
1181
1182                 self._eval_use_flags(cpv, d)
1183                 return d
1184
1185         def _new_pkgindex(self):
1186                 return portage.getbinpkg.PackageIndex(
1187                         allowed_pkg_keys=self._pkgindex_allowed_pkg_keys,
1188                         default_header_data=self._pkgindex_default_header_data,
1189                         default_pkg_data=self._pkgindex_default_pkg_data,
1190                         inherited_keys=self._pkgindex_inherited_keys,
1191                         translated_keys=self._pkgindex_translated_keys)
1192
1193         def _update_pkgindex_header(self, header):
1194                 portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"]))
1195                 profiles_base = os.path.join(portdir, "profiles") + os.path.sep
1196                 if self.settings.profile_path:
1197                         profile_path = normalize_path(
1198                                 os.path.realpath(self.settings.profile_path))
1199                         if profile_path.startswith(profiles_base):
1200                                 profile_path = profile_path[len(profiles_base):]
1201                         header["PROFILE"] = profile_path
1202                 header["VERSION"] = str(self._pkgindex_version)
1203                 base_uri = self.settings.get("PORTAGE_BINHOST_HEADER_URI")
1204                 if base_uri:
1205                         header["URI"] = base_uri
1206                 else:
1207                         header.pop("URI", None)
1208                 for k in self._pkgindex_header_keys:
1209                         v = self.settings.get(k, None)
1210                         if v:
1211                                 header[k] = v
1212                         else:
1213                                 header.pop(k, None)
1214
1215         def _pkgindex_version_supported(self, pkgindex):
1216                 version = pkgindex.header.get("VERSION")
1217                 if version:
1218                         try:
1219                                 if int(version) <= self._pkgindex_version:
1220                                         return True
1221                         except ValueError:
1222                                 pass
1223                 return False
1224
1225         def _eval_use_flags(self, cpv, metadata):
1226                 use = frozenset(metadata["USE"].split())
1227                 raw_use = use
1228                 iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split())
1229                 use = [f for f in use if f in iuse]
1230                 use.sort()
1231                 metadata["USE"] = " ".join(use)
1232                 for k in self._pkgindex_use_evaluated_keys:
1233                         if k.endswith('DEPEND'):
1234                                 token_class = Atom
1235                         else:
1236                                 token_class = None
1237
1238                         try:
1239                                 deps = metadata[k]
1240                                 deps = use_reduce(deps, uselist=raw_use, token_class=token_class)
1241                                 deps = paren_enclose(deps)
1242                         except portage.exception.InvalidDependString as e:
1243                                 writemsg("%s: %s\n" % (k, str(e)),
1244                                         noiselevel=-1)
1245                                 raise
1246                         metadata[k] = deps
1247
1248         def exists_specific(self, cpv):
1249                 if not self.populated:
1250                         self.populate()
1251                 return self.dbapi.match(
1252                         dep_expand("="+cpv, mydb=self.dbapi, settings=self.settings))
1253
1254         def dep_bestmatch(self, mydep):
1255                 "compatibility method -- all matches, not just visible ones"
1256                 if not self.populated:
1257                         self.populate()
1258                 writemsg("\n\n", 1)
1259                 writemsg("mydep: %s\n" % mydep, 1)
1260                 mydep = dep_expand(mydep, mydb=self.dbapi, settings=self.settings)
1261                 writemsg("mydep: %s\n" % mydep, 1)
1262                 mykey = dep_getkey(mydep)
1263                 writemsg("mykey: %s\n" % mykey, 1)
1264                 mymatch = best(match_from_list(mydep,self.dbapi.cp_list(mykey)))
1265                 writemsg("mymatch: %s\n" % mymatch, 1)
1266                 if mymatch is None:
1267                         return ""
1268                 return mymatch
1269
1270         def getname(self, pkgname):
1271                 """Returns a file location for this package.  The default location is
1272                 ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2
1273                 in the rare event of a collision.  The prevent_collision() method can
1274                 be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a
1275                 specific cpv."""
1276                 if not self.populated:
1277                         self.populate()
1278                 mycpv = pkgname
1279                 mypath = self._pkg_paths.get(mycpv, None)
1280                 if mypath:
1281                         return os.path.join(self.pkgdir, mypath)
1282                 mycat, mypkg = catsplit(mycpv)
1283                 if self._all_directory:
1284                         mypath = os.path.join("All", mypkg + ".tbz2")
1285                         if mypath in self._pkg_paths.values():
1286                                 mypath = os.path.join(mycat, mypkg + ".tbz2")
1287                 else:
1288                         mypath = os.path.join(mycat, mypkg + ".tbz2")
1289                 self._pkg_paths[mycpv] = mypath # cache for future lookups
1290                 return os.path.join(self.pkgdir, mypath)
1291
1292         def isremote(self, pkgname):
1293                 """Returns true if the package is kept remotely and it has not been
1294                 downloaded (or it is only partially downloaded)."""
1295                 if self._remotepkgs is None or pkgname not in self._remotepkgs:
1296                         return False
1297                 # Presence in self._remotepkgs implies that it's remote. When a
1298                 # package is downloaded, state is updated by self.inject().
1299                 return True
1300
1301         def get_pkgindex_uri(self, pkgname):
1302                 """Returns the URI to the Packages file for a given package."""
1303                 return self._pkgindex_uri.get(pkgname)
1304
1305         def gettbz2(self, pkgname):
1306                 """Fetches the package from a remote site, if necessary.  Attempts to
1307                 resume if the file appears to be partially downloaded."""
1308                 tbz2_path = self.getname(pkgname)
1309                 tbz2name = os.path.basename(tbz2_path)
1310                 resume = False
1311                 if os.path.exists(tbz2_path):
1312                         if (tbz2name not in self.invalids):
1313                                 return
1314                         else:
1315                                 resume = True
1316                                 writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"),
1317                                         noiselevel=-1)
1318
1319                 mydest = os.path.dirname(self.getname(pkgname))
1320                 self._ensure_dir(mydest)
1321                 # urljoin doesn't work correctly with unrecognized protocols like sftp
1322                 if self._remote_has_index:
1323                         rel_url = self._remotepkgs[pkgname].get("PATH")
1324                         if not rel_url:
1325                                 rel_url = pkgname+".tbz2"
1326                         remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"]
1327                         url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
1328                 else:
1329                         url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
1330                 protocol = urlparse(url)[0]
1331                 fcmd_prefix = "FETCHCOMMAND"
1332                 if resume:
1333                         fcmd_prefix = "RESUMECOMMAND"
1334                 fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
1335                 if not fcmd:
1336                         fcmd = self.settings.get(fcmd_prefix)
1337                 success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
1338                 if not success:
1339                         try:
1340                                 os.unlink(self.getname(pkgname))
1341                         except OSError:
1342                                 pass
1343                         raise portage.exception.FileNotFound(mydest)
1344                 self.inject(pkgname)
1345
1346         def _load_pkgindex(self):
1347                 pkgindex = self._new_pkgindex()
1348                 try:
1349                         f = io.open(_unicode_encode(self._pkgindex_file,
1350                                 encoding=_encodings['fs'], errors='strict'),
1351                                 mode='r', encoding=_encodings['repo.content'],
1352                                 errors='replace')
1353                 except EnvironmentError:
1354                         pass
1355                 else:
1356                         try:
1357                                 pkgindex.read(f)
1358                         finally:
1359                                 f.close()
1360                 return pkgindex
1361
1362         def digestCheck(self, pkg):
1363                 """
1364                 Verify digests for the given package and raise DigestException
1365                 if verification fails.
1366                 @rtype: bool
1367                 @return: True if digests could be located, False otherwise.
1368                 """
1369                 cpv = pkg
1370                 if not isinstance(cpv, basestring):
1371                         cpv = pkg.cpv
1372                         pkg = None
1373
1374                 pkg_path = self.getname(cpv)
1375                 metadata = None
1376                 if self._remotepkgs is None or cpv not in self._remotepkgs:
1377                         for d in self._load_pkgindex().packages:
1378                                 if d["CPV"] == cpv:
1379                                         metadata = d
1380                                         break
1381                 else:
1382                         metadata = self._remotepkgs[cpv]
1383                 if metadata is None:
1384                         return False
1385
1386                 digests = {}
1387                 for k in hashfunc_map:
1388                         v = metadata.get(k)
1389                         if not v:
1390                                 continue
1391                         digests[k] = v
1392
1393                 if "SIZE" in metadata:
1394                         try:
1395                                 digests["size"] = int(metadata["SIZE"])
1396                         except ValueError:
1397                                 writemsg(_("!!! Malformed SIZE attribute in remote " \
1398                                 "metadata for '%s'\n") % cpv)
1399
1400                 if not digests:
1401                         return False
1402
1403                 eout = EOutput()
1404                 eout.quiet = self.settings.get("PORTAGE_QUIET") == "1"
1405                 ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0)
1406                 if not ok:
1407                         ok, reason = verify_all(pkg_path, digests)
1408                         if not ok:
1409                                 raise portage.exception.DigestException(
1410                                         (pkg_path,) + tuple(reason))
1411
1412                 return True
1413
1414         def getslot(self, mycatpkg):
1415                 "Get a slot for a catpkg; assume it exists."
1416                 myslot = ""
1417                 try:
1418                         myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0]
1419                 except SystemExit as e:
1420                         raise
1421                 except Exception as e:
1422                         pass
1423                 return myslot