pym/portage/dbapi/bintree.py

   1 # Copyright 1998-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 __all__ = ["bindbapi", "binarytree"]
   7
   8 import portage
   9 portage.proxy.lazyimport.lazyimport(globals(),
  10         'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
  11                 'verify_all,_apply_hash_filter,_hash_filter',
  12         'portage.dbapi.dep_expand:dep_expand',
  13         'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list',
  14         'portage.output:EOutput,colorize',
  15         'portage.locks:lockfile,unlockfile',
  16         'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd',
  17         'portage.update:update_dbentries',
  18         'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
  19                 'writemsg,writemsg_stdout',
  20         'portage.util.listdir:listdir',
  21         'portage.util._urlopen:urlopen@_urlopen',
  22         'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
  23 )
  24
  25 from portage.cache.mappings import slot_dict_class
  26 from portage.const import CACHE_PATH
  27 from portage.dbapi.virtual import fakedbapi
  28 from portage.dep import Atom, use_reduce, paren_enclose
  29 from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
  30         PermissionDenied, PortageException
  31 from portage.localization import _
  32 from portage import _movefile
  33 from portage import os
  34 from portage import _encodings
  35 from portage import _unicode_decode
  36 from portage import _unicode_encode
  37
  38 import codecs
  39 import errno
  40 import io
  41 import stat
  42 import subprocess
  43 import sys
  44 import tempfile
  45 import textwrap
  46 import warnings
  47 from gzip import GzipFile
  48 from itertools import chain
  49 try:
  50         from urllib.parse import urlparse
  51 except ImportError:
  52         from urlparse import urlparse
  53
  54 if sys.hexversion >= 0x3000000:
  55         _unicode = str
  56         basestring = str
  57         long = int
  58 else:
  59         _unicode = unicode
  60
  61 class UseCachedCopyOfRemoteIndex(Exception):
  62         # If the local copy is recent enough
  63         # then fetching the remote index can be skipped.
  64         pass
  65
  66 class bindbapi(fakedbapi):
  67         _known_keys = frozenset(list(fakedbapi._known_keys) + \
  68                 ["CHOST", "repository", "USE"])
  69         def __init__(self, mybintree=None, **kwargs):
  70                 fakedbapi.__init__(self, **kwargs)
  71                 self.bintree = mybintree
  72                 self.move_ent = mybintree.move_ent
  73                 self.cpvdict={}
  74                 self.cpdict={}
  75                 # Selectively cache metadata in order to optimize dep matching.
  76                 self._aux_cache_keys = set(
  77                         ["BUILD_TIME", "CHOST", "DEPEND", "EAPI",
  78                         "HDEPEND", "IUSE", "KEYWORDS",
  79                         "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE",
  80                         "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES"
  81                         ])
  82                 self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
  83                 self._aux_cache = {}
  84
  85         def match(self, *pargs, **kwargs):
  86                 if self.bintree and not self.bintree.populated:
  87                         self.bintree.populate()
  88                 return fakedbapi.match(self, *pargs, **kwargs)
  89
  90         def cpv_exists(self, cpv, myrepo=None):
  91                 if self.bintree and not self.bintree.populated:
  92                         self.bintree.populate()
  93                 return fakedbapi.cpv_exists(self, cpv)
  94
  95         def cpv_inject(self, cpv, **kwargs):
  96                 self._aux_cache.pop(cpv, None)
  97                 fakedbapi.cpv_inject(self, cpv, **kwargs)
  98
  99         def cpv_remove(self, cpv):
 100                 self._aux_cache.pop(cpv, None)
 101                 fakedbapi.cpv_remove(self, cpv)
 102
 103         def aux_get(self, mycpv, wants, myrepo=None):
 104                 if self.bintree and not self.bintree.populated:
 105                         self.bintree.populate()
 106                 cache_me = False
 107                 if not self._known_keys.intersection(
 108                         wants).difference(self._aux_cache_keys):
 109                         aux_cache = self._aux_cache.get(mycpv)
 110                         if aux_cache is not None:
 111                                 return [aux_cache.get(x, "") for x in wants]
 112                         cache_me = True
 113                 mysplit = mycpv.split("/")
 114                 mylist = []
 115                 tbz2name = mysplit[1]+".tbz2"
 116                 if not self.bintree._remotepkgs or \
 117                         not self.bintree.isremote(mycpv):
 118                         tbz2_path = self.bintree.getname(mycpv)
 119                         if not os.path.exists(tbz2_path):
 120                                 raise KeyError(mycpv)
 121                         metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data()
 122                         def getitem(k):
 123                                 v = metadata_bytes.get(_unicode_encode(k,
 124                                         encoding=_encodings['repo.content'],
 125                                         errors='backslashreplace'))
 126                                 if v is not None:
 127                                         v = _unicode_decode(v,
 128                                                 encoding=_encodings['repo.content'], errors='replace')
 129                                 return v
 130                 else:
 131                         getitem = self.bintree._remotepkgs[mycpv].get
 132                 mydata = {}
 133                 mykeys = wants
 134                 if cache_me:
 135                         mykeys = self._aux_cache_keys.union(wants)
 136                 for x in mykeys:
 137                         myval = getitem(x)
 138                         # myval is None if the key doesn't exist
 139                         # or the tbz2 is corrupt.
 140                         if myval:
 141                                 mydata[x] = " ".join(myval.split())
 142
 143                 if not mydata.setdefault('EAPI', '0'):
 144                         mydata['EAPI'] = '0'
 145
 146                 if cache_me:
 147                         aux_cache = self._aux_cache_slot_dict()
 148                         for x in self._aux_cache_keys:
 149                                 aux_cache[x] = mydata.get(x, '')
 150                         self._aux_cache[mycpv] = aux_cache
 151                 return [mydata.get(x, '') for x in wants]
 152
 153         def aux_update(self, cpv, values):
 154                 if not self.bintree.populated:
 155                         self.bintree.populate()
 156                 tbz2path = self.bintree.getname(cpv)
 157                 if not os.path.exists(tbz2path):
 158                         raise KeyError(cpv)
 159                 mytbz2 = portage.xpak.tbz2(tbz2path)
 160                 mydata = mytbz2.get_data()
 161
 162                 for k, v in values.items():
 163                         k = _unicode_encode(k,
 164                                 encoding=_encodings['repo.content'], errors='backslashreplace')
 165                         v = _unicode_encode(v,
 166                                 encoding=_encodings['repo.content'], errors='backslashreplace')
 167                         mydata[k] = v
 168
 169                 for k, v in list(mydata.items()):
 170                         if not v:
 171                                 del mydata[k]
 172                 mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 173                 # inject will clear stale caches via cpv_inject.
 174                 self.bintree.inject(cpv)
 175
 176         def cp_list(self, *pargs, **kwargs):
 177                 if not self.bintree.populated:
 178                         self.bintree.populate()
 179                 return fakedbapi.cp_list(self, *pargs, **kwargs)
 180
 181         def cp_all(self):
 182                 if not self.bintree.populated:
 183                         self.bintree.populate()
 184                 return fakedbapi.cp_all(self)
 185
 186         def cpv_all(self):
 187                 if not self.bintree.populated:
 188                         self.bintree.populate()
 189                 return fakedbapi.cpv_all(self)
 190
 191         def getfetchsizes(self, pkg):
 192                 """
 193                 This will raise MissingSignature if SIZE signature is not available,
 194                 or InvalidSignature if SIZE signature is invalid.
 195                 """
 196
 197                 if not self.bintree.populated:
 198                         self.bintree.populate()
 199
 200                 pkg = getattr(pkg, 'cpv', pkg)
 201
 202                 filesdict = {}
 203                 if not self.bintree.isremote(pkg):
 204                         pass
 205                 else:
 206                         metadata = self.bintree._remotepkgs[pkg]
 207                         try:
 208                                 size = int(metadata["SIZE"])
 209                         except KeyError:
 210                                 raise portage.exception.MissingSignature("SIZE")
 211                         except ValueError:
 212                                 raise portage.exception.InvalidSignature(
 213                                         "SIZE: %s" % metadata["SIZE"])
 214                         else:
 215                                 filesdict[os.path.basename(self.bintree.getname(pkg))] = size
 216
 217                 return filesdict
 218
 219 def _pkgindex_cpv_map_latest_build(pkgindex):
 220         """
 221         Given a PackageIndex instance, create a dict of cpv -> metadata map.
 222         If multiple packages have identical CPV values, prefer the package
 223         with latest BUILD_TIME value.
 224         @param pkgindex: A PackageIndex instance.
 225         @type pkgindex: PackageIndex
 226         @rtype: dict
 227         @return: a dict containing entry for the give cpv.
 228         """
 229         cpv_map = {}
 230
 231         for d in pkgindex.packages:
 232                 cpv = d["CPV"]
 233
 234                 try:
 235                         cpv = _pkg_str(cpv)
 236                 except InvalidData:
 237                         writemsg(_("!!! Invalid remote binary package: %s\n") % cpv,
 238                                 noiselevel=-1)
 239                         continue
 240
 241                 btime = d.get('BUILD_TIME', '')
 242                 try:
 243                         btime = int(btime)
 244                 except ValueError:
 245                         btime = None
 246
 247                 other_d = cpv_map.get(cpv)
 248                 if other_d is not None:
 249                         other_btime = other_d.get('BUILD_TIME', '')
 250                         try:
 251                                 other_btime = int(other_btime)
 252                         except ValueError:
 253                                 other_btime = None
 254                         if other_btime and (not btime or other_btime > btime):
 255                                 continue
 256
 257                 cpv_map[_pkg_str(cpv)] = d
 258
 259         return cpv_map
 260
 261 class binarytree(object):
 262         "this tree scans for a list of all packages available in PKGDIR"
 263         def __init__(self, _unused=DeprecationWarning, pkgdir=None,
 264                 virtual=DeprecationWarning, settings=None):
 265
 266                 if pkgdir is None:
 267                         raise TypeError("pkgdir parameter is required")
 268
 269                 if settings is None:
 270                         raise TypeError("settings parameter is required")
 271
 272                 if _unused is not DeprecationWarning:
 273                         warnings.warn("The first parameter of the "
 274                                 "portage.dbapi.bintree.binarytree"
 275                                 " constructor is now unused. Instead "
 276                                 "settings['ROOT'] is used.",
 277                                 DeprecationWarning, stacklevel=2)
 278
 279                 if virtual is not DeprecationWarning:
 280                         warnings.warn("The 'virtual' parameter of the "
 281                                 "portage.dbapi.bintree.binarytree"
 282                                 " constructor is unused",
 283                                 DeprecationWarning, stacklevel=2)
 284
 285                 if True:
 286                         self.pkgdir = normalize_path(pkgdir)
 287                         self.dbapi = bindbapi(self, settings=settings)
 288                         self.update_ents = self.dbapi.update_ents
 289                         self.move_slot_ent = self.dbapi.move_slot_ent
 290                         self.populated = 0
 291                         self.tree = {}
 292                         self._remote_has_index = False
 293                         self._remotepkgs = None # remote metadata indexed by cpv
 294                         self.invalids = []
 295                         self.settings = settings
 296                         self._pkg_paths = {}
 297                         self._pkgindex_uri = {}
 298                         self._populating = False
 299                         self._all_directory = os.path.isdir(
 300                                 os.path.join(self.pkgdir, "All"))
 301                         self._pkgindex_version = 0
 302                         self._pkgindex_hashes = ["MD5","SHA1"]
 303                         self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
 304                         self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
 305                         self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"])
 306                         self._pkgindex_aux_keys = \
 307                                 ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
 308                                 "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
 309                                 "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
 310                                 "BASE_URI"]
 311                         self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 312                         self._pkgindex_use_evaluated_keys = \
 313                                 ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND",
 314                                 "PDEPEND", "PROPERTIES", "PROVIDE")
 315                         self._pkgindex_header_keys = set([
 316                                 "ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 317                                 "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD",
 318                                 "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
 319                                 "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"])
 320                         self._pkgindex_default_pkg_data = {
 321                                 "BUILD_TIME"         : "",
 322                                 "DEFINED_PHASES"     : "",
 323                                 "DEPEND"  : "",
 324                                 "EAPI"    : "0",
 325                                 "HDEPEND" : "",
 326                                 "IUSE"    : "",
 327                                 "KEYWORDS": "",
 328                                 "LICENSE" : "",
 329                                 "PATH"    : "",
 330                                 "PDEPEND" : "",
 331                                 "PROPERTIES" : "",
 332                                 "PROVIDE" : "",
 333                                 "RDEPEND" : "",
 334                                 "RESTRICT": "",
 335                                 "SLOT"    : "0",
 336                                 "USE"     : "",
 337                         }
 338                         self._pkgindex_inherited_keys = ["CHOST", "repository"]
 339
 340                         # Populate the header with appropriate defaults.
 341                         self._pkgindex_default_header_data = {
 342                                 "CHOST"        : self.settings.get("CHOST", ""),
 343                                 "repository"   : "",
 344                         }
 345
 346                         # It is especially important to populate keys like
 347                         # "repository" that save space when entries can
 348                         # inherit them from the header. If an existing
 349                         # pkgindex header already defines these keys, then
 350                         # they will appropriately override our defaults.
 351                         main_repo = self.settings.repositories.mainRepo()
 352                         if main_repo is not None and not main_repo.missing_repo_name:
 353                                 self._pkgindex_default_header_data["repository"] = \
 354                                         main_repo.name
 355
 356                         self._pkgindex_translated_keys = (
 357                                 ("DESCRIPTION"   ,   "DESC"),
 358                                 ("repository"    ,   "REPO"),
 359                         )
 360
 361                         self._pkgindex_allowed_pkg_keys = set(chain(
 362                                 self._pkgindex_keys,
 363                                 self._pkgindex_aux_keys,
 364                                 self._pkgindex_hashes,
 365                                 self._pkgindex_default_pkg_data,
 366                                 self._pkgindex_inherited_keys,
 367                                 chain(*self._pkgindex_translated_keys)
 368                         ))
 369
 370         @property
 371         def root(self):
 372                 warnings.warn("The root attribute of "
 373                         "portage.dbapi.bintree.binarytree"
 374                         " is deprecated. Use "
 375                         "settings['ROOT'] instead.",
 376                         DeprecationWarning, stacklevel=3)
 377                 return self.settings['ROOT']
 378
 379         def move_ent(self, mylist, repo_match=None):
 380                 if not self.populated:
 381                         self.populate()
 382                 origcp = mylist[1]
 383                 newcp = mylist[2]
 384                 # sanity check
 385                 for atom in (origcp, newcp):
 386                         if not isjustname(atom):
 387                                 raise InvalidPackageName(str(atom))
 388                 mynewcat = catsplit(newcp)[0]
 389                 origmatches=self.dbapi.cp_list(origcp)
 390                 moves = 0
 391                 if not origmatches:
 392                         return moves
 393                 for mycpv in origmatches:
 394                         try:
 395                                 mycpv = self.dbapi._pkg_str(mycpv, None)
 396                         except (KeyError, InvalidData):
 397                                 continue
 398                         mycpv_cp = portage.cpv_getkey(mycpv)
 399                         if mycpv_cp != origcp:
 400                                 # Ignore PROVIDE virtual match.
 401                                 continue
 402                         if repo_match is not None \
 403                                 and not repo_match(mycpv.repo):
 404                                 continue
 405
 406                         # Use isvalidatom() to check if this move is valid for the
 407                         # EAPI (characters allowed in package names may vary).
 408                         if not isvalidatom(newcp, eapi=mycpv.eapi):
 409                                 continue
 410
 411                         mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1)
 412                         myoldpkg = catsplit(mycpv)[1]
 413                         mynewpkg = catsplit(mynewcpv)[1]
 414
 415                         if (mynewpkg != myoldpkg) and os.path.exists(self.getname(mynewcpv)):
 416                                 writemsg(_("!!! Cannot update binary: Destination exists.\n"),
 417                                         noiselevel=-1)
 418                                 writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", noiselevel=-1)
 419                                 continue
 420
 421                         tbz2path = self.getname(mycpv)
 422                         if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK):
 423                                 writemsg(_("!!! Cannot update readonly binary: %s\n") % mycpv,
 424                                         noiselevel=-1)
 425                                 continue
 426
 427                         moves += 1
 428                         mytbz2 = portage.xpak.tbz2(tbz2path)
 429                         mydata = mytbz2.get_data()
 430                         updated_items = update_dbentries([mylist], mydata, parent=mycpv)
 431                         mydata.update(updated_items)
 432                         mydata[b'PF'] = \
 433                                 _unicode_encode(mynewpkg + "\n",
 434                                 encoding=_encodings['repo.content'])
 435                         mydata[b'CATEGORY'] = \
 436                                 _unicode_encode(mynewcat + "\n",
 437                                 encoding=_encodings['repo.content'])
 438                         if mynewpkg != myoldpkg:
 439                                 ebuild_data = mydata.pop(_unicode_encode(myoldpkg + '.ebuild',
 440                                         encoding=_encodings['repo.content']), None)
 441                                 if ebuild_data is not None:
 442                                         mydata[_unicode_encode(mynewpkg + '.ebuild',
 443                                                 encoding=_encodings['repo.content'])] = ebuild_data
 444
 445                         mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 446
 447                         self.dbapi.cpv_remove(mycpv)
 448                         del self._pkg_paths[mycpv]
 449                         new_path = self.getname(mynewcpv)
 450                         self._pkg_paths[mynewcpv] = os.path.join(
 451                                 *new_path.split(os.path.sep)[-2:])
 452                         if new_path != mytbz2:
 453                                 self._ensure_dir(os.path.dirname(new_path))
 454                                 _movefile(tbz2path, new_path, mysettings=self.settings)
 455                                 self._remove_symlink(mycpv)
 456                                 if new_path.split(os.path.sep)[-2] == "All":
 457                                         self._create_symlink(mynewcpv)
 458                         self.inject(mynewcpv)
 459
 460                 return moves
 461
 462         def _remove_symlink(self, cpv):
 463                 """Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove
 464                 the ${PKGDIR}/${CATEGORY} directory if empty.  The file will not be
 465                 removed if os.path.islink() returns False."""
 466                 mycat, mypkg = catsplit(cpv)
 467                 mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
 468                 if os.path.islink(mylink):
 469                         """Only remove it if it's really a link so that this method never
 470                         removes a real package that was placed here to avoid a collision."""
 471                         os.unlink(mylink)
 472                 try:
 473                         os.rmdir(os.path.join(self.pkgdir, mycat))
 474                 except OSError as e:
 475                         if e.errno not in (errno.ENOENT,
 476                                 errno.ENOTEMPTY, errno.EEXIST):
 477                                 raise
 478                         del e
 479
 480         def _create_symlink(self, cpv):
 481                 """Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and
 482                 ${PKGDIR}/${CATEGORY} directory, if necessary).  Any file that may
 483                 exist in the location of the symlink will first be removed."""
 484                 mycat, mypkg = catsplit(cpv)
 485                 full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
 486                 self._ensure_dir(os.path.dirname(full_path))
 487                 try:
 488                         os.unlink(full_path)
 489                 except OSError as e:
 490                         if e.errno != errno.ENOENT:
 491                                 raise
 492                         del e
 493                 os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path)
 494
 495         def prevent_collision(self, cpv):
 496                 """Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to
 497                 use for a given cpv.  If a collision will occur with an existing
 498                 package from another category, the existing package will be bumped to
 499                 ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist."""
 500                 if not self._all_directory:
 501                         return
 502
 503                 # Copy group permissions for new directories that
 504                 # may have been created.
 505                 for path in ("All", catsplit(cpv)[0]):
 506                         path = os.path.join(self.pkgdir, path)
 507                         self._ensure_dir(path)
 508                         if not os.access(path, os.W_OK):
 509                                 raise PermissionDenied("access('%s', W_OK)" % path)
 510
 511                 full_path = self.getname(cpv)
 512                 if "All" == full_path.split(os.path.sep)[-2]:
 513                         return
 514                 """Move a colliding package if it exists.  Code below this point only
 515                 executes in rare cases."""
 516                 mycat, mypkg = catsplit(cpv)
 517                 myfile = mypkg + ".tbz2"
 518                 mypath = os.path.join("All", myfile)
 519                 dest_path = os.path.join(self.pkgdir, mypath)
 520
 521                 try:
 522                         st = os.lstat(dest_path)
 523                 except OSError:
 524                         st = None
 525                 else:
 526                         if stat.S_ISLNK(st.st_mode):
 527                                 st = None
 528                                 try:
 529                                         os.unlink(dest_path)
 530                                 except OSError:
 531                                         if os.path.exists(dest_path):
 532                                                 raise
 533
 534                 if st is not None:
 535                         # For invalid packages, other_cat could be None.
 536                         other_cat = portage.xpak.tbz2(dest_path).getfile(b"CATEGORY")
 537                         if other_cat:
 538                                 other_cat = _unicode_decode(other_cat,
 539                                         encoding=_encodings['repo.content'], errors='replace')
 540                                 other_cat = other_cat.strip()
 541                                 other_cpv = other_cat + "/" + mypkg
 542                                 self._move_from_all(other_cpv)
 543                                 self.inject(other_cpv)
 544                 self._move_to_all(cpv)
 545
 546         def _ensure_dir(self, path):
 547                 """
 548                 Create the specified directory. Also, copy gid and group mode
 549                 bits from self.pkgdir if possible.
 550                 @param cat_dir: Absolute path of the directory to be created.
 551                 @type cat_dir: String
 552                 """
 553                 try:
 554                         pkgdir_st = os.stat(self.pkgdir)
 555                 except OSError:
 556                         ensure_dirs(path)
 557                         return
 558                 pkgdir_gid = pkgdir_st.st_gid
 559                 pkgdir_grp_mode = 0o2070 & pkgdir_st.st_mode
 560                 try:
 561                         ensure_dirs(path, gid=pkgdir_gid, mode=pkgdir_grp_mode, mask=0)
 562                 except PortageException:
 563                         if not os.path.isdir(path):
 564                                 raise
 565
 566         def _file_permissions(self, path):
 567                 try:
 568                         pkgdir_st = os.stat(self.pkgdir)
 569                 except OSError:
 570                         pass
 571                 else:
 572                         pkgdir_gid = pkgdir_st.st_gid
 573                         pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode
 574                         try:
 575                                 portage.util.apply_permissions(path, gid=pkgdir_gid,
 576                                         mode=pkgdir_grp_mode, mask=0)
 577                         except PortageException:
 578                                 pass
 579
 580         def _move_to_all(self, cpv):
 581                 """If the file exists, move it.  Whether or not it exists, update state
 582                 for future getname() calls."""
 583                 mycat, mypkg = catsplit(cpv)
 584                 myfile = mypkg + ".tbz2"
 585                 self._pkg_paths[cpv] = os.path.join("All", myfile)
 586                 src_path = os.path.join(self.pkgdir, mycat, myfile)
 587                 try:
 588                         mystat = os.lstat(src_path)
 589                 except OSError as e:
 590                         mystat = None
 591                 if mystat and stat.S_ISREG(mystat.st_mode):
 592                         self._ensure_dir(os.path.join(self.pkgdir, "All"))
 593                         dest_path = os.path.join(self.pkgdir, "All", myfile)
 594                         _movefile(src_path, dest_path, mysettings=self.settings)
 595                         self._create_symlink(cpv)
 596                         self.inject(cpv)
 597
 598         def _move_from_all(self, cpv):
 599                 """Move a package from ${PKGDIR}/All/${PF}.tbz2 to
 600                 ${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls."""
 601                 self._remove_symlink(cpv)
 602                 mycat, mypkg = catsplit(cpv)
 603                 myfile = mypkg + ".tbz2"
 604                 mypath = os.path.join(mycat, myfile)
 605                 dest_path = os.path.join(self.pkgdir, mypath)
 606                 self._ensure_dir(os.path.dirname(dest_path))
 607                 src_path = os.path.join(self.pkgdir, "All", myfile)
 608                 _movefile(src_path, dest_path, mysettings=self.settings)
 609                 self._pkg_paths[cpv] = mypath
 610
 611         def populate(self, getbinpkgs=0):
 612                 "populates the binarytree"
 613
 614                 if self._populating:
 615                         return
 616
 617                 pkgindex_lock = None
 618                 try:
 619                         if os.access(self.pkgdir, os.W_OK):
 620                                 pkgindex_lock = lockfile(self._pkgindex_file,
 621                                         wantnewlockfile=1)
 622                         self._populating = True
 623                         self._populate(getbinpkgs)
 624                 finally:
 625                         if pkgindex_lock:
 626                                 unlockfile(pkgindex_lock)
 627                         self._populating = False
 628
 629         def _populate(self, getbinpkgs=0):
 630                 if (not os.path.isdir(self.pkgdir) and not getbinpkgs):
 631                         return 0
 632
 633                 # Clear all caches in case populate is called multiple times
 634                 # as may be the case when _global_updates calls populate()
 635                 # prior to performing package moves since it only wants to
 636                 # operate on local packages (getbinpkgs=0).
 637                 self._remotepkgs = None
 638                 self.dbapi._clear_cache()
 639                 self.dbapi._aux_cache.clear()
 640                 if True:
 641                         pkg_paths = {}
 642                         self._pkg_paths = pkg_paths
 643                         dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True)
 644                         if "All" in dirs:
 645                                 dirs.remove("All")
 646                         dirs.sort()
 647                         dirs.insert(0, "All")
 648                         pkgindex = self._load_pkgindex()
 649                         pf_index = None
 650                         if not self._pkgindex_version_supported(pkgindex):
 651                                 pkgindex = self._new_pkgindex()
 652                         header = pkgindex.header
 653                         metadata = {}
 654                         for d in pkgindex.packages:
 655                                 metadata[d["CPV"]] = d
 656                         update_pkgindex = False
 657                         for mydir in dirs:
 658                                 for myfile in listdir(os.path.join(self.pkgdir, mydir)):
 659                                         if not myfile.endswith(".tbz2"):
 660                                                 continue
 661                                         mypath = os.path.join(mydir, myfile)
 662                                         full_path = os.path.join(self.pkgdir, mypath)
 663                                         s = os.lstat(full_path)
 664                                         if stat.S_ISLNK(s.st_mode):
 665                                                 continue
 666
 667                                         # Validate data from the package index and try to avoid
 668                                         # reading the xpak if possible.
 669                                         if mydir != "All":
 670                                                 possibilities = None
 671                                                 d = metadata.get(mydir+"/"+myfile[:-5])
 672                                                 if d:
 673                                                         possibilities = [d]
 674                                         else:
 675                                                 if pf_index is None:
 676                                                         pf_index = {}
 677                                                         for mycpv in metadata:
 678                                                                 mycat, mypf = catsplit(mycpv)
 679                                                                 pf_index.setdefault(
 680                                                                         mypf, []).append(metadata[mycpv])
 681                                                 possibilities = pf_index.get(myfile[:-5])
 682                                         if possibilities:
 683                                                 match = None
 684                                                 for d in possibilities:
 685                                                         try:
 686                                                                 if long(d["MTIME"]) != s[stat.ST_MTIME]:
 687                                                                         continue
 688                                                         except (KeyError, ValueError):
 689                                                                 continue
 690                                                         try:
 691                                                                 if long(d["SIZE"]) != long(s.st_size):
 692                                                                         continue
 693                                                         except (KeyError, ValueError):
 694                                                                 continue
 695                                                         if not self._pkgindex_keys.difference(d):
 696                                                                 match = d
 697                                                                 break
 698                                                 if match:
 699                                                         mycpv = match["CPV"]
 700                                                         if mycpv in pkg_paths:
 701                                                                 # discard duplicates (All/ is preferred)
 702                                                                 continue
 703                                                         mycpv = _pkg_str(mycpv)
 704                                                         pkg_paths[mycpv] = mypath
 705                                                         # update the path if the package has been moved
 706                                                         oldpath = d.get("PATH")
 707                                                         if oldpath and oldpath != mypath:
 708                                                                 update_pkgindex = True
 709                                                         if mypath != mycpv + ".tbz2":
 710                                                                 d["PATH"] = mypath
 711                                                                 if not oldpath:
 712                                                                         update_pkgindex = True
 713                                                         else:
 714                                                                 d.pop("PATH", None)
 715                                                                 if oldpath:
 716                                                                         update_pkgindex = True
 717                                                         self.dbapi.cpv_inject(mycpv)
 718                                                         if not self.dbapi._aux_cache_keys.difference(d):
 719                                                                 aux_cache = self.dbapi._aux_cache_slot_dict()
 720                                                                 for k in self.dbapi._aux_cache_keys:
 721                                                                         aux_cache[k] = d[k]
 722                                                                 self.dbapi._aux_cache[mycpv] = aux_cache
 723                                                         continue
 724                                         if not os.access(full_path, os.R_OK):
 725                                                 writemsg(_("!!! Permission denied to read " \
 726                                                         "binary package: '%s'\n") % full_path,
 727                                                         noiselevel=-1)
 728                                                 self.invalids.append(myfile[:-5])
 729                                                 continue
 730                                         metadata_bytes = portage.xpak.tbz2(full_path).get_data()
 731                                         mycat = _unicode_decode(metadata_bytes.get(b"CATEGORY", ""),
 732                                                 encoding=_encodings['repo.content'], errors='replace')
 733                                         mypf = _unicode_decode(metadata_bytes.get(b"PF", ""),
 734                                                 encoding=_encodings['repo.content'], errors='replace')
 735                                         slot = _unicode_decode(metadata_bytes.get(b"SLOT", ""),
 736                                                 encoding=_encodings['repo.content'], errors='replace')
 737                                         mypkg = myfile[:-5]
 738                                         if not mycat or not mypf or not slot:
 739                                                 #old-style or corrupt package
 740                                                 writemsg(_("\n!!! Invalid binary package: '%s'\n") % full_path,
 741                                                         noiselevel=-1)
 742                                                 missing_keys = []
 743                                                 if not mycat:
 744                                                         missing_keys.append("CATEGORY")
 745                                                 if not mypf:
 746                                                         missing_keys.append("PF")
 747                                                 if not slot:
 748                                                         missing_keys.append("SLOT")
 749                                                 msg = []
 750                                                 if missing_keys:
 751                                                         missing_keys.sort()
 752                                                         msg.append(_("Missing metadata key(s): %s.") % \
 753                                                                 ", ".join(missing_keys))
 754                                                 msg.append(_(" This binary package is not " \
 755                                                         "recoverable and should be deleted."))
 756                                                 for line in textwrap.wrap("".join(msg), 72):
 757                                                         writemsg("!!! %s\n" % line, noiselevel=-1)
 758                                                 self.invalids.append(mypkg)
 759                                                 continue
 760                                         mycat = mycat.strip()
 761                                         slot = slot.strip()
 762                                         if mycat != mydir and mydir != "All":
 763                                                 continue
 764                                         if mypkg != mypf.strip():
 765                                                 continue
 766                                         mycpv = mycat + "/" + mypkg
 767                                         if mycpv in pkg_paths:
 768                                                 # All is first, so it's preferred.
 769                                                 continue
 770                                         if not self.dbapi._category_re.match(mycat):
 771                                                 writemsg(_("!!! Binary package has an " \
 772                                                         "unrecognized category: '%s'\n") % full_path,
 773                                                         noiselevel=-1)
 774                                                 writemsg(_("!!! '%s' has a category that is not" \
 775                                                         " listed in %setc/portage/categories\n") % \
 776                                                         (mycpv, self.settings["PORTAGE_CONFIGROOT"]),
 777                                                         noiselevel=-1)
 778                                                 continue
 779                                         mycpv = _pkg_str(mycpv)
 780                                         pkg_paths[mycpv] = mypath
 781                                         self.dbapi.cpv_inject(mycpv)
 782                                         update_pkgindex = True
 783                                         d = metadata.get(mycpv, {})
 784                                         if d:
 785                                                 try:
 786                                                         if long(d["MTIME"]) != s[stat.ST_MTIME]:
 787                                                                 d.clear()
 788                                                 except (KeyError, ValueError):
 789                                                         d.clear()
 790                                         if d:
 791                                                 try:
 792                                                         if long(d["SIZE"]) != long(s.st_size):
 793                                                                 d.clear()
 794                                                 except (KeyError, ValueError):
 795                                                         d.clear()
 796
 797                                         d["CPV"] = mycpv
 798                                         d["SLOT"] = slot
 799                                         d["MTIME"] = str(s[stat.ST_MTIME])
 800                                         d["SIZE"] = str(s.st_size)
 801
 802                                         d.update(zip(self._pkgindex_aux_keys,
 803                                                 self.dbapi.aux_get(mycpv, self._pkgindex_aux_keys)))
 804                                         try:
 805                                                 self._eval_use_flags(mycpv, d)
 806                                         except portage.exception.InvalidDependString:
 807                                                 writemsg(_("!!! Invalid binary package: '%s'\n") % \
 808                                                         self.getname(mycpv), noiselevel=-1)
 809                                                 self.dbapi.cpv_remove(mycpv)
 810                                                 del pkg_paths[mycpv]
 811
 812                                         # record location if it's non-default
 813                                         if mypath != mycpv + ".tbz2":
 814                                                 d["PATH"] = mypath
 815                                         else:
 816                                                 d.pop("PATH", None)
 817                                         metadata[mycpv] = d
 818                                         if not self.dbapi._aux_cache_keys.difference(d):
 819                                                 aux_cache = self.dbapi._aux_cache_slot_dict()
 820                                                 for k in self.dbapi._aux_cache_keys:
 821                                                         aux_cache[k] = d[k]
 822                                                 self.dbapi._aux_cache[mycpv] = aux_cache
 823
 824                         for cpv in list(metadata):
 825                                 if cpv not in pkg_paths:
 826                                         del metadata[cpv]
 827
 828                         # Do not bother to write the Packages index if $PKGDIR/All/ exists
 829                         # since it will provide no benefit due to the need to read CATEGORY
 830                         # from xpak.
 831                         if update_pkgindex and os.access(self.pkgdir, os.W_OK):
 832                                 del pkgindex.packages[:]
 833                                 pkgindex.packages.extend(iter(metadata.values()))
 834                                 self._update_pkgindex_header(pkgindex.header)
 835                                 self._pkgindex_write(pkgindex)
 836
 837                 if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
 838                         writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
 839                                 noiselevel=-1)
 840
 841                 if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
 842                         self.populated=1
 843                         return
 844                 self._remotepkgs = {}
 845                 for base_url in self.settings["PORTAGE_BINHOST"].split():
 846                         parsed_url = urlparse(base_url)
 847                         host = parsed_url.netloc
 848                         port = parsed_url.port
 849                         user = None
 850                         passwd = None
 851                         user_passwd = ""
 852                         if "@" in host:
 853                                 user, host = host.split("@", 1)
 854                                 user_passwd = user + "@"
 855                                 if ":" in user:
 856                                         user, passwd = user.split(":", 1)
 857                         port_args = []
 858                         if port is not None:
 859                                 port_str = ":%s" % (port,)
 860                                 if host.endswith(port_str):
 861                                         host = host[:-len(port_str)]
 862                         pkgindex_file = os.path.join(self.settings["EROOT"], CACHE_PATH, "binhost",
 863                                 host, parsed_url.path.lstrip("/"), "Packages")
 864                         pkgindex = self._new_pkgindex()
 865                         try:
 866                                 f = io.open(_unicode_encode(pkgindex_file,
 867                                         encoding=_encodings['fs'], errors='strict'),
 868                                         mode='r', encoding=_encodings['repo.content'],
 869                                         errors='replace')
 870                                 try:
 871                                         pkgindex.read(f)
 872                                 finally:
 873                                         f.close()
 874                         except EnvironmentError as e:
 875                                 if e.errno != errno.ENOENT:
 876                                         raise
 877                         local_timestamp = pkgindex.header.get("TIMESTAMP", None)
 878                         remote_timestamp = None
 879                         rmt_idx = self._new_pkgindex()
 880                         proc = None
 881                         tmp_filename = None
 882                         try:
 883                                 # urlparse.urljoin() only works correctly with recognized
 884                                 # protocols and requires the base url to have a trailing
 885                                 # slash, so join manually...
 886                                 url = base_url.rstrip("/") + "/Packages"
 887                                 try:
 888                                         f = _urlopen(url, if_modified_since=local_timestamp)
 889                                         if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
 890                                                 remote_timestamp = f.headers.get('timestamp')
 891                                 except IOError as err:
 892                                         if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
 893                                                 raise UseCachedCopyOfRemoteIndex()
 894
 895                                         path = parsed_url.path.rstrip("/") + "/Packages"
 896
 897                                         if parsed_url.scheme == 'sftp':
 898                                                 # The sftp command complains about 'Illegal seek' if
 899                                                 # we try to make it write to /dev/stdout, so use a
 900                                                 # temp file instead.
 901                                                 fd, tmp_filename = tempfile.mkstemp()
 902                                                 os.close(fd)
 903                                                 if port is not None:
 904                                                         port_args = ['-P', "%s" % (port,)]
 905                                                 proc = subprocess.Popen(['sftp'] + port_args + \
 906                                                         [user_passwd + host + ":" + path, tmp_filename])
 907                                                 if proc.wait() != os.EX_OK:
 908                                                         raise
 909                                                 f = open(tmp_filename, 'rb')
 910                                         elif parsed_url.scheme == 'ssh':
 911                                                 if port is not None:
 912                                                         port_args = ['-p', "%s" % (port,)]
 913                                                 proc = subprocess.Popen(['ssh'] + port_args + \
 914                                                         [user_passwd + host, '--', 'cat', path],
 915                                                         stdout=subprocess.PIPE)
 916                                                 f = proc.stdout
 917                                         else:
 918                                                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
 919                                                 fcmd = self.settings.get(setting)
 920                                                 if not fcmd:
 921                                                         raise
 922                                                 fd, tmp_filename = tempfile.mkstemp()
 923                                                 tmp_dirname, tmp_basename = os.path.split(tmp_filename)
 924                                                 os.close(fd)
 925                                                 success = portage.getbinpkg.file_get(url,
 926                                                      tmp_dirname, fcmd=fcmd, filename=tmp_basename)
 927                                                 if not success:
 928                                                         raise EnvironmentError("%s failed" % (setting,))
 929                                                 f = open(tmp_filename, 'rb')
 930
 931                                 f_dec = codecs.iterdecode(f,
 932                                         _encodings['repo.content'], errors='replace')
 933                                 try:
 934                                         rmt_idx.readHeader(f_dec)
 935                                         if not remote_timestamp: # in case it had not been read from HTTP header
 936                                                 remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
 937                                         if not remote_timestamp:
 938                                                 # no timestamp in the header, something's wrong
 939                                                 pkgindex = None
 940                                                 writemsg(_("\n\n!!! Binhost package index " \
 941                                                 " has no TIMESTAMP field.\n"), noiselevel=-1)
 942                                         else:
 943                                                 if not self._pkgindex_version_supported(rmt_idx):
 944                                                         writemsg(_("\n\n!!! Binhost package index version" \
 945                                                         " is not supported: '%s'\n") % \
 946                                                         rmt_idx.header.get("VERSION"), noiselevel=-1)
 947                                                         pkgindex = None
 948                                                 elif local_timestamp != remote_timestamp:
 949                                                         rmt_idx.readBody(f_dec)
 950                                                         pkgindex = rmt_idx
 951                                 finally:
 952                                         # Timeout after 5 seconds, in case close() blocks
 953                                         # indefinitely (see bug #350139).
 954                                         try:
 955                                                 try:
 956                                                         AlarmSignal.register(5)
 957                                                         f.close()
 958                                                 finally:
 959                                                         AlarmSignal.unregister()
 960                                         except AlarmSignal:
 961                                                 writemsg("\n\n!!! %s\n" % \
 962                                                         _("Timed out while closing connection to binhost"),
 963                                                         noiselevel=-1)
 964                         except UseCachedCopyOfRemoteIndex:
 965                                 writemsg_stdout("\n")
 966                                 writemsg_stdout(
 967                                         colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \
 968                                         "\n")
 969                                 rmt_idx = pkgindex
 970                         except EnvironmentError as e:
 971                                 writemsg(_("\n\n!!! Error fetching binhost package" \
 972                                         " info from '%s'\n") % _hide_url_passwd(base_url))
 973                                 writemsg("!!! %s\n\n" % str(e))
 974                                 del e
 975                                 pkgindex = None
 976                         if proc is not None:
 977                                 if proc.poll() is None:
 978                                         proc.kill()
 979                                         proc.wait()
 980                                 proc = None
 981                         if tmp_filename is not None:
 982                                 try:
 983                                         os.unlink(tmp_filename)
 984                                 except OSError:
 985                                         pass
 986                         if pkgindex is rmt_idx:
 987                                 pkgindex.modified = False # don't update the header
 988                                 try:
 989                                         ensure_dirs(os.path.dirname(pkgindex_file))
 990                                         f = atomic_ofstream(pkgindex_file)
 991                                         pkgindex.write(f)
 992                                         f.close()
 993                                 except (IOError, PortageException):
 994                                         if os.access(os.path.dirname(pkgindex_file), os.W_OK):
 995                                                 raise
 996                                         # The current user doesn't have permission to cache the
 997                                         # file, but that's alright.
 998                         if pkgindex:
 999                                 # Organize remote package list as a cpv -> metadata map.
1000                                 remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
1001                                 remote_base_uri = pkgindex.header.get("URI", base_url)
1002                                 for cpv, remote_metadata in remotepkgs.items():
1003                                         remote_metadata["BASE_URI"] = remote_base_uri
1004                                         self._pkgindex_uri[cpv] = url
1005                                 self._remotepkgs.update(remotepkgs)
1006                                 self._remote_has_index = True
1007                                 for cpv in remotepkgs:
1008                                         self.dbapi.cpv_inject(cpv)
1009                                 if True:
1010                                         # Remote package instances override local package
1011                                         # if they are not identical.
1012                                         hash_names = ["SIZE"] + self._pkgindex_hashes
1013                                         for cpv, local_metadata in metadata.items():
1014                                                 remote_metadata = self._remotepkgs.get(cpv)
1015                                                 if remote_metadata is None:
1016                                                         continue
1017                                                 # Use digests to compare identity.
1018                                                 identical = True
1019                                                 for hash_name in hash_names:
1020                                                         local_value = local_metadata.get(hash_name)
1021                                                         if local_value is None:
1022                                                                 continue
1023                                                         remote_value = remote_metadata.get(hash_name)
1024                                                         if remote_value is None:
1025                                                                 continue
1026                                                         if local_value != remote_value:
1027                                                                 identical = False
1028                                                                 break
1029                                                 if identical:
1030                                                         del self._remotepkgs[cpv]
1031                                                 else:
1032                                                         # Override the local package in the aux_get cache.
1033                                                         self.dbapi._aux_cache[cpv] = remote_metadata
1034                                 else:
1035                                         # Local package instances override remote instances.
1036                                         for cpv in metadata:
1037                                                 self._remotepkgs.pop(cpv, None)
1038
1039                 self.populated=1
1040
1041         def inject(self, cpv, filename=None):
1042                 """Add a freshly built package to the database.  This updates
1043                 $PKGDIR/Packages with the new package metadata (including MD5).
1044                 @param cpv: The cpv of the new package to inject
1045                 @type cpv: string
1046                 @param filename: File path of the package to inject, or None if it's
1047                         already in the location returned by getname()
1048                 @type filename: string
1049                 @rtype: None
1050                 """
1051                 mycat, mypkg = catsplit(cpv)
1052                 if not self.populated:
1053                         self.populate()
1054                 if filename is None:
1055                         full_path = self.getname(cpv)
1056                 else:
1057                         full_path = filename
1058                 try:
1059                         s = os.stat(full_path)
1060                 except OSError as e:
1061                         if e.errno != errno.ENOENT:
1062                                 raise
1063                         del e
1064                         writemsg(_("!!! Binary package does not exist: '%s'\n") % full_path,
1065                                 noiselevel=-1)
1066                         return
1067                 mytbz2 = portage.xpak.tbz2(full_path)
1068                 slot = mytbz2.getfile("SLOT")
1069                 if slot is None:
1070                         writemsg(_("!!! Invalid binary package: '%s'\n") % full_path,
1071                                 noiselevel=-1)
1072                         return
1073                 slot = slot.strip()
1074                 self.dbapi.cpv_inject(cpv)
1075
1076                 # Reread the Packages index (in case it's been changed by another
1077                 # process) and then updated it, all while holding a lock.
1078                 pkgindex_lock = None
1079                 created_symlink = False
1080                 try:
1081                         pkgindex_lock = lockfile(self._pkgindex_file,
1082                                 wantnewlockfile=1)
1083                         if filename is not None:
1084                                 new_filename = self.getname(cpv)
1085                                 try:
1086                                         samefile = os.path.samefile(filename, new_filename)
1087                                 except OSError:
1088                                         samefile = False
1089                                 if not samefile:
1090                                         self._ensure_dir(os.path.dirname(new_filename))
1091                                         _movefile(filename, new_filename, mysettings=self.settings)
1092                                 full_path = new_filename
1093
1094                         self._file_permissions(full_path)
1095
1096                         if self._all_directory and \
1097                                 self.getname(cpv).split(os.path.sep)[-2] == "All":
1098                                 self._create_symlink(cpv)
1099                                 created_symlink = True
1100                         pkgindex = self._load_pkgindex()
1101
1102                         if not self._pkgindex_version_supported(pkgindex):
1103                                 pkgindex = self._new_pkgindex()
1104
1105                         # Discard remote metadata to ensure that _pkgindex_entry
1106                         # gets the local metadata. This also updates state for future
1107                         # isremote calls.
1108                         if self._remotepkgs is not None:
1109                                 self._remotepkgs.pop(cpv, None)
1110
1111                         # Discard cached metadata to ensure that _pkgindex_entry
1112                         # doesn't return stale metadata.
1113                         self.dbapi._aux_cache.pop(cpv, None)
1114
1115                         try:
1116                                 d = self._pkgindex_entry(cpv)
1117                         except portage.exception.InvalidDependString:
1118                                 writemsg(_("!!! Invalid binary package: '%s'\n") % \
1119                                         self.getname(cpv), noiselevel=-1)
1120                                 self.dbapi.cpv_remove(cpv)
1121                                 del self._pkg_paths[cpv]
1122                                 return
1123
1124                         # If found, remove package(s) with duplicate path.
1125                         path = d.get("PATH", "")
1126                         for i in range(len(pkgindex.packages) - 1, -1, -1):
1127                                 d2 = pkgindex.packages[i]
1128                                 if path and path == d2.get("PATH"):
1129                                         # Handle path collisions in $PKGDIR/All
1130                                         # when CPV is not identical.
1131                                         del pkgindex.packages[i]
1132                                 elif cpv == d2.get("CPV"):
1133                                         if path == d2.get("PATH", ""):
1134                                                 del pkgindex.packages[i]
1135                                         elif created_symlink and not d2.get("PATH", ""):
1136                                                 # Delete entry for the package that was just
1137                                                 # overwritten by a symlink to this package.
1138                                                 del pkgindex.packages[i]
1139
1140                         pkgindex.packages.append(d)
1141
1142                         self._update_pkgindex_header(pkgindex.header)
1143                         self._pkgindex_write(pkgindex)
1144
1145                 finally:
1146                         if pkgindex_lock:
1147                                 unlockfile(pkgindex_lock)
1148
1149         def _pkgindex_write(self, pkgindex):
1150                 contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
1151                 pkgindex.write(contents)
1152                 contents = contents.getvalue()
1153                 atime = mtime = long(pkgindex.header["TIMESTAMP"])
1154                 output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"),
1155                         self._pkgindex_file, None)]
1156
1157                 if "compress-index" in self.settings.features:
1158                         gz_fname = self._pkgindex_file + ".gz"
1159                         fileobj = atomic_ofstream(gz_fname, mode="wb")
1160                         output_files.append((GzipFile(filename='', mode="wb",
1161                                 fileobj=fileobj, mtime=mtime), gz_fname, fileobj))
1162
1163                 for f, fname, f_close in output_files:
1164                         f.write(contents)
1165                         f.close()
1166                         if f_close is not None:
1167                                 f_close.close()
1168                         self._file_permissions(fname)
1169                         # some seconds might have elapsed since TIMESTAMP
1170                         os.utime(fname, (atime, mtime))
1171
1172         def _pkgindex_entry(self, cpv):
1173                 """
1174                 Performs checksums and evaluates USE flag conditionals.
1175                 Raises InvalidDependString if necessary.
1176                 @rtype: dict
1177                 @return: a dict containing entry for the give cpv.
1178                 """
1179
1180                 pkg_path = self.getname(cpv)
1181
1182                 d = dict(zip(self._pkgindex_aux_keys,
1183                         self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
1184
1185                 d.update(perform_multiple_checksums(
1186                         pkg_path, hashes=self._pkgindex_hashes))
1187
1188                 d["CPV"] = cpv
1189                 st = os.stat(pkg_path)
1190                 d["MTIME"] = str(st[stat.ST_MTIME])
1191                 d["SIZE"] = str(st.st_size)
1192
1193                 rel_path = self._pkg_paths[cpv]
1194                 # record location if it's non-default
1195                 if rel_path != cpv + ".tbz2":
1196                         d["PATH"] = rel_path
1197
1198                 self._eval_use_flags(cpv, d)
1199                 return d
1200
1201         def _new_pkgindex(self):
1202                 return portage.getbinpkg.PackageIndex(
1203                         allowed_pkg_keys=self._pkgindex_allowed_pkg_keys,
1204                         default_header_data=self._pkgindex_default_header_data,
1205                         default_pkg_data=self._pkgindex_default_pkg_data,
1206                         inherited_keys=self._pkgindex_inherited_keys,
1207                         translated_keys=self._pkgindex_translated_keys)
1208
1209         def _update_pkgindex_header(self, header):
1210                 portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"]))
1211                 profiles_base = os.path.join(portdir, "profiles") + os.path.sep
1212                 if self.settings.profile_path:
1213                         profile_path = normalize_path(
1214                                 os.path.realpath(self.settings.profile_path))
1215                         if profile_path.startswith(profiles_base):
1216                                 profile_path = profile_path[len(profiles_base):]
1217                         header["PROFILE"] = profile_path
1218                 header["VERSION"] = str(self._pkgindex_version)
1219                 base_uri = self.settings.get("PORTAGE_BINHOST_HEADER_URI")
1220                 if base_uri:
1221                         header["URI"] = base_uri
1222                 else:
1223                         header.pop("URI", None)
1224                 for k in self._pkgindex_header_keys:
1225                         v = self.settings.get(k, None)
1226                         if v:
1227                                 header[k] = v
1228                         else:
1229                                 header.pop(k, None)
1230
1231         def _pkgindex_version_supported(self, pkgindex):
1232                 version = pkgindex.header.get("VERSION")
1233                 if version:
1234                         try:
1235                                 if int(version) <= self._pkgindex_version:
1236                                         return True
1237                         except ValueError:
1238                                 pass
1239                 return False
1240
1241         def _eval_use_flags(self, cpv, metadata):
1242                 use = frozenset(metadata["USE"].split())
1243                 raw_use = use
1244                 iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split())
1245                 use = [f for f in use if f in iuse]
1246                 use.sort()
1247                 metadata["USE"] = " ".join(use)
1248                 for k in self._pkgindex_use_evaluated_keys:
1249                         if k.endswith('DEPEND'):
1250                                 token_class = Atom
1251                         else:
1252                                 token_class = None
1253
1254                         try:
1255                                 deps = metadata[k]
1256                                 deps = use_reduce(deps, uselist=raw_use, token_class=token_class)
1257                                 deps = paren_enclose(deps)
1258                         except portage.exception.InvalidDependString as e:
1259                                 writemsg("%s: %s\n" % (k, str(e)),
1260                                         noiselevel=-1)
1261                                 raise
1262                         metadata[k] = deps
1263
1264         def exists_specific(self, cpv):
1265                 if not self.populated:
1266                         self.populate()
1267                 return self.dbapi.match(
1268                         dep_expand("="+cpv, mydb=self.dbapi, settings=self.settings))
1269
1270         def dep_bestmatch(self, mydep):
1271                 "compatibility method -- all matches, not just visible ones"
1272                 if not self.populated:
1273                         self.populate()
1274                 writemsg("\n\n", 1)
1275                 writemsg("mydep: %s\n" % mydep, 1)
1276                 mydep = dep_expand(mydep, mydb=self.dbapi, settings=self.settings)
1277                 writemsg("mydep: %s\n" % mydep, 1)
1278                 mykey = dep_getkey(mydep)
1279                 writemsg("mykey: %s\n" % mykey, 1)
1280                 mymatch = best(match_from_list(mydep,self.dbapi.cp_list(mykey)))
1281                 writemsg("mymatch: %s\n" % mymatch, 1)
1282                 if mymatch is None:
1283                         return ""
1284                 return mymatch
1285
1286         def getname(self, pkgname):
1287                 """Returns a file location for this package.  The default location is
1288                 ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2
1289                 in the rare event of a collision.  The prevent_collision() method can
1290                 be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a
1291                 specific cpv."""
1292                 if not self.populated:
1293                         self.populate()
1294                 mycpv = pkgname
1295                 mypath = self._pkg_paths.get(mycpv, None)
1296                 if mypath:
1297                         return os.path.join(self.pkgdir, mypath)
1298                 mycat, mypkg = catsplit(mycpv)
1299                 if self._all_directory:
1300                         mypath = os.path.join("All", mypkg + ".tbz2")
1301                         if mypath in self._pkg_paths.values():
1302                                 mypath = os.path.join(mycat, mypkg + ".tbz2")
1303                 else:
1304                         mypath = os.path.join(mycat, mypkg + ".tbz2")
1305                 self._pkg_paths[mycpv] = mypath # cache for future lookups
1306                 return os.path.join(self.pkgdir, mypath)
1307
1308         def isremote(self, pkgname):
1309                 """Returns true if the package is kept remotely and it has not been
1310                 downloaded (or it is only partially downloaded)."""
1311                 if self._remotepkgs is None or pkgname not in self._remotepkgs:
1312                         return False
1313                 # Presence in self._remotepkgs implies that it's remote. When a
1314                 # package is downloaded, state is updated by self.inject().
1315                 return True
1316
1317         def get_pkgindex_uri(self, pkgname):
1318                 """Returns the URI to the Packages file for a given package."""
1319                 return self._pkgindex_uri.get(pkgname)
1320
1321
1322
1323         def gettbz2(self, pkgname):
1324                 """Fetches the package from a remote site, if necessary.  Attempts to
1325                 resume if the file appears to be partially downloaded."""
1326                 tbz2_path = self.getname(pkgname)
1327                 tbz2name = os.path.basename(tbz2_path)
1328                 resume = False
1329                 if os.path.exists(tbz2_path):
1330                         if tbz2name[:-5] not in self.invalids:
1331                                 return
1332                         else:
1333                                 resume = True
1334                                 writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"),
1335                                         noiselevel=-1)
1336
1337                 mydest = os.path.dirname(self.getname(pkgname))
1338                 self._ensure_dir(mydest)
1339                 # urljoin doesn't work correctly with unrecognized protocols like sftp
1340                 if self._remote_has_index:
1341                         rel_url = self._remotepkgs[pkgname].get("PATH")
1342                         if not rel_url:
1343                                 rel_url = pkgname+".tbz2"
1344                         remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"]
1345                         url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
1346                 else:
1347                         url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
1348                 protocol = urlparse(url)[0]
1349                 fcmd_prefix = "FETCHCOMMAND"
1350                 if resume:
1351                         fcmd_prefix = "RESUMECOMMAND"
1352                 fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
1353                 if not fcmd:
1354                         fcmd = self.settings.get(fcmd_prefix)
1355                 success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
1356                 if not success:
1357                         try:
1358                                 os.unlink(self.getname(pkgname))
1359                         except OSError:
1360                                 pass
1361                         raise portage.exception.FileNotFound(mydest)
1362                 self.inject(pkgname)
1363
1364         def _load_pkgindex(self):
1365                 pkgindex = self._new_pkgindex()
1366                 try:
1367                         f = io.open(_unicode_encode(self._pkgindex_file,
1368                                 encoding=_encodings['fs'], errors='strict'),
1369                                 mode='r', encoding=_encodings['repo.content'],
1370                                 errors='replace')
1371                 except EnvironmentError:
1372                         pass
1373                 else:
1374                         try:
1375                                 pkgindex.read(f)
1376                         finally:
1377                                 f.close()
1378                 return pkgindex
1379
1380         def _get_digests(self, pkg):
1381
1382                 try:
1383                         cpv = pkg.cpv
1384                 except AttributeError:
1385                         cpv = pkg
1386
1387                 digests = {}
1388                 metadata = None
1389                 if self._remotepkgs is None or cpv not in self._remotepkgs:
1390                         for d in self._load_pkgindex().packages:
1391                                 if d["CPV"] == cpv:
1392                                         metadata = d
1393                                         break
1394                 else:
1395                         metadata = self._remotepkgs[cpv]
1396                 if metadata is None:
1397                         return digests
1398
1399                 for k in hashfunc_map:
1400                         v = metadata.get(k)
1401                         if not v:
1402                                 continue
1403                         digests[k] = v
1404
1405                 if "SIZE" in metadata:
1406                         try:
1407                                 digests["size"] = int(metadata["SIZE"])
1408                         except ValueError:
1409                                 writemsg(_("!!! Malformed SIZE attribute in remote " \
1410                                 "metadata for '%s'\n") % cpv)
1411
1412                 return digests
1413
1414         def digestCheck(self, pkg):
1415                 """
1416                 Verify digests for the given package and raise DigestException
1417                 if verification fails.
1418                 @rtype: bool
1419                 @return: True if digests could be located, False otherwise.
1420                 """
1421
1422                 digests = self._get_digests(pkg)
1423
1424                 if not digests:
1425                         return False
1426
1427                 try:
1428                         cpv = pkg.cpv
1429                 except AttributeError:
1430                         cpv = pkg
1431
1432                 pkg_path = self.getname(cpv)
1433                 hash_filter = _hash_filter(
1434                         self.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
1435                 if not hash_filter.transparent:
1436                         digests = _apply_hash_filter(digests, hash_filter)
1437                 eout = EOutput()
1438                 eout.quiet = self.settings.get("PORTAGE_QUIET") == "1"
1439                 ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0)
1440                 if not ok:
1441                         ok, reason = verify_all(pkg_path, digests)
1442                         if not ok:
1443                                 raise portage.exception.DigestException(
1444                                         (pkg_path,) + tuple(reason))
1445
1446                 return True
1447
1448         def getslot(self, mycatpkg):
1449                 "Get a slot for a catpkg; assume it exists."
1450                 myslot = ""
1451                 try:
1452                         myslot = self.dbapi._pkg_str(mycatpkg, None).slot
1453                 except KeyError:
1454                         pass
1455                 return myslot