-# Copyright 1998-2009 Gentoo Foundation
+# Copyright 1998-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import unicode_literals
+
__all__ = ["bindbapi", "binarytree"]
import portage
portage.proxy.lazyimport.lazyimport(globals(),
+ 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
+ 'verify_all,_apply_hash_filter,_hash_filter',
'portage.dbapi.dep_expand:dep_expand',
- 'portage.dep:dep_getkey,isjustname,match_from_list',
+ 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list',
'portage.output:EOutput,colorize',
- 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms',
+ 'portage.locks:lockfile,unlockfile',
+ 'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd',
'portage.update:update_dbentries',
'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
'writemsg,writemsg_stdout',
'portage.util.listdir:listdir',
- 'portage.versions:best,catpkgsplit,catsplit',
+ 'portage.util._urlopen:urlopen@_urlopen',
+ 'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
)
from portage.cache.mappings import slot_dict_class
+from portage.const import CACHE_PATH
from portage.dbapi.virtual import fakedbapi
-from portage.dep import use_reduce, paren_enclose
-from portage.exception import InvalidPackageName, \
+from portage.dep import Atom, use_reduce, paren_enclose
+from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
PermissionDenied, PortageException
from portage.localization import _
-
from portage import _movefile
from portage import os
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
-from portage.package.ebuild.fetch import _check_distfile
import codecs
import errno
-import re
+import io
import stat
+import subprocess
import sys
+import tempfile
+import textwrap
+import traceback
+import warnings
+from gzip import GzipFile
from itertools import chain
+try:
+ from urllib.parse import urlparse
+except ImportError:
+ from urlparse import urlparse
if sys.hexversion >= 0x3000000:
+ _unicode = str
basestring = str
long = int
+else:
+ _unicode = unicode
+
+class UseCachedCopyOfRemoteIndex(Exception):
+ # If the local copy is recent enough
+ # then fetching the remote index can be skipped.
+ pass
class bindbapi(fakedbapi):
_known_keys = frozenset(list(fakedbapi._known_keys) + \
self.cpdict={}
# Selectively cache metadata in order to optimize dep matching.
self._aux_cache_keys = set(
- ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS",
+ ["BUILD_TIME", "CHOST", "DEPEND", "EAPI",
+ "HDEPEND", "IUSE", "KEYWORDS",
"LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE",
- "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES",
- "REQUIRED_USE"])
+ "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES"
+ ])
self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
self._aux_cache = {}
self.bintree.populate()
return fakedbapi.match(self, *pargs, **kwargs)
+ def cpv_exists(self, cpv, myrepo=None):
+ if self.bintree and not self.bintree.populated:
+ self.bintree.populate()
+ return fakedbapi.cpv_exists(self, cpv)
+
def cpv_inject(self, cpv, **kwargs):
self._aux_cache.pop(cpv, None)
fakedbapi.cpv_inject(self, cpv, **kwargs)
self._aux_cache.pop(cpv, None)
fakedbapi.cpv_remove(self, cpv)
- def aux_get(self, mycpv, wants):
+ def aux_get(self, mycpv, wants, myrepo=None):
if self.bintree and not self.bintree.populated:
self.bintree.populate()
cache_me = False
if myval:
mydata[x] = " ".join(myval.split())
- if not mydata.setdefault('EAPI', _unicode_decode('0')):
- mydata['EAPI'] = _unicode_decode('0')
+ if not mydata.setdefault('EAPI', '0'):
+ mydata['EAPI'] = '0'
if cache_me:
aux_cache = self._aux_cache_slot_dict()
for x in self._aux_cache_keys:
- aux_cache[x] = mydata.get(x, _unicode_decode(''))
+ aux_cache[x] = mydata.get(x, '')
self._aux_cache[mycpv] = aux_cache
- return [mydata.get(x, _unicode_decode('')) for x in wants]
+ return [mydata.get(x, '') for x in wants]
def aux_update(self, cpv, values):
if not self.bintree.populated:
self.bintree.populate()
return fakedbapi.cpv_all(self)
+ def getfetchsizes(self, pkg):
+ """
+ This will raise MissingSignature if SIZE signature is not available,
+ or InvalidSignature if SIZE signature is invalid.
+ """
+
+ if not self.bintree.populated:
+ self.bintree.populate()
+
+ pkg = getattr(pkg, 'cpv', pkg)
+
+ filesdict = {}
+ if not self.bintree.isremote(pkg):
+ pass
+ else:
+ metadata = self.bintree._remotepkgs[pkg]
+ try:
+ size = int(metadata["SIZE"])
+ except KeyError:
+ raise portage.exception.MissingSignature("SIZE")
+ except ValueError:
+ raise portage.exception.InvalidSignature(
+ "SIZE: %s" % metadata["SIZE"])
+ else:
+ filesdict[os.path.basename(self.bintree.getname(pkg))] = size
+
+ return filesdict
+
def _pkgindex_cpv_map_latest_build(pkgindex):
"""
Given a PackageIndex instance, create a dict of cpv -> metadata map.
@param pkgindex: A PackageIndex instance.
@type pkgindex: PackageIndex
@rtype: dict
- @returns: a dict containing entry for the give cpv.
+ @return: a dict containing entry for the give cpv.
"""
cpv_map = {}
for d in pkgindex.packages:
cpv = d["CPV"]
+ try:
+ cpv = _pkg_str(cpv)
+ except InvalidData:
+ writemsg(_("!!! Invalid remote binary package: %s\n") % cpv,
+ noiselevel=-1)
+ continue
+
btime = d.get('BUILD_TIME', '')
try:
btime = int(btime)
if other_btime and (not btime or other_btime > btime):
continue
- cpv_map[cpv] = d
+ cpv_map[_pkg_str(cpv)] = d
return cpv_map
class binarytree(object):
"this tree scans for a list of all packages available in PKGDIR"
- def __init__(self, root, pkgdir, virtual=None, settings=None):
+ def __init__(self, _unused=DeprecationWarning, pkgdir=None,
+ virtual=DeprecationWarning, settings=None):
+
+ if pkgdir is None:
+ raise TypeError("pkgdir parameter is required")
+
+ if settings is None:
+ raise TypeError("settings parameter is required")
+
+ if _unused is not DeprecationWarning:
+ warnings.warn("The first parameter of the "
+ "portage.dbapi.bintree.binarytree"
+ " constructor is now unused. Instead "
+ "settings['ROOT'] is used.",
+ DeprecationWarning, stacklevel=2)
+
+ if virtual is not DeprecationWarning:
+ warnings.warn("The 'virtual' parameter of the "
+ "portage.dbapi.bintree.binarytree"
+ " constructor is unused",
+ DeprecationWarning, stacklevel=2)
+
if True:
- self.root = root
- #self.pkgdir=settings["PKGDIR"]
self.pkgdir = normalize_path(pkgdir)
self.dbapi = bindbapi(self, settings=settings)
self.update_ents = self.dbapi.update_ents
self.populated = 0
self.tree = {}
self._remote_has_index = False
- self._remote_base_uri = None
self._remotepkgs = None # remote metadata indexed by cpv
- self.__remotepkgs = {} # indexed by tbz2 name (deprecated)
self.invalids = []
self.settings = settings
self._pkg_paths = {}
+ self._pkgindex_uri = {}
self._populating = False
self._all_directory = os.path.isdir(
os.path.join(self.pkgdir, "All"))
self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"])
self._pkgindex_aux_keys = \
["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
- "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
- "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
- "REQUIRED_USE"]
+ "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
+ "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
+ "BASE_URI"]
self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
self._pkgindex_use_evaluated_keys = \
- ("LICENSE", "RDEPEND", "DEPEND",
- "PDEPEND", "PROPERTIES", "PROVIDE")
+ ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND",
+ "PDEPEND", "PROPERTIES", "PROVIDE", "RESTRICT")
self._pkgindex_header_keys = set([
"ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
- "ACCEPT_PROPERTIES", "CBUILD",
- "CHOST", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
- "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"])
+ "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD",
+ "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
+ "GENTOO_MIRRORS", "INSTALL_MASK", "IUSE_IMPLICIT", "USE",
+ "USE_EXPAND", "USE_EXPAND_HIDDEN", "USE_EXPAND_IMPLICIT",
+ "USE_EXPAND_UNPREFIXED"])
self._pkgindex_default_pkg_data = {
"BUILD_TIME" : "",
+ "DEFINED_PHASES" : "",
"DEPEND" : "",
"EAPI" : "0",
+ "HDEPEND" : "",
"IUSE" : "",
"KEYWORDS": "",
"LICENSE" : "",
"RESTRICT": "",
"SLOT" : "0",
"USE" : "",
- "DEFINED_PHASES" : "",
- "REQUIRED_USE" : ""
}
self._pkgindex_inherited_keys = ["CHOST", "repository"]
+
+ # Populate the header with appropriate defaults.
self._pkgindex_default_header_data = {
- "repository":""
+ "CHOST" : self.settings.get("CHOST", ""),
+ "repository" : "",
}
+
+ # It is especially important to populate keys like
+ # "repository" that save space when entries can
+ # inherit them from the header. If an existing
+ # pkgindex header already defines these keys, then
+ # they will appropriately override our defaults.
+ main_repo = self.settings.repositories.mainRepo()
+ if main_repo is not None and not main_repo.missing_repo_name:
+ self._pkgindex_default_header_data["repository"] = \
+ main_repo.name
+
self._pkgindex_translated_keys = (
("DESCRIPTION" , "DESC"),
("repository" , "REPO"),
self._pkgindex_hashes,
self._pkgindex_default_pkg_data,
self._pkgindex_inherited_keys,
- self._pkgindex_default_header_data,
chain(*self._pkgindex_translated_keys)
))
+ @property
+ def root(self):
+ warnings.warn("The root attribute of "
+ "portage.dbapi.bintree.binarytree"
+ " is deprecated. Use "
+ "settings['ROOT'] instead.",
+ DeprecationWarning, stacklevel=3)
+ return self.settings['ROOT']
+
def move_ent(self, mylist, repo_match=None):
if not self.populated:
self.populate()
if not origmatches:
return moves
for mycpv in origmatches:
+ try:
+ mycpv = self.dbapi._pkg_str(mycpv, None)
+ except (KeyError, InvalidData):
+ continue
mycpv_cp = portage.cpv_getkey(mycpv)
if mycpv_cp != origcp:
# Ignore PROVIDE virtual match.
continue
if repo_match is not None \
- and not repo_match(self.aux_get(mycpv, ['repository'])[0]):
+ and not repo_match(mycpv.repo):
+ continue
+
+ # Use isvalidatom() to check if this move is valid for the
+ # EAPI (characters allowed in package names may vary).
+ if not isvalidatom(newcp, eapi=mycpv.eapi):
continue
- mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1)
+
+ mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1)
myoldpkg = catsplit(mycpv)[1]
mynewpkg = catsplit(mynewcpv)[1]
moves += 1
mytbz2 = portage.xpak.tbz2(tbz2path)
mydata = mytbz2.get_data()
- updated_items = update_dbentries([mylist], mydata)
+ updated_items = update_dbentries([mylist], mydata, parent=mycpv)
mydata.update(updated_items)
- mydata[_unicode_encode('PF',
- encoding=_encodings['repo.content'])] = \
+ mydata[b'PF'] = \
_unicode_encode(mynewpkg + "\n",
encoding=_encodings['repo.content'])
- mydata[_unicode_encode('CATEGORY',
- encoding=_encodings['repo.content'])] = \
+ mydata[b'CATEGORY'] = \
_unicode_encode(mynewcat + "\n",
encoding=_encodings['repo.content'])
if mynewpkg != myoldpkg:
if st is not None:
# For invalid packages, other_cat could be None.
- other_cat = portage.xpak.tbz2(dest_path).getfile(
- _unicode_encode("CATEGORY",
- encoding=_encodings['repo.content']))
+ other_cat = portage.xpak.tbz2(dest_path).getfile(b"CATEGORY")
if other_cat:
other_cat = _unicode_decode(other_cat,
encoding=_encodings['repo.content'], errors='replace')
if not os.path.isdir(path):
raise
+ def _file_permissions(self, path):
+ try:
+ pkgdir_st = os.stat(self.pkgdir)
+ except OSError:
+ pass
+ else:
+ pkgdir_gid = pkgdir_st.st_gid
+ pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode
+ try:
+ portage.util.apply_permissions(path, gid=pkgdir_gid,
+ mode=pkgdir_grp_mode, mask=0)
+ except PortageException:
+ pass
+
def _move_to_all(self, cpv):
"""If the file exists, move it. Whether or not it exists, update state
for future getname() calls."""
if self._populating:
return
- from portage.locks import lockfile, unlockfile
+
pkgindex_lock = None
try:
if os.access(self.pkgdir, os.W_OK):
if mycpv in pkg_paths:
# discard duplicates (All/ is preferred)
continue
+ mycpv = _pkg_str(mycpv)
pkg_paths[mycpv] = mypath
# update the path if the package has been moved
oldpath = d.get("PATH")
self.invalids.append(myfile[:-5])
continue
metadata_bytes = portage.xpak.tbz2(full_path).get_data()
- mycat = _unicode_decode(metadata_bytes.get(
- _unicode_encode("CATEGORY",
- encoding=_encodings['repo.content']), ""),
+ mycat = _unicode_decode(metadata_bytes.get(b"CATEGORY", ""),
encoding=_encodings['repo.content'], errors='replace')
- mypf = _unicode_decode(metadata_bytes.get(
- _unicode_encode("PF",
- encoding=_encodings['repo.content']), ""),
+ mypf = _unicode_decode(metadata_bytes.get(b"PF", ""),
encoding=_encodings['repo.content'], errors='replace')
- slot = _unicode_decode(metadata_bytes.get(
- _unicode_encode("SLOT",
- encoding=_encodings['repo.content']), ""),
+ slot = _unicode_decode(metadata_bytes.get(b"SLOT", ""),
encoding=_encodings['repo.content'], errors='replace')
mypkg = myfile[:-5]
if not mycat or not mypf or not slot:
", ".join(missing_keys))
msg.append(_(" This binary package is not " \
"recoverable and should be deleted."))
- from textwrap import wrap
- for line in wrap("".join(msg), 72):
+ for line in textwrap.wrap("".join(msg), 72):
writemsg("!!! %s\n" % line, noiselevel=-1)
self.invalids.append(mypkg)
continue
(mycpv, self.settings["PORTAGE_CONFIGROOT"]),
noiselevel=-1)
continue
+ mycpv = _pkg_str(mycpv)
pkg_paths[mycpv] = mypath
self.dbapi.cpv_inject(mycpv)
update_pkgindex = True
del pkgindex.packages[:]
pkgindex.packages.extend(iter(metadata.values()))
self._update_pkgindex_header(pkgindex.header)
- f = atomic_ofstream(self._pkgindex_file)
- pkgindex.write(f)
- f.close()
+ self._pkgindex_write(pkgindex)
if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
noiselevel=-1)
- if getbinpkgs and 'PORTAGE_BINHOST' in self.settings:
- base_url = self.settings["PORTAGE_BINHOST"]
- from portage.const import CACHE_PATH
- try:
- from urllib.parse import urlparse
- except ImportError:
- from urlparse import urlparse
- urldata = urlparse(base_url)
- pkgindex_file = os.path.join(self.settings["ROOT"], CACHE_PATH, "binhost",
- urldata[1] + urldata[2], "Packages")
+ if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
+ self.populated=1
+ return
+ self._remotepkgs = {}
+ for base_url in self.settings["PORTAGE_BINHOST"].split():
+ parsed_url = urlparse(base_url)
+ host = parsed_url.netloc
+ port = parsed_url.port
+ user = None
+ passwd = None
+ user_passwd = ""
+ if "@" in host:
+ user, host = host.split("@", 1)
+ user_passwd = user + "@"
+ if ":" in user:
+ user, passwd = user.split(":", 1)
+ port_args = []
+ if port is not None:
+ port_str = ":%s" % (port,)
+ if host.endswith(port_str):
+ host = host[:-len(port_str)]
+ pkgindex_file = os.path.join(self.settings["EROOT"], CACHE_PATH, "binhost",
+ host, parsed_url.path.lstrip("/"), "Packages")
pkgindex = self._new_pkgindex()
try:
- f = codecs.open(_unicode_encode(pkgindex_file,
+ f = io.open(_unicode_encode(pkgindex_file,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace')
if e.errno != errno.ENOENT:
raise
local_timestamp = pkgindex.header.get("TIMESTAMP", None)
- try:
- from urllib.request import urlopen as urllib_request_urlopen
- except ImportError:
- from urllib import urlopen as urllib_request_urlopen
+ remote_timestamp = None
rmt_idx = self._new_pkgindex()
+ proc = None
+ tmp_filename = None
try:
# urlparse.urljoin() only works correctly with recognized
# protocols and requires the base url to have a trailing
# slash, so join manually...
- f = urllib_request_urlopen(base_url.rstrip("/") + "/Packages")
+ url = base_url.rstrip("/") + "/Packages"
+ f = None
+
+ # Don't use urlopen for https, since it doesn't support
+ # certificate/hostname verification (bug #469888).
+ if parsed_url.scheme not in ('https',):
+ try:
+ f = _urlopen(url, if_modified_since=local_timestamp)
+ if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
+ remote_timestamp = f.headers.get('timestamp')
+ except IOError as err:
+ if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
+ raise UseCachedCopyOfRemoteIndex()
+
+ if parsed_url.scheme in ('ftp', 'http', 'https'):
+ # This protocol is supposedly supported by urlopen,
+ # so apparently there's a problem with the url
+ # or a bug in urlopen.
+ if self.settings.get("PORTAGE_DEBUG", "0") != "0":
+ traceback.print_exc()
+
+ raise
+
+ if f is None:
+
+ path = parsed_url.path.rstrip("/") + "/Packages"
+
+ if parsed_url.scheme == 'ssh':
+ # Use a pipe so that we can terminate the download
+ # early if we detect that the TIMESTAMP header
+ # matches that of the cached Packages file.
+ ssh_args = ['ssh']
+ if port is not None:
+ ssh_args.append("-p%s" % (port,))
+ # NOTE: shlex evaluates embedded quotes
+ ssh_args.extend(portage.util.shlex_split(
+ self.settings.get("PORTAGE_SSH_OPTS", "")))
+ ssh_args.append(user_passwd + host)
+ ssh_args.append('--')
+ ssh_args.append('cat')
+ ssh_args.append(path)
+
+ proc = subprocess.Popen(ssh_args,
+ stdout=subprocess.PIPE)
+ f = proc.stdout
+ else:
+ setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
+ fcmd = self.settings.get(setting)
+ if not fcmd:
+ fcmd = self.settings.get('FETCHCOMMAND')
+ if not fcmd:
+ raise EnvironmentError("FETCHCOMMAND is unset")
+
+ fd, tmp_filename = tempfile.mkstemp()
+ tmp_dirname, tmp_basename = os.path.split(tmp_filename)
+ os.close(fd)
+
+ fcmd_vars = {
+ "DISTDIR": tmp_dirname,
+ "FILE": tmp_basename,
+ "URI": url
+ }
+
+ for k in ("PORTAGE_SSH_OPTS",):
+ try:
+ fcmd_vars[k] = self.settings[k]
+ except KeyError:
+ pass
+
+ success = portage.getbinpkg.file_get(
+ fcmd=fcmd, fcmd_vars=fcmd_vars)
+ if not success:
+ raise EnvironmentError("%s failed" % (setting,))
+ f = open(tmp_filename, 'rb')
+
f_dec = codecs.iterdecode(f,
_encodings['repo.content'], errors='replace')
try:
rmt_idx.readHeader(f_dec)
- remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
+ if not remote_timestamp: # in case it had not been read from HTTP header
+ remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
if not remote_timestamp:
# no timestamp in the header, something's wrong
pkgindex = None
+ writemsg(_("\n\n!!! Binhost package index " \
+ " has no TIMESTAMP field.\n"), noiselevel=-1)
else:
if not self._pkgindex_version_supported(rmt_idx):
writemsg(_("\n\n!!! Binhost package index version" \
rmt_idx.readBody(f_dec)
pkgindex = rmt_idx
finally:
- f.close()
+ # Timeout after 5 seconds, in case close() blocks
+ # indefinitely (see bug #350139).
+ try:
+ try:
+ AlarmSignal.register(5)
+ f.close()
+ finally:
+ AlarmSignal.unregister()
+ except AlarmSignal:
+ writemsg("\n\n!!! %s\n" % \
+ _("Timed out while closing connection to binhost"),
+ noiselevel=-1)
+ except UseCachedCopyOfRemoteIndex:
+ writemsg_stdout("\n")
+ writemsg_stdout(
+ colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \
+ "\n")
+ rmt_idx = pkgindex
except EnvironmentError as e:
writemsg(_("\n\n!!! Error fetching binhost package" \
- " info from '%s'\n") % base_url)
+ " info from '%s'\n") % _hide_url_passwd(base_url))
writemsg("!!! %s\n\n" % str(e))
del e
pkgindex = None
+ if proc is not None:
+ if proc.poll() is None:
+ proc.kill()
+ proc.wait()
+ proc = None
+ if tmp_filename is not None:
+ try:
+ os.unlink(tmp_filename)
+ except OSError:
+ pass
if pkgindex is rmt_idx:
pkgindex.modified = False # don't update the header
try:
# file, but that's alright.
if pkgindex:
# Organize remote package list as a cpv -> metadata map.
- self._remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
+ remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
+ remote_base_uri = pkgindex.header.get("URI", base_url)
+ for cpv, remote_metadata in remotepkgs.items():
+ remote_metadata["BASE_URI"] = remote_base_uri
+ self._pkgindex_uri[cpv] = url
+ self._remotepkgs.update(remotepkgs)
self._remote_has_index = True
- self._remote_base_uri = pkgindex.header.get("URI", base_url)
- self.__remotepkgs = {}
- for cpv in self._remotepkgs:
+ for cpv in remotepkgs:
self.dbapi.cpv_inject(cpv)
- self.populated = 1
if True:
# Remote package instances override local package
# if they are not identical.
# Local package instances override remote instances.
for cpv in metadata:
self._remotepkgs.pop(cpv, None)
- return
- self._remotepkgs = {}
- try:
- chunk_size = long(self.settings["PORTAGE_BINHOST_CHUNKSIZE"])
- if chunk_size < 8:
- chunk_size = 8
- except (ValueError, KeyError):
- chunk_size = 3000
- writemsg_stdout("\n")
- writemsg_stdout(
- colorize("GOOD", _("Fetching bininfo from ")) + \
- re.sub(r'//(.+):.+@(.+)/', r'//\1:*password*@\2/', base_url) + "\n")
- self.__remotepkgs = portage.getbinpkg.dir_get_metadata(
- self.settings["PORTAGE_BINHOST"], chunk_size=chunk_size)
- #writemsg(green(" -- DONE!\n\n"))
-
- for mypkg in list(self.__remotepkgs):
- if "CATEGORY" not in self.__remotepkgs[mypkg]:
- #old-style or corrupt package
- writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg,
- noiselevel=-1)
- del self.__remotepkgs[mypkg]
- continue
- mycat = self.__remotepkgs[mypkg]["CATEGORY"].strip()
- fullpkg = mycat+"/"+mypkg[:-5]
-
- if fullpkg in metadata:
- # When using this old protocol, comparison with the remote
- # package isn't supported, so the local package is always
- # preferred even if getbinpkgsonly is enabled.
- continue
-
- if not self.dbapi._category_re.match(mycat):
- writemsg(_("!!! Remote binary package has an " \
- "unrecognized category: '%s'\n") % fullpkg,
- noiselevel=-1)
- writemsg(_("!!! '%s' has a category that is not" \
- " listed in %setc/portage/categories\n") % \
- (fullpkg, self.settings["PORTAGE_CONFIGROOT"]),
- noiselevel=-1)
- continue
- mykey = portage.cpv_getkey(fullpkg)
- try:
- # invalid tbz2's can hurt things.
- self.dbapi.cpv_inject(fullpkg)
- remote_metadata = self.__remotepkgs[mypkg]
- for k, v in remote_metadata.items():
- remote_metadata[k] = v.strip()
-
- # Eliminate metadata values with names that digestCheck
- # uses, since they are not valid when using the old
- # protocol. Typically this is needed for SIZE metadata
- # which corresponds to the size of the unpacked files
- # rather than the binpkg file size, triggering digest
- # verification failures as reported in bug #303211.
- remote_metadata.pop('SIZE', None)
- for k in portage.checksum.hashfunc_map:
- remote_metadata.pop(k, None)
-
- self._remotepkgs[fullpkg] = remote_metadata
- except SystemExit as e:
- raise
- except:
- writemsg(_("!!! Failed to inject remote binary package: %s\n") % fullpkg,
- noiselevel=-1)
- del self.__remotepkgs[mypkg]
- continue
+
self.populated=1
def inject(self, cpv, filename=None):
# Reread the Packages index (in case it's been changed by another
# process) and then updated it, all while holding a lock.
- from portage.locks import lockfile, unlockfile
pkgindex_lock = None
created_symlink = False
try:
wantnewlockfile=1)
if filename is not None:
new_filename = self.getname(cpv)
- self._ensure_dir(os.path.dirname(new_filename))
- _movefile(filename, new_filename, mysettings=self.settings)
+ try:
+ samefile = os.path.samefile(filename, new_filename)
+ except OSError:
+ samefile = False
+ if not samefile:
+ self._ensure_dir(os.path.dirname(new_filename))
+ _movefile(filename, new_filename, mysettings=self.settings)
+ full_path = new_filename
+
+ self._file_permissions(full_path)
+
if self._all_directory and \
self.getname(cpv).split(os.path.sep)[-2] == "All":
self._create_symlink(cpv)
pkgindex.packages.append(d)
self._update_pkgindex_header(pkgindex.header)
- f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
- pkgindex.write(f)
- f.close()
+ self._pkgindex_write(pkgindex)
+
finally:
if pkgindex_lock:
unlockfile(pkgindex_lock)
+ def _pkgindex_write(self, pkgindex):
+ contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
+ pkgindex.write(contents)
+ contents = contents.getvalue()
+ atime = mtime = long(pkgindex.header["TIMESTAMP"])
+ output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"),
+ self._pkgindex_file, None)]
+
+ if "compress-index" in self.settings.features:
+ gz_fname = self._pkgindex_file + ".gz"
+ fileobj = atomic_ofstream(gz_fname, mode="wb")
+ output_files.append((GzipFile(filename='', mode="wb",
+ fileobj=fileobj, mtime=mtime), gz_fname, fileobj))
+
+ for f, fname, f_close in output_files:
+ f.write(contents)
+ f.close()
+ if f_close is not None:
+ f_close.close()
+ self._file_permissions(fname)
+ # some seconds might have elapsed since TIMESTAMP
+ os.utime(fname, (atime, mtime))
+
def _pkgindex_entry(self, cpv):
"""
Performs checksums and evaluates USE flag conditionals.
Raises InvalidDependString if necessary.
@rtype: dict
- @returns: a dict containing entry for the give cpv.
+ @return: a dict containing entry for the give cpv.
"""
pkg_path = self.getname(cpv)
- from portage.checksum import perform_multiple_checksums
d = dict(zip(self._pkgindex_aux_keys,
self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
else:
header.pop(k, None)
+ # These values may be useful for using a binhost without
+ # having a local copy of the profile (bug #470006).
+ for k in self.settings.get("USE_EXPAND_IMPLICIT", "").split():
+ k = "USE_EXPAND_VALUES_" + k
+ v = self.settings.get(k)
+ if v:
+ header[k] = v
+ else:
+ header.pop(k, None)
+
def _pkgindex_version_supported(self, pkgindex):
version = pkgindex.header.get("VERSION")
if version:
use.sort()
metadata["USE"] = " ".join(use)
for k in self._pkgindex_use_evaluated_keys:
+ if k.endswith('DEPEND'):
+ token_class = Atom
+ else:
+ token_class = None
+
try:
deps = metadata[k]
- deps = use_reduce(deps, uselist=raw_use)
+ deps = use_reduce(deps, uselist=raw_use, token_class=token_class)
deps = paren_enclose(deps)
except portage.exception.InvalidDependString as e:
writemsg("%s: %s\n" % (k, str(e)),
noiselevel=-1)
raise
- if k in _vdb_use_conditional_atoms:
- v_split = []
- for x in deps.split():
- try:
- x = portage.dep.Atom(x)
- except portage.exception.InvalidAtom:
- v_split.append(x)
- else:
- v_split.append(str(x.evaluate_conditionals(raw_use)))
- deps = ' '.join(v_split)
metadata[k] = deps
def exists_specific(self, cpv):
# package is downloaded, state is updated by self.inject().
return True
+ def get_pkgindex_uri(self, pkgname):
+ """Returns the URI to the Packages file for a given package."""
+ return self._pkgindex_uri.get(pkgname)
+
+
+
def gettbz2(self, pkgname):
"""Fetches the package from a remote site, if necessary. Attempts to
resume if the file appears to be partially downloaded."""
tbz2name = os.path.basename(tbz2_path)
resume = False
if os.path.exists(tbz2_path):
- if (tbz2name not in self.invalids):
+ if tbz2name[:-5] not in self.invalids:
return
else:
resume = True
mydest = os.path.dirname(self.getname(pkgname))
self._ensure_dir(mydest)
- try:
- from urllib.parse import urlparse
- except ImportError:
- from urlparse import urlparse
# urljoin doesn't work correctly with unrecognized protocols like sftp
if self._remote_has_index:
rel_url = self._remotepkgs[pkgname].get("PATH")
if not rel_url:
rel_url = pkgname+".tbz2"
- url = self._remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
+ remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"]
+ url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
else:
url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
protocol = urlparse(url)[0]
def _load_pkgindex(self):
pkgindex = self._new_pkgindex()
try:
- f = codecs.open(_unicode_encode(self._pkgindex_file,
+ f = io.open(_unicode_encode(self._pkgindex_file,
encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'],
errors='replace')
f.close()
return pkgindex
- def digestCheck(self, pkg):
- """
- Verify digests for the given package and raise DigestException
- if verification fails.
- @rtype: bool
- @returns: True if digests could be located, False otherwise.
- """
- cpv = pkg
- if not isinstance(cpv, basestring):
+ def _get_digests(self, pkg):
+
+ try:
cpv = pkg.cpv
- pkg = None
+ except AttributeError:
+ cpv = pkg
- pkg_path = self.getname(cpv)
+ digests = {}
metadata = None
if self._remotepkgs is None or cpv not in self._remotepkgs:
for d in self._load_pkgindex().packages:
else:
metadata = self._remotepkgs[cpv]
if metadata is None:
- return False
+ return digests
- digests = {}
- from portage.checksum import hashfunc_map, verify_all
for k in hashfunc_map:
v = metadata.get(k)
if not v:
writemsg(_("!!! Malformed SIZE attribute in remote " \
"metadata for '%s'\n") % cpv)
+ return digests
+
+ def digestCheck(self, pkg):
+ """
+ Verify digests for the given package and raise DigestException
+ if verification fails.
+ @rtype: bool
+ @return: True if digests could be located, False otherwise.
+ """
+
+ digests = self._get_digests(pkg)
+
if not digests:
return False
+ try:
+ cpv = pkg.cpv
+ except AttributeError:
+ cpv = pkg
+
+ pkg_path = self.getname(cpv)
+ hash_filter = _hash_filter(
+ self.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
+ if not hash_filter.transparent:
+ digests = _apply_hash_filter(digests, hash_filter)
eout = EOutput()
eout.quiet = self.settings.get("PORTAGE_QUIET") == "1"
ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0)
"Get a slot for a catpkg; assume it exists."
myslot = ""
try:
- myslot = self.dbapi.aux_get(mycatpkg,["SLOT"])[0]
- except SystemExit as e:
- raise
- except Exception as e:
+ myslot = self.dbapi._pkg_str(mycatpkg, None).slot
+ except KeyError:
pass
return myslot
projects / portage.git / blobdiff