DEPRECATED_PROFILE_FILE, USER_VIRTUALS_FILE, EBUILD_SH_ENV_FILE, \
INVALID_ENV_FILE, CUSTOM_MIRRORS_FILE, CONFIG_MEMORY_FILE,\
INCREMENTALS, EAPI, MISC_SH_BINARY, REPO_NAME_LOC, REPO_NAME_FILE, \
- EPREFIX, EPREFIX_LSTRIP, BPREFIX, rootuid, rootgid
+ EPREFIX, EPREFIX_LSTRIP, BPREFIX, rootuid
from portage.localization import _
return retval >> 8
return retval
-<<<<<<< .working
-_userpriv_spawn_kwargs = (
- ("uid", portage_uid),
- ("gid", portage_gid),
- ("groups", userpriv_groups),
- ("umask", 0o02),
-)
-
-def _spawn_fetch(settings, args, **kwargs):
- """
- Spawn a process with appropriate settings for fetching, including
- userfetch and selinux support.
- """
-
- global _userpriv_spawn_kwargs
-
- # Redirect all output to stdout since some fetchers like
- # wget pollute stderr (if portage detects a problem then it
- # can send it's own message to stderr).
- if "fd_pipes" not in kwargs:
-
- kwargs["fd_pipes"] = {
- 0 : sys.stdin.fileno(),
- 1 : sys.stdout.fileno(),
- 2 : sys.stdout.fileno(),
- }
-
- if "userfetch" in settings.features and \
- os.getuid() == 0 and portage_gid and portage_uid:
- kwargs.update(_userpriv_spawn_kwargs)
-
- spawn_func = portage.process.spawn
-
- if settings.selinux_enabled():
- spawn_func = selinux.spawn_wrapper(spawn_func,
- settings["PORTAGE_FETCH_T"])
-
- # bash is an allowed entrypoint, while most binaries are not
- if args[0] != BASH_BINARY:
- args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
-
- rval = spawn_func(args, env=settings.environ(), **kwargs)
-
- return rval
-
-_userpriv_test_write_file_cache = {}
-_userpriv_test_write_cmd_script = "touch %(file_path)s 2>/dev/null ; rval=$? ; " + \
- "rm -f %(file_path)s ; exit $rval"
-
-def _userpriv_test_write_file(settings, file_path):
- """
- Drop privileges and try to open a file for writing. The file may or
- may not exist, and the parent directory is assumed to exist. The file
- is removed before returning.
-
- @param settings: A config instance which is passed to _spawn_fetch()
- @param file_path: A file path to open and write.
- @return: True if write succeeds, False otherwise.
- """
-
- global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
- rval = _userpriv_test_write_file_cache.get(file_path)
- if rval is not None:
- return rval
-
- args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
- {"file_path" : _shell_quote(file_path)}]
-
- returncode = _spawn_fetch(settings, args)
-
- rval = returncode == os.EX_OK
- _userpriv_test_write_file_cache[file_path] = rval
- return rval
-
-def _checksum_failure_temp_file(distdir, basename):
- """
- First try to find a duplicate temp file with the same checksum and return
- that filename if available. Otherwise, use mkstemp to create a new unique
- filename._checksum_failure_.$RANDOM, rename the given file, and return the
- new filename. In any case, filename will be renamed or removed before this
- function returns a temp filename.
- """
-
- filename = os.path.join(distdir, basename)
- size = os.stat(filename).st_size
- checksum = None
- tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
- for temp_filename in os.listdir(distdir):
- if not tempfile_re.match(temp_filename):
- continue
- temp_filename = os.path.join(distdir, temp_filename)
- try:
- if size != os.stat(temp_filename).st_size:
- continue
- except OSError:
- continue
- try:
- temp_checksum = portage.checksum.perform_md5(temp_filename)
- except portage.exception.FileNotFound:
- # Apparently the temp file disappeared. Let it go.
- continue
- if checksum is None:
- checksum = portage.checksum.perform_md5(filename)
- if checksum == temp_checksum:
- os.unlink(filename)
- return temp_filename
-
- from tempfile import mkstemp
- fd, temp_filename = mkstemp("", basename + "._checksum_failure_.", distdir)
- os.close(fd)
- os.rename(filename, temp_filename)
- return temp_filename
-
-def _check_digests(filename, digests, show_errors=1):
- """
- Check digests and display a message if an error occurs.
- @return True if all digests match, False otherwise.
- """
- verified_ok, reason = portage.checksum.verify_all(filename, digests)
- if not verified_ok:
- if show_errors:
- writemsg(_("!!! Previously fetched"
- " file: '%s'\n") % filename, noiselevel=-1)
- writemsg(_("!!! Reason: %s\n") % reason[0],
- noiselevel=-1)
- writemsg(_("!!! Got: %s\n"
- "!!! Expected: %s\n") % \
- (reason[1], reason[2]), noiselevel=-1)
- return False
- return True
-
-def _check_distfile(filename, digests, eout, show_errors=1):
- """
- @return a tuple of (match, stat_obj) where match is True if filename
- matches all given digests (if any) and stat_obj is a stat result, or
- None if the file does not exist.
- """
- if digests is None:
- digests = {}
- size = digests.get("size")
- if size is not None and len(digests) == 1:
- digests = None
-
- try:
- st = os.stat(filename)
- except OSError:
- return (False, None)
- if size is not None and size != st.st_size:
- return (False, st)
- if not digests:
- if size is not None:
- eout.ebegin(_("%s size ;-)") % os.path.basename(filename))
- eout.eend(0)
- elif st.st_size == 0:
- # Zero-byte distfiles are always invalid.
- return (False, st)
- else:
- if _check_digests(filename, digests, show_errors=show_errors):
- eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
- " ".join(sorted(digests))))
- eout.eend(0)
- else:
- return (False, st)
- return (True, st)
-
-_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
-
-_size_suffix_map = {
- '' : 0,
- 'K' : 10,
- 'M' : 20,
- 'G' : 30,
- 'T' : 40,
- 'P' : 50,
- 'E' : 60,
- 'Z' : 70,
- 'Y' : 80,
-}
-
-def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
- "fetch files. Will use digest file if available."
-
- if not myuris:
- return 1
-
- features = mysettings.features
- restrict = mysettings.get("PORTAGE_RESTRICT","").split()
-
- from portage.data import secpass
- userfetch = secpass >= 2 and "userfetch" in features
- userpriv = secpass >= 2 and "userpriv" in features
-
- # 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
- if "mirror" in restrict or \
- "nomirror" in restrict:
- if ("mirror" in features) and ("lmirror" not in features):
- # lmirror should allow you to bypass mirror restrictions.
- # XXX: This is not a good thing, and is temporary at best.
- print(_(">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."))
- return 1
-
- # Generally, downloading the same file repeatedly from
- # every single available mirror is a waste of bandwidth
- # and time, so there needs to be a cap.
- checksum_failure_max_tries = 5
- v = checksum_failure_max_tries
- try:
- v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS",
- checksum_failure_max_tries))
- except (ValueError, OverflowError):
- writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
- " contains non-integer value: '%s'\n") % \
- mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1)
- writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
- "default value: %s\n") % checksum_failure_max_tries,
- noiselevel=-1)
- v = checksum_failure_max_tries
- if v < 1:
- writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
- " contains value less than 1: '%s'\n") % v, noiselevel=-1)
- writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
- "default value: %s\n") % checksum_failure_max_tries,
- noiselevel=-1)
- v = checksum_failure_max_tries
- checksum_failure_max_tries = v
- del v
-
- fetch_resume_size_default = "350K"
- fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
- if fetch_resume_size is not None:
- fetch_resume_size = "".join(fetch_resume_size.split())
- if not fetch_resume_size:
- # If it's undefined or empty, silently use the default.
- fetch_resume_size = fetch_resume_size_default
- match = _fetch_resume_size_re.match(fetch_resume_size)
- if match is None or \
- (match.group(2).upper() not in _size_suffix_map):
- writemsg(_("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE"
- " contains an unrecognized format: '%s'\n") % \
- mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
- writemsg(_("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE "
- "default value: %s\n") % fetch_resume_size_default,
- noiselevel=-1)
- fetch_resume_size = None
- if fetch_resume_size is None:
- fetch_resume_size = fetch_resume_size_default
- match = _fetch_resume_size_re.match(fetch_resume_size)
- fetch_resume_size = int(match.group(1)) * \
- 2 ** _size_suffix_map[match.group(2).upper()]
-
- # Behave like the package has RESTRICT="primaryuri" after a
- # couple of checksum failures, to increase the probablility
- # of success before checksum_failure_max_tries is reached.
- checksum_failure_primaryuri = 2
- thirdpartymirrors = mysettings.thirdpartymirrors()
-
- # In the background parallel-fetch process, it's safe to skip checksum
- # verification of pre-existing files in $DISTDIR that have the correct
- # file size. The parent process will verify their checksums prior to
- # the unpack phase.
-
- parallel_fetchonly = "PORTAGE_PARALLEL_FETCHONLY" in mysettings
- if parallel_fetchonly:
- fetchonly = 1
-
- check_config_instance(mysettings)
-
- custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
- CUSTOM_MIRRORS_FILE), recursive=1)
-
- mymirrors=[]
-
- if listonly or ("distlocks" not in features):
- use_locks = 0
-
- fetch_to_ro = 0
- if "skiprocheck" in features:
- fetch_to_ro = 1
-
- if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro:
- if use_locks:
- writemsg(colorize("BAD",
- _("!!! For fetching to a read-only filesystem, "
- "locking should be turned off.\n")), noiselevel=-1)
- writemsg(_("!!! This can be done by adding -distlocks to "
- "FEATURES in /etc/make.conf\n"), noiselevel=-1)
-# use_locks = 0
-
- # local mirrors are always added
- if "local" in custommirrors:
- mymirrors += custommirrors["local"]
-
- if "nomirror" in restrict or \
- "mirror" in restrict:
- # We don't add any mirrors.
- pass
- else:
- if try_mirrors:
- mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
-
- skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
- pkgdir = mysettings.get("O")
- if not (pkgdir is None or skip_manifest):
- mydigests = Manifest(
- pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
- else:
- # no digests because fetch was not called for a specific package
- mydigests = {}
-
- ro_distdirs = [x for x in \
- util.shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
- if os.path.isdir(x)]
-
- fsmirrors = []
- for x in range(len(mymirrors)-1,-1,-1):
- if mymirrors[x] and mymirrors[x][0]=='/':
- fsmirrors += [mymirrors[x]]
- del mymirrors[x]
-
- restrict_fetch = "fetch" in restrict
- custom_local_mirrors = custommirrors.get("local", [])
- if restrict_fetch:
- # With fetch restriction, a normal uri may only be fetched from
- # custom local mirrors (if available). A mirror:// uri may also
- # be fetched from specific mirrors (effectively overriding fetch
- # restriction, but only for specific mirrors).
- locations = custom_local_mirrors
- else:
- locations = mymirrors
-
- file_uri_tuples = []
- # Check for 'items' attribute since OrderedDict is not a dict.
- if hasattr(myuris, 'items'):
- for myfile, uri_set in myuris.items():
- for myuri in uri_set:
- file_uri_tuples.append((myfile, myuri))
- else:
- for myuri in myuris:
- file_uri_tuples.append((os.path.basename(myuri), myuri))
-
- filedict = OrderedDict()
- primaryuri_indexes={}
- primaryuri_dict = {}
- thirdpartymirror_uris = {}
- for myfile, myuri in file_uri_tuples:
- if myfile not in filedict:
- filedict[myfile]=[]
- for y in range(0,len(locations)):
- filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
- if myuri[:9]=="mirror://":
- eidx = myuri.find("/", 9)
- if eidx != -1:
- mirrorname = myuri[9:eidx]
- path = myuri[eidx+1:]
-
- # Try user-defined mirrors first
- if mirrorname in custommirrors:
- for cmirr in custommirrors[mirrorname]:
- filedict[myfile].append(
- cmirr.rstrip("/") + "/" + path)
-
- # now try the official mirrors
- if mirrorname in thirdpartymirrors:
- shuffle(thirdpartymirrors[mirrorname])
-
- uris = [locmirr.rstrip("/") + "/" + path \
- for locmirr in thirdpartymirrors[mirrorname]]
- filedict[myfile].extend(uris)
- thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
-
- if not filedict[myfile]:
- writemsg(_("No known mirror by the name: %s\n") % (mirrorname))
- else:
- writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
- writemsg(" %s\n" % (myuri), noiselevel=-1)
- else:
- if restrict_fetch:
- # Only fetch from specific mirrors is allowed.
- continue
- if "primaryuri" in restrict:
- # Use the source site first.
- if myfile in primaryuri_indexes:
- primaryuri_indexes[myfile] += 1
- else:
- primaryuri_indexes[myfile] = 0
- filedict[myfile].insert(primaryuri_indexes[myfile], myuri)
- else:
- filedict[myfile].append(myuri)
- primaryuris = primaryuri_dict.get(myfile)
- if primaryuris is None:
- primaryuris = []
- primaryuri_dict[myfile] = primaryuris
- primaryuris.append(myuri)
-
- # Prefer thirdpartymirrors over normal mirrors in cases when
- # the file does not yet exist on the normal mirrors.
- for myfile, uris in thirdpartymirror_uris.items():
- primaryuri_dict.setdefault(myfile, []).extend(uris)
-
- can_fetch=True
-
- if listonly:
- can_fetch = False
-
- if can_fetch and not fetch_to_ro:
- global _userpriv_test_write_file_cache
- dirmode = 0o2070
- filemode = 0o60
- modemask = 0o2
- dir_gid = portage_gid
- if "FAKED_MODE" in mysettings:
- # When inside fakeroot, directories with portage's gid appear
- # to have root's gid. Therefore, use root's gid instead of
- # portage's gid to avoid spurrious permissions adjustments
- # when inside fakeroot.
- dir_gid = rootgid
- distdir_dirs = [""]
- if "distlocks" in features:
- distdir_dirs.append(".locks")
- try:
-
- for x in distdir_dirs:
- mydir = os.path.join(mysettings["DISTDIR"], x)
- write_test_file = os.path.join(
- mydir, ".__portage_test_write__")
-
- try:
- st = os.stat(mydir)
- except OSError:
- st = None
-
- if st is not None and stat.S_ISDIR(st.st_mode):
- if not (userfetch or userpriv):
- continue
- if _userpriv_test_write_file(mysettings, write_test_file):
- continue
-
- _userpriv_test_write_file_cache.pop(write_test_file, None)
- if portage.util.ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask):
- if st is None:
- # The directory has just been created
- # and therefore it must be empty.
- continue
- writemsg(_("Adjusting permissions recursively: '%s'\n") % mydir,
- noiselevel=-1)
- def onerror(e):
- raise # bail out on the first error that occurs during recursion
- if not apply_recursive_permissions(mydir,
- gid=dir_gid, dirmode=dirmode, dirmask=modemask,
- filemode=filemode, filemask=modemask, onerror=onerror):
- raise portage.exception.OperationNotPermitted(
- _("Failed to apply recursive permissions for the portage group."))
- except portage.exception.PortageException as e:
- if not os.path.isdir(mysettings["DISTDIR"]):
- writemsg("!!! %s\n" % str(e), noiselevel=-1)
- writemsg(_("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], noiselevel=-1)
- writemsg(_("!!! Fetching will fail!\n"), noiselevel=-1)
-
- if can_fetch and \
- not fetch_to_ro and \
- not os.access(mysettings["DISTDIR"], os.W_OK):
- writemsg(_("!!! No write access to '%s'\n") % mysettings["DISTDIR"],
- noiselevel=-1)
- can_fetch = False
-
- if can_fetch and use_locks and locks_in_subdir:
- distlocks_subdir = os.path.join(mysettings["DISTDIR"], locks_in_subdir)
- if not os.access(distlocks_subdir, os.W_OK):
- writemsg(_("!!! No write access to write to %s. Aborting.\n") % distlocks_subdir,
- noiselevel=-1)
- return 0
- del distlocks_subdir
-
- distdir_writable = can_fetch and not fetch_to_ro
- failed_files = set()
- restrict_fetch_msg = False
-
- for myfile in filedict:
- """
- fetched status
- 0 nonexistent
- 1 partially downloaded
- 2 completely downloaded
- """
- fetched = 0
-
- orig_digests = mydigests.get(myfile, {})
- size = orig_digests.get("size")
- if size == 0:
- # Zero-byte distfiles are always invalid, so discard their digests.
- del mydigests[myfile]
- orig_digests.clear()
- size = None
- pruned_digests = orig_digests
- if parallel_fetchonly:
- pruned_digests = {}
- if size is not None:
- pruned_digests["size"] = size
-
- myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
- has_space = True
- has_space_superuser = True
- file_lock = None
- if listonly:
- writemsg_stdout("\n", noiselevel=-1)
- else:
- # check if there is enough space in DISTDIR to completely store myfile
- # overestimate the filesize so we aren't bitten by FS overhead
- if size is not None and hasattr(os, "statvfs"):
- vfs_stat = os.statvfs(mysettings["DISTDIR"])
- try:
- mysize = os.stat(myfile_path).st_size
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
- mysize = 0
- if (size - mysize + vfs_stat.f_bsize) >= \
- (vfs_stat.f_bsize * vfs_stat.f_bavail):
-
- if (size - mysize + vfs_stat.f_bsize) >= \
- (vfs_stat.f_bsize * vfs_stat.f_bfree):
- has_space_superuser = False
-
- if not has_space_superuser:
- has_space = False
- elif secpass < 2:
- has_space = False
- elif userfetch:
- has_space = False
-
- if not has_space:
- writemsg(_("!!! Insufficient space to store %s in %s\n") % \
- (myfile, mysettings["DISTDIR"]), noiselevel=-1)
-
- if has_space_superuser:
- writemsg(_("!!! Insufficient privileges to use "
- "remaining space.\n"), noiselevel=-1)
- if userfetch:
- writemsg(_("!!! You may set FEATURES=\"-userfetch\""
- " in /etc/make.conf in order to fetch with\n"
- "!!! superuser privileges.\n"), noiselevel=-1)
-
- if distdir_writable and use_locks:
-
- if locks_in_subdir:
- lock_file = os.path.join(mysettings["DISTDIR"],
- locks_in_subdir, myfile)
- else:
- lock_file = myfile_path
-
- lock_kwargs = {}
- if fetchonly:
- lock_kwargs["flags"] = os.O_NONBLOCK
-
- try:
- file_lock = portage.locks.lockfile(myfile_path,
- wantnewlockfile=1, **lock_kwargs)
- except portage.exception.TryAgain:
- writemsg(_(">>> File '%s' is already locked by "
- "another fetcher. Continuing...\n") % myfile,
- noiselevel=-1)
- continue
- try:
- if not listonly:
-
- eout = portage.output.EOutput()
- eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
- match, mystat = _check_distfile(
- myfile_path, pruned_digests, eout)
- if match:
- if distdir_writable:
- try:
- apply_secpass_permissions(myfile_path,
- gid=portage_gid, mode=0o664, mask=0o2,
- stat_cached=mystat)
- except portage.exception.PortageException as e:
- if not os.access(myfile_path, os.R_OK):
- writemsg(_("!!! Failed to adjust permissions:"
- " %s\n") % str(e), noiselevel=-1)
- del e
- continue
-
- if distdir_writable and mystat is None:
- # Remove broken symlinks if necessary.
- try:
- os.unlink(myfile_path)
- except OSError:
- pass
-
- if mystat is not None:
- if stat.S_ISDIR(mystat.st_mode):
- portage.util.writemsg_level(
- _("!!! Unable to fetch file since "
- "a directory is in the way: \n"
- "!!! %s\n") % myfile_path,
- level=logging.ERROR, noiselevel=-1)
- return 0
-
- if mystat.st_size == 0:
- if distdir_writable:
- try:
- os.unlink(myfile_path)
- except OSError:
- pass
- elif distdir_writable:
- if mystat.st_size < fetch_resume_size and \
- mystat.st_size < size:
- # If the file already exists and the size does not
- # match the existing digests, it may be that the
- # user is attempting to update the digest. In this
- # case, the digestgen() function will advise the
- # user to use `ebuild --force foo.ebuild manifest`
- # in order to force the old digests to be replaced.
- # Since the user may want to keep this file, rename
- # it instead of deleting it.
- writemsg(_(">>> Renaming distfile with size "
- "%d (smaller than " "PORTAGE_FETCH_RESU"
- "ME_MIN_SIZE)\n") % mystat.st_size)
- temp_filename = \
- _checksum_failure_temp_file(
- mysettings["DISTDIR"], myfile)
- writemsg_stdout(_("Refetching... "
- "File renamed to '%s'\n\n") % \
- temp_filename, noiselevel=-1)
- elif mystat.st_size >= size:
- temp_filename = \
- _checksum_failure_temp_file(
- mysettings["DISTDIR"], myfile)
- writemsg_stdout(_("Refetching... "
- "File renamed to '%s'\n\n") % \
- temp_filename, noiselevel=-1)
-
- if distdir_writable and ro_distdirs:
- readonly_file = None
- for x in ro_distdirs:
- filename = os.path.join(x, myfile)
- match, mystat = _check_distfile(
- filename, pruned_digests, eout)
- if match:
- readonly_file = filename
- break
- if readonly_file is not None:
- try:
- os.unlink(myfile_path)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
- os.symlink(readonly_file, myfile_path)
- continue
-
- if fsmirrors and not os.path.exists(myfile_path) and has_space:
- for mydir in fsmirrors:
- mirror_file = os.path.join(mydir, myfile)
- try:
- shutil.copyfile(mirror_file, myfile_path)
- writemsg(_("Local mirror has file: %s\n") % myfile)
- break
- except (IOError, OSError) as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
-
- try:
- mystat = os.stat(myfile_path)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
- else:
- try:
- apply_secpass_permissions(
- myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
- stat_cached=mystat)
- except portage.exception.PortageException as e:
- if not os.access(myfile_path, os.R_OK):
- writemsg(_("!!! Failed to adjust permissions:"
- " %s\n") % str(e), noiselevel=-1)
-
- # If the file is empty then it's obviously invalid. Remove
- # the empty file and try to download if possible.
- if mystat.st_size == 0:
- if distdir_writable:
- try:
- os.unlink(myfile_path)
- except EnvironmentError:
- pass
- elif myfile not in mydigests:
- # We don't have a digest, but the file exists. We must
- # assume that it is fully downloaded.
- continue
- else:
- if mystat.st_size < mydigests[myfile]["size"] and \
- not restrict_fetch:
- fetched = 1 # Try to resume this download.
- elif parallel_fetchonly and \
- mystat.st_size == mydigests[myfile]["size"]:
- eout = portage.output.EOutput()
- eout.quiet = \
- mysettings.get("PORTAGE_QUIET") == "1"
- eout.ebegin(
- "%s size ;-)" % (myfile, ))
- eout.eend(0)
- continue
- else:
- verified_ok, reason = portage.checksum.verify_all(
- myfile_path, mydigests[myfile])
- if not verified_ok:
- writemsg(_("!!! Previously fetched"
- " file: '%s'\n") % myfile, noiselevel=-1)
- writemsg(_("!!! Reason: %s\n") % reason[0],
- noiselevel=-1)
- writemsg(_("!!! Got: %s\n"
- "!!! Expected: %s\n") % \
- (reason[1], reason[2]), noiselevel=-1)
- if reason[0] == _("Insufficient data for checksum verification"):
- return 0
- if distdir_writable:
- temp_filename = \
- _checksum_failure_temp_file(
- mysettings["DISTDIR"], myfile)
- writemsg_stdout(_("Refetching... "
- "File renamed to '%s'\n\n") % \
- temp_filename, noiselevel=-1)
- else:
- eout = portage.output.EOutput()
- eout.quiet = \
- mysettings.get("PORTAGE_QUIET", None) == "1"
- digests = mydigests.get(myfile)
- if digests:
- digests = list(digests)
- digests.sort()
- eout.ebegin(
- "%s %s ;-)" % (myfile, " ".join(digests)))
- eout.eend(0)
- continue # fetch any remaining files
-
- # Create a reversed list since that is optimal for list.pop().
- uri_list = filedict[myfile][:]
- uri_list.reverse()
- checksum_failure_count = 0
- tried_locations = set()
- while uri_list:
- loc = uri_list.pop()
- # Eliminate duplicates here in case we've switched to
- # "primaryuri" mode on the fly due to a checksum failure.
- if loc in tried_locations:
- continue
- tried_locations.add(loc)
- if listonly:
- writemsg_stdout(loc+" ", noiselevel=-1)
- continue
- # allow different fetchcommands per protocol
- protocol = loc[0:loc.find("://")]
-
- missing_file_param = False
- fetchcommand_var = "FETCHCOMMAND_" + protocol.upper()
- fetchcommand = mysettings.get(fetchcommand_var)
- if fetchcommand is None:
- fetchcommand_var = "FETCHCOMMAND"
- fetchcommand = mysettings.get(fetchcommand_var)
- if fetchcommand is None:
- portage.util.writemsg_level(
- _("!!! %s is unset. It should "
- "have been defined in\n!!! %s/make.globals.\n") \
- % (fetchcommand_var,
- portage.const.GLOBAL_CONFIG_PATH),
- level=logging.ERROR, noiselevel=-1)
- return 0
- if "${FILE}" not in fetchcommand:
- portage.util.writemsg_level(
- _("!!! %s does not contain the required ${FILE}"
- " parameter.\n") % fetchcommand_var,
- level=logging.ERROR, noiselevel=-1)
- missing_file_param = True
-
- resumecommand_var = "RESUMECOMMAND_" + protocol.upper()
- resumecommand = mysettings.get(resumecommand_var)
- if resumecommand is None:
- resumecommand_var = "RESUMECOMMAND"
- resumecommand = mysettings.get(resumecommand_var)
- if resumecommand is None:
- portage.util.writemsg_level(
- _("!!! %s is unset. It should "
- "have been defined in\n!!! %s/make.globals.\n") \
- % (resumecommand_var,
- portage.const.GLOBAL_CONFIG_PATH),
- level=logging.ERROR, noiselevel=-1)
- return 0
- if "${FILE}" not in resumecommand:
- portage.util.writemsg_level(
- _("!!! %s does not contain the required ${FILE}"
- " parameter.\n") % resumecommand_var,
- level=logging.ERROR, noiselevel=-1)
- missing_file_param = True
-
- if missing_file_param:
- portage.util.writemsg_level(
- _("!!! Refer to the make.conf(5) man page for "
- "information about how to\n!!! correctly specify "
- "FETCHCOMMAND and RESUMECOMMAND.\n"),
- level=logging.ERROR, noiselevel=-1)
- if myfile != os.path.basename(loc):
- return 0
-
- if not can_fetch:
- if fetched != 2:
- try:
- mysize = os.stat(myfile_path).st_size
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
- mysize = 0
-
- if mysize == 0:
- writemsg(_("!!! File %s isn't fetched but unable to get it.\n") % myfile,
- noiselevel=-1)
- elif size is None or size > mysize:
- writemsg(_("!!! File %s isn't fully fetched, but unable to complete it\n") % myfile,
- noiselevel=-1)
- else:
- writemsg(_("!!! File %s is incorrect size, "
- "but unable to retry.\n") % myfile, noiselevel=-1)
- return 0
- else:
- continue
-
- if fetched != 2 and has_space:
- #we either need to resume or start the download
- if fetched == 1:
- try:
- mystat = os.stat(myfile_path)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
- fetched = 0
- else:
- if mystat.st_size < fetch_resume_size:
- writemsg(_(">>> Deleting distfile with size "
- "%d (smaller than " "PORTAGE_FETCH_RESU"
- "ME_MIN_SIZE)\n") % mystat.st_size)
- try:
- os.unlink(myfile_path)
- except OSError as e:
- if e.errno not in \
- (errno.ENOENT, errno.ESTALE):
- raise
- del e
- fetched = 0
- if fetched == 1:
- #resume mode:
- writemsg(_(">>> Resuming download...\n"))
- locfetch=resumecommand
- command_var = resumecommand_var
- else:
- #normal mode:
- locfetch=fetchcommand
- command_var = fetchcommand_var
- writemsg_stdout(_(">>> Downloading '%s'\n") % \
- re.sub(r'//(.+):.+@(.+)/',r'//\1:*password*@\2/', loc))
- variables = {
- "DISTDIR": mysettings["DISTDIR"],
- "URI": loc,
- "FILE": myfile
- }
-
- myfetch = util.shlex_split(locfetch)
- myfetch = [varexpand(x, mydict=variables) for x in myfetch]
- myret = -1
- try:
-
- myret = _spawn_fetch(mysettings, myfetch)
-
- finally:
- try:
- apply_secpass_permissions(myfile_path,
- gid=portage_gid, mode=0o664, mask=0o2)
- except portage.exception.FileNotFound as e:
- pass
- except portage.exception.PortageException as e:
- if not os.access(myfile_path, os.R_OK):
- writemsg(_("!!! Failed to adjust permissions:"
- " %s\n") % str(e), noiselevel=-1)
-
- # If the file is empty then it's obviously invalid. Don't
- # trust the return value from the fetcher. Remove the
- # empty file and try to download again.
- try:
- if os.stat(myfile_path).st_size == 0:
- os.unlink(myfile_path)
- fetched = 0
- continue
- except EnvironmentError:
- pass
-
- if mydigests is not None and myfile in mydigests:
- try:
- mystat = os.stat(myfile_path)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- del e
- fetched = 0
- else:
-
- if stat.S_ISDIR(mystat.st_mode):
- # This can happen if FETCHCOMMAND erroneously
- # contains wget's -P option where it should
- # instead have -O.
- portage.util.writemsg_level(
- _("!!! The command specified in the "
- "%s variable appears to have\n!!! "
- "created a directory instead of a "
- "normal file.\n") % command_var,
- level=logging.ERROR, noiselevel=-1)
- portage.util.writemsg_level(
- _("!!! Refer to the make.conf(5) "
- "man page for information about how "
- "to\n!!! correctly specify "
- "FETCHCOMMAND and RESUMECOMMAND.\n"),
- level=logging.ERROR, noiselevel=-1)
- return 0
-
- # no exception? file exists. let digestcheck() report
- # an appropriately for size or checksum errors
-
- # If the fetcher reported success and the file is
- # too small, it's probably because the digest is
- # bad (upstream changed the distfile). In this
- # case we don't want to attempt to resume. Show a
- # digest verification failure to that the user gets
- # a clue about what just happened.
- if myret != os.EX_OK and \
- mystat.st_size < mydigests[myfile]["size"]:
- # Fetch failed... Try the next one... Kill 404 files though.
- if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
- html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
- if html404.search(codecs.open(
- _unicode_encode(myfile_path,
- encoding=_encodings['fs'], errors='strict'),
- mode='r', encoding=_encodings['content'], errors='replace'
- ).read()):
- try:
- os.unlink(mysettings["DISTDIR"]+"/"+myfile)
- writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n"))
- fetched = 0
- continue
- except (IOError, OSError):
- pass
- fetched = 1
- continue
- if True:
- # File is the correct size--check the checksums for the fetched
- # file NOW, for those users who don't have a stable/continuous
- # net connection. This way we have a chance to try to download
- # from another mirror...
- verified_ok,reason = portage.checksum.verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile])
- if not verified_ok:
- print(reason)
- writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
- noiselevel=-1)
- writemsg(_("!!! Reason: %s\n") % reason[0],
- noiselevel=-1)
- writemsg(_("!!! Got: %s\n!!! Expected: %s\n") % \
- (reason[1], reason[2]), noiselevel=-1)
- if reason[0] == _("Insufficient data for checksum verification"):
- return 0
- temp_filename = \
- _checksum_failure_temp_file(
- mysettings["DISTDIR"], myfile)
- writemsg_stdout(_("Refetching... "
- "File renamed to '%s'\n\n") % \
- temp_filename, noiselevel=-1)
- fetched=0
- checksum_failure_count += 1
- if checksum_failure_count == \
- checksum_failure_primaryuri:
- # Switch to "primaryuri" mode in order
- # to increase the probablility of
- # of success.
- primaryuris = \
- primaryuri_dict.get(myfile)
- if primaryuris:
- uri_list.extend(
- reversed(primaryuris))
- if checksum_failure_count >= \
- checksum_failure_max_tries:
- break
- else:
- eout = portage.output.EOutput()
- eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
- digests = mydigests.get(myfile)
- if digests:
- eout.ebegin("%s %s ;-)" % \
- (myfile, " ".join(sorted(digests))))
- eout.eend(0)
- fetched=2
- break
- else:
- if not myret:
- fetched=2
- break
- elif mydigests!=None:
- writemsg(_("No digest file available and download failed.\n\n"),
- noiselevel=-1)
- finally:
- if use_locks and file_lock:
- portage.locks.unlockfile(file_lock)
-
- if listonly:
- writemsg_stdout("\n", noiselevel=-1)
- if fetched != 2:
- if restrict_fetch and not restrict_fetch_msg:
- restrict_fetch_msg = True
- msg = _("\n!!! %s/%s"
- " has fetch restriction turned on.\n"
- "!!! This probably means that this "
- "ebuild's files must be downloaded\n"
- "!!! manually. See the comments in"
- " the ebuild for more information.\n\n") % \
- (mysettings["CATEGORY"], mysettings["PF"])
- portage.util.writemsg_level(msg,
- level=logging.ERROR, noiselevel=-1)
- have_builddir = "PORTAGE_BUILDDIR" in mysettings and \
- os.path.isdir(mysettings["PORTAGE_BUILDDIR"])
-
- global_tmpdir = mysettings["PORTAGE_TMPDIR"]
- private_tmpdir = None
- if not parallel_fetchonly and not have_builddir:
- # When called by digestgen(), it's normal that
- # PORTAGE_BUILDDIR doesn't exist. It's helpful
- # to show the pkg_nofetch output though, so go
- # ahead and create a temporary PORTAGE_BUILDDIR.
- # Use a temporary config instance to avoid altering
- # the state of the one that's been passed in.
- mysettings = config(clone=mysettings)
- from tempfile import mkdtemp
- try:
- private_tmpdir = mkdtemp("", "._portage_fetch_.",
- global_tmpdir)
- except OSError as e:
- if e.errno != portage.exception.PermissionDenied.errno:
- raise
- raise portage.exception.PermissionDenied(global_tmpdir)
- mysettings["PORTAGE_TMPDIR"] = private_tmpdir
- mysettings.backup_changes("PORTAGE_TMPDIR")
- debug = mysettings.get("PORTAGE_DEBUG") == "1"
- portage.doebuild_environment(mysettings["EBUILD"], "fetch",
- mysettings["ROOT"], mysettings, debug, 1, None)
- prepare_build_dirs(mysettings["ROOT"], mysettings, 0)
- have_builddir = True
-
- if not parallel_fetchonly and have_builddir:
- # To spawn pkg_nofetch requires PORTAGE_BUILDDIR for
- # ensuring sane $PWD (bug #239560) and storing elog
- # messages. Therefore, calling code needs to ensure that
- # PORTAGE_BUILDDIR is already clean and locked here.
-
- # All the pkg_nofetch goes to stderr since it's considered
- # to be an error message.
- fd_pipes = {
- 0 : sys.stdin.fileno(),
- 1 : sys.stderr.fileno(),
- 2 : sys.stderr.fileno(),
- }
-
- ebuild_phase = mysettings.get("EBUILD_PHASE")
- try:
- mysettings["EBUILD_PHASE"] = "nofetch"
- spawn(_shell_quote(EBUILD_SH_BINARY) + \
- " nofetch", mysettings, fd_pipes=fd_pipes)
- finally:
- if ebuild_phase is None:
- mysettings.pop("EBUILD_PHASE", None)
- else:
- mysettings["EBUILD_PHASE"] = ebuild_phase
- if private_tmpdir is not None:
- shutil.rmtree(private_tmpdir)
-
- elif restrict_fetch:
- pass
- elif listonly:
- pass
- elif not filedict[myfile]:
- writemsg(_("Warning: No mirrors available for file"
- " '%s'\n") % (myfile), noiselevel=-1)
- else:
- writemsg(_("!!! Couldn't download '%s'. Aborting.\n") % myfile,
- noiselevel=-1)
-
- if listonly:
- continue
- elif fetchonly:
- failed_files.add(myfile)
- continue
- return 0
- if failed_files:
- return 0
- return 1
-
-=======
->>>>>>> .merge-right.r15429
def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None):
"""
Generates a digest file if missing. Assumes all files are available.
projects / portage.git / commitdiff