pym/portage/package/ebuild/fetch.py

   1 # Copyright 2010-2011 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import print_function
   5
   6 __all__ = ['fetch']
   7
   8 import errno
   9 import io
  10 import logging
  11 import random
  12 import re
  13 import shutil
  14 import stat
  15 import sys
  16 import tempfile
  17
  18
  19 import portage
  20 portage.proxy.lazyimport.lazyimport(globals(),
  21         'portage.package.ebuild.config:check_config_instance,config',
  22         'portage.package.ebuild.doebuild:doebuild_environment,' + \
  23                 '_doebuild_spawn',
  24         'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
  25 )
  26
  27 from portage import OrderedDict, os, selinux, _encodings, \
  28         _shell_quote, _unicode_encode
  29 from portage.checksum import hashfunc_map, perform_md5, verify_all
  30 from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
  31         GLOBAL_CONFIG_PATH
  32 from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
  33 from portage.exception import FileNotFound, OperationNotPermitted, \
  34         PortageException, TryAgain
  35 from portage.localization import _
  36 from portage.locks import lockfile, unlockfile
  37 from portage.manifest import Manifest
  38 from portage.output import colorize, EOutput
  39 from portage.util import apply_recursive_permissions, \
  40         apply_secpass_permissions, ensure_dirs, grabdict, shlex_split, \
  41         varexpand, writemsg, writemsg_level, writemsg_stdout
  42 from portage.process import spawn
  43
  44 _userpriv_spawn_kwargs = (
  45         ("uid",    portage_uid),
  46         ("gid",    portage_gid),
  47         ("groups", userpriv_groups),
  48         ("umask",  0o02),
  49 )
  50
  51 def _spawn_fetch(settings, args, **kwargs):
  52         """
  53         Spawn a process with appropriate settings for fetching, including
  54         userfetch and selinux support.
  55         """
  56
  57         global _userpriv_spawn_kwargs
  58
  59         # Redirect all output to stdout since some fetchers like
  60         # wget pollute stderr (if portage detects a problem then it
  61         # can send it's own message to stderr).
  62         if "fd_pipes" not in kwargs:
  63
  64                 kwargs["fd_pipes"] = {
  65                         0 : sys.stdin.fileno(),
  66                         1 : sys.stdout.fileno(),
  67                         2 : sys.stdout.fileno(),
  68                 }
  69
  70         if "userfetch" in settings.features and \
  71                 os.getuid() == 0 and portage_gid and portage_uid:
  72                 kwargs.update(_userpriv_spawn_kwargs)
  73
  74         spawn_func = spawn
  75
  76         if settings.selinux_enabled():
  77                 spawn_func = selinux.spawn_wrapper(spawn_func,
  78                         settings["PORTAGE_FETCH_T"])
  79
  80                 # bash is an allowed entrypoint, while most binaries are not
  81                 if args[0] != BASH_BINARY:
  82                         args = [BASH_BINARY, "-c", "exec \"$@\"", args[0]] + args
  83
  84         # Ensure that EBUILD_PHASE is set to fetch, so that config.environ()
  85         # does not filter the calling environment (which may contain needed
  86         # proxy variables, as in bug #315421).
  87         phase_backup = settings.get('EBUILD_PHASE')
  88         settings['EBUILD_PHASE'] = 'fetch'
  89         try:
  90                 rval = spawn_func(args, env=settings.environ(), **kwargs)
  91         finally:
  92                 if phase_backup is None:
  93                         settings.pop('EBUILD_PHASE', None)
  94                 else:
  95                         settings['EBUILD_PHASE'] = phase_backup
  96
  97         return rval
  98
  99 _userpriv_test_write_file_cache = {}
 100 _userpriv_test_write_cmd_script = ">> %(file_path)s 2>/dev/null ; rval=$? ; " + \
 101         "rm -f  %(file_path)s ; exit $rval"
 102
 103 def _userpriv_test_write_file(settings, file_path):
 104         """
 105         Drop privileges and try to open a file for writing. The file may or
 106         may not exist, and the parent directory is assumed to exist. The file
 107         is removed before returning.
 108
 109         @param settings: A config instance which is passed to _spawn_fetch()
 110         @param file_path: A file path to open and write.
 111         @return: True if write succeeds, False otherwise.
 112         """
 113
 114         global _userpriv_test_write_file_cache, _userpriv_test_write_cmd_script
 115         rval = _userpriv_test_write_file_cache.get(file_path)
 116         if rval is not None:
 117                 return rval
 118
 119         args = [BASH_BINARY, "-c", _userpriv_test_write_cmd_script % \
 120                 {"file_path" : _shell_quote(file_path)}]
 121
 122         returncode = _spawn_fetch(settings, args)
 123
 124         rval = returncode == os.EX_OK
 125         _userpriv_test_write_file_cache[file_path] = rval
 126         return rval
 127
 128 def _checksum_failure_temp_file(distdir, basename):
 129         """
 130         First try to find a duplicate temp file with the same checksum and return
 131         that filename if available. Otherwise, use mkstemp to create a new unique
 132         filename._checksum_failure_.$RANDOM, rename the given file, and return the
 133         new filename. In any case, filename will be renamed or removed before this
 134         function returns a temp filename.
 135         """
 136
 137         filename = os.path.join(distdir, basename)
 138         size = os.stat(filename).st_size
 139         checksum = None
 140         tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
 141         for temp_filename in os.listdir(distdir):
 142                 if not tempfile_re.match(temp_filename):
 143                         continue
 144                 temp_filename = os.path.join(distdir, temp_filename)
 145                 try:
 146                         if size != os.stat(temp_filename).st_size:
 147                                 continue
 148                 except OSError:
 149                         continue
 150                 try:
 151                         temp_checksum = perform_md5(temp_filename)
 152                 except FileNotFound:
 153                         # Apparently the temp file disappeared. Let it go.
 154                         continue
 155                 if checksum is None:
 156                         checksum = perform_md5(filename)
 157                 if checksum == temp_checksum:
 158                         os.unlink(filename)
 159                         return temp_filename
 160
 161         fd, temp_filename = \
 162                 tempfile.mkstemp("", basename + "._checksum_failure_.", distdir)
 163         os.close(fd)
 164         os.rename(filename, temp_filename)
 165         return temp_filename
 166
 167 def _check_digests(filename, digests, show_errors=1):
 168         """
 169         Check digests and display a message if an error occurs.
 170         @return True if all digests match, False otherwise.
 171         """
 172         verified_ok, reason = verify_all(filename, digests)
 173         if not verified_ok:
 174                 if show_errors:
 175                         writemsg(_("!!! Previously fetched"
 176                                 " file: '%s'\n") % filename, noiselevel=-1)
 177                         writemsg(_("!!! Reason: %s\n") % reason[0],
 178                                 noiselevel=-1)
 179                         writemsg(_("!!! Got:      %s\n"
 180                                 "!!! Expected: %s\n") % \
 181                                 (reason[1], reason[2]), noiselevel=-1)
 182                 return False
 183         return True
 184
 185 def _check_distfile(filename, digests, eout, show_errors=1):
 186         """
 187         @return a tuple of (match, stat_obj) where match is True if filename
 188         matches all given digests (if any) and stat_obj is a stat result, or
 189         None if the file does not exist.
 190         """
 191         if digests is None:
 192                 digests = {}
 193         size = digests.get("size")
 194         if size is not None and len(digests) == 1:
 195                 digests = None
 196
 197         try:
 198                 st = os.stat(filename)
 199         except OSError:
 200                 return (False, None)
 201         if size is not None and size != st.st_size:
 202                 return (False, st)
 203         if not digests:
 204                 if size is not None:
 205                         eout.ebegin(_("%s size ;-)") % os.path.basename(filename))
 206                         eout.eend(0)
 207                 elif st.st_size == 0:
 208                         # Zero-byte distfiles are always invalid.
 209                         return (False, st)
 210         else:
 211                 if _check_digests(filename, digests, show_errors=show_errors):
 212                         eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
 213                                 " ".join(sorted(digests))))
 214                         eout.eend(0)
 215                 else:
 216                         return (False, st)
 217         return (True, st)
 218
 219 _fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
 220
 221 _size_suffix_map = {
 222         ''  : 0,
 223         'K' : 10,
 224         'M' : 20,
 225         'G' : 30,
 226         'T' : 40,
 227         'P' : 50,
 228         'E' : 60,
 229         'Z' : 70,
 230         'Y' : 80,
 231 }
 232
 233 def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 234         locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
 235         allow_missing_digests=True):
 236         "fetch files.  Will use digest file if available."
 237
 238         if not myuris:
 239                 return 1
 240
 241         features = mysettings.features
 242         restrict = mysettings.get("PORTAGE_RESTRICT","").split()
 243
 244         userfetch = secpass >= 2 and "userfetch" in features
 245         userpriv = secpass >= 2 and "userpriv" in features
 246
 247         # 'nomirror' is bad/negative logic. You Restrict mirroring, not no-mirroring.
 248         restrict_mirror = "mirror" in restrict or "nomirror" in restrict
 249         if restrict_mirror:
 250                 if ("mirror" in features) and ("lmirror" not in features):
 251                         # lmirror should allow you to bypass mirror restrictions.
 252                         # XXX: This is not a good thing, and is temporary at best.
 253                         print(_(">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."))
 254                         return 1
 255
 256         # Generally, downloading the same file repeatedly from
 257         # every single available mirror is a waste of bandwidth
 258         # and time, so there needs to be a cap.
 259         checksum_failure_max_tries = 5
 260         v = checksum_failure_max_tries
 261         try:
 262                 v = int(mysettings.get("PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS",
 263                         checksum_failure_max_tries))
 264         except (ValueError, OverflowError):
 265                 writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
 266                         " contains non-integer value: '%s'\n") % \
 267                         mysettings["PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"], noiselevel=-1)
 268                 writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
 269                         "default value: %s\n") % checksum_failure_max_tries,
 270                         noiselevel=-1)
 271                 v = checksum_failure_max_tries
 272         if v < 1:
 273                 writemsg(_("!!! Variable PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS"
 274                         " contains value less than 1: '%s'\n") % v, noiselevel=-1)
 275                 writemsg(_("!!! Using PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS "
 276                         "default value: %s\n") % checksum_failure_max_tries,
 277                         noiselevel=-1)
 278                 v = checksum_failure_max_tries
 279         checksum_failure_max_tries = v
 280         del v
 281
 282         fetch_resume_size_default = "350K"
 283         fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
 284         if fetch_resume_size is not None:
 285                 fetch_resume_size = "".join(fetch_resume_size.split())
 286                 if not fetch_resume_size:
 287                         # If it's undefined or empty, silently use the default.
 288                         fetch_resume_size = fetch_resume_size_default
 289                 match = _fetch_resume_size_re.match(fetch_resume_size)
 290                 if match is None or \
 291                         (match.group(2).upper() not in _size_suffix_map):
 292                         writemsg(_("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE"
 293                                 " contains an unrecognized format: '%s'\n") % \
 294                                 mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
 295                         writemsg(_("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE "
 296                                 "default value: %s\n") % fetch_resume_size_default,
 297                                 noiselevel=-1)
 298                         fetch_resume_size = None
 299         if fetch_resume_size is None:
 300                 fetch_resume_size = fetch_resume_size_default
 301                 match = _fetch_resume_size_re.match(fetch_resume_size)
 302         fetch_resume_size = int(match.group(1)) * \
 303                 2 ** _size_suffix_map[match.group(2).upper()]
 304
 305         # Behave like the package has RESTRICT="primaryuri" after a
 306         # couple of checksum failures, to increase the probablility
 307         # of success before checksum_failure_max_tries is reached.
 308         checksum_failure_primaryuri = 2
 309         thirdpartymirrors = mysettings.thirdpartymirrors()
 310
 311         # In the background parallel-fetch process, it's safe to skip checksum
 312         # verification of pre-existing files in $DISTDIR that have the correct
 313         # file size. The parent process will verify their checksums prior to
 314         # the unpack phase.
 315
 316         parallel_fetchonly = "PORTAGE_PARALLEL_FETCHONLY" in mysettings
 317         if parallel_fetchonly:
 318                 fetchonly = 1
 319
 320         check_config_instance(mysettings)
 321
 322         custommirrors = grabdict(os.path.join(mysettings["PORTAGE_CONFIGROOT"],
 323                 CUSTOM_MIRRORS_FILE), recursive=1)
 324
 325         mymirrors=[]
 326
 327         if listonly or ("distlocks" not in features):
 328                 use_locks = 0
 329
 330         fetch_to_ro = 0
 331         if "skiprocheck" in features:
 332                 fetch_to_ro = 1
 333
 334         if not os.access(mysettings["DISTDIR"],os.W_OK) and fetch_to_ro:
 335                 if use_locks:
 336                         writemsg(colorize("BAD",
 337                                 _("!!! For fetching to a read-only filesystem, "
 338                                 "locking should be turned off.\n")), noiselevel=-1)
 339                         writemsg(_("!!! This can be done by adding -distlocks to "
 340                                 "FEATURES in /etc/make.conf\n"), noiselevel=-1)
 341 #                       use_locks = 0
 342
 343         # local mirrors are always added
 344         if "local" in custommirrors:
 345                 mymirrors += custommirrors["local"]
 346
 347         if restrict_mirror:
 348                 # We don't add any mirrors.
 349                 pass
 350         else:
 351                 if try_mirrors:
 352                         mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
 353
 354         skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
 355         if skip_manifest:
 356                 allow_missing_digests = True
 357         pkgdir = mysettings.get("O")
 358         if digests is None and not (pkgdir is None or skip_manifest):
 359                 mydigests = mysettings.repositories.get_repo_for_location(
 360                         os.path.dirname(os.path.dirname(pkgdir))).load_manifest(
 361                         pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST")
 362         elif digests is None or skip_manifest:
 363                 # no digests because fetch was not called for a specific package
 364                 mydigests = {}
 365         else:
 366                 mydigests = digests
 367
 368         ro_distdirs = [x for x in \
 369                 shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) \
 370                 if os.path.isdir(x)]
 371
 372         fsmirrors = []
 373         for x in range(len(mymirrors)-1,-1,-1):
 374                 if mymirrors[x] and mymirrors[x][0]=='/':
 375                         fsmirrors += [mymirrors[x]]
 376                         del mymirrors[x]
 377
 378         restrict_fetch = "fetch" in restrict
 379         force_mirror = "force-mirror" in features and not restrict_mirror
 380         custom_local_mirrors = custommirrors.get("local", [])
 381         if restrict_fetch:
 382                 # With fetch restriction, a normal uri may only be fetched from
 383                 # custom local mirrors (if available).  A mirror:// uri may also
 384                 # be fetched from specific mirrors (effectively overriding fetch
 385                 # restriction, but only for specific mirrors).
 386                 locations = custom_local_mirrors
 387         else:
 388                 locations = mymirrors
 389
 390         file_uri_tuples = []
 391         # Check for 'items' attribute since OrderedDict is not a dict.
 392         if hasattr(myuris, 'items'):
 393                 for myfile, uri_set in myuris.items():
 394                         for myuri in uri_set:
 395                                 file_uri_tuples.append((myfile, myuri))
 396         else:
 397                 for myuri in myuris:
 398                         file_uri_tuples.append((os.path.basename(myuri), myuri))
 399
 400         filedict = OrderedDict()
 401         primaryuri_indexes={}
 402         primaryuri_dict = {}
 403         thirdpartymirror_uris = {}
 404         for myfile, myuri in file_uri_tuples:
 405                 if myfile not in filedict:
 406                         filedict[myfile]=[]
 407                         for y in range(0,len(locations)):
 408                                 filedict[myfile].append(locations[y]+"/distfiles/"+myfile)
 409                 if myuri[:9]=="mirror://":
 410                         eidx = myuri.find("/", 9)
 411                         if eidx != -1:
 412                                 mirrorname = myuri[9:eidx]
 413                                 path = myuri[eidx+1:]
 414
 415                                 # Try user-defined mirrors first
 416                                 if mirrorname in custommirrors:
 417                                         for cmirr in custommirrors[mirrorname]:
 418                                                 filedict[myfile].append(
 419                                                         cmirr.rstrip("/") + "/" + path)
 420
 421                                 # now try the official mirrors
 422                                 if mirrorname in thirdpartymirrors:
 423                                         random.shuffle(thirdpartymirrors[mirrorname])
 424
 425                                         uris = [locmirr.rstrip("/") + "/" + path \
 426                                                 for locmirr in thirdpartymirrors[mirrorname]]
 427                                         filedict[myfile].extend(uris)
 428                                         thirdpartymirror_uris.setdefault(myfile, []).extend(uris)
 429
 430                                 if not filedict[myfile]:
 431                                         writemsg(_("No known mirror by the name: %s\n") % (mirrorname))
 432                         else:
 433                                 writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1)
 434                                 writemsg("  %s\n" % (myuri), noiselevel=-1)
 435                 else:
 436                         if restrict_fetch or force_mirror:
 437                                 # Only fetch from specific mirrors is allowed.
 438                                 continue
 439                         if "primaryuri" in restrict:
 440                                 # Use the source site first.
 441                                 if myfile in primaryuri_indexes:
 442                                         primaryuri_indexes[myfile] += 1
 443                                 else:
 444                                         primaryuri_indexes[myfile] = 0
 445                                 filedict[myfile].insert(primaryuri_indexes[myfile], myuri)
 446                         else:
 447                                 filedict[myfile].append(myuri)
 448                         primaryuris = primaryuri_dict.get(myfile)
 449                         if primaryuris is None:
 450                                 primaryuris = []
 451                                 primaryuri_dict[myfile] = primaryuris
 452                         primaryuris.append(myuri)
 453
 454         # Prefer thirdpartymirrors over normal mirrors in cases when
 455         # the file does not yet exist on the normal mirrors.
 456         for myfile, uris in thirdpartymirror_uris.items():
 457                 primaryuri_dict.setdefault(myfile, []).extend(uris)
 458
 459         can_fetch=True
 460
 461         if listonly:
 462                 can_fetch = False
 463
 464         if can_fetch and not fetch_to_ro:
 465                 global _userpriv_test_write_file_cache
 466                 dirmode  = 0o070
 467                 filemode =   0o60
 468                 modemask =    0o2
 469                 dir_gid = portage_gid
 470                 if "FAKED_MODE" in mysettings:
 471                         # When inside fakeroot, directories with portage's gid appear
 472                         # to have root's gid. Therefore, use root's gid instead of
 473                         # portage's gid to avoid spurrious permissions adjustments
 474                         # when inside fakeroot.
 475                         dir_gid = 0
 476                 distdir_dirs = [""]
 477                 try:
 478
 479                         for x in distdir_dirs:
 480                                 mydir = os.path.join(mysettings["DISTDIR"], x)
 481                                 write_test_file = os.path.join(
 482                                         mydir, ".__portage_test_write__")
 483
 484                                 try:
 485                                         st = os.stat(mydir)
 486                                 except OSError:
 487                                         st = None
 488
 489                                 if st is not None and stat.S_ISDIR(st.st_mode):
 490                                         if not (userfetch or userpriv):
 491                                                 continue
 492                                         if _userpriv_test_write_file(mysettings, write_test_file):
 493                                                 continue
 494
 495                                 _userpriv_test_write_file_cache.pop(write_test_file, None)
 496                                 if ensure_dirs(mydir, gid=dir_gid, mode=dirmode, mask=modemask):
 497                                         if st is None:
 498                                                 # The directory has just been created
 499                                                 # and therefore it must be empty.
 500                                                 continue
 501                                         writemsg(_("Adjusting permissions recursively: '%s'\n") % mydir,
 502                                                 noiselevel=-1)
 503                                         def onerror(e):
 504                                                 raise # bail out on the first error that occurs during recursion
 505                                         if not apply_recursive_permissions(mydir,
 506                                                 gid=dir_gid, dirmode=dirmode, dirmask=modemask,
 507                                                 filemode=filemode, filemask=modemask, onerror=onerror):
 508                                                 raise OperationNotPermitted(
 509                                                         _("Failed to apply recursive permissions for the portage group."))
 510                 except PortageException as e:
 511                         if not os.path.isdir(mysettings["DISTDIR"]):
 512                                 writemsg("!!! %s\n" % str(e), noiselevel=-1)
 513                                 writemsg(_("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], noiselevel=-1)
 514                                 writemsg(_("!!! Fetching will fail!\n"), noiselevel=-1)
 515
 516         if can_fetch and \
 517                 not fetch_to_ro and \
 518                 not os.access(mysettings["DISTDIR"], os.W_OK):
 519                 writemsg(_("!!! No write access to '%s'\n") % mysettings["DISTDIR"],
 520                         noiselevel=-1)
 521                 can_fetch = False
 522
 523         distdir_writable = can_fetch and not fetch_to_ro
 524         failed_files = set()
 525         restrict_fetch_msg = False
 526
 527         for myfile in filedict:
 528                 """
 529                 fetched  status
 530                 0        nonexistent
 531                 1        partially downloaded
 532                 2        completely downloaded
 533                 """
 534                 fetched = 0
 535
 536                 orig_digests = mydigests.get(myfile, {})
 537
 538                 if not (allow_missing_digests or listonly):
 539                         verifiable_hash_types = set(orig_digests).intersection(hashfunc_map)
 540                         verifiable_hash_types.discard("size")
 541                         if not verifiable_hash_types:
 542                                 expected = set(hashfunc_map)
 543                                 expected.discard("size")
 544                                 expected = " ".join(sorted(expected))
 545                                 got = set(orig_digests)
 546                                 got.discard("size")
 547                                 got = " ".join(sorted(got))
 548                                 reason = (_("Insufficient data for checksum verification"),
 549                                         got, expected)
 550                                 writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
 551                                         noiselevel=-1)
 552                                 writemsg(_("!!! Reason: %s\n") % reason[0],
 553                                         noiselevel=-1)
 554                                 writemsg(_("!!! Got:      %s\n!!! Expected: %s\n") % \
 555                                         (reason[1], reason[2]), noiselevel=-1)
 556
 557                                 if fetchonly:
 558                                         failed_files.add(myfile)
 559                                         continue
 560                                 else:
 561                                         return 0
 562
 563                 size = orig_digests.get("size")
 564                 if size == 0:
 565                         # Zero-byte distfiles are always invalid, so discard their digests.
 566                         del mydigests[myfile]
 567                         orig_digests.clear()
 568                         size = None
 569                 pruned_digests = orig_digests
 570                 if parallel_fetchonly:
 571                         pruned_digests = {}
 572                         if size is not None:
 573                                 pruned_digests["size"] = size
 574
 575                 myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
 576                 has_space = True
 577                 has_space_superuser = True
 578                 file_lock = None
 579                 if listonly:
 580                         writemsg_stdout("\n", noiselevel=-1)
 581                 else:
 582                         # check if there is enough space in DISTDIR to completely store myfile
 583                         # overestimate the filesize so we aren't bitten by FS overhead
 584                         vfs_stat = None
 585                         if size is not None and hasattr(os, "statvfs"):
 586                                 try:
 587                                         vfs_stat = os.statvfs(mysettings["DISTDIR"])
 588                                 except OSError as e:
 589                                         writemsg_level("!!! statvfs('%s'): %s\n" %
 590                                                 (mysettings["DISTDIR"], e),
 591                                                 noiselevel=-1, level=logging.ERROR)
 592                                         del e
 593
 594                         if vfs_stat is not None:
 595                                 try:
 596                                         mysize = os.stat(myfile_path).st_size
 597                                 except OSError as e:
 598                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 599                                                 raise
 600                                         del e
 601                                         mysize = 0
 602                                 if (size - mysize + vfs_stat.f_bsize) >= \
 603                                         (vfs_stat.f_bsize * vfs_stat.f_bavail):
 604
 605                                         if (size - mysize + vfs_stat.f_bsize) >= \
 606                                                 (vfs_stat.f_bsize * vfs_stat.f_bfree):
 607                                                 has_space_superuser = False
 608
 609                                         if not has_space_superuser:
 610                                                 has_space = False
 611                                         elif secpass < 2:
 612                                                 has_space = False
 613                                         elif userfetch:
 614                                                 has_space = False
 615
 616                         if not has_space:
 617                                 writemsg(_("!!! Insufficient space to store %s in %s\n") % \
 618                                         (myfile, mysettings["DISTDIR"]), noiselevel=-1)
 619
 620                                 if has_space_superuser:
 621                                         writemsg(_("!!! Insufficient privileges to use "
 622                                                 "remaining space.\n"), noiselevel=-1)
 623                                         if userfetch:
 624                                                 writemsg(_("!!! You may set FEATURES=\"-userfetch\""
 625                                                         " in /etc/make.conf in order to fetch with\n"
 626                                                         "!!! superuser privileges.\n"), noiselevel=-1)
 627
 628                         if distdir_writable and use_locks:
 629
 630                                 lock_kwargs = {}
 631                                 if fetchonly:
 632                                         lock_kwargs["flags"] = os.O_NONBLOCK
 633
 634                                 try:
 635                                         file_lock = lockfile(myfile_path,
 636                                                 wantnewlockfile=1, **lock_kwargs)
 637                                 except TryAgain:
 638                                         writemsg(_(">>> File '%s' is already locked by "
 639                                                 "another fetcher. Continuing...\n") % myfile,
 640                                                 noiselevel=-1)
 641                                         continue
 642                 try:
 643                         if not listonly:
 644
 645                                 eout = EOutput()
 646                                 eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
 647                                 match, mystat = _check_distfile(
 648                                         myfile_path, pruned_digests, eout)
 649                                 if match:
 650                                         if distdir_writable:
 651                                                 try:
 652                                                         apply_secpass_permissions(myfile_path,
 653                                                                 gid=portage_gid, mode=0o664, mask=0o2,
 654                                                                 stat_cached=mystat)
 655                                                 except PortageException as e:
 656                                                         if not os.access(myfile_path, os.R_OK):
 657                                                                 writemsg(_("!!! Failed to adjust permissions:"
 658                                                                         " %s\n") % str(e), noiselevel=-1)
 659                                                         del e
 660                                         continue
 661
 662                                 if distdir_writable and mystat is None:
 663                                         # Remove broken symlinks if necessary.
 664                                         try:
 665                                                 os.unlink(myfile_path)
 666                                         except OSError:
 667                                                 pass
 668
 669                                 if mystat is not None:
 670                                         if stat.S_ISDIR(mystat.st_mode):
 671                                                 writemsg_level(
 672                                                         _("!!! Unable to fetch file since "
 673                                                         "a directory is in the way: \n"
 674                                                         "!!!   %s\n") % myfile_path,
 675                                                         level=logging.ERROR, noiselevel=-1)
 676                                                 return 0
 677
 678                                         if mystat.st_size == 0:
 679                                                 if distdir_writable:
 680                                                         try:
 681                                                                 os.unlink(myfile_path)
 682                                                         except OSError:
 683                                                                 pass
 684                                         elif distdir_writable:
 685                                                 if mystat.st_size < fetch_resume_size and \
 686                                                         mystat.st_size < size:
 687                                                         # If the file already exists and the size does not
 688                                                         # match the existing digests, it may be that the
 689                                                         # user is attempting to update the digest. In this
 690                                                         # case, the digestgen() function will advise the
 691                                                         # user to use `ebuild --force foo.ebuild manifest`
 692                                                         # in order to force the old digests to be replaced.
 693                                                         # Since the user may want to keep this file, rename
 694                                                         # it instead of deleting it.
 695                                                         writemsg(_(">>> Renaming distfile with size "
 696                                                                 "%d (smaller than " "PORTAGE_FETCH_RESU"
 697                                                                 "ME_MIN_SIZE)\n") % mystat.st_size)
 698                                                         temp_filename = \
 699                                                                 _checksum_failure_temp_file(
 700                                                                 mysettings["DISTDIR"], myfile)
 701                                                         writemsg_stdout(_("Refetching... "
 702                                                                 "File renamed to '%s'\n\n") % \
 703                                                                 temp_filename, noiselevel=-1)
 704                                                 elif mystat.st_size >= size:
 705                                                         temp_filename = \
 706                                                                 _checksum_failure_temp_file(
 707                                                                 mysettings["DISTDIR"], myfile)
 708                                                         writemsg_stdout(_("Refetching... "
 709                                                                 "File renamed to '%s'\n\n") % \
 710                                                                 temp_filename, noiselevel=-1)
 711
 712                                 if distdir_writable and ro_distdirs:
 713                                         readonly_file = None
 714                                         for x in ro_distdirs:
 715                                                 filename = os.path.join(x, myfile)
 716                                                 match, mystat = _check_distfile(
 717                                                         filename, pruned_digests, eout)
 718                                                 if match:
 719                                                         readonly_file = filename
 720                                                         break
 721                                         if readonly_file is not None:
 722                                                 try:
 723                                                         os.unlink(myfile_path)
 724                                                 except OSError as e:
 725                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 726                                                                 raise
 727                                                         del e
 728                                                 os.symlink(readonly_file, myfile_path)
 729                                                 continue
 730
 731                                 if fsmirrors and not os.path.exists(myfile_path) and has_space:
 732                                         for mydir in fsmirrors:
 733                                                 mirror_file = os.path.join(mydir, myfile)
 734                                                 try:
 735                                                         shutil.copyfile(mirror_file, myfile_path)
 736                                                         writemsg(_("Local mirror has file: %s\n") % myfile)
 737                                                         break
 738                                                 except (IOError, OSError) as e:
 739                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 740                                                                 raise
 741                                                         del e
 742
 743                                 try:
 744                                         mystat = os.stat(myfile_path)
 745                                 except OSError as e:
 746                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 747                                                 raise
 748                                         del e
 749                                 else:
 750                                         try:
 751                                                 apply_secpass_permissions(
 752                                                         myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
 753                                                         stat_cached=mystat)
 754                                         except PortageException as e:
 755                                                 if not os.access(myfile_path, os.R_OK):
 756                                                         writemsg(_("!!! Failed to adjust permissions:"
 757                                                                 " %s\n") % str(e), noiselevel=-1)
 758
 759                                         # If the file is empty then it's obviously invalid. Remove
 760                                         # the empty file and try to download if possible.
 761                                         if mystat.st_size == 0:
 762                                                 if distdir_writable:
 763                                                         try:
 764                                                                 os.unlink(myfile_path)
 765                                                         except EnvironmentError:
 766                                                                 pass
 767                                         elif myfile not in mydigests:
 768                                                 # We don't have a digest, but the file exists.  We must
 769                                                 # assume that it is fully downloaded.
 770                                                 continue
 771                                         else:
 772                                                 if mystat.st_size < mydigests[myfile]["size"] and \
 773                                                         not restrict_fetch:
 774                                                         fetched = 1 # Try to resume this download.
 775                                                 elif parallel_fetchonly and \
 776                                                         mystat.st_size == mydigests[myfile]["size"]:
 777                                                         eout = EOutput()
 778                                                         eout.quiet = \
 779                                                                 mysettings.get("PORTAGE_QUIET") == "1"
 780                                                         eout.ebegin(
 781                                                                 "%s size ;-)" % (myfile, ))
 782                                                         eout.eend(0)
 783                                                         continue
 784                                                 else:
 785                                                         verified_ok, reason = verify_all(
 786                                                                 myfile_path, mydigests[myfile])
 787                                                         if not verified_ok:
 788                                                                 writemsg(_("!!! Previously fetched"
 789                                                                         " file: '%s'\n") % myfile, noiselevel=-1)
 790                                                                 writemsg(_("!!! Reason: %s\n") % reason[0],
 791                                                                         noiselevel=-1)
 792                                                                 writemsg(_("!!! Got:      %s\n"
 793                                                                         "!!! Expected: %s\n") % \
 794                                                                         (reason[1], reason[2]), noiselevel=-1)
 795                                                                 if reason[0] == _("Insufficient data for checksum verification"):
 796                                                                         return 0
 797                                                                 if distdir_writable:
 798                                                                         temp_filename = \
 799                                                                                 _checksum_failure_temp_file(
 800                                                                                 mysettings["DISTDIR"], myfile)
 801                                                                         writemsg_stdout(_("Refetching... "
 802                                                                                 "File renamed to '%s'\n\n") % \
 803                                                                                 temp_filename, noiselevel=-1)
 804                                                         else:
 805                                                                 eout = EOutput()
 806                                                                 eout.quiet = \
 807                                                                         mysettings.get("PORTAGE_QUIET", None) == "1"
 808                                                                 digests = mydigests.get(myfile)
 809                                                                 if digests:
 810                                                                         digests = list(digests)
 811                                                                         digests.sort()
 812                                                                         eout.ebegin(
 813                                                                                 "%s %s ;-)" % (myfile, " ".join(digests)))
 814                                                                         eout.eend(0)
 815                                                                 continue # fetch any remaining files
 816
 817                         # Create a reversed list since that is optimal for list.pop().
 818                         uri_list = filedict[myfile][:]
 819                         uri_list.reverse()
 820                         checksum_failure_count = 0
 821                         tried_locations = set()
 822                         while uri_list:
 823                                 loc = uri_list.pop()
 824                                 # Eliminate duplicates here in case we've switched to
 825                                 # "primaryuri" mode on the fly due to a checksum failure.
 826                                 if loc in tried_locations:
 827                                         continue
 828                                 tried_locations.add(loc)
 829                                 if listonly:
 830                                         writemsg_stdout(loc+" ", noiselevel=-1)
 831                                         continue
 832                                 # allow different fetchcommands per protocol
 833                                 protocol = loc[0:loc.find("://")]
 834
 835                                 global_config_path = GLOBAL_CONFIG_PATH
 836                                 if mysettings['EPREFIX']:
 837                                         global_config_path = os.path.join(mysettings['EPREFIX'],
 838                                                         GLOBAL_CONFIG_PATH.lstrip(os.sep))
 839
 840                                 missing_file_param = False
 841                                 fetchcommand_var = "FETCHCOMMAND_" + protocol.upper()
 842                                 fetchcommand = mysettings.get(fetchcommand_var)
 843                                 if fetchcommand is None:
 844                                         fetchcommand_var = "FETCHCOMMAND"
 845                                         fetchcommand = mysettings.get(fetchcommand_var)
 846                                         if fetchcommand is None:
 847                                                 writemsg_level(
 848                                                         _("!!! %s is unset. It should "
 849                                                         "have been defined in\n!!! %s/make.globals.\n") \
 850                                                         % (fetchcommand_var, global_config_path),
 851                                                         level=logging.ERROR, noiselevel=-1)
 852                                                 return 0
 853                                 if "${FILE}" not in fetchcommand:
 854                                         writemsg_level(
 855                                                 _("!!! %s does not contain the required ${FILE}"
 856                                                 " parameter.\n") % fetchcommand_var,
 857                                                 level=logging.ERROR, noiselevel=-1)
 858                                         missing_file_param = True
 859
 860                                 resumecommand_var = "RESUMECOMMAND_" + protocol.upper()
 861                                 resumecommand = mysettings.get(resumecommand_var)
 862                                 if resumecommand is None:
 863                                         resumecommand_var = "RESUMECOMMAND"
 864                                         resumecommand = mysettings.get(resumecommand_var)
 865                                         if resumecommand is None:
 866                                                 writemsg_level(
 867                                                         _("!!! %s is unset. It should "
 868                                                         "have been defined in\n!!! %s/make.globals.\n") \
 869                                                         % (resumecommand_var, global_config_path),
 870                                                         level=logging.ERROR, noiselevel=-1)
 871                                                 return 0
 872                                 if "${FILE}" not in resumecommand:
 873                                         writemsg_level(
 874                                                 _("!!! %s does not contain the required ${FILE}"
 875                                                 " parameter.\n") % resumecommand_var,
 876                                                 level=logging.ERROR, noiselevel=-1)
 877                                         missing_file_param = True
 878
 879                                 if missing_file_param:
 880                                         writemsg_level(
 881                                                 _("!!! Refer to the make.conf(5) man page for "
 882                                                 "information about how to\n!!! correctly specify "
 883                                                 "FETCHCOMMAND and RESUMECOMMAND.\n"),
 884                                                 level=logging.ERROR, noiselevel=-1)
 885                                         if myfile != os.path.basename(loc):
 886                                                 return 0
 887
 888                                 if not can_fetch:
 889                                         if fetched != 2:
 890                                                 try:
 891                                                         mysize = os.stat(myfile_path).st_size
 892                                                 except OSError as e:
 893                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 894                                                                 raise
 895                                                         del e
 896                                                         mysize = 0
 897
 898                                                 if mysize == 0:
 899                                                         writemsg(_("!!! File %s isn't fetched but unable to get it.\n") % myfile,
 900                                                                 noiselevel=-1)
 901                                                 elif size is None or size > mysize:
 902                                                         writemsg(_("!!! File %s isn't fully fetched, but unable to complete it\n") % myfile,
 903                                                                 noiselevel=-1)
 904                                                 else:
 905                                                         writemsg(_("!!! File %s is incorrect size, "
 906                                                                 "but unable to retry.\n") % myfile, noiselevel=-1)
 907                                                 return 0
 908                                         else:
 909                                                 continue
 910
 911                                 if fetched != 2 and has_space:
 912                                         #we either need to resume or start the download
 913                                         if fetched == 1:
 914                                                 try:
 915                                                         mystat = os.stat(myfile_path)
 916                                                 except OSError as e:
 917                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 918                                                                 raise
 919                                                         del e
 920                                                         fetched = 0
 921                                                 else:
 922                                                         if mystat.st_size < fetch_resume_size:
 923                                                                 writemsg(_(">>> Deleting distfile with size "
 924                                                                         "%d (smaller than " "PORTAGE_FETCH_RESU"
 925                                                                         "ME_MIN_SIZE)\n") % mystat.st_size)
 926                                                                 try:
 927                                                                         os.unlink(myfile_path)
 928                                                                 except OSError as e:
 929                                                                         if e.errno not in \
 930                                                                                 (errno.ENOENT, errno.ESTALE):
 931                                                                                 raise
 932                                                                         del e
 933                                                                 fetched = 0
 934                                         if fetched == 1:
 935                                                 #resume mode:
 936                                                 writemsg(_(">>> Resuming download...\n"))
 937                                                 locfetch=resumecommand
 938                                                 command_var = resumecommand_var
 939                                         else:
 940                                                 #normal mode:
 941                                                 locfetch=fetchcommand
 942                                                 command_var = fetchcommand_var
 943                                         writemsg_stdout(_(">>> Downloading '%s'\n") % \
 944                                                 re.sub(r'//(.+):.+@(.+)/',r'//\1:*password*@\2/', loc))
 945                                         variables = {
 946                                                 "DISTDIR": mysettings["DISTDIR"],
 947                                                 "URI":     loc,
 948                                                 "FILE":    myfile
 949                                         }
 950
 951                                         myfetch = shlex_split(locfetch)
 952                                         myfetch = [varexpand(x, mydict=variables) for x in myfetch]
 953                                         myret = -1
 954                                         try:
 955
 956                                                 myret = _spawn_fetch(mysettings, myfetch)
 957
 958                                         finally:
 959                                                 try:
 960                                                         apply_secpass_permissions(myfile_path,
 961                                                                 gid=portage_gid, mode=0o664, mask=0o2)
 962                                                 except FileNotFound:
 963                                                         pass
 964                                                 except PortageException as e:
 965                                                         if not os.access(myfile_path, os.R_OK):
 966                                                                 writemsg(_("!!! Failed to adjust permissions:"
 967                                                                         " %s\n") % str(e), noiselevel=-1)
 968                                                         del e
 969
 970                                         # If the file is empty then it's obviously invalid.  Don't
 971                                         # trust the return value from the fetcher.  Remove the
 972                                         # empty file and try to download again.
 973                                         try:
 974                                                 if os.stat(myfile_path).st_size == 0:
 975                                                         os.unlink(myfile_path)
 976                                                         fetched = 0
 977                                                         continue
 978                                         except EnvironmentError:
 979                                                 pass
 980
 981                                         if mydigests is not None and myfile in mydigests:
 982                                                 try:
 983                                                         mystat = os.stat(myfile_path)
 984                                                 except OSError as e:
 985                                                         if e.errno not in (errno.ENOENT, errno.ESTALE):
 986                                                                 raise
 987                                                         del e
 988                                                         fetched = 0
 989                                                 else:
 990
 991                                                         if stat.S_ISDIR(mystat.st_mode):
 992                                                                 # This can happen if FETCHCOMMAND erroneously
 993                                                                 # contains wget's -P option where it should
 994                                                                 # instead have -O.
 995                                                                 writemsg_level(
 996                                                                         _("!!! The command specified in the "
 997                                                                         "%s variable appears to have\n!!! "
 998                                                                         "created a directory instead of a "
 999                                                                         "normal file.\n") % command_var,
1000                                                                         level=logging.ERROR, noiselevel=-1)
1001                                                                 writemsg_level(
1002                                                                         _("!!! Refer to the make.conf(5) "
1003                                                                         "man page for information about how "
1004                                                                         "to\n!!! correctly specify "
1005                                                                         "FETCHCOMMAND and RESUMECOMMAND.\n"),
1006                                                                         level=logging.ERROR, noiselevel=-1)
1007                                                                 return 0
1008
1009                                                         # no exception?  file exists. let digestcheck() report
1010                                                         # an appropriately for size or checksum errors
1011
1012                                                         # If the fetcher reported success and the file is
1013                                                         # too small, it's probably because the digest is
1014                                                         # bad (upstream changed the distfile).  In this
1015                                                         # case we don't want to attempt to resume. Show a
1016                                                         # digest verification failure to that the user gets
1017                                                         # a clue about what just happened.
1018                                                         if myret != os.EX_OK and \
1019                                                                 mystat.st_size < mydigests[myfile]["size"]:
1020                                                                 # Fetch failed... Try the next one... Kill 404 files though.
1021                                                                 if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")):
1022                                                                         html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M)
1023                                                                         if html404.search(io.open(
1024                                                                                 _unicode_encode(myfile_path,
1025                                                                                 encoding=_encodings['fs'], errors='strict'),
1026                                                                                 mode='r', encoding=_encodings['content'], errors='replace'
1027                                                                                 ).read()):
1028                                                                                 try:
1029                                                                                         os.unlink(mysettings["DISTDIR"]+"/"+myfile)
1030                                                                                         writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n"))
1031                                                                                         fetched = 0
1032                                                                                         continue
1033                                                                                 except (IOError, OSError):
1034                                                                                         pass
1035                                                                 fetched = 1
1036                                                                 continue
1037                                                         if True:
1038                                                                 # File is the correct size--check the checksums for the fetched
1039                                                                 # file NOW, for those users who don't have a stable/continuous
1040                                                                 # net connection. This way we have a chance to try to download
1041                                                                 # from another mirror...
1042                                                                 verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile])
1043                                                                 if not verified_ok:
1044                                                                         print(reason)
1045                                                                         writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,
1046                                                                                 noiselevel=-1)
1047                                                                         writemsg(_("!!! Reason: %s\n") % reason[0],
1048                                                                                 noiselevel=-1)
1049                                                                         writemsg(_("!!! Got:      %s\n!!! Expected: %s\n") % \
1050                                                                                 (reason[1], reason[2]), noiselevel=-1)
1051                                                                         if reason[0] == _("Insufficient data for checksum verification"):
1052                                                                                 return 0
1053                                                                         temp_filename = \
1054                                                                                 _checksum_failure_temp_file(
1055                                                                                 mysettings["DISTDIR"], myfile)
1056                                                                         writemsg_stdout(_("Refetching... "
1057                                                                                 "File renamed to '%s'\n\n") % \
1058                                                                                 temp_filename, noiselevel=-1)
1059                                                                         fetched=0
1060                                                                         checksum_failure_count += 1
1061                                                                         if checksum_failure_count == \
1062                                                                                 checksum_failure_primaryuri:
1063                                                                                 # Switch to "primaryuri" mode in order
1064                                                                                 # to increase the probablility of
1065                                                                                 # of success.
1066                                                                                 primaryuris = \
1067                                                                                         primaryuri_dict.get(myfile)
1068                                                                                 if primaryuris:
1069                                                                                         uri_list.extend(
1070                                                                                                 reversed(primaryuris))
1071                                                                         if checksum_failure_count >= \
1072                                                                                 checksum_failure_max_tries:
1073                                                                                 break
1074                                                                 else:
1075                                                                         eout = EOutput()
1076                                                                         eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
1077                                                                         digests = mydigests.get(myfile)
1078                                                                         if digests:
1079                                                                                 eout.ebegin("%s %s ;-)" % \
1080                                                                                         (myfile, " ".join(sorted(digests))))
1081                                                                                 eout.eend(0)
1082                                                                         fetched=2
1083                                                                         break
1084                                         else:
1085                                                 if not myret:
1086                                                         fetched=2
1087                                                         break
1088                                                 elif mydigests!=None:
1089                                                         writemsg(_("No digest file available and download failed.\n\n"),
1090                                                                 noiselevel=-1)
1091                 finally:
1092                         if use_locks and file_lock:
1093                                 unlockfile(file_lock)
1094                                 file_lock = None
1095
1096                 if listonly:
1097                         writemsg_stdout("\n", noiselevel=-1)
1098                 if fetched != 2:
1099                         if restrict_fetch and not restrict_fetch_msg:
1100                                 restrict_fetch_msg = True
1101                                 msg = _("\n!!! %s/%s"
1102                                         " has fetch restriction turned on.\n"
1103                                         "!!! This probably means that this "
1104                                         "ebuild's files must be downloaded\n"
1105                                         "!!! manually.  See the comments in"
1106                                         " the ebuild for more information.\n\n") % \
1107                                         (mysettings["CATEGORY"], mysettings["PF"])
1108                                 writemsg_level(msg,
1109                                         level=logging.ERROR, noiselevel=-1)
1110                         elif restrict_fetch:
1111                                 pass
1112                         elif listonly:
1113                                 pass
1114                         elif not filedict[myfile]:
1115                                 writemsg(_("Warning: No mirrors available for file"
1116                                         " '%s'\n") % (myfile), noiselevel=-1)
1117                         else:
1118                                 writemsg(_("!!! Couldn't download '%s'. Aborting.\n") % myfile,
1119                                         noiselevel=-1)
1120
1121                         if listonly:
1122                                 failed_files.add(myfile)
1123                                 continue
1124                         elif fetchonly:
1125                                 failed_files.add(myfile)
1126                                 continue
1127                         return 0
1128         if failed_files:
1129                 return 0
1130         return 1