match=0
cp = dep_getkey(cpv)
pkgdict = self.pkeywordsdict.get(cp)
+ matches = False
if pkgdict:
- cpv_slot = "%s:%s" % (cpv, metadata["SLOT"])
- matches = match_to_list(cpv_slot, pkgdict.keys())
- for atom in matches:
- pgroups.extend(pkgdict[atom])
+ cpv_slot_list = ["%s:%s" % (cpv, metadata["SLOT"])]
+ for atom, pkgkeywords in pkgdict.iteritems():
+ if match_from_list(atom, cpv_slot_list):
+ matches = True
+ pgroups.extend(pkgkeywords)
+ if matches or egroups:
pgroups.extend(egroups)
- if matches:
- # normalize pgroups with incrementals logic so it
- # matches ACCEPT_KEYWORDS behavior
- inc_pgroups = set()
- for x in pgroups:
+ inc_pgroups = set()
+ for x in pgroups:
+ if x.startswith("-"):
if x == "-*":
inc_pgroups.clear()
- elif x.startswith("-"):
+ else:
inc_pgroups.discard(x[1:])
- elif x not in inc_pgroups:
- inc_pgroups.add(x)
- pgroups = inc_pgroups
- del inc_pgroups
+ else:
+ inc_pgroups.add(x)
+ pgroups = inc_pgroups
+ del inc_pgroups
hasstable = False
hastesting = False
for gp in mygroups:
return retval >> 8
return retval
+def _checksum_failure_temp_file(distdir, basename):
+ """
+ First try to find a duplicate temp file with the same checksum and return
+ that filename if available. Otherwise, use mkstemp to create a new unique
+ filename._checksum_failure_.$RANDOM, rename the given file, and return the
+ new filename. In any case, filename will be renamed or removed before this
+ function returns a temp filename.
+ """
+
+ filename = os.path.join(distdir, basename)
+ size = os.stat(filename).st_size
+ checksum = None
+ tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*')
+ for temp_filename in os.listdir(distdir):
+ if not tempfile_re.match(temp_filename):
+ continue
+ temp_filename = os.path.join(distdir, temp_filename)
+ try:
+ if size != os.stat(temp_filename).st_size:
+ continue
+ except OSError:
+ continue
+ try:
+ temp_checksum = portage.checksum.perform_md5(temp_filename)
+ except portage.exception.FileNotFound:
+ # Apparently the temp file disappeared. Let it go.
+ continue
+ if checksum is None:
+ checksum = portage.checksum.perform_md5(filename)
+ if checksum == temp_checksum:
+ os.unlink(filename)
+ return temp_filename
+
+ from tempfile import mkstemp
+ fd, temp_filename = mkstemp("", basename + "._checksum_failure_.", distdir)
+ os.close(fd)
+ os.rename(filename, temp_filename)
+ return temp_filename
+
def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
"fetch files. Will use digest file if available."
print ">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch."
return 1
+ # Generally, downloading the same file repeatedly from
+ # every single available mirror is a waste of bandwidth
+ # and time, so there needs to be a cap.
+ checksum_failure_max_tries = 5
+ # Behave like the package has RESTRICT="primaryuri" after a
+ # couple of checksum failures, to increase the probablility
+ # of success before checksum_failure_max_tries is reached.
+ checksum_failure_primaryuri = 2
thirdpartymirrors = mysettings.thirdpartymirrors()
check_config_instance(mysettings)
filedict={}
primaryuri_indexes={}
+ primaryuri_dict = {}
for myuri in myuris:
myfile=os.path.basename(myuri)
if not filedict.has_key(myfile):
filedict[myfile].insert(primaryuri_indexes[myfile], myuri)
else:
filedict[myfile].append(myuri)
+ primaryuris = primaryuri_dict.get(myfile)
+ if primaryuris is None:
+ primaryuris = []
+ primaryuri_dict[myfile] = primaryuris
+ primaryuris.append(myuri)
can_fetch=True
noiselevel=-1)
return 0
del distlocks_subdir
+
for myfile in filedict:
"""
fetched status
if reason[0] == "Insufficient data for checksum verification":
return 0
if can_fetch and not restrict_fetch:
- from tempfile import mkstemp
- fd, temp_filename = mkstemp("",
- myfile + "._checksum_failure_.",
- mysettings["DISTDIR"])
- os.close(fd)
- os.rename(myfile_path, temp_filename)
+ temp_filename = \
+ _checksum_failure_temp_file(
+ mysettings["DISTDIR"], myfile)
writemsg_stdout("Refetching... " + \
"File renamed to '%s'\n\n" % \
temp_filename, noiselevel=-1)
eout.eend(0)
continue # fetch any remaining files
- for loc in filedict[myfile]:
+ # Create a reversed list since that is optimal for list.pop().
+ uri_list = filedict[myfile][:]
+ uri_list.reverse()
+ checksum_failure_count = 0
+ tried_locations = set()
+ while uri_list:
+ loc = uri_list.pop()
+ # Eliminate duplicates here in case we've switched to
+ # "primaryuri" mode on the fly due to a checksum failure.
+ if loc in tried_locations:
+ continue
+ tried_locations.add(loc)
if listonly:
writemsg_stdout(loc+" ", noiselevel=-1)
continue
(reason[1], reason[2]), noiselevel=-1)
if reason[0] == "Insufficient data for checksum verification":
return 0
- from tempfile import mkstemp
- fd, temp_filename = mkstemp("",
- myfile + "._checksum_failure_.",
- mysettings["DISTDIR"])
- os.close(fd)
- os.rename(myfile_path, temp_filename)
+ temp_filename = \
+ _checksum_failure_temp_file(
+ mysettings["DISTDIR"], myfile)
writemsg_stdout("Refetching... " + \
"File renamed to '%s'\n\n" % \
temp_filename, noiselevel=-1)
fetched=0
+ checksum_failure_count += 1
+ if checksum_failure_count == \
+ checksum_failure_primaryuri:
+ # Switch to "primaryuri" mode in order
+ # to increase the probablility of
+ # of success.
+ primaryuris = \
+ primaryuri_dict.get(myfile)
+ if primaryuris:
+ uri_list.extend(
+ reversed(primaryuris))
+ if checksum_failure_count >= \
+ checksum_failure_max_tries:
+ break
else:
eout = portage.output.EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
metadata["USE"] = settings.get("USE", "")
else:
metadata["USE"] = ""
- cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])]
mycp=mysplit[0]+"/"+mysplit[1]
rValue = []
eapi = eapi[1:]
if not eapi_is_supported(eapi):
return ["required EAPI %s, supported EAPI %s" % (eapi, portage.const.EAPI)]
+ egroups = settings.configdict["backupenv"].get(
+ "ACCEPT_KEYWORDS", "").split()
mygroups = mygroups.split()
pgroups = settings["ACCEPT_KEYWORDS"].split()
myarch = settings["ARCH"]
"""For operating systems other than Linux, ARCH is not necessarily a
valid keyword."""
myarch = pgroups[0].lstrip("~")
- pkgdict = settings.pkeywordsdict
cp = dep_getkey(mycpv)
- if pkgdict.has_key(cp):
- matches = []
- for match in pkgdict[cp]:
- if match_from_list(match, cpv_slot_list):
- matches.append(match)
- for match in matches:
- pgroups.extend(pkgdict[cp][match])
- if matches:
- inc_pgroups = []
- for x in pgroups:
+ pkgdict = settings.pkeywordsdict.get(cp)
+ matches = False
+ if pkgdict:
+ cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])]
+ for atom, pkgkeywords in pkgdict.iteritems():
+ if match_from_list(atom, cpv_slot_list):
+ matches = True
+ pgroups.extend(pkgkeywords)
+ if matches or egroups:
+ pgroups.extend(egroups)
+ inc_pgroups = set()
+ for x in pgroups:
+ if x.startswith("-"):
if x == "-*":
- inc_pgroups = []
- elif x[0] == "-":
- try:
- inc_pgroups.remove(x[1:])
- except ValueError:
- pass
- elif x not in inc_pgroups:
- inc_pgroups.append(x)
- pgroups = inc_pgroups
- del inc_pgroups
+ inc_pgroups.clear()
+ else:
+ inc_pgroups.discard(x[1:])
+ else:
+ inc_pgroups.add(x)
+ pgroups = inc_pgroups
+ del inc_pgroups
kmask = "missing"
projects / portage.git / commitdiff