From: Fabian Groffen Date: Mon, 5 Nov 2007 21:03:53 +0000 (-0000) Subject: Merged from trunk 8400:8418 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=2a871b9a32aa4c28437a30fe15e0630b1fc4d89e;p=portage.git Merged from trunk 8400:8418 | 8401 | Bug #197965 - When fetch() encounters a checksum failure, | | zmedico | look for an identical file from a previous checksum failure | | | in order to avoid consuming unnecessary disk space with | | | duplicate files. | | 8402 | if mode=1 and `dialog` is not usable, abort #197990 by | | SpankMan | DominikBuerkle | | 8403 | Bug #197965 - Make fetch() bail out after 5 checksum | | zmedico | failures for a particular file since downloading the same | | | file repeatedly from every single available mirror is a | | | waste of bandwidth and time. | | 8405 | Call die() so that the $TMP directory is always properly | | zmedico | cleaned up. | | 8408 | Bug #197797 - Do not actively clear o+rx permission bits on | | zmedico | the /var/lib/portage directory. Only clear the o+w bits. | | | This allows users who aren't in the portage group to have | | | read access to the world file by default, but | | | administrators can explicitly chmod o-rx that directory if | | | necessary, and portage will leave those bits alone. This | | | makes the permission settings in dblink.treewalk() | | | consistent with those that are already defined in | | | config._init_dirs(). | | 8410 | For sanity, pass an actual list of keys into aux_get() | | zmedico | instead of iterating over a set that happens to be mutable | | | (though it remains constant in practice). | | 8412 | Make fetch() behave like the ebuild has | | zmedico | RESTRICT="primaryuri" after 2 checksum failures for a | | | fetched file. This should increase the probability of | | | fetching the correct file before the maximum number of | | | tries is reached. | | 8414 | Generate a usage.obsolete warning whenever -* is found in | | zmedico | KEYWORDS. | | 8416 | In fetch(), replace checksum_failure_counts dict with | | zmedico | single int counter. | | 8418 | Clean up ACCEPT_KEYWORDS incremental stacking logic and fix | | zmedico | some minor inconsistencies. | svn path=/main/branches/prefix/; revision=8434 --- diff --git a/bin/etc-update b/bin/etc-update index f0e544d7b..68bc065af 100755 --- a/bin/etc-update +++ b/bin/etc-update @@ -444,11 +444,13 @@ Please select from the menu above (-1 to exit, losing this merge): " die() { trap "" TERM trap "" KILL - echo "Exiting: ${1}" if [ ${2} -eq 0 ]; then + echo "Exiting: ${1}" scan > /dev/null [ ${count} -gt 0 ] && echo "NOTE: ${count} updates remaining" + else + echo "ERROR: ${1}" fi rm -rf "${TMP}" @@ -534,13 +536,17 @@ if [ "${using_editor}" == 0 ]; then echo > "${TMP}"/.diff-test-2 if ! diff_command "${TMP}"/.diff-test-1 "${TMP}"/.diff-test-2 ; then - echo "ERROR: '${diff_command}' does not seem to work, aborting" - exit 1 + die "'${diff_command}' does not seem to work, aborting" 1 fi else if ! type ${diff_command%% *} >/dev/null; then - echo "ERROR: '${diff_command}' does not seem to work, aborting" - exit 1 + die "'${diff_command}' does not seem to work, aborting" 1 + fi +fi + +if [[ ${mode} == "1" ]] ; then + if ! type dialog >/dev/null || ! dialog --help >/dev/null ; then + die "mode=1 and 'dialog' not found or not executable, aborting" 1 fi fi diff --git a/bin/repoman b/bin/repoman index 97a8d3b5c..fd9637ac3 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1243,9 +1243,12 @@ for x in scanlist: " ".join(stable_keywords)) # KEYWORDS="-*" is a stupid replacement for package.mask and screws general KEYWORDS semantics - if "-*" in myaux["KEYWORDS"].split(): + if "-*" in keywords: + stats["usage.obsolete"] += 1 + fails["usage.obsolete"].append( + x+'/'+y+'.ebuild: KEYWORDS contain -*') haskeyword = False - for kw in myaux["KEYWORDS"].split(): + for kw in keywords: if kw[0] == "~": kw = kw[1:] if kw in kwlist: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 5ba87e64e..405847b25 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1946,25 +1946,26 @@ class config(object): match=0 cp = dep_getkey(cpv) pkgdict = self.pkeywordsdict.get(cp) + matches = False if pkgdict: - cpv_slot = "%s:%s" % (cpv, metadata["SLOT"]) - matches = match_to_list(cpv_slot, pkgdict.keys()) - for atom in matches: - pgroups.extend(pkgdict[atom]) + cpv_slot_list = ["%s:%s" % (cpv, metadata["SLOT"])] + for atom, pkgkeywords in pkgdict.iteritems(): + if match_from_list(atom, cpv_slot_list): + matches = True + pgroups.extend(pkgkeywords) + if matches or egroups: pgroups.extend(egroups) - if matches: - # normalize pgroups with incrementals logic so it - # matches ACCEPT_KEYWORDS behavior - inc_pgroups = set() - for x in pgroups: + inc_pgroups = set() + for x in pgroups: + if x.startswith("-"): if x == "-*": inc_pgroups.clear() - elif x.startswith("-"): + else: inc_pgroups.discard(x[1:]) - elif x not in inc_pgroups: - inc_pgroups.add(x) - pgroups = inc_pgroups - del inc_pgroups + else: + inc_pgroups.add(x) + pgroups = inc_pgroups + del inc_pgroups hasstable = False hastesting = False for gp in mygroups: @@ -2730,6 +2731,45 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero return retval >> 8 return retval +def _checksum_failure_temp_file(distdir, basename): + """ + First try to find a duplicate temp file with the same checksum and return + that filename if available. Otherwise, use mkstemp to create a new unique + filename._checksum_failure_.$RANDOM, rename the given file, and return the + new filename. In any case, filename will be renamed or removed before this + function returns a temp filename. + """ + + filename = os.path.join(distdir, basename) + size = os.stat(filename).st_size + checksum = None + tempfile_re = re.compile(re.escape(basename) + r'\._checksum_failure_\..*') + for temp_filename in os.listdir(distdir): + if not tempfile_re.match(temp_filename): + continue + temp_filename = os.path.join(distdir, temp_filename) + try: + if size != os.stat(temp_filename).st_size: + continue + except OSError: + continue + try: + temp_checksum = portage.checksum.perform_md5(temp_filename) + except portage.exception.FileNotFound: + # Apparently the temp file disappeared. Let it go. + continue + if checksum is None: + checksum = portage.checksum.perform_md5(filename) + if checksum == temp_checksum: + os.unlink(filename) + return temp_filename + + from tempfile import mkstemp + fd, temp_filename = mkstemp("", basename + "._checksum_failure_.", distdir) + os.close(fd) + os.rename(filename, temp_filename) + return temp_filename + def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1): "fetch files. Will use digest file if available." @@ -2744,6 +2784,14 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", print ">>> \"mirror\" mode desired and \"mirror\" restriction found; skipping fetch." return 1 + # Generally, downloading the same file repeatedly from + # every single available mirror is a waste of bandwidth + # and time, so there needs to be a cap. + checksum_failure_max_tries = 5 + # Behave like the package has RESTRICT="primaryuri" after a + # couple of checksum failures, to increase the probablility + # of success before checksum_failure_max_tries is reached. + checksum_failure_primaryuri = 2 thirdpartymirrors = mysettings.thirdpartymirrors() check_config_instance(mysettings) @@ -2807,6 +2855,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", filedict={} primaryuri_indexes={} + primaryuri_dict = {} for myuri in myuris: myfile=os.path.basename(myuri) if not filedict.has_key(myfile): @@ -2850,6 +2899,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", filedict[myfile].insert(primaryuri_indexes[myfile], myuri) else: filedict[myfile].append(myuri) + primaryuris = primaryuri_dict.get(myfile) + if primaryuris is None: + primaryuris = [] + primaryuri_dict[myfile] = primaryuris + primaryuris.append(myuri) can_fetch=True @@ -2901,6 +2955,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", noiselevel=-1) return 0 del distlocks_subdir + for myfile in filedict: """ fetched status @@ -2989,12 +3044,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if reason[0] == "Insufficient data for checksum verification": return 0 if can_fetch and not restrict_fetch: - from tempfile import mkstemp - fd, temp_filename = mkstemp("", - myfile + "._checksum_failure_.", - mysettings["DISTDIR"]) - os.close(fd) - os.rename(myfile_path, temp_filename) + temp_filename = \ + _checksum_failure_temp_file( + mysettings["DISTDIR"], myfile) writemsg_stdout("Refetching... " + \ "File renamed to '%s'\n\n" % \ temp_filename, noiselevel=-1) @@ -3011,7 +3063,18 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", eout.eend(0) continue # fetch any remaining files - for loc in filedict[myfile]: + # Create a reversed list since that is optimal for list.pop(). + uri_list = filedict[myfile][:] + uri_list.reverse() + checksum_failure_count = 0 + tried_locations = set() + while uri_list: + loc = uri_list.pop() + # Eliminate duplicates here in case we've switched to + # "primaryuri" mode on the fly due to a checksum failure. + if loc in tried_locations: + continue + tried_locations.add(loc) if listonly: writemsg_stdout(loc+" ", noiselevel=-1) continue @@ -3171,16 +3234,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", (reason[1], reason[2]), noiselevel=-1) if reason[0] == "Insufficient data for checksum verification": return 0 - from tempfile import mkstemp - fd, temp_filename = mkstemp("", - myfile + "._checksum_failure_.", - mysettings["DISTDIR"]) - os.close(fd) - os.rename(myfile_path, temp_filename) + temp_filename = \ + _checksum_failure_temp_file( + mysettings["DISTDIR"], myfile) writemsg_stdout("Refetching... " + \ "File renamed to '%s'\n\n" % \ temp_filename, noiselevel=-1) fetched=0 + checksum_failure_count += 1 + if checksum_failure_count == \ + checksum_failure_primaryuri: + # Switch to "primaryuri" mode in order + # to increase the probablility of + # of success. + primaryuris = \ + primaryuri_dict.get(myfile) + if primaryuris: + uri_list.extend( + reversed(primaryuris)) + if checksum_failure_count >= \ + checksum_failure_max_tries: + break else: eout = portage.output.EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" @@ -5211,7 +5285,6 @@ def getmaskingstatus(mycpv, metadata=None, settings=None, portdb=None): metadata["USE"] = settings.get("USE", "") else: metadata["USE"] = "" - cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])] mycp=mysplit[0]+"/"+mysplit[1] rValue = [] @@ -5233,6 +5306,8 @@ def getmaskingstatus(mycpv, metadata=None, settings=None, portdb=None): eapi = eapi[1:] if not eapi_is_supported(eapi): return ["required EAPI %s, supported EAPI %s" % (eapi, portage.const.EAPI)] + egroups = settings.configdict["backupenv"].get( + "ACCEPT_KEYWORDS", "").split() mygroups = mygroups.split() pgroups = settings["ACCEPT_KEYWORDS"].split() myarch = settings["ARCH"] @@ -5240,30 +5315,29 @@ def getmaskingstatus(mycpv, metadata=None, settings=None, portdb=None): """For operating systems other than Linux, ARCH is not necessarily a valid keyword.""" myarch = pgroups[0].lstrip("~") - pkgdict = settings.pkeywordsdict cp = dep_getkey(mycpv) - if pkgdict.has_key(cp): - matches = [] - for match in pkgdict[cp]: - if match_from_list(match, cpv_slot_list): - matches.append(match) - for match in matches: - pgroups.extend(pkgdict[cp][match]) - if matches: - inc_pgroups = [] - for x in pgroups: + pkgdict = settings.pkeywordsdict.get(cp) + matches = False + if pkgdict: + cpv_slot_list = ["%s:%s" % (mycpv, metadata["SLOT"])] + for atom, pkgkeywords in pkgdict.iteritems(): + if match_from_list(atom, cpv_slot_list): + matches = True + pgroups.extend(pkgkeywords) + if matches or egroups: + pgroups.extend(egroups) + inc_pgroups = set() + for x in pgroups: + if x.startswith("-"): if x == "-*": - inc_pgroups = [] - elif x[0] == "-": - try: - inc_pgroups.remove(x[1:]) - except ValueError: - pass - elif x not in inc_pgroups: - inc_pgroups.append(x) - pgroups = inc_pgroups - del inc_pgroups + inc_pgroups.clear() + else: + inc_pgroups.discard(x[1:]) + else: + inc_pgroups.add(x) + pgroups = inc_pgroups + del inc_pgroups kmask = "missing" diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index e6238700d..ea51f5d7e 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -604,14 +604,15 @@ class portdbapi(dbapi): myval = "" settings = self.mysettings local_config = settings.local_config + aux_keys = list(self._aux_cache_keys) if level == "minimum-visible": iterfunc = iter else: iterfunc = reversed for cpv in iterfunc(mylist): try: - metadata = dict(izip(self._aux_cache_keys, - self.aux_get(cpv, self._aux_cache_keys))) + metadata = dict(izip(aux_keys, + self.aux_get(cpv, aux_keys))) except KeyError: # ebuild masked by corruption continue @@ -709,7 +710,7 @@ class portdbapi(dbapi): if mylist is None: return [] newlist=[] - aux_keys = self._aux_cache_keys + aux_keys = list(self._aux_cache_keys) metadata = {} local_config = self.mysettings.local_config for mycpv in mylist: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 393896982..5d6025037 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -2000,10 +2000,7 @@ class dblink(object): del cfgfiledict["IGNORE"] my_private_path = os.path.join(destroot, PRIVATE_PATH) - if not os.path.exists(my_private_path): - os.makedirs(my_private_path) - os.chown(my_private_path, os.getuid(), portage_gid) - os.chmod(my_private_path, 02770) + ensure_dirs(my_private_path, gid=portage_gid, mode=02750, mask=02) writedict(cfgfiledict, conf_mem_file) del conf_mem_file diff --git a/src/chpathtool.c b/src/chpathtool.c index 54f3fe45d..6f6960afe 100644 --- a/src/chpathtool.c +++ b/src/chpathtool.c @@ -283,7 +283,7 @@ int dirwalk(char *src, char *srcp, char *trg, char *trgp) { #else ub.actime = s.ATIME_SEC; ub.modtime = s.MTIME_SEC; - if(utime(trg, &ub) != 0) { + if (utime(trg, &ub) != 0) { fprintf(stderr, "failed to set utime of %s: %s\n", trg, strerror(errno)); return(-1);