From 9b3e14653d27207a30274903ce60d5d0c32af642 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Thu, 10 Apr 2008 16:41:19 +0000 Subject: [PATCH] Merged from trunk 9668:9697 | 9669 | Remove some boolean logic that is obsolete now that the | | zmedico | "selective" parameter is passed into dep_zapdeps(). | | 9672 | Use pkgcmp() instead of == and best() for correctness in | | zmedico | case of implicit -r0. | | 9674 | Bug #215673 - Make ebuild.sh leave IUSE defaults intact | | zmedico | instead of filtering them out. The built_with_use() function | | | in eutils.eclass needs to be updated for compatibility. | | 9675 | Remove the digraph._parent_child_digraph since it's no | | zmedico | longer needed. It's main purpose was to track PDEPEND | | | relationships but those are handled just like the other deps | | | now. | | 9677 | Handle a potential FileNotFound exception in | | zmedico | new_protect_filename() when the last ._cfg* file happens to | | | be a broken symlink. Thanks to bonsaikitten for reporting. | | 9678 | Eliminate a redundant os.path.join() call. | | zmedico | | | 9680 | Use Package.cp instead of dep_getkey(cpv). | | zmedico | | | 9681 | Fix the previous commit so that the Package instance is | | zmedico | created in the correct scope. | | 9683 | In cpv_expand(), if the virtuals file contains a versioned | | zmedico | atom (like the base profile currently has for virtual/pam), | | | properly remove the operator and version from the atom | | | before it is passed into dbapi.cp_list(). | | 9685 | For old-style virtuals, we need to repeat the | | zmedico | package.provided check against the selected package. | | 9687 | Implement a PORTAGE_FETCH_RESUME_MIN_SIZE variable that is | | zmedico | useful for helping to ensure that small garbage files such | | | as html 404 pages are properly discarded. | | 9689 | If PORTAGE_FETCH_RESUME_MIN_SIZE is undefined or empty, | | zmedico | silently use the default. | | 9691 | Add missing str.upper() call for | | zmedico | PORTAGE_FETCH_RESUME_MIN_SIZE. | | 9693 | Fix package selection logic so that it always properly finds | | zmedico | the highest available version in a new slot even though the | | | graph already contains a matching version in a lower slot. | | 9695 | Eliminate a redundant atom from the --update greedy slot | | zmedico | expansion code. | | 9696 | Add missing DepPriority instances to the graph so that the | | zmedico | --buildpkgonly hasallzeros() calculation works correctly | | | again. | | 9697 | Decrease priority on --onlydpes dependencies so that | | zmedico | --buildpkgonly hasallzeros() works correctly. | svn path=/main/branches/prefix/; revision=9803 --- bin/ebuild.sh | 14 ------ cnf/make.globals | 3 ++ man/make.conf.5 | 8 ++++ pym/_emerge/__init__.py | 100 +++++++++++++++++++++++++++++++--------- pym/portage/__init__.py | 80 ++++++++++++++++++++++++++++---- pym/portage/util.py | 11 +++-- 6 files changed, 168 insertions(+), 48 deletions(-) diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 1cd140e3d..54e1904d6 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1728,20 +1728,6 @@ if ! hasq ${EBUILD_PHASE} clean && \ PDEPEND="${PDEPEND} ${E_PDEPEND}" unset ECLASS E_IUSE E_DEPEND E_RDEPEND E_PDEPEND - - if [ "${EBUILD_PHASE}" != "depend" ] ; then - # Make IUSE defaults backward compatible with all the old shell code. - iuse_temp="" - for x in ${IUSE} ; do - if [[ ${x} == +* ]] || [[ ${x} == -* ]] ; then - iuse_temp="${iuse_temp} ${x:1}" - else - iuse_temp="${iuse_temp} ${x}" - fi - done - export IUSE=${iuse_temp} - unset x iuse_temp - fi set +f fi diff --git a/cnf/make.globals b/cnf/make.globals index 1c848a749..0851d53a0 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -59,6 +59,9 @@ AUTOCLEAN="yes" # Number of mirrors to try when a downloaded file has an incorrect checksum. PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5" +# Minimum size of existing file for RESUMECOMMAND to be called. +PORTAGE_FETCH_RESUME_MIN_SIZE="350K" + # Number of times 'emerge --sync' will run before giving up. PORTAGE_RSYNC_RETRIES="3" diff --git a/man/make.conf.5 b/man/make.conf.5 index 7f9a49618..90e419afc 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -412,6 +412,14 @@ Please see /etc/make.conf.example for elog documentation. \fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR Number of mirrors to try when a downloaded file has an incorrect checksum. .TP +\PORTAGE_FETCH_RESUME_MIN_SIZE\fR = \fI350K\fR +Minimum size of existing file for RESUMECOMMAND to be called. Files smaller +than this size will be removed and FETCHCOMMAND will be called to download +the file from the beginning. This is useful for helping to ensure that small +garbage files such as html 404 pages are properly discarded. The variable +should contain an integer number of bytes and may have a suffix such as +K, M, or G. +.TP \fBPORTAGE_NICENESS\fR = \fI[number]\fR The value of this variable will be added to the current nice level that emerge is running at. In other words, this will not set the nice level, diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 06727799b..b8aad59aa 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1228,6 +1228,24 @@ class Package(object): status = "nomerge" self._digraph_node = (self.type_name, self.root, self.cpv, status) + def __lt__(self, other): + other_split = portage.catpkgsplit(other.cpv) + self_split = portage.catpkgsplit(self.cpv) + if other_split[:2] != self_split[:2]: + return False + if portage.pkgcmp(self_split[1:], other_split[1:]) < 0: + return True + return False + + def __gt__(self, other): + other_split = portage.catpkgsplit(other.cpv) + self_split = portage.catpkgsplit(self.cpv) + if other_split[:2] != self_split[:2]: + return False + if portage.pkgcmp(self_split[1:], other_split[1:]) > 0: + return True + return False + def __eq__(self, other): return self._digraph_node == other def __ne__(self, other): @@ -1532,9 +1550,6 @@ class depgraph(object): del trees self.digraph=portage.digraph() - # Tracks simple parent/child relationships (PDEPEND relationships are - # not reversed). - self._parent_child_digraph = digraph() # contains all sets added to the graph self._sets = {} # contains atoms given as arguments @@ -1592,7 +1607,7 @@ class depgraph(object): for node in slot_nodes: msg.append(indent) msg.append(str(node)) - parents = self._parent_child_digraph.parent_nodes(node) + parents = self.digraph.parent_nodes(node) if parents: omitted_parents = 0 if len(parents) > max_parents: @@ -1788,10 +1803,10 @@ class depgraph(object): if existing_node: if pkg.cpv == existing_node.cpv: # The existing node can be reused. - self._parent_child_digraph.add(existing_node, myparent) if args: for arg in args: - self._parent_child_digraph.add(existing_node, arg) + self.digraph.add(existing_node, arg, + priority=priority) # If a direct circular dependency is not an unsatisfied # buildtime dependency then drop it here since otherwise # it can skew the merge order calculation in an unwanted @@ -1859,10 +1874,11 @@ class depgraph(object): # Do this even when addme is False (--onlydeps) so that the # parent/child relationship is always known in case # self._show_slot_collision_notice() needs to be called later. - self._parent_child_digraph.add(pkg, myparent) + if pkg.onlydeps: + self.digraph.add(pkg, myparent, priority=priority) if args: for arg in args: - self._parent_child_digraph.add(pkg, arg) + self.digraph.add(pkg, arg, priority=priority) """ This section determines whether we go deeper into dependencies or not. We want to go deeper on a few occasions: @@ -1932,6 +1948,10 @@ class depgraph(object): strict = mytype != "installed" try: for dep_root, dep_string, dep_priority in deps: + if pkg.onlydeps: + # Decrease priority so that --buildpkgonly + # hasallzeros() works correctly. + dep_priority = DepPriority() if not dep_string: continue if debug: @@ -1996,9 +2016,6 @@ class depgraph(object): myslots.add(vardb.aux_get(cpv, ["SLOT"])[0]) for myslot in myslots: yield "%s:%s" % (mykey, myslot) - # In addition to any installed slots, also try to pull - # in the latest new slot that may be available. - yield atom def _iter_args_for_pkg(self, pkg): # TODO: add multiple $ROOT support @@ -2227,6 +2244,8 @@ class depgraph(object): # atoms could be a property of the set itself. greedy_atoms = [] for arg in args: + # In addition to any installed slots, also try to pull + # in the latest new slot that may be available. greedy_atoms.append(arg) if not isinstance(arg, (AtomArg, PackageArg)): continue @@ -2267,6 +2286,7 @@ class depgraph(object): while args: arg = args.pop() for atom in arg.set: + atom_cp = portage.dep_getkey(atom) try: pprovided = pprovideddict.get(portage.dep_getkey(atom)) if pprovided and portage.match_from_list(atom, pprovided): @@ -2290,6 +2310,17 @@ class depgraph(object): return 0, myfavorites self._missing_args.append((arg, atom)) continue + if atom_cp != pkg.cp: + # For old-style virtuals, we need to repeat the + # package.provided check against the selected package. + expanded_atom = atom.replace(atom_cp, pkg.cp) + pprovided = pprovideddict.get(pkg.cp) + if pprovided and \ + portage.match_from_list(expanded_atom, pprovided): + # A provided package has been + # specified on the command line. + self._pprovided_args.append((arg, atom)) + continue if pkg.installed and "selective" not in self.myparams: self._unsatisfied_deps_for_display.append( ((myroot, atom), {})) @@ -2578,6 +2609,8 @@ class depgraph(object): portdb = self.roots[root].trees["porttree"].dbapi # List of acceptable packages, ordered by type preference. matched_packages = [] + highest_version = None + atom_cp = portage.dep_getkey(atom) existing_node = None myeb = None usepkgonly = "--usepkgonly" in self.myopts @@ -2642,13 +2675,18 @@ class depgraph(object): if not installed: # masked by corruption continue + pkg = Package(built=built, cpv=cpv, installed=installed, + metadata=metadata, type_name=pkg_type) if not installed: if myarg: found_available_arg = True - if not visible(pkgsettings, Package(built=built, - cpv=cpv, installed=installed, metadata=metadata, - type_name=pkg_type)): + if not visible(pkgsettings, pkg): continue + if pkg.cp == atom_cp: + if highest_version is None: + highest_version = pkg + elif pkg > highest_version: + highest_version = pkg # At this point, we've found the highest visible # match from the current repo. Any lower versions # from this repo are ignored, so this so the loop @@ -2663,8 +2701,17 @@ class depgraph(object): cpv_slot = "%s:%s" % \ (e_pkg.cpv, e_pkg.metadata["SLOT"]) if portage.dep.match_from_list(atom, [cpv_slot]): - matched_packages.append(e_pkg) - existing_node = e_pkg + if highest_version and \ + e_pkg.cp == atom_cp and \ + e_pkg < highest_version and \ + e_pkg.slot_atom != highest_version.slot_atom: + # There is a higher version available in a + # different slot, so this existing node is + # irrelevant. + pass + else: + matched_packages.append(e_pkg) + existing_node = e_pkg break # Compare built package to current config and # reject the built package if necessary. @@ -2837,7 +2884,7 @@ class depgraph(object): setconfig = root_config.setconfig args = [] # Reuse existing SetArg instances when available. - for arg in self._parent_child_digraph.root_nodes(): + for arg in self.digraph.root_nodes(): if not isinstance(arg, SetArg): continue if arg.root_config != root_config: @@ -3519,7 +3566,7 @@ class depgraph(object): tree_nodes = [] display_list = [] - mygraph = self._parent_child_digraph + mygraph = self.digraph i = 0 depth = 0 shown_edges = set() @@ -3662,6 +3709,8 @@ class depgraph(object): repo_name = self.roots[myroot].settings.get("PORTAGE_BINHOST") else: repo_name = metadata["repository"] + built = pkg_type != "ebuild" + installed = pkg_type == "installed" if pkg_type == "ebuild": ebuild_path = portdb.findname(pkg_key) if not ebuild_path: # shouldn't happen @@ -3674,6 +3723,9 @@ class depgraph(object): repo_path_real = repo_name else: repo_path_real = portdb.getRepositoryPath(repo_name) + pkg_node = Package(type_name=pkg_type, root=myroot, + cpv=pkg_key, built=built, installed=installed, + metadata=metadata) pkg_use = metadata["USE"].split() try: restrict = flatten(use_reduce(paren_reduce( @@ -4568,8 +4620,10 @@ class MergeTask(object): mergecount=0 for x in mymergelist: - mergecount+=1 pkg_type = x[0] + if pkg_type == "blocks": + continue + mergecount+=1 myroot=x[1] pkg_key = x[2] pkgindex=2 @@ -4593,6 +4647,11 @@ class MergeTask(object): raise AssertionError("Package type: '%s'" % pkg_type) metadata.update(izip(metadata_keys, mydbapi.aux_get(pkg_key, metadata_keys))) + built = pkg_type != "ebuild" + installed = pkg_type == "installed" + pkg = Package(type_name=pkg_type, root=myroot, + cpv=pkg_key, built=built, installed=installed, + metadata=metadata) if x[0]=="blocks": pkgindex=3 y = portdb.findname(pkg_key) @@ -4823,8 +4882,7 @@ class MergeTask(object): "--fetch-all-uri" not in self.myopts: # Figure out if we need a restart. - if myroot == "/" and \ - portage.dep_getkey(pkg_key) == "sys-apps/portage": + if myroot == "/" and pkg.cp == "sys-apps/portage": if len(mymergelist) > mergecount and EPREFIX == BPREFIX: emergelog(xterm_titles, " ::: completed emerge ("+ \ diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index fa9012f7f..3cecd9ef7 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -949,7 +949,8 @@ class config(object): "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", - "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR", + "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE", + "PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR", @@ -3032,6 +3033,20 @@ def _checksum_failure_temp_file(distdir, basename): os.rename(filename, temp_filename) return temp_filename +_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)') + +_size_suffix_map = { + '' : 0, + 'K' : 10, + 'M' : 20, + 'G' : 30, + 'T' : 40, + 'P' : 50, + 'E' : 60, + 'Z' : 70, + 'Y' : 80, +} + def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1): "fetch files. Will use digest file if available." @@ -3072,6 +3087,29 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", checksum_failure_max_tries = v del v + fetch_resume_size_default = "350K" + fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE") + if fetch_resume_size is not None: + fetch_resume_size = "".join(fetch_resume_size.split()) + if not fetch_resume_size: + # If it's undefined or empty, silently use the default. + fetch_resume_size = fetch_resume_size_default + match = _fetch_resume_size_re.match(fetch_resume_size) + if match is None or \ + (match.group(2).upper() not in _size_suffix_map): + writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \ + " contains an unrecognized format: '%s'\n" % \ + mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1) + writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \ + "default value: %s\n" % fetch_resume_size_default, + noiselevel=-1) + fetch_resume_size = None + if fetch_resume_size is None: + fetch_resume_size = fetch_resume_size_default + match = _fetch_resume_size_re.match(fetch_resume_size) + fetch_resume_size = int(match.group(1)) * \ + 2 ** _size_suffix_map[match.group(2).upper()] + # Behave like the package has RESTRICT="primaryuri" after a # couple of checksum failures, to increase the probablility # of success before checksum_failure_max_tries is reached. @@ -3427,8 +3465,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if fetched != 2 and has_space: #we either need to resume or start the download - #you can't use "continue" when you're inside a "try" block - if fetched==1: + if fetched == 1: + try: + mystat = os.stat(myfile_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + fetched = 0 + else: + if mystat.st_size < fetch_resume_size: + writemsg((">>> Deleting distfile with size " + \ + "%d (smaller than " "PORTAGE_FETCH_RESU" + \ + "ME_MIN_SIZE)\n") % mystat.st_size) + try: + os.unlink(myfile_path) + except OSError, e: + if e.errno != errno.ENOENT: + raise + del e + fetched = 0 + if fetched == 1: #resume mode: writemsg(">>> Resuming download...\n") locfetch=resumecommand @@ -5435,10 +5492,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_slot = "%s:%s" % (dep_getkey(atom), mydbapi.aux_get(avail_pkg, ["SLOT"])[0]) elif not avail_pkg: - has_mask = False - if hasattr(mydbapi, "xmatch"): - has_mask = bool(mydbapi.xmatch("match-all", atom)) - if (selective or use_binaries or not has_mask): + if selective: avail_pkg = vardb.match(atom) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) @@ -5505,8 +5559,10 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for myslot in intersecting_slots: myversion = versions[myslot] o_version = o_versions[myslot] - if myversion != o_version: - if myversion == best([myversion, o_version]): + difference = pkgcmp(catpkgsplit(myversion)[1:], + catpkgsplit(o_version)[1:]) + if difference: + if difference > 0: has_upgrade = True else: has_downgrade = True @@ -5732,7 +5788,11 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): writemsg("virts[%s]: %s\n" % (str(mykey),virts[mykey]), 1) mykey_orig = mykey[:] for vkey in virts[mykey]: - if mydb.cp_list(vkey,use_cache=use_cache): + # The virtuals file can contain a versioned atom, so + # it may be necessary to remove the operator and + # version from the atom before it is passed into + # dbapi.cp_list(). + if mydb.cp_list(dep_getkey(vkey), use_cache=use_cache): mykey = vkey writemsg("virts chosen: %s\n" % (mykey), 1) break diff --git a/pym/portage/util.py b/pym/portage/util.py index 2398bad87..4a53a3611 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -1073,9 +1073,14 @@ def new_protect_filename(mydest, newmd5=None): old_pfile = normalize_path(os.path.join(real_dirname, last_pfile)) if last_pfile and newmd5: import portage.checksum - if portage.checksum.perform_md5( - os.path.join(real_dirname, last_pfile)) == newmd5: - return old_pfile + try: + last_pfile_md5 = portage.checksum.perform_md5(old_pfile) + except FileNotFound: + # The file suddenly disappeared or it's a broken symlink. + pass + else: + if last_pfile_md5 == newmd5: + return old_pfile return new_pfile def getlibpaths(): -- 2.26.2