Merged from trunk 9668:9697
authorFabian Groffen <grobian@gentoo.org>
Thu, 10 Apr 2008 16:41:19 +0000 (16:41 -0000)
committerFabian Groffen <grobian@gentoo.org>
Thu, 10 Apr 2008 16:41:19 +0000 (16:41 -0000)
   | 9669    | Remove some boolean logic that is obsolete now that the      |
   | zmedico | "selective" parameter is passed into dep_zapdeps().          |

   | 9672    | Use pkgcmp() instead of == and best() for correctness in     |
   | zmedico | case of implicit -r0.                                        |

   | 9674    | Bug #215673 - Make ebuild.sh leave IUSE defaults intact      |
   | zmedico | instead of filtering them out. The built_with_use() function |
   |         | in eutils.eclass needs to be updated for compatibility.      |

   | 9675    | Remove the digraph._parent_child_digraph since it's no       |
   | zmedico | longer needed. It's main purpose was to track PDEPEND        |
   |         | relationships but those are handled just like the other deps |
   |         | now.                                                         |

   | 9677    | Handle a potential FileNotFound exception in                 |
   | zmedico | new_protect_filename() when the last ._cfg* file happens to  |
   |         | be a broken symlink. Thanks to bonsaikitten for reporting.   |

   | 9678    | Eliminate a redundant os.path.join() call.                   |
   | zmedico |                                                              |

   | 9680    | Use Package.cp instead of dep_getkey(cpv).                   |
   | zmedico |                                                              |

   | 9681    | Fix the previous commit so that the Package instance is      |
   | zmedico | created in the correct scope.                                |

   | 9683    | In cpv_expand(), if the virtuals file contains a versioned   |
   | zmedico | atom (like the base profile currently has for virtual/pam),  |
   |         | properly remove the operator and version from the atom       |
   |         | before it is passed into dbapi.cp_list().                    |

   | 9685    | For old-style virtuals, we need to repeat the                |
   | zmedico | package.provided check against the selected package.         |

   | 9687    | Implement a PORTAGE_FETCH_RESUME_MIN_SIZE variable that is   |
   | zmedico | useful for helping to ensure that small garbage files such   |
   |         | as html 404 pages are properly discarded.                    |

   | 9689    | If PORTAGE_FETCH_RESUME_MIN_SIZE is undefined or empty,      |
   | zmedico | silently use the default.                                    |

   | 9691    | Add missing str.upper() call for                             |
   | zmedico | PORTAGE_FETCH_RESUME_MIN_SIZE.                               |

   | 9693    | Fix package selection logic so that it always properly finds |
   | zmedico | the highest available version in a new slot even though the  |
   |         | graph already contains a matching version in a lower slot.   |

   | 9695    | Eliminate a redundant atom from the --update greedy slot     |
   | zmedico | expansion code.                                              |

   | 9696    | Add missing DepPriority instances to the graph so that the   |
   | zmedico | --buildpkgonly hasallzeros() calculation works correctly     |
   |         | again.                                                       |

   | 9697    | Decrease priority on --onlydpes dependencies so that         |
   | zmedico | --buildpkgonly hasallzeros() works correctly.                |

svn path=/main/branches/prefix/; revision=9803

bin/ebuild.sh
cnf/make.globals
man/make.conf.5
pym/_emerge/__init__.py
pym/portage/__init__.py
pym/portage/util.py

index 1cd140e3d0431cb9a86ade94b5b29181d630d3b2..54e1904d6afdcfa2e378fd9b06d202f34c83ed46 100755 (executable)
@@ -1728,20 +1728,6 @@ if ! hasq ${EBUILD_PHASE} clean && \
        PDEPEND="${PDEPEND} ${E_PDEPEND}"
 
        unset ECLASS E_IUSE E_DEPEND E_RDEPEND E_PDEPEND
-
-       if [ "${EBUILD_PHASE}" != "depend" ] ; then
-               # Make IUSE defaults backward compatible with all the old shell code.
-               iuse_temp=""
-               for x in ${IUSE} ; do
-                       if [[ ${x} == +* ]] || [[ ${x} == -* ]] ; then
-                               iuse_temp="${iuse_temp} ${x:1}"
-                       else
-                               iuse_temp="${iuse_temp} ${x}"
-                       fi
-               done
-               export IUSE=${iuse_temp}
-               unset x iuse_temp
-       fi
        set +f
 fi
 
index 1c848a749036ce6c18600d694d2de20a7d232a02..0851d53a0aa65fc5f47011c6ef2e70cd49232d06 100644 (file)
@@ -59,6 +59,9 @@ AUTOCLEAN="yes"
 # Number of mirrors to try when a downloaded file has an incorrect checksum.
 PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS="5"
 
+# Minimum size of existing file for RESUMECOMMAND to be called.
+PORTAGE_FETCH_RESUME_MIN_SIZE="350K"
+
 # Number of times 'emerge --sync' will run before giving up.
 PORTAGE_RSYNC_RETRIES="3"
 
index 7f9a496189710bcea40203fcbd51ab4f197688b5..90e419afc8220ac5aedb2e5124ffe9e064daad95 100644 (file)
@@ -412,6 +412,14 @@ Please see /etc/make.conf.example for elog documentation.
 \fBPORTAGE_FETCH_CHECKSUM_TRY_MIRRORS\fR = \fI5\fR
 Number of mirrors to try when a downloaded file has an incorrect checksum.
 .TP
+\PORTAGE_FETCH_RESUME_MIN_SIZE\fR = \fI350K\fR
+Minimum size of existing file for RESUMECOMMAND to be called. Files smaller
+than this size will be removed and FETCHCOMMAND will be called to download
+the file from the beginning. This is useful for helping to ensure that small
+garbage files such as html 404 pages are properly discarded. The variable
+should contain an integer number of bytes and may have a suffix such as
+K, M, or G.
+.TP
 \fBPORTAGE_NICENESS\fR = \fI[number]\fR
 The value of this variable will be added to the current nice level that
 emerge is running at.  In other words, this will not set the nice level,
index 06727799b9b355abe4f862943851c8a3a3917340..b8aad59aa116283a39956a4b0b7e76ee95e64988 100644 (file)
@@ -1228,6 +1228,24 @@ class Package(object):
                        status = "nomerge"
                self._digraph_node = (self.type_name, self.root, self.cpv, status)
 
+       def __lt__(self, other):
+               other_split = portage.catpkgsplit(other.cpv)
+               self_split = portage.catpkgsplit(self.cpv)
+               if other_split[:2] != self_split[:2]:
+                       return False
+               if portage.pkgcmp(self_split[1:], other_split[1:]) < 0:
+                       return True
+               return False
+
+       def __gt__(self, other):
+               other_split = portage.catpkgsplit(other.cpv)
+               self_split = portage.catpkgsplit(self.cpv)
+               if other_split[:2] != self_split[:2]:
+                       return False
+               if portage.pkgcmp(self_split[1:], other_split[1:]) > 0:
+                       return True
+               return False
+
        def __eq__(self, other):
                return self._digraph_node == other
        def __ne__(self, other):
@@ -1532,9 +1550,6 @@ class depgraph(object):
                del trees
 
                self.digraph=portage.digraph()
-               # Tracks simple parent/child relationships (PDEPEND relationships are
-               # not reversed).
-               self._parent_child_digraph = digraph()
                # contains all sets added to the graph
                self._sets = {}
                # contains atoms given as arguments
@@ -1592,7 +1607,7 @@ class depgraph(object):
                        for node in slot_nodes:
                                msg.append(indent)
                                msg.append(str(node))
-                               parents = self._parent_child_digraph.parent_nodes(node)
+                               parents = self.digraph.parent_nodes(node)
                                if parents:
                                        omitted_parents = 0
                                        if len(parents) > max_parents:
@@ -1788,10 +1803,10 @@ class depgraph(object):
                        if existing_node:
                                if pkg.cpv == existing_node.cpv:
                                        # The existing node can be reused.
-                                       self._parent_child_digraph.add(existing_node, myparent)
                                        if args:
                                                for arg in args:
-                                                       self._parent_child_digraph.add(existing_node, arg)
+                                                       self.digraph.add(existing_node, arg,
+                                                               priority=priority)
                                        # If a direct circular dependency is not an unsatisfied
                                        # buildtime dependency then drop it here since otherwise
                                        # it can skew the merge order calculation in an unwanted
@@ -1859,10 +1874,11 @@ class depgraph(object):
                # Do this even when addme is False (--onlydeps) so that the
                # parent/child relationship is always known in case
                # self._show_slot_collision_notice() needs to be called later.
-               self._parent_child_digraph.add(pkg, myparent)
+               if pkg.onlydeps:
+                       self.digraph.add(pkg, myparent, priority=priority)
                if args:
                        for arg in args:
-                               self._parent_child_digraph.add(pkg, arg)
+                               self.digraph.add(pkg, arg, priority=priority)
 
                """ This section determines whether we go deeper into dependencies or not.
                    We want to go deeper on a few occasions:
@@ -1932,6 +1948,10 @@ class depgraph(object):
                strict = mytype != "installed"
                try:
                        for dep_root, dep_string, dep_priority in deps:
+                               if pkg.onlydeps:
+                                       # Decrease priority so that --buildpkgonly
+                                       # hasallzeros() works correctly.
+                                       dep_priority = DepPriority()
                                if not dep_string:
                                        continue
                                if debug:
@@ -1996,9 +2016,6 @@ class depgraph(object):
                        myslots.add(vardb.aux_get(cpv, ["SLOT"])[0])
                for myslot in myslots:
                        yield "%s:%s" % (mykey, myslot)
-               # In addition to any installed slots, also try to pull
-               # in the latest new slot that may be available.
-               yield atom
 
        def _iter_args_for_pkg(self, pkg):
                # TODO: add multiple $ROOT support
@@ -2227,6 +2244,8 @@ class depgraph(object):
                        # atoms could be a property of the set itself.
                        greedy_atoms = []
                        for arg in args:
+                               # In addition to any installed slots, also try to pull
+                               # in the latest new slot that may be available.
                                greedy_atoms.append(arg)
                                if not isinstance(arg, (AtomArg, PackageArg)):
                                        continue
@@ -2267,6 +2286,7 @@ class depgraph(object):
                while args:
                        arg = args.pop()
                        for atom in arg.set:
+                               atom_cp = portage.dep_getkey(atom)
                                try:
                                        pprovided = pprovideddict.get(portage.dep_getkey(atom))
                                        if pprovided and portage.match_from_list(atom, pprovided):
@@ -2290,6 +2310,17 @@ class depgraph(object):
                                                        return 0, myfavorites
                                                self._missing_args.append((arg, atom))
                                                continue
+                                       if atom_cp != pkg.cp:
+                                               # For old-style virtuals, we need to repeat the
+                                               # package.provided check against the selected package.
+                                               expanded_atom = atom.replace(atom_cp, pkg.cp)
+                                               pprovided = pprovideddict.get(pkg.cp)
+                                               if pprovided and \
+                                                       portage.match_from_list(expanded_atom, pprovided):
+                                                       # A provided package has been
+                                                       # specified on the command line.
+                                                       self._pprovided_args.append((arg, atom))
+                                                       continue
                                        if pkg.installed and "selective" not in self.myparams:
                                                self._unsatisfied_deps_for_display.append(
                                                        ((myroot, atom), {}))
@@ -2578,6 +2609,8 @@ class depgraph(object):
                portdb = self.roots[root].trees["porttree"].dbapi
                # List of acceptable packages, ordered by type preference.
                matched_packages = []
+               highest_version = None
+               atom_cp = portage.dep_getkey(atom)
                existing_node = None
                myeb = None
                usepkgonly = "--usepkgonly" in self.myopts
@@ -2642,13 +2675,18 @@ class depgraph(object):
                                                        if not installed:
                                                                # masked by corruption
                                                                continue
+                                       pkg = Package(built=built, cpv=cpv, installed=installed,
+                                               metadata=metadata, type_name=pkg_type)
                                        if not installed:
                                                if myarg:
                                                        found_available_arg = True
-                                               if not visible(pkgsettings, Package(built=built,
-                                                       cpv=cpv, installed=installed, metadata=metadata,
-                                                       type_name=pkg_type)):
+                                               if not visible(pkgsettings, pkg):
                                                        continue
+                                       if pkg.cp == atom_cp:
+                                               if highest_version is None:
+                                                       highest_version = pkg
+                                               elif pkg > highest_version:
+                                                       highest_version = pkg
                                        # At this point, we've found the highest visible
                                        # match from the current repo. Any lower versions
                                        # from this repo are ignored, so this so the loop
@@ -2663,8 +2701,17 @@ class depgraph(object):
                                                cpv_slot = "%s:%s" % \
                                                        (e_pkg.cpv, e_pkg.metadata["SLOT"])
                                                if portage.dep.match_from_list(atom, [cpv_slot]):
-                                                       matched_packages.append(e_pkg)
-                                                       existing_node = e_pkg
+                                                       if highest_version and \
+                                                               e_pkg.cp == atom_cp and \
+                                                               e_pkg < highest_version and \
+                                                               e_pkg.slot_atom != highest_version.slot_atom:
+                                                               # There is a higher version available in a
+                                                               # different slot, so this existing node is
+                                                               # irrelevant.
+                                                               pass
+                                                       else:
+                                                               matched_packages.append(e_pkg)
+                                                               existing_node = e_pkg
                                                break
                                        # Compare built package to current config and
                                        # reject the built package if necessary.
@@ -2837,7 +2884,7 @@ class depgraph(object):
                        setconfig = root_config.setconfig
                        args = []
                        # Reuse existing SetArg instances when available.
-                       for arg in self._parent_child_digraph.root_nodes():
+                       for arg in self.digraph.root_nodes():
                                if not isinstance(arg, SetArg):
                                        continue
                                if arg.root_config != root_config:
@@ -3519,7 +3566,7 @@ class depgraph(object):
 
                tree_nodes = []
                display_list = []
-               mygraph = self._parent_child_digraph
+               mygraph = self.digraph
                i = 0
                depth = 0
                shown_edges = set()
@@ -3662,6 +3709,8 @@ class depgraph(object):
                                        repo_name = self.roots[myroot].settings.get("PORTAGE_BINHOST")
                                else:
                                        repo_name = metadata["repository"]
+                               built = pkg_type != "ebuild"
+                               installed = pkg_type == "installed"
                                if pkg_type == "ebuild":
                                        ebuild_path = portdb.findname(pkg_key)
                                        if not ebuild_path: # shouldn't happen
@@ -3674,6 +3723,9 @@ class depgraph(object):
                                        repo_path_real = repo_name
                                else:
                                        repo_path_real = portdb.getRepositoryPath(repo_name)
+                               pkg_node = Package(type_name=pkg_type, root=myroot,
+                                       cpv=pkg_key, built=built, installed=installed,
+                                       metadata=metadata)
                                pkg_use = metadata["USE"].split()
                                try:
                                        restrict = flatten(use_reduce(paren_reduce(
@@ -4568,8 +4620,10 @@ class MergeTask(object):
 
                mergecount=0
                for x in mymergelist:
-                       mergecount+=1
                        pkg_type = x[0]
+                       if pkg_type == "blocks":
+                               continue
+                       mergecount+=1
                        myroot=x[1]
                        pkg_key = x[2]
                        pkgindex=2
@@ -4593,6 +4647,11 @@ class MergeTask(object):
                                        raise AssertionError("Package type: '%s'" % pkg_type)
                                metadata.update(izip(metadata_keys,
                                        mydbapi.aux_get(pkg_key, metadata_keys)))
+                       built = pkg_type != "ebuild"
+                       installed = pkg_type == "installed"
+                       pkg = Package(type_name=pkg_type, root=myroot,
+                               cpv=pkg_key, built=built, installed=installed,
+                               metadata=metadata)
                        if x[0]=="blocks":
                                pkgindex=3
                        y = portdb.findname(pkg_key)
@@ -4823,8 +4882,7 @@ class MergeTask(object):
                                        "--fetch-all-uri" not in self.myopts:
 
                                        # Figure out if we need a restart.
-                                       if myroot == "/" and \
-                                               portage.dep_getkey(pkg_key) == "sys-apps/portage":
+                                       if myroot == "/" and pkg.cp == "sys-apps/portage":
                                                if len(mymergelist) > mergecount and EPREFIX == BPREFIX:
                                                        emergelog(xterm_titles,
                                                                " ::: completed emerge ("+ \
index fa9012f7ff5b44881a0aa3c8f4648dc5558d45fa..3cecd9ef7cded258033835283a358280c213299c 100644 (file)
@@ -949,7 +949,8 @@ class config(object):
                "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES",
                "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
                "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
-               "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_GPG_DIR",
+               "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", "PORTAGE_FETCH_RESUME_MIN_SIZE",
+               "PORTAGE_GPG_DIR",
                "PORTAGE_GPG_KEY", "PORTAGE_PACKAGE_EMPTY_ABORT",
                "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
                "PORTAGE_RSYNC_RETRIES", "PORTAGE_USE", "PORT_LOGDIR",
@@ -3032,6 +3033,20 @@ def _checksum_failure_temp_file(distdir, basename):
        os.rename(filename, temp_filename)
        return temp_filename
 
+_fetch_resume_size_re = re.compile('(^[\d]+)([KMGTPEZY]?$)')
+
+_size_suffix_map = {
+       ''  : 0,
+       'K' : 10,
+       'M' : 20,
+       'G' : 30,
+       'T' : 40,
+       'P' : 50,
+       'E' : 60,
+       'Z' : 70,
+       'Y' : 80,
+}
+
 def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1):
        "fetch files.  Will use digest file if available."
 
@@ -3072,6 +3087,29 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
        checksum_failure_max_tries = v
        del v
 
+       fetch_resume_size_default = "350K"
+       fetch_resume_size = mysettings.get("PORTAGE_FETCH_RESUME_MIN_SIZE")
+       if fetch_resume_size is not None:
+               fetch_resume_size = "".join(fetch_resume_size.split())
+               if not fetch_resume_size:
+                       # If it's undefined or empty, silently use the default.
+                       fetch_resume_size = fetch_resume_size_default
+               match = _fetch_resume_size_re.match(fetch_resume_size)
+               if match is None or \
+                       (match.group(2).upper() not in _size_suffix_map):
+                       writemsg("!!! Variable PORTAGE_FETCH_RESUME_MIN_SIZE" + \
+                               " contains an unrecognized format: '%s'\n" % \
+                               mysettings["PORTAGE_FETCH_RESUME_MIN_SIZE"], noiselevel=-1)
+                       writemsg("!!! Using PORTAGE_FETCH_RESUME_MIN_SIZE " + \
+                               "default value: %s\n" % fetch_resume_size_default,
+                               noiselevel=-1)
+                       fetch_resume_size = None
+       if fetch_resume_size is None:
+               fetch_resume_size = fetch_resume_size_default
+               match = _fetch_resume_size_re.match(fetch_resume_size)
+       fetch_resume_size = int(match.group(1)) * \
+               2 ** _size_suffix_map[match.group(2).upper()]
+
        # Behave like the package has RESTRICT="primaryuri" after a
        # couple of checksum failures, to increase the probablility
        # of success before checksum_failure_max_tries is reached.
@@ -3427,8 +3465,27 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",
 
                                if fetched != 2 and has_space:
                                        #we either need to resume or start the download
-                                       #you can't use "continue" when you're inside a "try" block
-                                       if fetched==1:
+                                       if fetched == 1:
+                                               try:
+                                                       mystat = os.stat(myfile_path)
+                                               except OSError, e:
+                                                       if e.errno != errno.ENOENT:
+                                                               raise
+                                                       del e
+                                                       fetched = 0
+                                               else:
+                                                       if mystat.st_size < fetch_resume_size:
+                                                               writemsg((">>> Deleting distfile with size " + \
+                                                                       "%d (smaller than " "PORTAGE_FETCH_RESU" + \
+                                                                       "ME_MIN_SIZE)\n") % mystat.st_size)
+                                                               try:
+                                                                       os.unlink(myfile_path)
+                                                               except OSError, e:
+                                                                       if e.errno != errno.ENOENT:
+                                                                               raise
+                                                                       del e
+                                                               fetched = 0
+                                       if fetched == 1:
                                                #resume mode:
                                                writemsg(">>> Resuming download...\n")
                                                locfetch=resumecommand
@@ -5435,10 +5492,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None):
                                avail_slot = "%s:%s" % (dep_getkey(atom),
                                        mydbapi.aux_get(avail_pkg, ["SLOT"])[0])
                        elif not avail_pkg:
-                               has_mask = False
-                               if hasattr(mydbapi, "xmatch"):
-                                       has_mask = bool(mydbapi.xmatch("match-all", atom))
-                               if (selective or use_binaries or not has_mask):
+                               if selective:
                                        avail_pkg = vardb.match(atom)
                                        if avail_pkg:
                                                avail_pkg = avail_pkg[-1] # highest (ascending order)
@@ -5505,8 +5559,10 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None):
                        for myslot in intersecting_slots:
                                myversion = versions[myslot]
                                o_version = o_versions[myslot]
-                               if myversion != o_version:
-                                       if myversion == best([myversion, o_version]):
+                               difference = pkgcmp(catpkgsplit(myversion)[1:],
+                                       catpkgsplit(o_version)[1:])
+                               if difference:
+                                       if difference > 0:
                                                has_upgrade = True
                                        else:
                                                has_downgrade = True
@@ -5732,7 +5788,11 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None):
                                        writemsg("virts[%s]: %s\n" % (str(mykey),virts[mykey]), 1)
                                        mykey_orig = mykey[:]
                                        for vkey in virts[mykey]:
-                                               if mydb.cp_list(vkey,use_cache=use_cache):
+                                               # The virtuals file can contain a versioned atom, so
+                                               # it may be necessary to remove the operator and
+                                               # version from the atom before it is passed into
+                                               # dbapi.cp_list().
+                                               if mydb.cp_list(dep_getkey(vkey), use_cache=use_cache):
                                                        mykey = vkey
                                                        writemsg("virts chosen: %s\n" % (mykey), 1)
                                                        break
index 2398bad8785c63f58e9ab13fee6d0cb0e5c19d59..4a53a3611adc5b60925e01672b6c94e0c1340fe0 100644 (file)
@@ -1073,9 +1073,14 @@ def new_protect_filename(mydest, newmd5=None):
        old_pfile = normalize_path(os.path.join(real_dirname, last_pfile))
        if last_pfile and newmd5:
                import portage.checksum
-               if portage.checksum.perform_md5(
-                       os.path.join(real_dirname, last_pfile)) == newmd5:
-                       return old_pfile
+               try:
+                       last_pfile_md5 = portage.checksum.perform_md5(old_pfile)
+               except FileNotFound:
+                       # The file suddenly disappeared or it's a broken symlink.
+                       pass
+               else:
+                       if last_pfile_md5 == newmd5:
+                               return old_pfile
        return new_pfile
 
 def getlibpaths():