From 85b447a82ee6f46d4bbc483ea695c06d8b9aa26f Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 11 Apr 2008 10:13:26 +0000 Subject: [PATCH] Merged from trunk 9814:9827 | 9815 | Bug #217122 - When waiting for a parallel-fetch download to | | zmedico | complete, show a more friendly and informative message. | | 9817 | Make the CHOST visibility check tolerant of missing CHOST | | zmedico | metadata. | | 9819 | Fix "pkg" variable name collisions in depgraph.display(). | | zmedico | | | 9821 | Rename variable pkg_cpv to pkg_str. | | zmedico | | | 9823 | Nest the DepcheckCompositeDB class inside the depgraph class | | zmedico | since it's not intended for use elsewhere. | | 9825 | Tweak --buildpkgonly and fakeroot logic so that | | zmedico | --buildpkgonly can be used by a non-root user to build | | | binary packages with fakeroot. | | 9827 | Copy group mode bits from $PKGDIR when creating | | zmedico | subdirectories. | svn path=/main/branches/prefix/; revision=9829 --- pym/_emerge/__init__.py | 230 ++++++++++++++++++----------------- pym/portage/__init__.py | 17 ++- pym/portage/dbapi/bintree.py | 61 +++++----- 3 files changed, 161 insertions(+), 147 deletions(-) diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index d5ff2a52b..7045c2448 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1104,9 +1104,10 @@ def visible(pkgsettings, pkg): """ if not pkg.metadata["SLOT"]: return False - if pkg.built and not pkg.installed and \ - pkg.metadata["CHOST"] != pkgsettings["CHOST"]: - return False + if pkg.built and not pkg.installed: + pkg_chost = pkg.metadata.get("CHOST") + if pkg_chost and pkg_chost != pkgsettings["CHOST"]: + return False if pkg.built and not pkg.installed: # we can have an old binary which has no EPREFIX information if "EPREFIX" not in pkg.metadata or not pkg.metadata["EPREFIX"]: @@ -1135,10 +1136,11 @@ def get_masking_status(pkg, pkgsettings, root_config): pkg, settings=pkgsettings, portdb=root_config.trees["porttree"].dbapi) - if pkg.built and not pkg.installed and \ - pkg.metadata["CHOST"] != root_config.settings["CHOST"]: - mreasons.append("CHOST: %s" % \ - pkg.metadata["CHOST"]) + if pkg.built and not pkg.installed: + pkg_chost = pkg.metadata.get("CHOST") + if pkg_chost and pkg_chost != pkgsettings["CHOST"]: + mreasons.append("CHOST: %s" % \ + pkg.metadata["CHOST"]) if pkg.built and not pkg.installed: if not "EPREFIX" in metadata or not metadata["EPREFIX"]: @@ -1459,91 +1461,6 @@ def show_invalid_depstring_notice(parent_node, depstring, error_msg): f.add_flowing_data(x) f.end_paragraph(1) -class DepcheckCompositeDB(object): - """ - A dbapi-like interface that is optimized for use in dep_check() calls. - This is built on top of the existing depgraph package selection logic. - Some packages that have been added to the graph may be masked from this - view in order to control the atom preference selection that occurs via - dep_check(). - """ - def __init__(self, depgraph, root): - self._depgraph = depgraph - self._root = root - self._match_cache = {} - self._cpv_pkg_map = {} - - def match(self, atom): - ret = self._match_cache.get(atom) - if ret is not None: - return ret[:] - orig_atom = atom - if "/" not in atom: - atom = self._dep_expand(atom) - pkg, existing = self._depgraph._select_package(self._root, atom) - if not pkg: - ret = [] - else: - if pkg.installed and "selective" not in self._depgraph.myparams: - try: - arg = self._depgraph._iter_atoms_for_pkg(pkg).next() - except (StopIteration, portage.exception.InvalidDependString): - arg = None - if arg: - ret = [] - if ret is None and pkg.installed and \ - not visible(self._depgraph.pkgsettings[pkg.root], pkg): - # For disjunctive || deps, this will cause alternative - # atoms or packages to be selected if available. - ret = [] - if ret is None: - self._cpv_pkg_map[pkg.cpv] = pkg - ret = [pkg.cpv] - self._match_cache[orig_atom] = ret - return ret[:] - - def _dep_expand(self, atom): - """ - This is only needed for old installed packages that may - contain atoms that are not fully qualified with a specific - category. Emulate the cpv_expand() function that's used by - dbapi.match() in cases like this. If there are multiple - matches, it's often due to a new-style virtual that has - been added, so try to filter those out to avoid raising - a ValueError. - """ - root_config = self._depgraph.roots[self._root] - orig_atom = atom - expanded_atoms = self._depgraph._dep_expand(root_config, atom) - if len(expanded_atoms) > 1: - non_virtual_atoms = [] - for x in expanded_atoms: - if not portage.dep_getkey(x).startswith("virtual/"): - non_virtual_atoms.append(x) - if len(non_virtual_atoms) == 1: - expanded_atoms = non_virtual_atoms - if len(expanded_atoms) > 1: - # compatible with portage.cpv_expand() - raise ValueError([portage.dep_getkey(x) \ - for x in expanded_atoms]) - if expanded_atoms: - atom = expanded_atoms[0] - else: - null_atom = insert_category_into_atom(atom, "null") - null_cp = portage.dep_getkey(null_atom) - cat, atom_pn = portage.catsplit(null_cp) - virts_p = root_config.settings.get_virts_p().get(atom_pn) - if virts_p: - # Allow the resolver to choose which virtual. - atom = insert_category_into_atom(atom, "virtual") - else: - atom = insert_category_into_atom(atom, "null") - return atom - - def aux_get(self, cpv, wants): - metadata = self._cpv_pkg_map[cpv].metadata - return [metadata.get(x, "") for x in wants] - class PackageVirtualDbapi(portage.dbapi): """ A dbapi-like interface class that represents the state of the installed @@ -1710,7 +1627,7 @@ class depgraph(object): self._filtered_trees[myroot]["vartree"] = self.trees[myroot]["vartree"] def filtered_tree(): pass - filtered_tree.dbapi = DepcheckCompositeDB(self, myroot) + filtered_tree.dbapi = self._dep_check_composite_db(self, myroot) self._filtered_trees[myroot]["porttree"] = filtered_tree dbs = [] portdb = self.trees[myroot]["porttree"].dbapi @@ -3914,13 +3831,12 @@ class depgraph(object): cur_use = [flag for flag in cur_use if flag in cur_iuse] if myoldbest and myinslotlist: - pkg = myoldbest[0] + previous_cpv = myoldbest[0] else: - pkg = x[2] - if self.trees[x[1]]["vartree"].dbapi.cpv_exists(pkg): - old_iuse, old_use = \ - self.trees[x[1]]["vartree"].dbapi.aux_get( - pkg, ["IUSE", "USE"]) + previous_cpv = pkg.cpv + if vardb.cpv_exists(previous_cpv): + old_iuse, old_use = vardb.aux_get( + previous_cpv, ["IUSE", "USE"]) old_iuse = list(set( filter_iuse_defaults(old_iuse.split()))) old_iuse.sort() @@ -4108,21 +4024,21 @@ class depgraph(object): # This is reported elsewhere if relevant. pass - def pkgprint(pkg): + def pkgprint(pkg_str): if pkg_merge: if pkg_system: - return colorize("PKG_MERGE_SYSTEM", pkg) + return colorize("PKG_MERGE_SYSTEM", pkg_str) elif pkg_world: - return colorize("PKG_MERGE_WORLD", pkg) + return colorize("PKG_MERGE_WORLD", pkg_str) else: - return colorize("PKG_MERGE", pkg) + return colorize("PKG_MERGE", pkg_str) else: if pkg_system: - return colorize("PKG_NOMERGE_SYSTEM", pkg) + return colorize("PKG_NOMERGE_SYSTEM", pkg_str) elif pkg_world: - return colorize("PKG_NOMERGE_WORLD", pkg) + return colorize("PKG_NOMERGE_WORLD", pkg_str) else: - return colorize("PKG_NOMERGE", pkg) + return colorize("PKG_NOMERGE", pkg_str) if x[1]!="/": if myoldbest: @@ -4440,6 +4356,91 @@ class depgraph(object): fakedb[myroot].cpv_inject(pkg) self.spinner.update() + class _dep_check_composite_db(object): + """ + A dbapi-like interface that is optimized for use in dep_check() calls. + This is built on top of the existing depgraph package selection logic. + Some packages that have been added to the graph may be masked from this + view in order to influence the atom preference selection that occurs + via dep_check(). + """ + def __init__(self, depgraph, root): + self._depgraph = depgraph + self._root = root + self._match_cache = {} + self._cpv_pkg_map = {} + + def match(self, atom): + ret = self._match_cache.get(atom) + if ret is not None: + return ret[:] + orig_atom = atom + if "/" not in atom: + atom = self._dep_expand(atom) + pkg, existing = self._depgraph._select_package(self._root, atom) + if not pkg: + ret = [] + else: + if pkg.installed and "selective" not in self._depgraph.myparams: + try: + arg = self._depgraph._iter_atoms_for_pkg(pkg).next() + except (StopIteration, portage.exception.InvalidDependString): + arg = None + if arg: + ret = [] + if ret is None and pkg.installed and \ + not visible(self._depgraph.pkgsettings[pkg.root], pkg): + # For disjunctive || deps, this will cause alternative + # atoms or packages to be selected if available. + ret = [] + if ret is None: + self._cpv_pkg_map[pkg.cpv] = pkg + ret = [pkg.cpv] + self._match_cache[orig_atom] = ret + return ret[:] + + def _dep_expand(self, atom): + """ + This is only needed for old installed packages that may + contain atoms that are not fully qualified with a specific + category. Emulate the cpv_expand() function that's used by + dbapi.match() in cases like this. If there are multiple + matches, it's often due to a new-style virtual that has + been added, so try to filter those out to avoid raising + a ValueError. + """ + root_config = self._depgraph.roots[self._root] + orig_atom = atom + expanded_atoms = self._depgraph._dep_expand(root_config, atom) + if len(expanded_atoms) > 1: + non_virtual_atoms = [] + for x in expanded_atoms: + if not portage.dep_getkey(x).startswith("virtual/"): + non_virtual_atoms.append(x) + if len(non_virtual_atoms) == 1: + expanded_atoms = non_virtual_atoms + if len(expanded_atoms) > 1: + # compatible with portage.cpv_expand() + raise ValueError([portage.dep_getkey(x) \ + for x in expanded_atoms]) + if expanded_atoms: + atom = expanded_atoms[0] + else: + null_atom = insert_category_into_atom(atom, "null") + null_cp = portage.dep_getkey(null_atom) + cat, atom_pn = portage.catsplit(null_cp) + virts_p = root_config.settings.get_virts_p().get(atom_pn) + if virts_p: + # Allow the resolver to choose which virtual. + atom = insert_category_into_atom(atom, "virtual") + else: + atom = insert_category_into_atom(atom, "null") + return atom + + def aux_get(self, cpv, wants): + metadata = self._cpv_pkg_map[cpv].metadata + return [metadata.get(x, "") for x in wants] + class RepoDisplay(object): def __init__(self, roots): self._shown_repos = {} @@ -4803,7 +4804,7 @@ class MergeTask(object): try: catdir_lock = portage.locks.lockdir(catdir) portage.util.ensure_dirs(catdir, - uid=portage.portage_uid, gid=portage.portage_gid, + gid=portage.portage_gid, mode=070, mask=0) builddir_lock = portage.locks.lockdir( pkgsettings["PORTAGE_BUILDDIR"]) @@ -7041,6 +7042,7 @@ def action_build(settings, trees, mtimedb, ldpath_mtimes = mtimedb["ldpath"] favorites=[] merge_count = 0 + buildpkgonly = "--buildpkgonly" in myopts pretend = "--pretend" in myopts fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts ask = "--ask" in myopts @@ -7342,7 +7344,7 @@ def action_build(settings, trees, mtimedb, + " AUTOCLEAN is disabled. This can cause serious" + " problems due to overlapping packages.\n") - if merge_count and not (pretend or fetchonly): + if merge_count and not (buildpkgonly or fetchonly or pretend): post_emerge(trees, mtimedb, retval) return retval @@ -7838,13 +7840,17 @@ def emerge_main(): _emerge.help.help(myaction, myopts, portage.output.havecolor) return 1 + pretend = "--pretend" in myopts + fetchonly = "--fetchonly" in myopts or "--fetch-all-uri" in myopts + buildpkgonly = "--buildpkgonly" in myopts + # check if root user is the current user for the actions where emerge needs this if portage.secpass < 2: # We've already allowed "--version" and "--help" above. if "--pretend" not in myopts and myaction not in ("search","info"): need_superuser = not \ - ("--fetchonly" in myopts or \ - "--fetch-all-uri" in myopts or \ + (fetchonly or \ + (buildpkgonly and secpass >= 1) or \ myaction in ("metadata", "regen") or \ (myaction == "sync" and os.access(settings["PORTDIR"], os.W_OK))) if portage.secpass < 1 or \ @@ -7947,14 +7953,14 @@ def emerge_main(): root_config = trees[settings["ROOT"]]["root_config"] if 1 == unmerge(root_config, myopts, myaction, myfiles, mtimedb["ldpath"]): - if "--pretend" not in myopts: + if not (buildpkgonly or fetchonly or pretend): post_emerge(trees, mtimedb, os.EX_OK) elif myaction in ("depclean", "prune"): validate_ebuild_environment(trees) action_depclean(settings, trees, mtimedb["ldpath"], myopts, myaction, myfiles, spinner) - if "--pretend" not in myopts: + if not (buildpkgonly or fetchonly or pretend): post_emerge(trees, mtimedb, os.EX_OK) # "update", "system", or just process files: else: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 2e81a3b61..07d9f1210 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -2922,7 +2922,7 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero if not free: free=((droppriv and "usersandbox" not in features) or \ (not droppriv and "sandbox" not in features and \ - "usersandbox" not in features)) + "usersandbox" not in features and not fakeroot)) if free or "SANDBOX_ACTIVE" in os.environ: keywords["opt_name"] += " bash" @@ -3320,8 +3320,15 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", if use_locks and can_fetch: waiting_msg = None if not parallel_fetchonly and "parallel-fetch" in features: - waiting_msg = ("Downloading '%s'... " + \ - "see "+EPREFIX+"/var/log/emerge-fetch.log for details.") % myfile + waiting_msg = ("Fetching '%s' " + \ + "in the background. " + \ + "To view fetch progress, run `tail -f " + \ + EPREFIX + "/var/log/emerge-fetch.log` in another " + \ + "terminal.") % myfile + msg_prefix = colorize("GOOD", " * ") + from textwrap import wrap + waiting_msg = "\n".join(msg_prefix + line \ + for line in wrap(waiting_msg, 65)) if locks_in_subdir: file_lock = portage.locks.lockfile( os.path.join(mysettings["DISTDIR"], @@ -4512,6 +4519,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, vartree = db[myroot]["vartree"] features = mysettings.features + from portage.data import secpass validcommands = ["help","clean","prerm","postrm","cleanrm","preinst","postinst", "config","info","setup","depend","fetch","digest", @@ -5018,7 +5026,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, "sesandbox" in mysettings.features droppriv = "userpriv" in mysettings.features and \ - "userpriv" not in restrict + "userpriv" not in restrict and \ + secpass >= 2 fakeroot = "fakeroot" in mysettings.features diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 541de39cc..ebe49a798 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -4,9 +4,9 @@ from portage.dep import isvalidatom, isjustname, dep_getkey, match_from_list from portage.dbapi.virtual import fakedbapi -from portage.exception import InvalidPackageName, InvalidAtom +from portage.exception import InvalidPackageName, InvalidAtom, PortageException from portage.output import green -from portage.util import normalize_path, writemsg, writemsg_stdout +from portage.util import ensure_dirs, normalize_path, writemsg, writemsg_stdout from portage.versions import best, catpkgsplit, catsplit from portage.update import update_dbentries from portage.const import EAPI, EAPIPREFIX @@ -242,12 +242,7 @@ class binarytree(object): self._pkg_paths[mynewcpv] = os.path.join( *new_path.split(os.path.sep)[-2:]) if new_path != mytbz2: - try: - os.makedirs(os.path.dirname(new_path)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e + self._ensure_dir(os.path.dirname(new_path)) _movefile(tbz2path, new_path, mysettings=self.settings) self._remove_symlink(mycpv) if new_path.split(os.path.sep)[-2] == "All": @@ -280,12 +275,7 @@ class binarytree(object): exist in the location of the symlink will first be removed.""" mycat, mypkg = catsplit(cpv) full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2") - try: - os.makedirs(os.path.dirname(full_path)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e + self._ensure_dir(os.path.dirname(full_path)) try: os.unlink(full_path) except OSError, e: @@ -324,6 +314,26 @@ class binarytree(object): if os.path.exists(full_path): self.inject(cpv) + def _ensure_dir(self, path): + """ + Create the specified directory. Also, copy gid and group mode + bits from self.pkgdir if possible. + @param cat_dir: Absolute path of the directory to be created. + @type cat_dir: String + """ + try: + pkgdir_st = os.stat(self.pkgdir) + except OSError: + ensure_dirs(path) + return + pkgdir_gid = pkgdir_st.st_gid + pkgdir_grp_mode = 02070 & pkgdir_st.st_mode + try: + ensure_dirs(path, gid=pkgdir_gid, mode=pkgdir_grp_mode, mask=0) + except PortageException: + if not os.path.isdir(path): + raise + def _move_to_all(self, cpv): """If the file exists, move it. Whether or not it exists, update state for future getname() calls.""" @@ -335,12 +345,7 @@ class binarytree(object): except OSError, e: mystat = None if mystat and stat.S_ISREG(mystat.st_mode): - try: - os.makedirs(os.path.join(self.pkgdir, "All")) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e + self._ensure_dir(os.path.join(self.pkgdir, "All")) dest_path = os.path.join(self.pkgdir, "All", myfile) _movefile(src_path, dest_path, mysettings=self.settings) self._create_symlink(cpv) @@ -354,12 +359,7 @@ class binarytree(object): myfile = mypkg + ".tbz2" mypath = os.path.join(mycat, myfile) dest_path = os.path.join(self.pkgdir, mypath) - try: - os.makedirs(os.path.dirname(dest_path)) - except OSError, e: - if e.errno != errno.EEXIST: - raise - del e + self._ensure_dir(os.path.dirname(dest_path)) src_path = os.path.join(self.pkgdir, "All", myfile) _movefile(src_path, dest_path, mysettings=self.settings) self._pkg_paths[cpv] = mypath @@ -765,7 +765,9 @@ class binarytree(object): pkgindex_lock = lockfile(self._pkgindex_file, wantnewlockfile=1) if filename is not None: - _movefile(filename, self.getname(cpv), mysettings=self.settings) + new_filename = self.getname(cpv) + self._ensure_dir(os.path.dirname(new_filename)) + _movefile(filename, new_filename, mysettings=self.settings) if self._all_directory and \ self.getname(cpv).split(os.path.sep)[-2] == "All": self._create_symlink(cpv) @@ -957,10 +959,7 @@ class binarytree(object): noiselevel=-1) mydest = os.path.dirname(self.getname(pkgname)) - try: - os.makedirs(mydest, 0775) - except (OSError, IOError): - pass + self._ensure_dir(mydest) from urlparse import urlparse # urljoin doesn't work correctly with unrecognized protocols like sftp if self._remote_has_index: -- 2.26.2