From 1ae148e91af6d411f81f76332ff71a9b2a1d27ec Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 2 May 2008 14:39:08 +0000 Subject: [PATCH] Merged from trunk 10035:10069 | 10037 | Make unmerge() group packages by cat/pn when the order | | zmedico | doesn't matter, so the display appears like earlier versions | | | of portage. | | 10039 | Fix "deep" logic in depgraph._complete_graph() so that it | | zmedico | always properly recognizes whether or not "deep" was | | | previously enabled. | | 10041 | * Fix dep_check() so that it doesn't expand virtual blockers | | zmedico | since the un-expanded virtual atom is more useful for | | | maintaining a cache of blocker atoms. * Expand virtual | | | blockers in depgraph.validate_blockers(), since it's not | | | done by dep_check() anymore. * If blocker data from the | | | graph is available, use it to validate the blocker cache and | | | update the cache if it seems invalid. * Make | | | BlockerCache._load() more tolerant to installs/uninstalls so | | | so that cache isn't rebuilt every time. | | 10042 | Fix logic to re-use blockers from the graph when populating | | zmedico | the blocker cache. | | 10043 | Fix broken comparison, compare with None instead of nonzero. | | zmedico | | | 10045 | Handle a KeyError when validating the blocker cache. | | zmedico | | | 10047 | Bug #219837 - Adjust date calculations to improve accuracy. | | zmedico | | | 10050 | * Add a Package.operation attribute and use it to | | zmedico | distinguish "uninstall" operations. | | 10052 | Don't add blockers to the graph more times than necessary. | | zmedico | | | 10054 | Sort blockers when caching them. | | zmedico | | | 10056 | Detect an old version of tarsync and use bzip2 compression | | zmedico | in that case. | | 10062 | When adding blockers for installed packages to the graph, | | zmedico | use cached Package instances instead of constructing new | | | ones. | | 10065 | Fix variable rename breakage: pkg -> pkg.cpv | | zmedico | | | 10067 | Bug #219837 - Adjust date calculations some more and show an | | zmedico | informative ewarn message when bailing out due to a newer | | | snapshot being unavailable. | | 10069 | check if PORTAGE_TMPDIR is mounted readonly/noexec (bug | | genone | #219957) | svn path=/main/branches/prefix/; revision=10088 --- bin/emerge-webrsync | 88 +++++++++++---- pym/_emerge/__init__.py | 233 +++++++++++++++++++++++++++------------- pym/portage/__init__.py | 34 ++++++ 3 files changed, 258 insertions(+), 97 deletions(-) diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index d53ee4536..91a1375d6 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -173,8 +173,7 @@ sync_local() { vecho "Syncing local tree ..." - # tarsync-0.2.1 doesn't seem to support lzma compression. - if [ "${file##*.}" != "lzma" ] && type -P tarsync > /dev/null; then + if type -P tarsync > /dev/null ; then if ! tarsync $(vvecho -v) -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then eecho "tarsync failed; tarball is corrupt? (${file})" return 1 @@ -221,7 +220,14 @@ do_snapshot() { local mirror local compressions="" - type -P lzcat > /dev/null && compressions="${compressions} lzma" + # lzma is not supported in <=app-arch/tarsync-0.2.1, so use + # bz2 format if we have an old version of tarsync. + if type -P tarsync > /dev/null && \ + portageq has_version / '<=app-arch/tarsync-0.2.1' ; then + true + else + type -P lzcat > /dev/null && compressions="${compressions} lzma" + fi type -P bzcat > /dev/null && compressions="${compressions} bz2" type -P zcat > /dev/null && compressions="${compressions} gz" if [[ -z ${compressions} ]] ; then @@ -308,33 +314,75 @@ do_snapshot() { } do_latest_snapshot() { - local attempts=-1 + local attempts=0 local r=1 vecho "Fetching most recent snapshot ..." - while (( ${attempts} < 40 )) ; do - local day - local month - local year - local seconds - - attempts=$(( ${attempts} + 1 )) - - utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts}) + # The snapshot for a given day is generated at 01:45 UTC on the following + # day, so the current day's snapshot (going by UTC time) hasn't been + # generated yet. Therefore, always start by looking for the previous day's + # snapshot (for attempts=1, subtract 1 day from the current UTC time). + + # Timestamps that differ by less than 2 hours + # are considered to be approximately equal. + local min_time_diff=$(( 2 * 60 * 60 )) + + local existing_timestamp=$(get_portage_timestamp) + local timestamp_difference + local timestamp_problem + local approx_snapshot_time + local start_time=$(get_utc_date_in_seconds) + local start_hour=$(get_date_part ${start_time} "%H") + + # Daily snapshots are created at 1:45 AM and are not + # available until after 2 AM. Don't waste time trying + # to fetch a snapshot before it's been created. + if [ ${start_hour} -lt 2 ] ; then + (( start_time -= 86400 )) + fi + local snapshot_date=$(get_date_part ${start_time} "%Y%m%d") + local snapshot_date_seconds=$(get_utc_second_from_string ${snapshot_date}) - day=$(get_date_part ${utc_attempt} "%d") - month=$(get_date_part ${utc_attempt} "%m") - year=$(get_date_part ${utc_attempt} "%Y") - utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s") + while (( ${attempts} < 40 )) ; do + (( attempts++ )) + (( snapshot_date_seconds -= 86400 )) + # snapshots are created at 1:45 AM + (( approx_snapshot_time = snapshot_date_seconds + 86400 + 6300 )) + (( timestamp_difference = existing_timestamp - approx_snapshot_time )) + [ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference )) + snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d") + + timestamp_problem="" + if [ ${timestamp_difference} -eq 0 ]; then + timestamp_problem="is identical to" + elif [ ${timestamp_difference} -lt ${min_time_diff} ]; then + timestamp_problem="is possibly identical to" + elif [ ${approx_snapshot_time} -lt ${existing_timestamp} ] ; then + timestamp_problem="is newer than" + fi - if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then - wecho "portage content is newer than available snapshots (use --revert option to overide)" + if [ -n "${timestamp_problem}" ]; then + ewarn "Latest snapshot date: ${snapshot_date}" + ewarn + ewarn "Approximate snapshot timestamp: ${approx_snapshot_time}" + ewarn " Current local timestamp: ${existing_timestamp}" + ewarn + echo -e "The current local timestamp" \ + "${timestamp_problem} the" \ + "timestamp of the latest" \ + "snapshot. In order to force sync," \ + "use the --revert option or remove" \ + "the timestamp file located at" \ + "'${PORTDIR}/metadata/timestamp.x'." | fmt -w 70 | \ + while read line ; do + ewarn "${line}" + done r=0 break fi - if do_snapshot 0 "${year}${month}${day}"; then + if do_snapshot 0 "${snapshot_date}"; then r=0 break; fi diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 0f88c7c16..77281f3fc 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1274,7 +1274,11 @@ class Task(SlotObject): return str(self._get_hash_key()) class Blocker(Task): - __slots__ = ("root", "atom", "satisfied") + __slots__ = ("root", "atom", "cp", "satisfied") + + def __init__(self, **kwargs): + Task.__init__(self, **kwargs) + self.cp = portage.dep_getkey(self.atom) def _get_hash_key(self): hash_key = getattr(self, "_hash_key", None) @@ -1285,7 +1289,8 @@ class Blocker(Task): class Package(Task): __slots__ = ("built", "cpv", "depth", - "installed", "metadata", "root", "onlydeps", "type_name", + "installed", "metadata", "onlydeps", "operation", + "root", "type_name", "cp", "cpv_slot", "pv_split", "slot_atom") def __init__(self, **kwargs): Task.__init__(self, **kwargs) @@ -1297,11 +1302,12 @@ class Package(Task): def _get_hash_key(self): hash_key = getattr(self, "_hash_key", None) if hash_key is None: - operation = "merge" - if self.onlydeps or self.installed: - operation = "nomerge" + if self.operation is None: + self.operation = "merge" + if self.onlydeps or self.installed: + self.operation = "nomerge" self._hash_key = \ - (self.type_name, self.root, self.cpv, operation) + (self.type_name, self.root, self.cpv, self.operation) return self._hash_key def __lt__(self, other): @@ -1332,15 +1338,6 @@ class Package(Task): return True return False -class Uninstall(Package): - __slots__ = () - def _get_hash_key(self): - hash_key = getattr(self, "_hash_key", None) - if hash_key is None: - self._hash_key = \ - (self.type_name, self.root, self.cpv, "uninstall") - return self._hash_key - class DependencyArg(object): def __init__(self, arg=None, root_config=None): self.arg = arg @@ -1415,14 +1412,27 @@ class BlockerCache(DictMixin): cache_valid = self._cache_data and \ isinstance(self._cache_data, dict) and \ self._cache_data.get("version") == self._cache_version and \ - self._cache_data.get("virtuals") == self._virtuals and \ - set(self._cache_data.get("blockers", [])) == self._installed_pkgs + isinstance(self._cache_data.get("blockers"), dict) if cache_valid: - for pkg in self._installed_pkgs: - if long(self._vardb.aux_get(pkg, ["COUNTER"])[0]) != \ - self[pkg].counter: - cache_valid = False - break + invalid_cache = set() + for cpv, value \ + in self._cache_data["blockers"].iteritems(): + if not (isinstance(value, tuple) and len(value) == 2): + invalid_cache.add(cpv) + continue + counter, atoms = value + try: + if counter != long(self._vardb.aux_get(cpv, ["COUNTER"])[0]): + invalid_cache.add(cpv) + continue + except KeyError: + # The package is no longer installed. + invalid_cache.add(cpv) + continue + for cpv in invalid_cache: + del self._cache_data["blockers"][cpv] + if not self._cache_data["blockers"]: + cache_valid = False if not cache_valid: self._cache_data = {"version":self._cache_version} self._cache_data["blockers"] = {} @@ -3063,12 +3073,14 @@ class depgraph(object): # accounted for. self._select_atoms = self._select_atoms_from_graph self._select_package = self._select_pkg_from_graph - self.myparams.add("deep") + already_deep = "deep" in self.myparams + if not already_deep: + self.myparams.add("deep") for root in self.roots: required_set_names = self._required_set_names.copy() if root == self.target_root and \ - ("deep" in self.myparams or "empty" in self.myparams): + (already_deep or "empty" in self.myparams): required_set_names.difference_update(self._sets) if not required_set_names and not self._ignored_deps: continue @@ -3162,25 +3174,50 @@ class depgraph(object): final_db = self.mydbapi[myroot] cpv_all_installed = self.trees[myroot]["vartree"].dbapi.cpv_all() blocker_cache = BlockerCache(myroot, vardb) - for pkg in cpv_all_installed: + for cpv in cpv_all_installed: blocker_atoms = None - metadata = dict(izip(self._mydbapi_keys, - vardb.aux_get(pkg, self._mydbapi_keys))) - node = Package(cpv=pkg, built=True, - installed=True, metadata=metadata, - type_name="installed", root=myroot) - if self.digraph.contains(node): - continue + pkg = self._pkg_cache[ + ("installed", myroot, cpv, "nomerge")] + blockers = None + if self.digraph.contains(pkg): + try: + blockers = self._blocker_parents.child_nodes(pkg) + except KeyError: + blockers = [] + if blockers is not None: + blockers = set("!" + blocker.atom \ + for blocker in blockers) + # If this node has any blockers, create a "nomerge" # node for it so that they can be enforced. self.spinner.update() - blocker_data = blocker_cache.get(pkg) + blocker_data = blocker_cache.get(cpv) + + # If blocker data from the graph is available, use + # it to validate the cache and update the cache if + # it seems invalid. + if blocker_data is not None and \ + blockers is not None: + if not blockers.symmetric_difference( + blocker_data.atoms): + continue + blocker_data = None + + if blocker_data is None and \ + blockers is not None: + # Re-use the blockers from the graph. + blocker_atoms = sorted(blockers) + counter = long(node.metadata["COUNTER"]) + blocker_data = \ + blocker_cache.BlockerData(counter, blocker_atoms) + blocker_cache[pkg.cpv] = blocker_data + continue + if blocker_data: blocker_atoms = blocker_data.atoms else: - dep_vals = vardb.aux_get(pkg, dep_keys) - myuse = vardb.aux_get(pkg, ["USE"])[0].split() - depstr = " ".join(dep_vals) + myuse = pkg.metadata["USE"].split() + depstr = " ".join(pkg.metadata[k] for k in dep_keys) # It is crucial to pass in final_db here in order to # optimize dep_check calls by eliminating atoms via # dep_wordreduce and dep_eval calls. @@ -3198,41 +3235,64 @@ class depgraph(object): # matches (this can happen if an atom lacks a # category). show_invalid_depstring_notice( - node, depstr, str(e)) + pkg, depstr, str(e)) del e raise finally: portage.dep._dep_check_strict = True if not success: - slot_atom = "%s:%s" % (portage.dep_getkey(pkg), - vardb.aux_get(pkg, ["SLOT"])[0]) - if slot_atom in modified_slots[myroot]: + if pkg.slot_atom in modified_slots[myroot]: # This package is being replaced anyway, so # ignore invalid dependencies so as not to # annoy the user too much (otherwise they'd be # forced to manually unmerge it first). continue - show_invalid_depstring_notice(node, depstr, atoms) + show_invalid_depstring_notice(pkg, depstr, atoms) return False blocker_atoms = [myatom for myatom in atoms \ if myatom.startswith("!")] - counter = long(vardb.aux_get(pkg, ["COUNTER"])[0]) - blocker_cache[pkg] = \ + blocker_atoms.sort() + counter = long(pkg.metadata["COUNTER"]) + blocker_cache[cpv] = \ blocker_cache.BlockerData(counter, blocker_atoms) if blocker_atoms: for myatom in blocker_atoms: blocker = Blocker(atom=myatom[1:], root=myroot) - self._blocker_parents.add(blocker, node) + self._blocker_parents.add(blocker, pkg) blocker_cache.flush() del blocker_cache for blocker in self._blocker_parents.leaf_nodes(): self.spinner.update() + root_config = self.roots[blocker.root] + virtuals = root_config.settings.getvirtuals() mytype, myroot, mydep = blocker initial_db = self.trees[myroot]["vartree"].dbapi final_db = self.mydbapi[myroot] - blocked_initial = initial_db.match(mydep) - blocked_final = final_db.match(mydep) + + provider_virtual = False + if blocker.cp in virtuals and \ + not self._have_new_virt(blocker.root, blocker.cp): + provider_virtual = True + + if provider_virtual: + atoms = [] + for provider_entry in virtuals[blocker.cp]: + provider_cp = \ + portage.dep_getkey(provider_entry) + atoms.append(blocker.atom.replace( + blocker.cp, provider_cp)) + else: + atoms = [blocker.atom] + + blocked_initial = [] + for atom in atoms: + blocked_initial.extend(initial_db.match(atom)) + + blocked_final = [] + for atom in atoms: + blocked_final.extend(final_db.match(atom)) + if not blocked_initial and not blocked_final: parent_pkgs = self._blocker_parents.parent_nodes(blocker) self._blocker_parents.remove(blocker) @@ -3322,9 +3382,10 @@ class depgraph(object): if not unresolved_blocks and depends_on_order: for inst_pkg, inst_task in depends_on_order: - uninst_task = Uninstall(built=inst_pkg.built, + uninst_task = Package(built=inst_pkg.built, cpv=inst_pkg.cpv, installed=inst_pkg.installed, - metadata=inst_pkg.metadata, root=inst_pkg.root, + metadata=inst_pkg.metadata, + operation="uninstall", root=inst_pkg.root, type_name=inst_pkg.type_name) self._pkg_cache[uninst_task] = uninst_task # Enforce correct merge order with a hard dep. @@ -3437,7 +3498,8 @@ class depgraph(object): since those should be executed as late as possible. """ return [node for node in mygraph.leaf_nodes(**kwargs) \ - if not isinstance(node, Uninstall)] + if isinstance(node, Package) and \ + node.operation != "uninstall"] if True: for node in mygraph.order: if node.root == "/" and \ @@ -3764,7 +3826,8 @@ class depgraph(object): # and uninstallation tasks. solved_blockers = set() uninst_task = None - if isinstance(node, Uninstall): + if isinstance(node, Package) and \ + "uninstall" == node.operation: have_uninstall_task = True uninst_task = node else: @@ -3794,7 +3857,8 @@ class depgraph(object): if node[-1] != "nomerge": retlist.append(node) - if isinstance(node, Uninstall): + if isinstance(node, Package) and \ + "uninstall" == node.operation: # Include satisfied blockers in the merge list so # that the user can see why the package had to be # uninstalled in advance rather than through @@ -4753,13 +4817,10 @@ class depgraph(object): metadata["USE"] = pkgsettings["PORTAGE_USE"] installed = action == "uninstall" built = pkg_type != "ebuild" - if installed: - pkg_constructor = Uninstall - else: - pkg_constructor = Package - pkg = pkg_constructor(built=built, cpv=pkg_key, + pkg = Package(built=built, cpv=pkg_key, installed=installed, metadata=metadata, - root=myroot, type_name=pkg_type) + operation=action, root=myroot, + type_name=pkg_type) self._pkg_cache[pkg] = pkg fakedb[myroot].cpv_inject(pkg) self.spinner.update() @@ -5161,7 +5222,7 @@ class MergeTask(object): if "--resume" not in self.myopts: mymergelist = mylist mtimedb["resume"]["mergelist"] = [list(x) for x in mymergelist \ - if isinstance(x, (Package, Uninstall))] + if isinstance(x, Package)] mtimedb.commit() myfeat = self.settings.features[:] @@ -5218,11 +5279,9 @@ class MergeTask(object): mymergelist = [x for x in mymergelist if x[-1] == "merge"] mergecount=0 for x in task_list: - pkg_type = x[0] - if pkg_type == "blocks": + if x[0] == "blocks": continue - myroot=x[1] - pkg_key = x[2] + pkg_type, myroot, pkg_key, operation = x pkgindex=2 built = pkg_type != "ebuild" installed = pkg_type == "installed" @@ -5258,14 +5317,12 @@ class MergeTask(object): # isn't installed anymore. Since it's already # been uninstalled, move on to the next task. continue - if installed: - pkg_constructor = Uninstall - else: - pkg_constructor = Package + if not installed: mergecount += 1 - pkg = pkg_constructor(type_name=pkg_type, root=myroot, - cpv=pkg_key, built=built, installed=installed, - metadata=metadata) + pkg = Package(cpv=pkg_key, built=built, + installed=installed, metadata=metadata, + operation=operation, root=myroot, + type_name=pkg_type) if pkg.installed: if not (buildpkgonly or fetchonly or pretend): self._uninstall_queue.append(pkg) @@ -5590,7 +5647,7 @@ class MergeTask(object): return os.EX_OK def unmerge(root_config, myopts, unmerge_action, - unmerge_files, ldpath_mtimes, autoclean=0, clean_world=1): + unmerge_files, ldpath_mtimes, autoclean=0, clean_world=1, ordered=0): settings = root_config.settings sets = root_config.sets vartree = root_config.trees["vartree"] @@ -5864,7 +5921,25 @@ def unmerge(root_config, myopts, unmerge_action, pkgmap[cp]["protected"].add(cpv) del installed_sets - + + # Unmerge order only matters in some cases + if not ordered: + unordered = {} + for d in pkgmap: + selected = d["selected"] + if not selected: + continue + cp = portage.cpv_getkey(iter(selected).next()) + cp_dict = unordered.get(cp) + if cp_dict is None: + cp_dict = {} + unordered[cp] = cp_dict + for k in d: + cp_dict[k] = set() + for k, v in d.iteritems(): + cp_dict[k].update(v) + pkgmap = [unordered[cp] for cp in sorted(unordered)] + for x in xrange(len(pkgmap)): selected = pkgmap[x]["selected"] if not selected: @@ -7519,11 +7594,12 @@ def action_depclean(settings, trees, ldpath_mtimes, if cpv in clean_set: graph.add(cpv, node, priority=priority) + ordered = True if len(graph.order) == len(graph.root_nodes()): # If there are no dependencies between packages - # then just unmerge them alphabetically. - cleanlist = graph.order[:] - cleanlist.sort() + # let unmerge() group them by cat/pn. + ordered = False + cleanlist = graph.all_nodes() else: # Order nodes from lowest to highest overall reference count for # optimal root node selection. @@ -7553,8 +7629,8 @@ def action_depclean(settings, trees, ldpath_mtimes, graph.remove(node) cleanlist.append(node) - unmerge(root_config, myopts, - "unmerge", cleanlist, ldpath_mtimes) + unmerge(root_config, myopts, "unmerge", cleanlist, + ldpath_mtimes, ordered=ordered) if action == "prune": return @@ -8491,8 +8567,11 @@ def emerge_main(): (myaction == "prune" and "--nodeps" in myopts): validate_ebuild_environment(trees) root_config = trees[settings["ROOT"]]["root_config"] + # When given a list of atoms, unmerge + # them in the order given. + ordered = myaction == "unmerge" if 1 == unmerge(root_config, myopts, myaction, myfiles, - mtimedb["ldpath"]): + mtimedb["ldpath"], ordered=ordered): if not (buildpkgonly or fetchonly or pretend): post_emerge(trees, mtimedb, os.EX_OK) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 0818cdaff..22c026cbb 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -4774,6 +4774,34 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, writemsg("does not exist. Please create this directory or " + \ "correct your PORTAGE_TMPDIR setting.\n", noiselevel=-1) return 1 + + # as some people use a separate PORTAGE_TMPDIR mount + # we prefer that as the checks below would otherwise be pointless + # for those people. + if os.path.exists(os.path.join(mysettings["PORTAGE_TMPDIR"], "portage")): + checkdir = os.path.join(mysettings["PORTAGE_TMPDIR"], "portage") + else: + checkdir = mysettings["PORTAGE_TMPDIR"] + + if not os.access(checkdir, os.W_OK): + writemsg("%s is not writable.\n" % checkdir + \ + "Likely cause is that you've mounted it as readonly.\n" \ + , noiselevel=-1) + return 1 + else: + from tempfile import NamedTemporaryFile + fd = NamedTemporaryFile(prefix="exectest-", dir=checkdir) + os.chmod(fd.name, 0755) + if not os.access(fd.name, os.X_OK): + writemsg("Can not execute files in %s\n" % checkdir + \ + "Likely cause is that you've mounted it with one of the\n" + \ + "following mount options: 'noexec', 'user', 'users'\n\n" + \ + "Please make sure that portage can execute files in this direxctory.\n" \ + , noiselevel=-1) + fd.close() + return 1 + fd.close() + del checkdir if mydo == "unmerge": return unmerge(mysettings["CATEGORY"], @@ -5422,6 +5450,12 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", continue mychoices = myvirtuals.get(mykey, []) isblocker = x.startswith("!") + if isblocker: + # Virtual blockers are no longer expanded here since + # the un-expanded virtual atom is more useful for + # maintaining a cache of blocker atoms. + newsplit.append(x) + continue match_atom = x if isblocker: match_atom = x[1:] -- 2.26.2