From 0544ad2af89fcf0bd5d22e6cd46d665cd9ef3ed6 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 21 May 2008 17:30:12 +0000 Subject: [PATCH] Merged from trunk 10348:10359 | 10349 | Add back the "install new portage asap" behavior that got | | zmedico | lost when that area of code was rewritten. | | 10350 | Fix th logic from the previous commit. | | zmedico | | | 10351 | Add the environment and some more files to the existing | | zmedico | make.defaults variable substitution support. Variable | | | substitution occurs in the following order: * env.d * env * | | | make.globals * make.defaults * make.conf | | 10357 | Bug #222623 - Use "regardless" instead of "irregardless" | | zmedico | since the latter is considered to be incorrect. | | 10359 | * To avoid excessive IO on each call to | | zmedico | LinkageMap.rebuild(), cache NEEDED.* files via | | | vardbapi.aux_get(). * Make vardbapi.aux_get() preserve | | | newlines in multi-line metadata such as NEEDED.* and | | | CONTENTS. | | | Prefix: hacked in MACHO cache support svn path=/main/branches/prefix/; revision=10384 --- man/emerge.1 | 2 +- pym/_emerge/__init__.py | 4 ++ pym/portage/__init__.py | 89 ++++++++++++++++++++++-------------- pym/portage/dbapi/vartree.py | 52 +++++++++++++++++---- 4 files changed, 103 insertions(+), 44 deletions(-) diff --git a/man/emerge.1 b/man/emerge.1 index 04efc73a9..a09beee97 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -291,7 +291,7 @@ packages (fetch things from SRC_URI based upon USE setting). .TP .BR "\-\-fetch\-all\-uri " (\fB\-F\fR) Instead of doing any package building, just perform fetches for all -packages (fetch everything in SRC_URI irregardless of USE setting). +packages (fetch everything in SRC_URI regardless of USE setting). .TP .BR "\-\-getbinpkg " (\fB\-g\fR) Using the server and location defined in \fIPORTAGE_BINHOST\fR (see diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index fbaa6316f..7f2950179 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -3669,6 +3669,10 @@ class depgraph(object): if replacement_portage == running_portage: replacement_portage = None + if replacement_portage is not None: + # update from running_portage to replacement_portage asap + asap_nodes.append(replacement_portage) + if running_portage is not None: try: portage_rdepend = self._select_atoms_highest_available( diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 6172035eb..610d2a7ed 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1274,9 +1274,60 @@ class config(object): self.puseforce_list.append(cpdict) del rawpuseforce + make_conf = getconfig( + os.path.join(config_root, MAKE_CONF_FILE.lstrip(os.path.sep)), + tolerant=tolerant, allow_sourcing=True) + + # Allow ROOT setting to come from make.conf if it's not overridden + # by the constructor argument (from the calling environment). + if target_root is None and "ROOT" in make_conf: + target_root = make_conf["ROOT"] + if target_root is None: + target_root = "/" + + # The expand_map is used for variable substitution + # in getconfig() calls, and the getconfig() calls + # update expand_map with the value of each variable + # assignment that occurs. Variable substitution occurs + # in the following order: + # + # * env.d + # * env + # * make.globals + # * make.defaults + # * make.conf + # + expand_map = {} + + env_d = getconfig(os.path.join(target_root + EPREFIX_LSTRIP, "etc", "profile.env"), + expand=expand_map) + # env_d will be None if profile.env doesn't exist. + if env_d: + self.configdict["env.d"].update(env_d) + expand_map.update(env_d) + + # backupenv is used for calculating incremental variables. + self.backupenv = os.environ.copy() + expand_map.update(self.backupenv) + # make.globals should not be relative to config_root # because it only contains constants. - self.mygcfg = getconfig(os.path.join(BPREFIX, "etc", "make.globals")) + self.mygcfg = getconfig(os.path.join(BPREFIX, "etc", "make.globals"), + expand=expand_map) + + if env_d: + # Remove duplicate values so they don't override updated + # profile.env values later (profile.env is reloaded in each + # call to self.regenerate). + for k, v in env_d.iteritems(): + try: + if self.backupenv[k] == v: + del self.backupenv[k] + except KeyError: + pass + del k, v + + self.configdict["env"] = self.backupenv.copy() if self.mygcfg is None: self.mygcfg = {} @@ -1287,7 +1338,6 @@ class config(object): self.make_defaults_use = [] self.mygcfg = {} if self.profiles: - expand_map = {} mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), expand=expand_map) for x in self.profiles] @@ -1305,7 +1355,7 @@ class config(object): self.mygcfg = getconfig( os.path.join(config_root, MAKE_CONF_FILE.lstrip(os.path.sep)), - tolerant=tolerant, allow_sourcing=True) + tolerant=tolerant, allow_sourcing=True, expand=expand_map) if self.mygcfg is None: self.mygcfg = {} @@ -1314,12 +1364,7 @@ class config(object): "PROFILE_ONLY_VARIABLES", "").split() for k in profile_only_variables: self.mygcfg.pop(k, None) - - # Allow ROOT setting to come from make.conf if it's not overridden - # by the constructor argument (from the calling environment). - if target_root is None and "ROOT" in self.mygcfg: - target_root = self.mygcfg["ROOT"] - + self.configlist.append(self.mygcfg) self.configdict["conf"]=self.configlist[-1] @@ -1330,8 +1375,6 @@ class config(object): self.configlist.append({}) self.configdict["auto"]=self.configlist[-1] - # backupenv is used for calculating incremental variables. - self.backupenv = os.environ.copy() self.configlist.append(self.backupenv) # XXX Why though? self.configdict["backupenv"]=self.configlist[-1] @@ -1339,8 +1382,7 @@ class config(object): for k in profile_only_variables: self.backupenv.pop(k, None) - self.configlist.append(self.backupenv.copy()) - self.configdict["env"]=self.configlist[-1] + self.configlist.append(self.configdict["env"]) # make lookuplist for loading package.* self.lookuplist=self.configlist[:] @@ -1354,33 +1396,12 @@ class config(object): cfg.pop(blacklisted, None) del blacklisted, cfg - if target_root is None: - target_root = "/" - target_root = normalize_path(os.path.abspath( target_root)).rstrip(os.path.sep) + os.path.sep portage.util.ensure_dirs(target_root + EPREFIX_LSTRIP) check_var_directory("ROOT", target_root + EPREFIX_LSTRIP) - env_d = getconfig( - os.path.join(target_root, EPREFIX_LSTRIP, "etc", "profile.env"), expand=False) - # env_d will be None if profile.env doesn't exist. - if env_d: - self.configdict["env.d"].update(env_d) - # Remove duplicate values so they don't override updated - # profile.env values later (profile.env is reloaded in each - # call to self.regenerate). - for cfg in (self.configdict["backupenv"], - self.configdict["env"]): - for k, v in env_d.iteritems(): - try: - if cfg[k] == v: - del cfg[k] - except KeyError: - pass - del cfg, k, v - self["PORTAGE_CONFIGROOT"] = config_root self.backup_changes("PORTAGE_CONFIGROOT") self["ROOT"] = target_root diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index b2a1e8a4b..5f4e947a2 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -137,7 +137,9 @@ class LinkageMap(object): obj_properties = {} lines = [] for cpv in self._dbapi.cpv_all(): - lines += grabfile(self._dbapi.getpath(cpv, filename="NEEDED.ELF.2")) + lines += self._dbapi.aux_get(cpv, ["NEEDED.ELF.2"])[0].split('\n') + # Cache NEEDED.* files avoid doing excessive IO for every rebuild. + self._dbapi.flush_cache() if include_file: lines += grabfile(include_file) @@ -240,11 +242,24 @@ class LinkageMapMachO(object): obj_properties = {} lines = [] for cpv in self._dbapi.cpv_all(): - lines += grabfile(self._dbapi.getpath(cpv, filename="NEEDED.MACHO.2")) + lines += self._dbapi.aux_get(cpv, ["NEEDED.MACHO.2"])[0].split('\n') + # Cache NEEDED.* files avoid doing excessive IO for every rebuild. + self._dbapi.flush_cache() if include_file: lines += grabfile(include_file) + # have to call otool for preserved libs here as they aren't + # registered in NEEDED.MACHO.2 files +# TODO: FIXME: ETC. +# if self._dbapi.plib_registry and self._dbapi.plib_registry.getPreservedLibs(): +# args = [EPREFIX+"/usr/bin/scanelf", "-yqF", "%a;%F;%S;%r;%n"] +# for items in self._dbapi.plib_registry.getPreservedLibs().values(): +# args += items +# proc = subprocess.Popen(args, stdout=subprocess.PIPE) +# output = [l[3:] for l in proc.communicate()[0].split("\n")] +# lines += output + for l in lines: if l.strip() == "": continue @@ -323,6 +338,9 @@ class vardbapi(dbapi): _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ "|".join(_excluded_dirs) + r')$') + _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') + _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') + def __init__(self, root, categories=None, settings=None, vartree=None): """ The categories parameter is unused since the dbapi class @@ -659,8 +677,17 @@ class vardbapi(dbapi): unrecognized, the cache will simple be recreated from scratch (it is completely disposable). """ - if not self._aux_cache_keys.intersection(wants): + cache_these_wants = self._aux_cache_keys.intersection(wants) + for x in wants: + if self._aux_cache_keys_re.match(x) is not None: + cache_these_wants.add(x) + + if not cache_these_wants: return self._aux_get(mycpv, wants) + + cache_these = set(self._aux_cache_keys) + cache_these.update(cache_these_wants) + if self._aux_cache is None: try: f = open(self._aux_cache_filename) @@ -690,29 +717,33 @@ class vardbapi(dbapi): pkg_data = self._aux_cache["packages"].get(mycpv) mydata = {} cache_valid = False + cache_incomplete = False if pkg_data: cache_mtime, metadata = pkg_data cache_valid = cache_mtime == mydir_mtime if cache_valid: - cache_incomplete = self._aux_cache_keys.difference(metadata) + cache_incomplete = cache_these.difference(metadata) if cache_incomplete: # Allow self._aux_cache_keys to change without a cache version # bump and efficiently recycle partial cache whenever possible. cache_valid = False pull_me = cache_incomplete.union(wants) else: - pull_me = set(wants).difference(self._aux_cache_keys) + pull_me = set(wants).difference(cache_these) mydata.update(metadata) else: - pull_me = self._aux_cache_keys.union(wants) + pull_me = cache_these + if pull_me: # pull any needed data and cache it aux_keys = list(pull_me) for k, v in izip(aux_keys, self._aux_get(mycpv, aux_keys)): mydata[k] = v - if not cache_valid: + if not cache_valid or cache_incomplete: cache_data = {} - for aux_key in self._aux_cache_keys: + if cache_incomplete: + cache_data.update(metadata) + for aux_key in cache_these: cache_data[aux_key] = mydata[aux_key] self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) self._aux_cache["modified"] = True @@ -736,7 +767,10 @@ class vardbapi(dbapi): myd = myf.read() finally: myf.close() - myd = " ".join(myd.split()) + # Preserve \n for metadata that is known to + # contain multiple lines. + if self._aux_multi_line_re.match(x) is None: + myd = " ".join(myd.split()) except IOError: myd = "" if x == "EAPI" and not myd: -- 2.26.2