# sourced again even if $T/environment already exists.
ebuild_changed = False
if mytree == "porttree" and build_dir_phases.intersection(pargs):
- metadata, st, emtime = \
- portage.portdb._pull_valid_cache(cpv, ebuild, ebuild_portdir)
- if metadata is None:
- ebuild_changed = True
+ ebuild_changed = \
+ portage.portdb._pull_valid_cache(cpv, ebuild, ebuild_portdir)[0] is None
tmpsettings = portage.config(clone=portage.settings)
tmpsettings["PORTAGE_VERBOSE"] = "1"
consumer=self._metadata_callback,
max_jobs=max_jobs, max_load=max_load)
self.returncode = os.EX_OK
- self._trg_cache = metadata.database(portdb.porttrees[0],
- "metadata/cache", portage.auxdbkeys[:])
+ conf = portdb.repositories.get_repo_for_location(portdb.porttrees[0])
+ self._trg_cache = conf.get_pregenerated_cache(portage.auxdbkeys[:],
+ force=True, readonly=False)
+ if self._trg_cache is None:
+ raise Exception("cache format %s isn't supported" % (conf.cache_format,))
if rsync:
self._trg_cache.raise_stat_collision = True
try:
pass
self._existing_nodes = set()
- def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata):
+ def _metadata_callback(self, cpv, repo_path, metadata, ebuild_hash):
self._existing_nodes.add(cpv)
self._cp_missing.discard(cpv_getkey(cpv))
if metadata is not None:
if metadata.get('EAPI') == '0':
del metadata['EAPI']
try:
+ chf = self._trg_cache.validation_chf
+ if chf != 'mtime':
+ metadata['_%s_' % chf] = getattr(ebuild_hash, chf)
try:
self._trg_cache[cpv] = metadata
except StatCollision as sc:
max_mtime += 1
max_mtime = long(max_mtime)
try:
- os.utime(ebuild_path, (max_mtime, max_mtime))
+ os.utime(ebuild_hash.location, (max_mtime, max_mtime))
except OSError as e:
self.returncode |= 1
writemsg_level(
used to extract metadata from the ebuild.
"""
- __slots__ = ("cpv", "ebuild_path", "fd_pipes", "metadata_callback",
- "ebuild_mtime", "metadata", "portdb", "repo_path", "settings") + \
+ __slots__ = ("cpv", "ebuild_hash", "fd_pipes", "metadata_callback",
+ "metadata", "portdb", "repo_path", "settings") + \
("_raw_metadata",)
_file_names = ("ebuild",)
def _start(self):
settings = self.settings
settings.setcpv(self.cpv)
- ebuild_path = self.ebuild_path
+ ebuild_path = self.ebuild_hash.location
eapi = None
if eapi is None and \
if eapi is not None:
if not portage.eapi_is_supported(eapi):
- self.metadata_callback(self.cpv, self.ebuild_path,
- self.repo_path, {'EAPI' : eapi}, self.ebuild_mtime)
+ self.metadata_callback(self.cpv, ebuild_path,
+ self.repo_path, {'EAPI' : eapi}, self.ebuild_hash.mtime)
self._set_returncode((self.pid, os.EX_OK << 8))
self.wait()
return
else:
metadata = zip(portage.auxdbkeys, metadata_lines)
self.metadata = self.metadata_callback(self.cpv,
- self.ebuild_path, self.repo_path, metadata,
- self.ebuild_mtime)
+ self.repo_path, metadata, self.ebuild_hash)
import portage
from portage import os
+from portage.eclass_cache import hashed_path
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
from _emerge.PollScheduler import PollScheduler
ebuild_path, repo_path = portdb.findname2(cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % cpv)
- metadata, st, emtime = portdb._pull_valid_cache(
+ metadata, ebuild_hash = portdb._pull_valid_cache(
cpv, ebuild_path, repo_path)
if metadata is not None:
if consumer is not None:
- consumer(cpv, ebuild_path,
- repo_path, metadata)
+ consumer(cpv, repo_path, metadata, ebuild_hash)
continue
- yield EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path,
- ebuild_mtime=emtime,
+ yield EbuildMetadataPhase(cpv=cpv,
+ ebuild_hash=ebuild_hash,
metadata_callback=portdb._metadata_callback,
portdb=portdb, repo_path=repo_path,
settings=portdb.doebuild_settings)
# On failure, still notify the consumer (in this case the metadata
# argument is None).
self._consumer(metadata_process.cpv,
- metadata_process.ebuild_path,
metadata_process.repo_path,
- metadata_process.metadata)
+ metadata_process.metadata,
+ metadata_process.ebuild_hash)
self._schedule()
if dest is not None:
if not (dest['_mtime_'] == src['_mtime_'] and \
- tree_data.eclass_db.is_eclass_data_valid(
- dest['_eclasses_']) and \
+ tree_data.eclass_db.validate_and_rewrite_cache(
+ dest['_eclasses_'], tree_data.dest_db.validation_chf,
+ tree_data.dest_db.store_eclass_paths) and \
set(dest['_eclasses_']) == set(src['_eclasses_'])):
dest = None
else:
continue
if eclasses is not None:
- if not tree_data.eclass_db.is_eclass_data_valid(
- src['_eclasses_']):
+ if not tree_data.eclass_db.validate_and_rewrite_cache(
+ src['_eclasses_'], tree_data.src_db.validation_chf,
+ tree_data.src_db.store_eclass_paths):
continue
inherited = eclasses
else:
import re
import stat
import sys
+from operator import attrgetter
from portage import os
from portage import _encodings
from portage import _unicode_encode
if "INHERITED" in d:
if self.ec is None:
self.ec = portage.eclass_cache.cache(self.location[:-15])
+ getter = attrgetter(self.validation_chf)
try:
- d["_eclasses_"] = self.ec.get_eclass_data(
- d["INHERITED"].split())
+ ec_data = self.ec.get_eclass_data(d["INHERITED"].split())
+ d["_eclasses_"] = dict((k, (v.eclass_dir, getter(v)))
+ for k,v in ec_data.items())
except KeyError as e:
# INHERITED contains a non-existent eclass.
raise cache_errors.CacheCorruption(cpv, e)
from portage.cache.mappings import ProtectedDict
import sys
import warnings
+import operator
if sys.hexversion >= 0x3000000:
basestring = str
autocommits = False
cleanse_keys = False
serialize_eclasses = True
+ validation_chf = 'mtime'
+ store_eclass_paths = True
def __init__(self, location, label, auxdbkeys, readonly=False):
""" initialize the derived class; specifically, store label/keys"""
self.updates = 0
d=self._getitem(cpv)
if self.serialize_eclasses and "_eclasses_" in d:
- d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"])
+ d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"],
+ self.validation_chf, paths=self.store_eclass_paths)
elif "_eclasses_" not in d:
d["_eclasses_"] = {}
mtime = d.get('_mtime_')
if not v:
del d[k]
if self.serialize_eclasses and "_eclasses_" in values:
- d["_eclasses_"] = serialize_eclasses(d["_eclasses_"])
+ d["_eclasses_"] = serialize_eclasses(d["_eclasses_"],
+ self.validation_chf, paths=self.store_eclass_paths)
elif self.serialize_eclasses and "_eclasses_" in values:
d = ProtectedDict(values)
- d["_eclasses_"] = serialize_eclasses(d["_eclasses_"])
+ d["_eclasses_"] = serialize_eclasses(d["_eclasses_"],
+ self.validation_chf, paths=self.store_eclass_paths)
else:
d = values
self._setitem(cpv, d)
except KeyError:
return x
+ def validate_entry(self, entry, ebuild_hash, eclass_db):
+ hash_key = '_%s_' % self.validation_chf
+ if entry[hash_key] != getattr(ebuild_hash, self.validation_chf):
+ return False
+ update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf,
+ self.store_eclass_paths)
+ if update is None:
+ return False
+ if update:
+ entry['_eclasses_'] = update
+ return True
+
def get_matches(self, match_dict):
"""generic function for walking the entire cache db, matching restrictions to
filter what cpv's are returned. Derived classes should override this if they
keys = __iter__
items = iteritems
-def serialize_eclasses(eclass_dict):
+_keysorter = operator.itemgetter(0)
+
+def serialize_eclasses(eclass_dict, chf_type='mtime', paths=True):
"""takes a dict, returns a string representing said dict"""
"""The "new format", which causes older versions of <portage-2.1.2 to
traceback with a ValueError due to failed long() conversion. This format
"""
if not eclass_dict:
return ""
- return "\t".join(k + "\t%s\t%s" % eclass_dict[k] \
- for k in sorted(eclass_dict))
+ getter = operator.attrgetter(chf_type)
+ if paths:
+ return "\t".join("%s\t%s\t%s" % (k, v.eclass_dir, getter(v))
+ for k, v in sorted(eclass_dict.items(), key=_keysorter))
+ return "\t".join("%s\t%s" % (k, getter(v))
+ for k, v in sorted(eclass_dict.items(), key=_keysorter))
-def reconstruct_eclasses(cpv, eclass_string):
+
+def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True):
"""returns a dict when handed a string generated by serialize_eclasses"""
eclasses = eclass_string.rstrip().lstrip().split("\t")
if eclasses == [""]:
# occasionally this occurs in the fs backends. they suck.
return {}
-
- if len(eclasses) % 2 != 0 and len(eclasses) % 3 != 0:
+
+ converter = str
+ if chf_type == 'mtime':
+ converter = long
+
+ if paths:
+ if len(eclasses) % 3 != 0:
+ raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses))
+ elif len(eclasses) % 2 != 0:
raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses))
d={}
try:
- if eclasses[1].isdigit():
- for x in range(0, len(eclasses), 2):
- d[eclasses[x]] = ("", long(eclasses[x + 1]))
- else:
+ i = iter(eclasses)
+ if paths:
# The old format contains paths that will be discarded.
- for x in range(0, len(eclasses), 3):
- d[eclasses[x]] = (eclasses[x + 1], long(eclasses[x + 2]))
+ for name, path, val in zip(i, i, i):
+ d[name] = (path, converter(val))
+ else:
+ for name, val in zip(i, i):
+ d[name] = converter(val)
except IndexError:
raise cache_errors.CacheCorruption(cpv,
"_eclasses_ was of invalid len %i" % len(eclasses))
@returns: A new EbuildMetadataPhase instance, or None if the
metadata cache is already valid.
"""
- metadata, st, emtime = self._pull_valid_cache(cpv, ebuild_path, repo_path)
+ metadata, ebuild_hash = self._pull_valid_cache(cpv, ebuild_path, repo_path)
if metadata is not None:
return None
- process = EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path,
- ebuild_mtime=emtime, metadata_callback=self._metadata_callback,
+ process = EbuildMetadataPhase(cpv=cpv,
+ ebuild_hash=ebuild_hash, metadata_callback=self._metadata_callback,
portdb=self, repo_path=repo_path, settings=self.doebuild_settings)
return process
- def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata, mtime):
+ def _metadata_callback(self, cpv, repo_path, metadata, ebuild_hash):
i = metadata
if hasattr(metadata, "items"):
else:
metadata["_eclasses_"] = {}
+ try:
+ cache = self.auxdb[repo_path]
+ chf = cache.validation_chf
+ metadata['_%s_' % chf] = getattr(ebuild_hash, chf)
+ except CacheError:
+ # Normally this shouldn't happen, so we'll show
+ # a traceback for debugging purposes.
+ traceback.print_exc()
+ cache = None
+
metadata.pop("INHERITED", None)
- metadata["_mtime_"] = mtime
eapi = metadata.get("EAPI")
if not eapi or not eapi.strip():
metadata[k] = ""
metadata["EAPI"] = "-" + eapi.lstrip("-")
- try:
- self.auxdb[repo_path][cpv] = metadata
- except CacheError:
- # Normally this shouldn't happen, so we'll show
- # a traceback for debugging purposes.
- traceback.print_exc()
+ if cache is not None:
+ try:
+ cache[cpv] = metadata
+ except CacheError:
+ # Normally this shouldn't happen, so we'll show
+ # a traceback for debugging purposes.
+ traceback.print_exc()
return metadata
def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
try:
# Don't use unicode-wrapped os module, for better performance.
- st = _os.stat(_unicode_encode(ebuild_path,
- encoding=_encodings['fs'], errors='strict'))
- emtime = st[stat.ST_MTIME]
+ path = _unicode_encode(ebuild_path,
+ encoding=_encodings['fs'], errors='strict')
+ ebuild_hash = eclass_cache.hashed_path(path)
+ # snag mtime since we use it later, and to trigger stat failure
+ # if it doesn't exist
+ ebuild_hash.mtime
except OSError:
writemsg(_("!!! aux_get(): ebuild for " \
"'%s' does not exist at:\n") % (cpv,), noiselevel=-1)
auxdbs.append(self.auxdb[repo_path])
eclass_db = self._repo_info[repo_path].eclass_db
- doregen = True
for auxdb in auxdbs:
try:
metadata = auxdb[cpv]
except KeyError:
- pass
+ continue
except CacheError:
if not auxdb.readonly:
try:
del auxdb[cpv]
- except KeyError:
- pass
- except CacheError:
+ except (KeyError, CacheError):
pass
- else:
- eapi = metadata.get('EAPI', '').strip()
- if not eapi:
- eapi = '0'
- if not (eapi[:1] == '-' and eapi_is_supported(eapi[1:])) and \
- emtime == metadata['_mtime_'] and \
- eclass_db.is_eclass_data_valid(metadata['_eclasses_']):
- doregen = False
-
- if not doregen:
+ continue
+ eapi = metadata.get('EAPI', '').strip()
+ if not eapi:
+ eapi = '0'
+ if eapi[:1] == '-' and eapi_is_supported(eapi[1:]):
+ continue
+ if auxdb.validate_entry(metadata, ebuild_hash, eclass_db):
break
-
- if doregen:
+ else:
metadata = None
- return (metadata, st, emtime)
+ return (metadata, ebuild_hash)
def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
_("ebuild not found for '%s'") % mycpv, noiselevel=1)
raise KeyError(mycpv)
- mydata, st, emtime = self._pull_valid_cache(mycpv, myebuild, mylocation)
+ mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
doregen = mydata is None
if doregen:
if eapi is not None and not portage.eapi_is_supported(eapi):
mydata = self._metadata_callback(
- mycpv, myebuild, mylocation, {'EAPI':eapi}, emtime)
+ mycpv, ebuild_hash, mylocation, {'EAPI':eapi}, emtime)
else:
- proc = EbuildMetadataPhase(cpv=mycpv, ebuild_path=myebuild,
- ebuild_mtime=emtime,
+ proc = EbuildMetadataPhase(cpv=mycpv,
+ ebuild_hash=ebuild_hash,
metadata_callback=self._metadata_callback, portdb=self,
repo_path=mylocation,
scheduler=PollScheduler().sched_iface,
# do we have a origin repository name for the current package
mydata["repository"] = self.repositories.get_name_for_location(mylocation)
mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
- mydata["_mtime_"] = st[stat.ST_MTIME]
+ mydata["_mtime_"] = ebuild_hash.mtime
eapi = mydata.get("EAPI")
if not eapi:
eapi = "0"
mydata["EAPI"] = eapi
if not eapi_is_supported(eapi):
- for k in set(mydata).difference(("_mtime_", "_eclasses_")):
- mydata[k] = ""
+ keys = set(mydata)
+ keys.discard("_eclasses_")
+ keys.discard("_mtime_")
+ mydata.update((k, '') for k in keys)
mydata["EAPI"] = "-" + eapi.lstrip("-")
#finally, we look at our internal cache entry and return the requested data.
import stat
import sys
+import operator
from portage.util import normalize_path
import errno
from portage.exception import PermissionDenied
from portage import os
+from portage import checksum
if sys.hexversion >= 0x3000000:
long = int
+
+class hashed_path(object):
+
+ def __init__(self, location):
+ self.location = location
+
+ def __getattr__(self, attr):
+ if attr == 'mtime':
+ # use stat.ST_MTIME; accessing .st_mtime gets you a float
+ # depending on the python version, and long(float) introduces
+ # some rounding issues that aren't present for people using
+ # the straight c api.
+ # thus use the defacto python compatibility work around;
+ # access via index, which gurantees you get the raw long.
+ self.mtime = obj = os.stat(self.location)[stat.ST_MTIME]
+ return obj
+ if not attr.islower():
+ # we don't care to allow .mD5 as an alias for .md5
+ raise AttributeError(attr)
+ try:
+ val = checksum.perform_checksum(self.location, attr.upper())[0]
+ except KeyError:
+ raise AttributeError(attr)
+ setattr(self, attr, val)
+ return val
+
+
class cache(object):
"""
Maintains the cache information about eclasses used in ebuild.
"""
def __init__(self, porttree_root, overlays=[]):
- self.eclasses = {} # {"Name": ("location","_mtime_")}
+ self.eclasses = {} # {"Name": hashed_path}
self._eclass_locations = {}
# screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you.
for y in eclass_filenames:
if not y.endswith(".eclass"):
continue
+ obj = hashed_path(os.path.join(x, y))
+ obj.eclass_dir = x
try:
- mtime = os.stat(os.path.join(x, y))[stat.ST_MTIME]
+ mtime = obj.mtime
except OSError:
continue
ys=y[:-eclass_len]
if x == self._master_eclass_root:
master_eclasses[ys] = mtime
- self.eclasses[ys] = (x, mtime)
+ self.eclasses[ys] = obj
self._eclass_locations[ys] = x
continue
# so prefer the master entry.
continue
- self.eclasses[ys] = (x, mtime)
+ self.eclasses[ys] = obj
self._eclass_locations[ys] = x
- def is_eclass_data_valid(self, ec_dict):
+ def validate_and_rewrite_cache(self, ec_dict, chf_type, stores_paths):
if not isinstance(ec_dict, dict):
- return False
- for eclass, tup in ec_dict.items():
- cached_data = self.eclasses.get(eclass, None)
- """ Only use the mtime for validation since the probability of a
- collision is small and, depending on the cache implementation, the
- path may not be specified (cache from rsync mirrors, for example).
- """
- if cached_data is None or tup[1] != cached_data[1]:
- return False
-
- return True
+ return None
+ our_getter = operator.attrgetter(chf_type)
+ cache_getter = lambda x:x
+ if stores_paths:
+ key_getter = operator.itemgetter(1)
+ d = {}
+ for eclass, ec_data in ec_dict.items():
+ cached_data = self.eclasses.get(eclass)
+ if cached_data is None:
+ return None
+ if cache_getter(ec_data) != our_getter(cached_data):
+ return None
+ d[eclass] = cached_data
+ return d
def get_eclass_data(self, inherits):
ec_dict = {}
self.manifest_hashes = None
self.cache_format = None
- def get_pregenerated_cache(self, auxdbkeys, readonly=True):
- if self.cache_format is None:
- return None
- elif self.cache_format == 'pms':
+ def get_pregenerated_cache(self, auxdbkeys, readonly=True, force=False):
+ format = self.cache_format
+ if format is None:
+ if not force:
+ return None
+ format = 'pms'
+ if format == 'pms':
from portage.cache.metadata import database
- return database(self.location, 'metadata/cache',
- auxdbkeys, readonly=readonly)
- return None
+ else:
+ return None
+ return database(self.location, 'metadata/cache',
+ auxdbkeys, readonly=readonly)
def load_manifest(self, *args, **kwds):
kwds['thin'] = self.thin_manifest
projects / portage.git / commitdiff