From 89650a0905466baee5e8f6ed6f5d085c37e906a3 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 27 Feb 2010 19:21:09 +0000 Subject: [PATCH] Merged from trunk -r15451:15455 | 15452 | Move env_update to portage.util.env_update.envupdate. | | zmedico | | | 15453 | Move ExtractKernelVersion | | zmedico | portage.util.ExtractKernelVersion. | | 15454 | Move digestgen to portage.util.digestgen. | | zmedico | | | 15455 | Define 'long' for Python 3. | | arfrever | | svn path=/main/branches/prefix/; revision=15481 --- bin/repoman | 9 +- pym/_emerge/Scheduler.py | 3 +- pym/portage/__init__.py | 528 +---------------------- pym/portage/dbapi/vartree.py | 3 +- pym/portage/package/ebuild/doebuild.py | 24 +- pym/portage/util/ExtractKernelVersion.py | 77 ++++ pym/portage/util/digestgen.py | 204 +++++++++ pym/portage/util/env_update.py | 307 +++++++++++++ 8 files changed, 618 insertions(+), 537 deletions(-) create mode 100644 pym/portage/util/ExtractKernelVersion.py create mode 100644 pym/portage/util/digestgen.py create mode 100644 pym/portage/util/env_update.py diff --git a/bin/repoman b/bin/repoman index b05ae6ba1..4761a12b9 100755 --- a/bin/repoman +++ b/bin/repoman @@ -75,6 +75,7 @@ from portage.output import bold, create_color_func, darkgreen, \ green, nocolor, red, turquoise, yellow from portage.output import ConsoleStyleFile, StyleWriter from portage.util import cmp_sort_key, writemsg_level +from portage.util.digestgen import digestgen if sys.hexversion >= 0x3000000: basestring = str @@ -1011,7 +1012,7 @@ for x in scanlist: portage._doebuild_manifest_exempt_depend -= 1 repoman_settings["O"] = checkdir - if not portage.digestgen(mysettings=repoman_settings, myportdb=portdb): + if not digestgen(mysettings=repoman_settings, myportdb=portdb): print("Unable to generate manifest.") dofail = 1 if options.mode == "manifest": @@ -2339,7 +2340,7 @@ else: mydone=[] if repolevel==3: # In a package dir repoman_settings["O"] = startdir - portage.digestgen(mysettings=repoman_settings, myportdb=portdb) + digestgen(mysettings=repoman_settings, myportdb=portdb) elif repolevel==2: # In a category dir for x in myfiles: xs=x.split("/") @@ -2353,7 +2354,7 @@ else: repoman_settings["O"] = os.path.join(startdir, xs[0]) if not os.path.isdir(repoman_settings["O"]): continue - portage.digestgen(mysettings=repoman_settings, myportdb=portdb) + digestgen(mysettings=repoman_settings, myportdb=portdb) elif repolevel==1: # repo-cvsroot print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n") for x in myfiles: @@ -2368,7 +2369,7 @@ else: repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1]) if not os.path.isdir(repoman_settings["O"]): continue - portage.digestgen(mysettings=repoman_settings, myportdb=portdb) + digestgen(mysettings=repoman_settings, myportdb=portdb) else: print(red("I'm confused... I don't know where I am!")) sys.exit(1) diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index e508e233f..071b8e8d7 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -24,6 +24,7 @@ bad = create_color_func("BAD") from portage.sets import SETPREFIX from portage.sets.base import InternalPackageSet from portage.util import writemsg, writemsg_level +from portage.util.digestgen import digestgen from _emerge.BinpkgPrefetcher import BinpkgPrefetcher from _emerge.Blocker import Blocker @@ -591,7 +592,7 @@ class Scheduler(PollScheduler): if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % x.cpv) pkgsettings['O'] = os.path.dirname(ebuild_path) - if not portage.digestgen(mysettings=pkgsettings, myportdb=portdb): + if not digestgen(mysettings=pkgsettings, myportdb=portdb): writemsg_level( "!!! Unable to generate manifest for '%s'.\n" \ % x.cpv, level=logging.ERROR, noiselevel=-1) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index c4724d409..d066d3452 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -42,8 +42,6 @@ try: # which is unavailable. from StringIO import StringIO - from time import sleep - from itertools import chain import platform import warnings @@ -121,7 +119,10 @@ try: 'pickle_read,pickle_write,stack_dictlist,stack_dicts,' + \ 'stack_lists,unique_array,varexpand,writedict,writemsg,' + \ 'writemsg_stdout,write_atomic', + 'portage.util.digestgen:digestgen', 'portage.util.digraph:digraph', + 'portage.util.env_update:env_update', + 'portage.util.ExtractKernelVersion:ExtractKernelVersion', 'portage.util.listdir:cacheddir,listdir', 'portage.versions', 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,' + \ @@ -535,529 +536,6 @@ def abssymlink(symlink): mylink=mydir+"/"+mylink return os.path.normpath(mylink) -#parse /etc/env.d and generate /etc/profile.env - -def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, - env=None, writemsg_level=None): - if writemsg_level is None: - writemsg_level = portage.util.writemsg_level - if target_root is None: - global settings - target_root = settings["ROOT"] - if prev_mtimes is None: - global mtimedb - prev_mtimes = mtimedb["ldpath"] - if env is None: - env = os.environ - envd_dir = os.path.join(target_root, EPREFIX_LSTRIP, "etc", "env.d") - portage.util.ensure_dirs(envd_dir, mode=0o755) - fns = listdir(envd_dir, EmptyOnError=1) - fns.sort() - templist = [] - for x in fns: - if len(x) < 3: - continue - if not x[0].isdigit() or not x[1].isdigit(): - continue - if x.startswith(".") or x.endswith("~") or x.endswith(".bak"): - continue - templist.append(x) - fns = templist - del templist - - space_separated = set(["CONFIG_PROTECT", "CONFIG_PROTECT_MASK"]) - colon_separated = set(["ADA_INCLUDE_PATH", "ADA_OBJECTS_PATH", - "CLASSPATH", "INFODIR", "INFOPATH", "KDEDIRS", "LDPATH", "MANPATH", - "PATH", "PKG_CONFIG_PATH", "PRELINK_PATH", "PRELINK_PATH_MASK", - "PYTHONPATH", "ROOTPATH"]) - - config_list = [] - - for x in fns: - file_path = os.path.join(envd_dir, x) - try: - myconfig = getconfig(file_path, expand=False) - except portage.exception.ParseError as e: - writemsg("!!! '%s'\n" % str(e), noiselevel=-1) - del e - continue - if myconfig is None: - # broken symlink or file removed by a concurrent process - writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1) - continue - - config_list.append(myconfig) - if "SPACE_SEPARATED" in myconfig: - space_separated.update(myconfig["SPACE_SEPARATED"].split()) - del myconfig["SPACE_SEPARATED"] - if "COLON_SEPARATED" in myconfig: - colon_separated.update(myconfig["COLON_SEPARATED"].split()) - del myconfig["COLON_SEPARATED"] - - env = {} - specials = {} - for var in space_separated: - mylist = [] - for myconfig in config_list: - if var in myconfig: - for item in myconfig[var].split(): - if item and not item in mylist: - mylist.append(item) - del myconfig[var] # prepare for env.update(myconfig) - if mylist: - env[var] = " ".join(mylist) - specials[var] = mylist - - for var in colon_separated: - mylist = [] - for myconfig in config_list: - if var in myconfig: - for item in myconfig[var].split(":"): - if item and not item in mylist: - mylist.append(item) - del myconfig[var] # prepare for env.update(myconfig) - if mylist: - env[var] = ":".join(mylist) - specials[var] = mylist - - for myconfig in config_list: - """Cumulative variables have already been deleted from myconfig so that - they won't be overwritten by this dict.update call.""" - env.update(myconfig) - - if EPREFIX == '': - sleep_for_mtime_granularity = dolinkingstuff( - target_root, specials, prelink_capable, - makelinks, contents, prev_mtimes, env) - else: - sleep_for_mtime_granularity = False - writeshellprofile(target_root, env, sleep_for_mtime_granularity) - -def dolinkingstuff(target_root, specials, prelink_capable, makelinks, - contents, prev_mtimes, env): - # updating this stuff will never work in an offset, other than ROOT - # (e.g. not in Prefix), hence the EPREFIX is not taken into account - # here since this code should never be triggered on an offset install - ldsoconf_path = os.path.join(target_root, "etc", "ld.so.conf") - try: - myld = codecs.open(_unicode_encode(ldsoconf_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - myldlines=myld.readlines() - myld.close() - oldld=[] - for x in myldlines: - #each line has at least one char (a newline) - if x[0]=="#": - continue - oldld.append(x[:-1]) - except (IOError, OSError) as e: - if e.errno != errno.ENOENT: - raise - oldld = None - - ld_cache_update=False - - newld = specials["LDPATH"] - if (oldld!=newld): - #ld.so.conf needs updating and ldconfig needs to be run - myfd = atomic_ofstream(ldsoconf_path) - myfd.write("# ld.so.conf autogenerated by env-update; make all changes to\n") - myfd.write("# contents of /etc/env.d directory\n") - for x in specials["LDPATH"]: - myfd.write(x+"\n") - myfd.close() - ld_cache_update=True - - # Update prelink.conf if we are prelink-enabled - if prelink_capable: - newprelink = atomic_ofstream( - os.path.join(target_root, "etc", "prelink.conf")) - newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n") - newprelink.write("# contents of /etc/env.d directory\n") - - for x in ["/bin","/sbin","/usr/bin","/usr/sbin","/lib","/usr/lib"]: - newprelink.write("-l "+x+"\n"); - for x in specials["LDPATH"]+specials["PATH"]+specials["PRELINK_PATH"]: - if not x: - continue - if x[-1]!='/': - x=x+"/" - plmasked=0 - for y in specials["PRELINK_PATH_MASK"]: - if not y: - continue - if y[-1]!='/': - y=y+"/" - if y==x[0:len(y)]: - plmasked=1 - break - if not plmasked: - newprelink.write("-h "+x+"\n") - for x in specials["PRELINK_PATH_MASK"]: - newprelink.write("-b "+x+"\n") - newprelink.close() - - # Portage stores mtimes with 1 second granularity but in >=python-2.5 finer - # granularity is possible. In order to avoid the potential ambiguity of - # mtimes that differ by less than 1 second, sleep here if any of the - # directories have been modified during the current second. - sleep_for_mtime_granularity = False - current_time = long(time.time()) - mtime_changed = False - lib_dirs = set() - for lib_dir in portage.util.unique_array(specials["LDPATH"]+['usr/lib','usr/lib64','usr/lib32','lib','lib64','lib32']): - x = os.path.join(target_root, lib_dir.lstrip(os.sep)) - try: - newldpathtime = os.stat(x)[stat.ST_MTIME] - lib_dirs.add(normalize_path(x)) - except OSError as oe: - if oe.errno == errno.ENOENT: - try: - del prev_mtimes[x] - except KeyError: - pass - # ignore this path because it doesn't exist - continue - raise - if newldpathtime == current_time: - sleep_for_mtime_granularity = True - if x in prev_mtimes: - if prev_mtimes[x] == newldpathtime: - pass - else: - prev_mtimes[x] = newldpathtime - mtime_changed = True - else: - prev_mtimes[x] = newldpathtime - mtime_changed = True - - if mtime_changed: - ld_cache_update = True - - if makelinks and \ - not ld_cache_update and \ - contents is not None: - libdir_contents_changed = False - for mypath, mydata in contents.items(): - if mydata[0] not in ("obj","sym"): - continue - head, tail = os.path.split(mypath) - if head in lib_dirs: - libdir_contents_changed = True - break - if not libdir_contents_changed: - makelinks = False - - ldconfig = "/sbin/ldconfig" - if "CHOST" in env and "CBUILD" in env and \ - env["CHOST"] != env["CBUILD"]: - from portage.process import find_binary - ldconfig = find_binary("%s-ldconfig" % env["CHOST"]) - - # Only run ldconfig as needed - if (ld_cache_update or makelinks) and ldconfig: - # ldconfig has very different behaviour between FreeBSD and Linux - if ostype=="Linux" or ostype.lower().endswith("gnu"): - # We can't update links if we haven't cleaned other versions first, as - # an older package installed ON TOP of a newer version will cause ldconfig - # to overwrite the symlinks we just made. -X means no links. After 'clean' - # we can safely create links. - writemsg_level(_(">>> Regenerating %setc/ld.so.cache...\n") % \ - (target_root,)) - if makelinks: - os.system("cd / ; %s -r '%s'" % (ldconfig, target_root)) - else: - os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root)) - elif ostype in ("FreeBSD","DragonFly"): - writemsg_level(_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") % \ - target_root) - os.system(("cd / ; %s -elf -i " + \ - "-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'") % \ - (ldconfig, target_root, target_root)) - - del specials["LDPATH"] - - return sleep_for_mtime_granularity - -def writeshellprofile(target_root, env, sleep_for_mtime_granularity): - penvnotice = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n" - penvnotice += "# DO NOT EDIT THIS FILE. CHANGES TO STARTUP PROFILES\n" - cenvnotice = penvnotice[:] - penvnotice += "# GO INTO "+EPREFIX+"/etc/profile NOT "+EPREFIX+"/etc/profile.env\n\n" - cenvnotice += "# GO INTO "+EPREFIX+"/etc/csh.cshrc NOT "+EPREFIX+"/etc/csh.env\n\n" - - #create /etc/profile.env for bash support - outfile = atomic_ofstream(os.path.join(target_root, EPREFIX_LSTRIP, "etc", "profile.env")) - outfile.write(penvnotice) - - env_keys = [ x for x in env if x != "LDPATH" ] - env_keys.sort() - for k in env_keys: - v = env[k] - if v.startswith('$') and not v.startswith('${'): - outfile.write("export %s=$'%s'\n" % (k, v[1:])) - else: - outfile.write("export %s='%s'\n" % (k, v)) - outfile.close() - - #create /etc/csh.env for (t)csh support - outfile = atomic_ofstream(os.path.join(target_root, EPREFIX_LSTRIP, "etc", "csh.env")) - outfile.write(cenvnotice) - for x in env_keys: - outfile.write("setenv %s '%s'\n" % (x, env[x])) - outfile.close() - - if sleep_for_mtime_granularity: - while current_time == long(time.time()): - sleep(1) - -def ExtractKernelVersion(base_dir): - """ - Try to figure out what kernel version we are running - @param base_dir: Path to sources (usually /usr/src/linux) - @type base_dir: string - @rtype: tuple( version[string], error[string]) - @returns: - 1. tuple( version[string], error[string]) - Either version or error is populated (but never both) - - """ - lines = [] - pathname = os.path.join(base_dir, 'Makefile') - try: - f = codecs.open(_unicode_encode(pathname, - encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['content'], errors='replace') - except OSError as details: - return (None, str(details)) - except IOError as details: - return (None, str(details)) - - try: - for i in range(4): - lines.append(f.readline()) - except OSError as details: - return (None, str(details)) - except IOError as details: - return (None, str(details)) - - lines = [l.strip() for l in lines] - - version = '' - - #XXX: The following code relies on the ordering of vars within the Makefile - for line in lines: - # split on the '=' then remove annoying whitespace - items = line.split("=") - items = [i.strip() for i in items] - if items[0] == 'VERSION' or \ - items[0] == 'PATCHLEVEL': - version += items[1] - version += "." - elif items[0] == 'SUBLEVEL': - version += items[1] - elif items[0] == 'EXTRAVERSION' and \ - items[-1] != items[0]: - version += items[1] - - # Grab a list of files named localversion* and sort them - localversions = os.listdir(base_dir) - for x in range(len(localversions)-1,-1,-1): - if localversions[x][:12] != "localversion": - del localversions[x] - localversions.sort() - - # Append the contents of each to the version string, stripping ALL whitespace - for lv in localversions: - version += "".join( " ".join( grabfile( base_dir+ "/" + lv ) ).split() ) - - # Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace - kernelconfig = getconfig(base_dir+"/.config") - if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig: - version += "".join(kernelconfig["CONFIG_LOCALVERSION"].split()) - - return (version,None) - -def digestgen(myarchives=None, mysettings=None, - overwrite=None, manifestonly=None, myportdb=None): - """ - Generates a digest file if missing. Fetches files if necessary. - NOTE: myarchives and mysettings used to be positional arguments, - so their order must be preserved for backward compatibility. - @param mysettings: the ebuild config (mysettings["O"] must correspond - to the ebuild's parent directory) - @type mysettings: config - @param myportdb: a portdbapi instance - @type myportdb: portdbapi - @rtype: int - @returns: 1 on success and 0 on failure - """ - if mysettings is None: - raise TypeError("portage.digestgen(): missing" + \ - " required 'mysettings' parameter") - if myportdb is None: - warnings.warn("portage.digestgen() called without 'myportdb' parameter", - DeprecationWarning, stacklevel=2) - global portdb - myportdb = portdb - if overwrite is not None: - warnings.warn("portage.digestgen() called with " + \ - "deprecated 'overwrite' parameter", - DeprecationWarning, stacklevel=2) - if manifestonly is not None: - warnings.warn("portage.digestgen() called with " + \ - "deprecated 'manifestonly' parameter", - DeprecationWarning, stacklevel=2) - global _doebuild_manifest_exempt_depend - try: - _doebuild_manifest_exempt_depend += 1 - distfiles_map = {} - fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb) - for cpv in fetchlist_dict: - try: - for myfile in fetchlist_dict[cpv]: - distfiles_map.setdefault(myfile, []).append(cpv) - except portage.exception.InvalidDependString as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e - return 0 - mytree = os.path.dirname(os.path.dirname(mysettings["O"])) - manifest1_compat = False - mf = Manifest(mysettings["O"], mysettings["DISTDIR"], - fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) - # Don't require all hashes since that can trigger excessive - # fetches when sufficient digests already exist. To ease transition - # while Manifest 1 is being removed, only require hashes that will - # exist before and after the transition. - required_hash_types = set() - required_hash_types.add("size") - required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) - dist_hashes = mf.fhashdict.get("DIST", {}) - - # To avoid accidental regeneration of digests with the incorrect - # files (such as partially downloaded files), trigger the fetch - # code if the file exists and it's size doesn't match the current - # manifest entry. If there really is a legitimate reason for the - # digest to change, `ebuild --force digest` can be used to avoid - # triggering this code (or else the old digests can be manually - # removed from the Manifest). - missing_files = [] - for myfile in distfiles_map: - myhashes = dist_hashes.get(myfile) - if not myhashes: - try: - st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) - except OSError: - st = None - if st is None or st.st_size == 0: - missing_files.append(myfile) - continue - size = myhashes.get("size") - - try: - st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - if size == 0: - missing_files.append(myfile) - continue - if required_hash_types.difference(myhashes): - missing_files.append(myfile) - continue - else: - if st.st_size == 0 or size is not None and size != st.st_size: - missing_files.append(myfile) - continue - - if missing_files: - mytree = os.path.realpath(os.path.dirname( - os.path.dirname(mysettings["O"]))) - fetch_settings = config(clone=mysettings) - debug = mysettings.get("PORTAGE_DEBUG") == "1" - for myfile in missing_files: - uris = set() - for cpv in distfiles_map[myfile]: - myebuild = os.path.join(mysettings["O"], - catsplit(cpv)[1] + ".ebuild") - # for RESTRICT=fetch, mirror, etc... - doebuild_environment(myebuild, "fetch", - mysettings["ROOT"], fetch_settings, - debug, 1, myportdb) - uris.update(myportdb.getFetchMap( - cpv, mytree=mytree)[myfile]) - - fetch_settings["A"] = myfile # for use by pkg_nofetch() - - try: - st = os.stat(os.path.join( - mysettings["DISTDIR"],myfile)) - except OSError: - st = None - - if not fetch({myfile : uris}, fetch_settings): - writemsg(_("!!! Fetch failed for %s, can't update " - "Manifest\n") % myfile, noiselevel=-1) - if myfile in dist_hashes and \ - st is not None and st.st_size > 0: - # stat result is obtained before calling fetch(), - # since fetch may rename the existing file if the - # digest does not match. - writemsg(_("!!! If you would like to " - "forcefully replace the existing " - "Manifest entry\n!!! for %s, use " - "the following command:\n") % myfile + \ - "!!! " + colorize("INFORM", - "ebuild --force %s manifest" % \ - os.path.basename(myebuild)) + "\n", - noiselevel=-1) - return 0 - writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) - try: - mf.create(assumeDistHashesSometimes=True, - assumeDistHashesAlways=( - "assume-digests" in mysettings.features)) - except portage.exception.FileNotFound as e: - writemsg(_("!!! File %s doesn't exist, can't update " - "Manifest\n") % e, noiselevel=-1) - return 0 - except portage.exception.PortagePackageException as e: - writemsg(("!!! %s\n") % (e,), noiselevel=-1) - return 0 - try: - mf.write(sign=False) - except portage.exception.PermissionDenied as e: - writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1) - return 0 - if "assume-digests" not in mysettings.features: - distlist = list(mf.fhashdict.get("DIST", {})) - distlist.sort() - auto_assumed = [] - for filename in distlist: - if not os.path.exists( - os.path.join(mysettings["DISTDIR"], filename)): - auto_assumed.append(filename) - if auto_assumed: - mytree = os.path.realpath( - os.path.dirname(os.path.dirname(mysettings["O"]))) - cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:]) - pkgs = myportdb.cp_list(cp, mytree=mytree) - pkgs.sort() - writemsg_stdout(" digest.assumed" + portage.output.colorize("WARN", - str(len(auto_assumed)).rjust(18)) + "\n") - for pkg_key in pkgs: - fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree) - pv = pkg_key.split("/")[1] - for filename in auto_assumed: - if filename in fetchlist: - writemsg_stdout( - " %s::%s\n" % (pv, filename)) - return 1 - finally: - _doebuild_manifest_exempt_depend -= 1 - def digestParseFile(myfilename, mysettings=None): """(filename) -- Parses a given file for entries matching: diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 0ae1e5739..aefb53f96 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -25,6 +25,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabfile,grabdict,normalize_path,new_protect_filename,getlibpaths', 'portage.util.digraph:digraph', + 'portage.util.env_update:env_update', 'portage.util.listdir:dircache,listdir', 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \ '_pkgsplit@pkgsplit', @@ -39,7 +40,7 @@ from portage.exception import CommandNotFound, \ FileNotFound, PermissionDenied, UnsupportedAPIException from portage.localization import _ -from portage import dep_expand, env_update, \ +from portage import dep_expand, \ abssymlink, movefile, _movefile, bsd_chflags # This is a special version of the os module, wrapped for unicode support. diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index b2824d4cc..f77f7c869 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -21,23 +21,35 @@ import time import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.config:check_config_instance', + 'portage.util.digestgen:digestgen', + 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) -from portage import auxdbkeys, bsd_chflags, dep_check, digestcheck, digestgen, eapi_is_supported, ExtractKernelVersion, merge, os, selinux, StringIO, unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, _shell_quote, _split_ebuild_name_glep55, _unicode_decode, _unicode_encode -from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY -from portage.data import portage_gid, portage_uid, secpass, uid, userpriv_groups +from portage import auxdbkeys, bsd_chflags, dep_check, digestcheck, \ + eapi_is_supported, merge, os, selinux, StringIO, \ + unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \ + _shell_quote, _split_ebuild_name_glep55, _unicode_decode, _unicode_encode +from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_BINARY, \ + INVALID_ENV_FILE, MISC_SH_BINARY +from portage.data import portage_gid, portage_uid, secpass, \ + uid, userpriv_groups from portage.dbapi.virtual import fakedbapi -from portage.dep import Atom, paren_enclose, paren_normalize, paren_reduce, use_reduce +from portage.dep import Atom, paren_enclose, paren_normalize, \ + paren_reduce, use_reduce from portage.elog import elog_process from portage.elog.messages import eerror, eqawarn -from portage.exception import DigestException, FileNotFound, IncorrectParameter, InvalidAtom, InvalidDependString, PermissionDenied, UnsupportedAPIException +from portage.exception import DigestException, FileNotFound, \ + IncorrectParameter, InvalidAtom, InvalidDependString, PermissionDenied, \ + UnsupportedAPIException from portage.localization import _ from portage.manifest import Manifest from portage.output import style_to_ansi_code from portage.package.ebuild.fetch import fetch from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.package.ebuild._pty import _create_pty_or_pipe -from portage.util import apply_recursive_permissions, apply_secpass_permissions, noiselimit, normalize_path, writemsg, writemsg_stdout, write_atomic +from portage.util import apply_recursive_permissions, \ + apply_secpass_permissions, noiselimit, normalize_path, \ + writemsg, writemsg_stdout, write_atomic from portage.versions import _pkgsplit def doebuild_environment(myebuild, mydo, myroot, mysettings, diff --git a/pym/portage/util/ExtractKernelVersion.py b/pym/portage/util/ExtractKernelVersion.py new file mode 100644 index 000000000..9e6433b28 --- /dev/null +++ b/pym/portage/util/ExtractKernelVersion.py @@ -0,0 +1,77 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['ExtractKernelVersion'] + +import codecs + +from portage import os, _encodings, _unicode_encode +from portage.util import getconfig, grabfile + +def ExtractKernelVersion(base_dir): + """ + Try to figure out what kernel version we are running + @param base_dir: Path to sources (usually /usr/src/linux) + @type base_dir: string + @rtype: tuple( version[string], error[string]) + @returns: + 1. tuple( version[string], error[string]) + Either version or error is populated (but never both) + + """ + lines = [] + pathname = os.path.join(base_dir, 'Makefile') + try: + f = codecs.open(_unicode_encode(pathname, + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['content'], errors='replace') + except OSError as details: + return (None, str(details)) + except IOError as details: + return (None, str(details)) + + try: + for i in range(4): + lines.append(f.readline()) + except OSError as details: + return (None, str(details)) + except IOError as details: + return (None, str(details)) + + lines = [l.strip() for l in lines] + + version = '' + + #XXX: The following code relies on the ordering of vars within the Makefile + for line in lines: + # split on the '=' then remove annoying whitespace + items = line.split("=") + items = [i.strip() for i in items] + if items[0] == 'VERSION' or \ + items[0] == 'PATCHLEVEL': + version += items[1] + version += "." + elif items[0] == 'SUBLEVEL': + version += items[1] + elif items[0] == 'EXTRAVERSION' and \ + items[-1] != items[0]: + version += items[1] + + # Grab a list of files named localversion* and sort them + localversions = os.listdir(base_dir) + for x in range(len(localversions)-1,-1,-1): + if localversions[x][:12] != "localversion": + del localversions[x] + localversions.sort() + + # Append the contents of each to the version string, stripping ALL whitespace + for lv in localversions: + version += "".join( " ".join( grabfile( base_dir+ "/" + lv ) ).split() ) + + # Check the .config for a CONFIG_LOCALVERSION and append that too, also stripping whitespace + kernelconfig = getconfig(base_dir+"/.config") + if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig: + version += "".join(kernelconfig["CONFIG_LOCALVERSION"].split()) + + return (version,None) diff --git a/pym/portage/util/digestgen.py b/pym/portage/util/digestgen.py new file mode 100644 index 000000000..2dd0caef4 --- /dev/null +++ b/pym/portage/util/digestgen.py @@ -0,0 +1,204 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['digestgen'] + +import errno +import warnings + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.package.ebuild.doebuild:doebuild_environment', +) + +from portage import os +from portage.const import MANIFEST2_REQUIRED_HASH +from portage.dbapi.porttree import FetchlistDict +from portage.exception import InvalidDependString, FileNotFound, \ + PermissionDenied, PortagePackageException +from portage.localization import _ +from portage.manifest import Manifest +from portage.output import colorize +from portage.package.ebuild.config import config +from portage.package.ebuild.fetch import fetch +from portage.util import writemsg, writemsg_stdout +from portage.versions import catsplit + +def digestgen(myarchives=None, mysettings=None, + overwrite=None, manifestonly=None, myportdb=None): + """ + Generates a digest file if missing. Fetches files if necessary. + NOTE: myarchives and mysettings used to be positional arguments, + so their order must be preserved for backward compatibility. + @param mysettings: the ebuild config (mysettings["O"] must correspond + to the ebuild's parent directory) + @type mysettings: config + @param myportdb: a portdbapi instance + @type myportdb: portdbapi + @rtype: int + @returns: 1 on success and 0 on failure + """ + if mysettings is None: + raise TypeError("portage.digestgen(): missing" + \ + " required 'mysettings' parameter") + if myportdb is None: + warnings.warn("portage.digestgen() called without 'myportdb' parameter", + DeprecationWarning, stacklevel=2) + myportdb = portage.portdb + if overwrite is not None: + warnings.warn("portage.digestgen() called with " + \ + "deprecated 'overwrite' parameter", + DeprecationWarning, stacklevel=2) + if manifestonly is not None: + warnings.warn("portage.digestgen() called with " + \ + "deprecated 'manifestonly' parameter", + DeprecationWarning, stacklevel=2) + + try: + portage._doebuild_manifest_exempt_depend += 1 + distfiles_map = {} + fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb) + for cpv in fetchlist_dict: + try: + for myfile in fetchlist_dict[cpv]: + distfiles_map.setdefault(myfile, []).append(cpv) + except InvalidDependString as e: + writemsg("!!! %s\n" % str(e), noiselevel=-1) + del e + return 0 + mytree = os.path.dirname(os.path.dirname(mysettings["O"])) + manifest1_compat = False + mf = Manifest(mysettings["O"], mysettings["DISTDIR"], + fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) + # Don't require all hashes since that can trigger excessive + # fetches when sufficient digests already exist. To ease transition + # while Manifest 1 is being removed, only require hashes that will + # exist before and after the transition. + required_hash_types = set() + required_hash_types.add("size") + required_hash_types.add(MANIFEST2_REQUIRED_HASH) + dist_hashes = mf.fhashdict.get("DIST", {}) + + # To avoid accidental regeneration of digests with the incorrect + # files (such as partially downloaded files), trigger the fetch + # code if the file exists and it's size doesn't match the current + # manifest entry. If there really is a legitimate reason for the + # digest to change, `ebuild --force digest` can be used to avoid + # triggering this code (or else the old digests can be manually + # removed from the Manifest). + missing_files = [] + for myfile in distfiles_map: + myhashes = dist_hashes.get(myfile) + if not myhashes: + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError: + st = None + if st is None or st.st_size == 0: + missing_files.append(myfile) + continue + size = myhashes.get("size") + + try: + st = os.stat(os.path.join(mysettings["DISTDIR"], myfile)) + except OSError as e: + if e.errno != errno.ENOENT: + raise + del e + if size == 0: + missing_files.append(myfile) + continue + if required_hash_types.difference(myhashes): + missing_files.append(myfile) + continue + else: + if st.st_size == 0 or size is not None and size != st.st_size: + missing_files.append(myfile) + continue + + if missing_files: + mytree = os.path.realpath(os.path.dirname( + os.path.dirname(mysettings["O"]))) + fetch_settings = config(clone=mysettings) + debug = mysettings.get("PORTAGE_DEBUG") == "1" + for myfile in missing_files: + uris = set() + for cpv in distfiles_map[myfile]: + myebuild = os.path.join(mysettings["O"], + catsplit(cpv)[1] + ".ebuild") + # for RESTRICT=fetch, mirror, etc... + doebuild_environment(myebuild, "fetch", + mysettings["ROOT"], fetch_settings, + debug, 1, myportdb) + uris.update(myportdb.getFetchMap( + cpv, mytree=mytree)[myfile]) + + fetch_settings["A"] = myfile # for use by pkg_nofetch() + + try: + st = os.stat(os.path.join( + mysettings["DISTDIR"],myfile)) + except OSError: + st = None + + if not fetch({myfile : uris}, fetch_settings): + writemsg(_("!!! Fetch failed for %s, can't update " + "Manifest\n") % myfile, noiselevel=-1) + if myfile in dist_hashes and \ + st is not None and st.st_size > 0: + # stat result is obtained before calling fetch(), + # since fetch may rename the existing file if the + # digest does not match. + writemsg(_("!!! If you would like to " + "forcefully replace the existing " + "Manifest entry\n!!! for %s, use " + "the following command:\n") % myfile + \ + "!!! " + colorize("INFORM", + "ebuild --force %s manifest" % \ + os.path.basename(myebuild)) + "\n", + noiselevel=-1) + return 0 + writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"]) + try: + mf.create(assumeDistHashesSometimes=True, + assumeDistHashesAlways=( + "assume-digests" in mysettings.features)) + except FileNotFound as e: + writemsg(_("!!! File %s doesn't exist, can't update " + "Manifest\n") % e, noiselevel=-1) + return 0 + except PortagePackageException as e: + writemsg(("!!! %s\n") % (e,), noiselevel=-1) + return 0 + try: + mf.write(sign=False) + except PermissionDenied as e: + writemsg(_("!!! Permission Denied: %s\n") % (e,), noiselevel=-1) + return 0 + if "assume-digests" not in mysettings.features: + distlist = list(mf.fhashdict.get("DIST", {})) + distlist.sort() + auto_assumed = [] + for filename in distlist: + if not os.path.exists( + os.path.join(mysettings["DISTDIR"], filename)): + auto_assumed.append(filename) + if auto_assumed: + mytree = os.path.realpath( + os.path.dirname(os.path.dirname(mysettings["O"]))) + cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:]) + pkgs = myportdb.cp_list(cp, mytree=mytree) + pkgs.sort() + writemsg_stdout(" digest.assumed" + colorize("WARN", + str(len(auto_assumed)).rjust(18)) + "\n") + for pkg_key in pkgs: + fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree) + pv = pkg_key.split("/")[1] + for filename in auto_assumed: + if filename in fetchlist: + writemsg_stdout( + " %s::%s\n" % (pv, filename)) + return 1 + finally: + portage._doebuild_manifest_exempt_depend -= 1 diff --git a/pym/portage/util/env_update.py b/pym/portage/util/env_update.py new file mode 100644 index 000000000..ef3ad5a03 --- /dev/null +++ b/pym/portage/util/env_update.py @@ -0,0 +1,307 @@ +# Copyright 2010 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +__all__ = ['env_update'] + +import codecs +import errno +import stat +import sys +import time + +import portage +from portage import os, _encodings, _unicode_encode +from portage.checksum import prelink_capable +from portage.data import ostype +from portage.exception import ParseError +from portage.localization import _ +from portage.process import find_binary +from portage.util import atomic_ofstream, ensure_dirs, getconfig, \ + normalize_path, writemsg +from portage.util.listdir import listdir + +if sys.hexversion >= 0x3000000: + long = int + +def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, + env=None, writemsg_level=None): + """ + Parse /etc/env.d and use it to generate /etc/profile.env, csh.env, + ld.so.conf, and prelink.conf. Finally, run ldconfig. + """ + if writemsg_level is None: + writemsg_level = portage.util.writemsg_level + if target_root is None: + target_root = portage.settings["ROOT"] + if prev_mtimes is None: + prev_mtimes = portage.mtimedb["ldpath"] + if env is None: + env = os.environ + envd_dir = os.path.join(target_root, EPREFIX_LSTRIP, "etc", "env.d") + ensure_dirs(envd_dir, mode=0o755) + fns = listdir(envd_dir, EmptyOnError=1) + fns.sort() + templist = [] + for x in fns: + if len(x) < 3: + continue + if not x[0].isdigit() or not x[1].isdigit(): + continue + if x.startswith(".") or x.endswith("~") or x.endswith(".bak"): + continue + templist.append(x) + fns = templist + del templist + + space_separated = set(["CONFIG_PROTECT", "CONFIG_PROTECT_MASK"]) + colon_separated = set(["ADA_INCLUDE_PATH", "ADA_OBJECTS_PATH", + "CLASSPATH", "INFODIR", "INFOPATH", "KDEDIRS", "LDPATH", "MANPATH", + "PATH", "PKG_CONFIG_PATH", "PRELINK_PATH", "PRELINK_PATH_MASK", + "PYTHONPATH", "ROOTPATH"]) + + config_list = [] + + for x in fns: + file_path = os.path.join(envd_dir, x) + try: + myconfig = getconfig(file_path, expand=False) + except ParseError as e: + writemsg("!!! '%s'\n" % str(e), noiselevel=-1) + del e + continue + if myconfig is None: + # broken symlink or file removed by a concurrent process + writemsg("!!! File Not Found: '%s'\n" % file_path, noiselevel=-1) + continue + + config_list.append(myconfig) + if "SPACE_SEPARATED" in myconfig: + space_separated.update(myconfig["SPACE_SEPARATED"].split()) + del myconfig["SPACE_SEPARATED"] + if "COLON_SEPARATED" in myconfig: + colon_separated.update(myconfig["COLON_SEPARATED"].split()) + del myconfig["COLON_SEPARATED"] + + env = {} + specials = {} + for var in space_separated: + mylist = [] + for myconfig in config_list: + if var in myconfig: + for item in myconfig[var].split(): + if item and not item in mylist: + mylist.append(item) + del myconfig[var] # prepare for env.update(myconfig) + if mylist: + env[var] = " ".join(mylist) + specials[var] = mylist + + for var in colon_separated: + mylist = [] + for myconfig in config_list: + if var in myconfig: + for item in myconfig[var].split(":"): + if item and not item in mylist: + mylist.append(item) + del myconfig[var] # prepare for env.update(myconfig) + if mylist: + env[var] = ":".join(mylist) + specials[var] = mylist + + for myconfig in config_list: + """Cumulative variables have already been deleted from myconfig so that + they won't be overwritten by this dict.update call.""" + env.update(myconfig) + + if EPREFIX == '': + sleep_for_mtime_granularity = dolinkingstuff( + target_root, specials, prelink_capable, + makelinks, contents, prev_mtimes, env) + else: + sleep_for_mtime_granularity = False + writeshellprofile(target_root, env, sleep_for_mtime_granularity) + +def dolinkingstuff(target_root, specials, prelink_capable, makelinks, + contents, prev_mtimes, env): + # updating this stuff will never work in an offset, other than ROOT + # (e.g. not in Prefix), hence the EPREFIX is not taken into account + # here since this code should never be triggered on an offset install + ldsoconf_path = os.path.join(target_root, "etc", "ld.so.conf") + try: + myld = codecs.open(_unicode_encode(ldsoconf_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') + myldlines=myld.readlines() + myld.close() + oldld=[] + for x in myldlines: + #each line has at least one char (a newline) + if x[:1] == "#": + continue + oldld.append(x[:-1]) + except (IOError, OSError) as e: + if e.errno != errno.ENOENT: + raise + oldld = None + + ld_cache_update=False + + newld = specials["LDPATH"] + if (oldld != newld): + #ld.so.conf needs updating and ldconfig needs to be run + myfd = atomic_ofstream(ldsoconf_path) + myfd.write("# ld.so.conf autogenerated by env-update; make all changes to\n") + myfd.write("# contents of /etc/env.d directory\n") + for x in specials["LDPATH"]: + myfd.write(x + "\n") + myfd.close() + ld_cache_update=True + + # Update prelink.conf if we are prelink-enabled + if prelink_capable: + newprelink = atomic_ofstream( + os.path.join(target_root, "etc", "prelink.conf")) + newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n") + newprelink.write("# contents of /etc/env.d directory\n") + + for x in ["/bin","/sbin","/usr/bin","/usr/sbin","/lib","/usr/lib"]: + newprelink.write("-l %s\n" % (x,)); + prelink_paths = [] + prelink_paths += specials.get("LDPATH", []) + prelink_paths += specials.get("PATH", []) + prelink_paths += specials.get("PRELINK_PATH", []) + prelink_path_mask = specials.get("PRELINK_PATH_MASK", []) + for x in prelink_paths: + if not x: + continue + if x[-1:] != '/': + x += "/" + plmasked = 0 + for y in prelink_path_mask: + if not y: + continue + if y[-1] != '/': + y += "/" + if y == x[0:len(y)]: + plmasked = 1 + break + if not plmasked: + newprelink.write("-h %s\n" % (x,)) + for x in prelink_path_mask: + newprelink.write("-b %s\n" % (x,)) + newprelink.close() + + # Portage stores mtimes with 1 second granularity but in >=python-2.5 finer + # granularity is possible. In order to avoid the potential ambiguity of + # mtimes that differ by less than 1 second, sleep here if any of the + # directories have been modified during the current second. + sleep_for_mtime_granularity = False + current_time = long(time.time()) + mtime_changed = False + lib_dirs = set() + for lib_dir in set(specials["LDPATH"] + \ + ['usr/lib','usr/lib64','usr/lib32','lib','lib64','lib32']): + x = os.path.join(target_root, lib_dir.lstrip(os.sep)) + try: + newldpathtime = os.stat(x)[stat.ST_MTIME] + lib_dirs.add(normalize_path(x)) + except OSError as oe: + if oe.errno == errno.ENOENT: + try: + del prev_mtimes[x] + except KeyError: + pass + # ignore this path because it doesn't exist + continue + raise + if newldpathtime == current_time: + sleep_for_mtime_granularity = True + if x in prev_mtimes: + if prev_mtimes[x] == newldpathtime: + pass + else: + prev_mtimes[x] = newldpathtime + mtime_changed = True + else: + prev_mtimes[x] = newldpathtime + mtime_changed = True + + if mtime_changed: + ld_cache_update = True + + if makelinks and \ + not ld_cache_update and \ + contents is not None: + libdir_contents_changed = False + for mypath, mydata in contents.items(): + if mydata[0] not in ("obj", "sym"): + continue + head, tail = os.path.split(mypath) + if head in lib_dirs: + libdir_contents_changed = True + break + if not libdir_contents_changed: + makelinks = False + + ldconfig = "/sbin/ldconfig" + if "CHOST" in env and "CBUILD" in env and \ + env["CHOST"] != env["CBUILD"]: + ldconfig = find_binary("%s-ldconfig" % env["CHOST"]) + + # Only run ldconfig as needed + if (ld_cache_update or makelinks) and ldconfig: + # ldconfig has very different behaviour between FreeBSD and Linux + if ostype == "Linux" or ostype.lower().endswith("gnu"): + # We can't update links if we haven't cleaned other versions first, as + # an older package installed ON TOP of a newer version will cause ldconfig + # to overwrite the symlinks we just made. -X means no links. After 'clean' + # we can safely create links. + writemsg_level(_(">>> Regenerating %setc/ld.so.cache...\n") % \ + (target_root,)) + if makelinks: + os.system("cd / ; %s -r '%s'" % (ldconfig, target_root)) + else: + os.system("cd / ; %s -X -r '%s'" % (ldconfig, target_root)) + elif ostype in ("FreeBSD","DragonFly"): + writemsg_level(_(">>> Regenerating %svar/run/ld-elf.so.hints...\n") % \ + target_root) + os.system(("cd / ; %s -elf -i " + \ + "-f '%svar/run/ld-elf.so.hints' '%setc/ld.so.conf'") % \ + (ldconfig, target_root, target_root)) + + del specials["LDPATH"] + + return sleep_for_mtime_granularity + +def writeshellprofile(target_root, env, sleep_for_mtime_granularity): + penvnotice = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n" + penvnotice += "# DO NOT EDIT THIS FILE. CHANGES TO STARTUP PROFILES\n" + cenvnotice = penvnotice[:] + penvnotice += "# GO INTO " + EPREFIX + "/etc/profile NOT /etc/profile.env\n\n" + cenvnotice += "# GO INTO " + EPREFIX + "/etc/csh.cshrc NOT /etc/csh.env\n\n" + + #create /etc/profile.env for bash support + outfile = atomic_ofstream(os.path.join(target_root, EPREFIX_LSTRIP, "etc", "profile.env")) + outfile.write(penvnotice) + + env_keys = [ x for x in env if x != "LDPATH" ] + env_keys.sort() + for k in env_keys: + v = env[k] + if v.startswith('$') and not v.startswith('${'): + outfile.write("export %s=$'%s'\n" % (k, v[1:])) + else: + outfile.write("export %s='%s'\n" % (k, v)) + outfile.close() + + #create /etc/csh.env for (t)csh support + outfile = atomic_ofstream(os.path.join(target_root, EPREFIX_LSTRIP, "etc", "csh.env")) + outfile.write(cenvnotice) + for x in env_keys: + outfile.write("setenv %s '%s'\n" % (x, env[x])) + outfile.close() + + if sleep_for_mtime_granularity: + while current_time == long(time.time()): + time.sleep(1) -- 2.26.2