From: Fabian Groffen Date: Sat, 1 Mar 2008 17:36:46 +0000 (-0000) Subject: Merged from trunk 9386:9402 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=f8e4ee5f6e0321fccc93aa74d29b37ce654e76c5;p=portage.git Merged from trunk 9386:9402 | 9387 | Bug #211554 - Replace references to config["USE"] with | | zmedico | references to config["PORTAGE_USE"] (which is filtered for | | | intersection with IUSE) so that dependency calculations at | | | installation time are consistent with those at depclean | | | time. | | 9388 | Don't filter USE flags in emerge --info output. | | zmedico | | | 9389 | Bug #211294 - Make repoman ignore profiles labeled with | | zmedico | anything other than "stable" or "dev" in profiles.desc so | | | that we can extend it with new labels such as "exp". | | 9391 | Revert r9387 (the fix for bug #211554) until I figure out | | zmedico | why it's messing up USE calculations for world updates. | | 9392 | Bug #211554 - Replace references to config["USE"] with | | zmedico | references to config["PORTAGE_USE"] (which is filtered for | | | intersection with IUSE) so that dependency calculations at | | | installation time are consistent with those at depclean | | | time. | | 9393 | Fix some references to config["PORTAGE_USE"] to prevent | | zmedico | potential KeyErrors. | | 9394 | separate different messages with newlines in post-emerge | | genone | output | | 9395 | remove obsolete information from preserve-libs post-emerge | | genone | message | | 9397 | supress stupid line echo in deprecation notices | | genone | | | 9398 | remove code for manifest1 support | | genone | | | 9399 | In the suidctl code, move the addwrite call out of the loop. | | zmedico | Thanks to ferringb. | | 9400 | Make 'import portage' statements more tolerant to broken | | zmedico | source statements in make.conf since exceptions thrown | | | during 'import portage' statements can practically render | | | the api unusable for api consumers. Thanks to lxnay for the | | | suggestion. | | 9402 | Bug #211949 - Make filter_readonly_variables() remove any | | zmedico | variables with names containing a hyphen since bash does not | | | allow them. | svn path=/main/branches/prefix/; revision=9404 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 249855481..e81cedb53 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -1410,8 +1410,12 @@ PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME" # @FUNCTION: filter_readonly_variables # @DESCRIPTION: [--filter-sandbox] [--allow-extra-vars] -# Read an environment from stdin and echo to stdout while filtering readonly -# variables. +# Read an environment from stdin and echo to stdout while filtering variables +# with names that are known to cause interference: +# +# * some specific variables for which bash does not allow assignment +# * any variables with names containing a hyphen (not allowed by bash) +# * some specific variables that affect portage or sandbox behavior # # --filter-sandbox causes all SANDBOX_* variables to be filtered, which # is only desired in certain cases, such as during preprocessing or when @@ -1440,7 +1444,7 @@ filter_readonly_variables() { SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB SANDBOX_LOG SANDBOX_ON" filtered_vars="${readonly_bash_vars} ${READONLY_PORTAGE_VARS} - BASH_[_[:alnum:]]* PATH" + BASH_[_[:alnum:]]* PATH [-_[:alnum:]]*-[-_[:alnum:]]*" if hasq --filter-sandbox $* ; then filtered_vars="${filtered_vars} SANDBOX_[_[:alnum:]]*" else diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 891d6ef8c..aa666e478 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -588,6 +588,10 @@ preinst_suid_scan() { if hasq suidctl $FEATURES; then #TODO: not sure if PORTAGE_CONFIGROOT includes EPREFIX sfconf=${PORTAGE_CONFIGROOT}etc/portage/suidctl.conf + # sandbox prevents us from writing directly + # to files outside of the sandbox, but this + # can easly be bypassed using the addwrite() function + addwrite "${sfconf}" vecho ">>> Performing suid scan in ${D}" #note not space-safe for i in $(find "${ED}" -type f \( -perm -4000 -o -perm -2000 \) ); do @@ -602,10 +606,6 @@ preinst_suid_scan() { ls_ret=$(ls -ldh "${i}") chmod ugo-s "${i}" grep "^#${i/${D}}$" "${sfconf}" > /dev/null || { - # sandbox prevents us from writing directly - # to files outside of the sandbox, but this - # can easly be bypassed using the addwrite() function - addwrite "${sfconf}" vecho ">>> Appending commented out entry to ${sfconf} for ${PF}" echo "## ${ls_ret%${D}*}${ls_ret#*${D}}" >> "${sfconf}" echo "#${i/${D}}" >> "${sfconf}" diff --git a/bin/repoman b/bin/repoman index 2612cc78d..ae0a5ca32 100755 --- a/bin/repoman +++ b/bin/repoman @@ -234,13 +234,6 @@ def ParseArgs(args, qahelp): qahelp={ "CVS/Entries.IO_error":"Attempting to commit, and an IO error was encountered access the Entries file", "desktop.invalid":"desktop-file-validate reports errors in a *.desktop file", - "digest.partial":"Digest files do not contain all corresponding URI elements", - "digest.assumed":"Existing digest must be assumed correct (Package level only)", - "digestentry.unused":"Digest/Manifest entry has no matching SRC_URI entry", - "digest.fail":"Digest does not match the specified local file", - "digest.stray":"Digest files that do not have a corresponding ebuild", - "digest.missing":"Digest files that are missing (ebuild exists, digest doesn't)", - "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)", "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)", "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory", "changelog.missing":"Missing ChangeLog files", @@ -313,9 +306,6 @@ qawarnings=[ "ebuild.allmasked", "ebuild.nesteddie", "desktop.invalid", -"digest.assumed", -"digest.missing", -"digestentry.unused", "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked", "DEPEND.badindev","RDEPEND.badindev","PDEPEND.badindev", "DEPEND.badmaskedindev","RDEPEND.badmaskedindev","PDEPEND.badmaskedindev", @@ -544,7 +534,6 @@ if not kwlist: logging.fatal("Couldn't read KEYWORDS from arch.list") sys.exit(1) -manifest1_compat = False if portdir_overlay != portdir: kwlist.update(portage.grabfile( os.path.join(portdir_overlay, "profiles", "arch.list"))) @@ -782,9 +771,6 @@ for x in scanlist: has_filesdir = True if not os.path.isdir(os.path.join(checkdir, "files")): has_filesdir = False - if manifest1_compat: - stats["filedir.missing"] += 1 - fails["filedir.missing"].append(checkdir) if isVcs: try: @@ -904,71 +890,6 @@ for x in scanlist: if os.path.exists(checkdir+"/files"): filesdirlist=os.listdir(checkdir+"/files") - if manifest1_compat: - for y in filesdirlist: - if not y.startswith("digest-"): - continue - relative_path = os.path.join(x, "files", y) - full_path = os.path.join(repodir, relative_path) - if stat.S_IMODE(os.stat(full_path).st_mode) & 0111: - stats["file.executable"] += 1 - fails["file.executable"].append(x+"/files/"+y) - - mykey = catdir + "/" + y[7:] - if y[7:] not in ebuildlist: - #stray digest - if options.mode == "fix": - if vcs == "cvs": - vcsrm="cvs rm -f" - if vcs == "svn": - vcsrm="svn rm --force" - if options.pretend: - print "(cd "+repodir+"/"+x+"/files; "+vcsrm+" "+y+")" - else: - os.system("(cd "+repodir+"/"+x+"/files; "+vcsrm+" "+y+")") - else: - stats["digest.stray"]=stats["digest.stray"]+1 - fails["digest.stray"].append(x+"/files/"+y) - else: - # We have an ebuild - try: - myuris, myfiles = portdb.getfetchlist(mykey, all=True) - except portage.exception.InvalidDependString, e: - # Already handled above. - continue - - uri_dict = {} - for myu in myuris: - myubn = os.path.basename(myu) - if myubn not in uri_dict: - uri_dict[myubn] = [myu] - else: - uri_dict[myubn] += [myu] - - for myf in uri_dict: - myff = repoman_settings["DISTDIR"] + "/" + myf - if not mydigests.has_key(myf): - uri_settings = portage.config(clone=repoman_settings) - if options.mode == "fix": - if not portage.fetch(uri_dict[myf], uri_settings): - stats["digest.unmatch"] += 1 - fails["digest.unmatch"].append(y+"::"+myf) - else: - eb_name = portdb.findname2(mykey)[0] - portage.doebuild(eb_name, "digest", "/", - uri_settings, tree="porttree", - mydbapi=portdb) - else: - stats["digest.partial"] += 1 - fails["digest.partial"].append(y+"::"+myf) - elif "assume-digests" not in repoman_settings.features: - if os.path.exists(myff): - if not portage.checksum.verify_all(myff, mydigests[myf])[0]: - stats["digest.fail"] += 1 - fails["digest.fail"].append(y+"::"+myf) - elif repolevel == 3: - stats["digest.assumed"] += 1 - fails["digest.assumed"].append(y+"::"+myf) # recurse through files directory # use filesdirlist as a stack, appending directories as needed so people can't hide > 20k files in a subdirectory. @@ -1061,20 +982,6 @@ for x in scanlist: if y in dadded: stats["ebuild.disjointed"]=stats["ebuild.disjointed"]+1 fails["ebuild.disjointed"].append(x+"/"+y+".ebuild") - if manifest1_compat and \ - not os.path.exists(os.path.join(checkdir, "files", "digest-"+y)): - if options.mode == "fix": - if options.pretend: - print "You will need to run:" - print " "+portage.const.PORTAGE_BASE+"/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest" - else: - retval=os.system(portage.const.PORTAGE_BASE+"/bin/ebuild "+repodir+"/"+x+"/"+y+".ebuild digest") - if retval: - print "!!! Exiting on ebuild digest (shell) error code:",retval - sys.exit(retval) - else: - stats["digest.missing"]=stats["digest.missing"]+1 - fails["digest.missing"].append(x+"/files/digest-"+y) myesplit=portage.pkgsplit(y) if myesplit is None or myesplit[0] != x.split("/")[-1]: stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1 @@ -1361,6 +1268,9 @@ for x in scanlist: for prof in profiles[arch]: + if prof[1] not in ("stable", "dev"): + continue + profdir = portdir+"/profiles/"+prof[0] if prof[0] in arch_caches: @@ -1558,20 +1468,6 @@ else: if options.pretend: print green("RepoMan sez:"), "\"So, you want to play it safe. Good call.\"\n" - if fails["digest.missing"]: - print green("Creating missing digests...") - for x in fails["digest.missing"]: - xs=x.split("/") - del xs[-2] - myeb="/".join(xs[:-1])+"/"+xs[-1][7:] - if options.pretend: - print "(ebuild "+portdir+"/"+myeb+".ebuild digest)" - else: - retval=os.system("ebuild "+portdir+"/"+myeb+".ebuild digest") - if retval: - print "!!! Exiting on ebuild digest (shell) error code:",retval - sys.exit(retval) - if isVcs and vcs == "cvs": try: myvcstree=portage.cvstree.getentries("./",recursive=1) diff --git a/man/repoman.1 b/man/repoman.1 index f213b9adf..037f5db2e 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -174,24 +174,6 @@ ChangeLogs that exist but have not been added to cvs .B digest.assumed Existing digest must be assumed correct (Package level only) .TP -.B digest.fail -Digest does not match the specified local file -.TP -.B digest.missing -Digest files that are missing (ebuild exists, digest doesn't) -.TP -.B digest.partial -Digest files do not contain all corresponding URI elements -.TP -.B digest.stray -Digest files that do not have a corresponding ebuild -.TP -.B digest.unmatch -Digests which are incomplete (please check if your USE/ARCH includes all files) -.TP -.B digest.unused -Digest/Manifest entry has no matching SRC_URI entry -.TP .B ebuild.allmasked All ebuilds are masked for this package (Package level only) .TP diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 7d0ea9c4d..d74a92579 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -1130,7 +1130,7 @@ def get_mask_info(root_config, cpv, pkgsettings, metadata = None if metadata and not built: pkgsettings.setcpv(cpv, mydb=metadata) - metadata["USE"] = pkgsettings.get("USE", "") + metadata["USE"] = pkgsettings["PORTAGE_USE"] if metadata is None: mreasons = ["corruption"] else: @@ -2146,7 +2146,7 @@ class depgraph(object): metadata = dict(izip(self._mydbapi_keys, portdb.aux_get(mykey, self._mydbapi_keys))) pkgsettings.setcpv(mykey, mydb=metadata) - metadata["USE"] = pkgsettings["USE"] + metadata["USE"] = pkgsettings["PORTAGE_USE"] pkg = Package(type_name="ebuild", root=myroot, cpv=mykey, metadata=metadata, onlydeps=onlydeps) args.append(PackageArg(arg=x, package=pkg, @@ -2454,7 +2454,7 @@ class depgraph(object): if not built: if (is_virt or "?" in metadata["LICENSE"]): pkgsettings.setcpv(cpv, mydb=metadata) - metadata["USE"] = pkgsettings["USE"] + metadata["USE"] = pkgsettings["PORTAGE_USE"] else: metadata["USE"] = "" @@ -2646,7 +2646,7 @@ class depgraph(object): if not built: if "?" in metadata["LICENSE"]: pkgsettings.setcpv(cpv, mydb=metadata) - metadata["USE"] = pkgsettings.get("USE","") + metadata["USE"] = pkgsettings["PORTAGE_USE"] else: metadata["USE"] = "" myarg = None @@ -2703,7 +2703,7 @@ class depgraph(object): pkgsettings.setcpv(myeb, mydb=mydb) else: pkgsettings.setcpv(cpv, mydb=mydb) - now_use = pkgsettings["USE"].split() + now_use = pkgsettings["PORTAGE_USE"].split() forced_flags = set() forced_flags.update(pkgsettings.useforce) forced_flags.update(pkgsettings.usemask) @@ -2729,7 +2729,7 @@ class depgraph(object): old_use = vardb.aux_get(cpv, ["USE"])[0].split() old_iuse = set(filter_iuse_defaults( vardb.aux_get(cpv, ["IUSE"])[0].split())) - cur_use = pkgsettings["USE"].split() + cur_use = pkgsettings["PORTAGE_USE"].split() cur_iuse = set(filter_iuse_defaults( metadata["IUSE"].split())) reinstall_for_flags = \ @@ -2760,7 +2760,7 @@ class depgraph(object): db.aux_get(cpv, self._mydbapi_keys))) if not built: pkgsettings.setcpv(cpv, mydb=metadata) - metadata["USE"] = pkgsettings.get("USE","") + metadata["USE"] = pkgsettings["PORTAGE_USE"] myeb = cpv matched_packages.append( Package(type_name=pkg_type, root=root, @@ -3686,7 +3686,7 @@ class depgraph(object): repo_path_real = os.path.dirname(os.path.dirname( os.path.dirname(ebuild_path))) pkgsettings.setcpv(pkg_key, mydb=mydbapi) - metadata["USE"] = pkgsettings["USE"] + metadata["USE"] = pkgsettings["PORTAGE_USE"] elif pkg_type == "binary": repo_path_real = repo_name else: @@ -4279,7 +4279,8 @@ class depgraph(object): if pkg_type == "ebuild": pkgsettings = self.pkgsettings[myroot] pkgsettings.setcpv(pkg_key, mydb=fakedb[myroot]) - fakedb[myroot].aux_update(pkg_key, {"USE":pkgsettings["USE"]}) + fakedb[myroot].aux_update(pkg_key, + {"USE":pkgsettings["PORTAGE_USE"]}) self.spinner.update() class RepoDisplay(object): @@ -4574,7 +4575,7 @@ class MergeTask(object): metadata.update(izip(metadata_keys, mydbapi.aux_get(pkg_key, metadata_keys))) pkgsettings.setcpv(pkg_key, mydb=mydbapi) - metadata["USE"] = pkgsettings["USE"] + metadata["USE"] = pkgsettings["PORTAGE_USE"] else: if pkg_type == "binary": mydbapi = bindb @@ -5233,9 +5234,9 @@ def chk_updated_info_files(root, infodirs, prev_mtimes, retval): regen_infodirs.append(inforoot) if not regen_infodirs: - portage.writemsg_stdout(" "+green("*")+" GNU info directory index is up-to-date.\n") + portage.writemsg_stdout("\n "+green("*")+" GNU info directory index is up-to-date.\n") else: - portage.writemsg_stdout(" "+green("*")+" Regenerating GNU info directory index...\n") + portage.writemsg_stdout("\n "+green("*")+" Regenerating GNU info directory index...\n") dir_extensions = ("", ".gz", ".bz2") icount=0 @@ -5421,6 +5422,7 @@ def post_emerge(trees, mtimedb, retval): display_news_notification(trees) if vardbapi.plib_registry.hasEntries(): + print print colorize("WARN", "!!!") + " existing preserved libs:" plibdata = vardbapi.plib_registry.getPreservedLibs() for cpv in plibdata: @@ -5428,7 +5430,6 @@ def post_emerge(trees, mtimedb, retval): for f in plibdata[cpv]: print colorize("WARN", " * ") + " - %s" % f print "Use " + colorize("GOOD", "emerge @preserved-rebuild") + " to rebuild packages using these libraries" - print "and then remerge the packages listed above." sys.exit(retval) @@ -5475,7 +5476,7 @@ def chk_updated_cfg_files(target_root, config_protect): del files[-1] if files: procount += 1 - print colorize("WARN", " * IMPORTANT:"), + print "\n"+colorize("WARN", " * IMPORTANT:"), if stat.S_ISDIR(mymode): print "%d config files in '%s' need updating." % \ (len(files), x) @@ -6373,8 +6374,8 @@ def action_info(settings, trees, myopts, myfiles): if portdb.cpv_exists(pkg): mydb = portdb pkgsettings.setcpv(pkg, mydb=mydb) - if valuesmap["IUSE"].intersection(pkgsettings["USE"].split()) != \ - valuesmap["USE"]: + if valuesmap["IUSE"].intersection( + pkgsettings["PORTAGE_USE"].split()) != valuesmap["USE"]: diff_values["USE"] = valuesmap["USE"] # If a difference was found, print the info for # this package. diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index c92adedb2..926d50012 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1,10 +1,10 @@ # portage.py -- core Portage functionality # Copyright 1998-2004 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Id: $ +# $Id$ -VERSION="$Rev$"[6:-2] + "-svn" +VERSION="$Rev: 9373 $"[6:-2] + "-svn" # =========================================================================== # START OF IMPORTS -- START OF IMPORTS -- START OF IMPORTS -- START OF IMPORT @@ -980,7 +980,11 @@ class config(object): @type local_config: Boolean """ - debug = os.environ.get("PORTAGE_DEBUG") == "1" + # When initializing the global portage.settings instance, avoid + # raising exceptions whenever possible since exceptions thrown + # from 'import portage' or 'import portage.exceptions' statements + # can practically render the api unusable for api consumers. + tolerant = "_initializing_globals" in globals() self.already_in_regenerate = 0 @@ -1255,7 +1259,7 @@ class config(object): self.mygcfg = getconfig( os.path.join(config_root, MAKE_CONF_FILE.lstrip(os.path.sep)), - allow_sourcing=True) + tolerant=tolerant, allow_sourcing=True) if self.mygcfg is None: self.mygcfg = {} @@ -1303,7 +1307,7 @@ class config(object): # Blacklist vars that could interfere with portage internals. for blacklisted in "CATEGORY", "PKGUSE", "PORTAGE_CONFIGROOT", \ - "ROOT", "EPREFIX", "EROOT": + "PORTAGE_IUSE", "PORTAGE_USE", "ROOT", "EPREFIX", "EROOT": for cfg in self.lookuplist: cfg.pop(blacklisted, None) del blacklisted, cfg @@ -1855,6 +1859,7 @@ class config(object): self.modifying() if self.mycpv == mycpv: return + ebuild_phase = self.get("EBUILD_PHASE") has_changed = False self.mycpv = mycpv cp = dep_getkey(mycpv) @@ -1931,7 +1936,7 @@ class config(object): if "test" in self.features: test_use_changed = \ bool(re.search(r'(^|\s)[-+]?test(\s|$)', iuse)) != \ - ("test" in self.get("PORTAGE_USE","").split()) + ("test" in self["USE"].split()) if self.get("EBUILD_PHASE") or \ self._use_wildcards or \ test_use_changed: @@ -1943,6 +1948,66 @@ class config(object): if has_changed: self.reset(keeping_pkg=1,use_cache=use_cache) + # Filter out USE flags that aren't part of IUSE. This has to + # be done for every setcpv() call since practically every + # package has different IUSE. Some flags are considered to + # be implicit members of IUSE: + # + # * Flags derived from ARCH + # * Flags derived from USE_EXPAND_HIDDEN variables + # * Masked flags, such as those from {,package}use.mask + # * Forced flags, such as those from {,package}use.force + # * build and bootstrap flags used by bootstrap.sh + + usesplit = self["USE"].split() + iuse_implicit = set(x.lstrip("+-") for x in iuse.split()) + + # Flags derived from ARCH. + arch = self.configdict["defaults"].get("ARCH") + if arch: + iuse_implicit.add(arch) + iuse_implicit.update(self.get("PORTAGE_ARCHLIST", "").split()) + + # Flags derived from USE_EXPAND_HIDDEN variables + # such as ELIBC, KERNEL, and USERLAND. + use_expand_hidden = self.get("USE_EXPAND_HIDDEN", "").split() + use_expand_hidden_raw = use_expand_hidden + if use_expand_hidden: + use_expand_hidden = re.compile("^(%s)_.*" % \ + ("|".join(x.lower() for x in use_expand_hidden))) + for x in usesplit: + if use_expand_hidden.match(x): + iuse_implicit.add(x) + + # Flags that have been masked or forced. + iuse_implicit.update(self.usemask) + iuse_implicit.update(self.useforce) + + # build and bootstrap flags used by bootstrap.sh + iuse_implicit.add("build") + iuse_implicit.add("bootstrap") + + # prefix flag is used in Prefix + iuse_implicit.add("prefix") + + if ebuild_phase: + iuse_grep = iuse_implicit.copy() + if use_expand_hidden_raw: + for x in use_expand_hidden_raw: + iuse_grep.add(x.lower() + "_.*") + if iuse_grep: + iuse_grep = "^(%s)$" % "|".join(sorted(iuse_grep)) + else: + iuse_grep = "" + self.configdict["pkg"]["PORTAGE_IUSE"] = iuse_grep + + # Filtered for the ebuild environment. Store this in a separate + # attribute since we still want to be able to see global USE + # settings for things like emerge --info. + self.configdict["pkg"]["PORTAGE_USE"] = " ".join(sorted( + x for x in usesplit if \ + x in iuse_implicit)) + def getMaskAtom(self, cpv, metadata): """ Take a package and return a matching package.mask atom, or None if no @@ -2154,7 +2219,7 @@ class config(object): return if isinstance(mydbapi, portdbapi): self.setcpv(mycpv, mydb=mydbapi) - myuse = self["USE"] + myuse = self["PORTAGE_USE"] elif isinstance(mydbapi, dict): myuse = mydbapi["USE"] else: @@ -2434,67 +2499,9 @@ class config(object): if arch and arch not in usesplit: usesplit.append(arch) - # Filter out USE flags that aren't part of IUSE. Some - # flags are considered to be implicit members of IUSE: - # - # * Flags derived from ARCH - # * Flags derived from USE_EXPAND_HIDDEN variables - # * Masked flags, such as those from {,package}use.mask - # * Forced flags, such as those from {,package}use.force - # * build and bootstrap flags used by bootstrap.sh - - # Do this even when there's no package since setcpv() can - # optimize away regenerate() calls. - iuse_implicit = set(iuse) - - # Flags derived from ARCH. - if arch: - iuse_implicit.add(arch) - iuse_implicit.update(self.get("PORTAGE_ARCHLIST", "").split()) - - # Flags derived from USE_EXPAND_HIDDEN variables - # such as ELIBC, KERNEL, and USERLAND. - use_expand_hidden = self.get("USE_EXPAND_HIDDEN", "").split() - use_expand_hidden_raw = use_expand_hidden - if use_expand_hidden: - use_expand_hidden = re.compile("^(%s)_.*" % \ - ("|".join(x.lower() for x in use_expand_hidden))) - for x in usesplit: - if use_expand_hidden.match(x): - iuse_implicit.add(x) - - # Flags that have been masked or forced. - iuse_implicit.update(self.usemask) - iuse_implicit.update(self.useforce) - - # build and bootstrap flags used by bootstrap.sh - iuse_implicit.add("build") - iuse_implicit.add("bootstrap") - - # prefix flag is used in Prefix - iuse_implicit.add("prefix") - - iuse_grep = iuse_implicit.copy() - if use_expand_hidden_raw: - for x in use_expand_hidden_raw: - iuse_grep.add(x.lower() + "_.*") - if iuse_grep: - iuse_grep = "^(%s)$" % "|".join(sorted(iuse_grep)) - else: - iuse_grep = "" - self["PORTAGE_IUSE"] = iuse_grep - usesplit = [x for x in usesplit if \ x not in self.usemask] - # Filtered for the ebuild environment. Store this in a separate - # attribute since we still want to be able to see global USE - # settings for things like emerge --info. - self["PORTAGE_USE"] = " ".join(sorted( - x for x in usesplit if \ - x in iuse_implicit)) - self.backup_changes("PORTAGE_USE") - usesplit.sort() self.configlist[-1]["USE"]= " ".join(usesplit) @@ -2696,7 +2703,7 @@ class config(object): mydict[k] = v # Filtered by IUSE and implicit IUSE. - mydict["USE"] = self["PORTAGE_USE"] + mydict["USE"] = self.get("PORTAGE_USE", "") # sandbox's bashrc sources /etc/profile which unsets ROOTPATH, # so we have to back it up and restore it. @@ -3646,7 +3653,7 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None for filename in auto_assumed: if filename in fetchlist: writemsg_stdout( - " digest-%s::%s\n" % (pv, filename)) + " %s::%s\n" % (pv, filename)) return 1 finally: _doebuild_manifest_exempt_depend -= 1 @@ -5444,7 +5451,7 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, if use=="yes": if myuse is None: #default behavior - myusesplit = mysettings["USE"].split() + myusesplit = mysettings["PORTAGE_USE"].split() else: myusesplit = myuse # We've been given useflags to use. @@ -5769,7 +5776,7 @@ def getmaskingstatus(mycpv, settings=None, portdb=None): return ["corruption"] if "?" in metadata["LICENSE"]: settings.setcpv(p, mydb=metadata) - metadata["USE"] = settings.get("USE", "") + metadata["USE"] = settings["PORTAGE_USE"] else: metadata["USE"] = "" mycp=mysplit[0]+"/"+mysplit[1] @@ -6369,7 +6376,10 @@ def init_legacy_globals(): for k, envvar in (("config_root", "PORTAGE_CONFIGROOT"), ("target_root", "ROOT")): kwargs[k] = os.environ.get(envvar, "/") + global _initializing_globals + _initializing_globals = True db = create_trees(**kwargs) + del _initializing_globals settings = db["/"]["vartree"].settings portdb = db["/"]["porttree"].dbapi diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 1937c4e11..de5a03bfe 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -146,18 +146,6 @@ class portdbapi(dbapi): for x in self.auxdb.values(): x.sync() - def finddigest(self,mycpv): - try: - mydig = self.findname2(mycpv)[0] - if not mydig: - return "" - mydigs = mydig.split("/")[:-1] - mydig = "/".join(mydigs) - mysplit = mycpv.split("/") - except OSError: - return "" - return mydig+"/files/digest-"+mysplit[-1] - def findLicensePath(self, license_name): mytrees = self.porttrees[:] mytrees.reverse() diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 68e50cc97..359eb9565 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -50,21 +50,6 @@ def parseManifest2(mysplit): myentry = Manifest2Entry(type=mytype, name=myname, hashes=myhashes) return myentry -def parseManifest1(mysplit): - myentry = None - if len(mysplit) == 4 and mysplit[0] in ["size"] + portage.const.MANIFEST1_HASH_FUNCTIONS: - myname = mysplit[2] - mytype = None - mytype = guessManifestFileType(myname) - if mytype == "AUX": - if myname.startswith("files" + os.path.sep): - myname = myname[6:] - mysize = int(mysplit[3]) - myhashes = {mysplit[0]: mysplit[1]} - myhashes["size"] = mysize - myentry = Manifest1Entry(type=mytype, name=myname, hashes=myhashes) - return myentry - class ManifestEntry(object): __slots__ = ("type", "name", "hashes") def __init__(self, **kwargs): @@ -75,17 +60,6 @@ class ManifestEntry(object): return 0 return 1 -class Manifest1Entry(ManifestEntry): - def __str__(self): - for hashkey in self.hashes: - if hashkey != "size": - break - hashvalue = self.hashes[hashkey] - myname = self.name - if self.type == "AUX" and not myname.startswith("files" + os.sep): - myname = os.path.join("files", myname) - return " ".join([hashkey, str(hashvalue), myname, str(self.hashes["size"])]) - class Manifest2Entry(ManifestEntry): def __str__(self): myline = " ".join([self.type, self.name, str(self.hashes["size"])]) @@ -97,7 +71,7 @@ class Manifest2Entry(ManifestEntry): return myline class Manifest(object): - parsers = (parseManifest2, parseManifest1) + parsers = (parseManifest2,) def __init__(self, pkgdir, distdir, fetchlist_dict=None, manifest1_compat=False, from_scratch=False): """ create new Manifest instance for package in pkgdir @@ -110,16 +84,15 @@ class Manifest(object): self.hashes = set() self.hashes.update(portage.const.MANIFEST2_HASH_FUNCTIONS) if manifest1_compat: - self.hashes.update(portage.const.MANIFEST1_HASH_FUNCTIONS) + raise NotImplementedError("manifest1 support has been removed") self.hashes.difference_update(hashname for hashname in \ list(self.hashes) if hashname not in hashfunc_map) self.hashes.add("size") if manifest1_compat: - self.hashes.add(portage.const.MANIFEST1_REQUIRED_HASH) + raise NotImplementedError("manifest1 support has been removed") self.hashes.add(portage.const.MANIFEST2_REQUIRED_HASH) for t in portage.const.MANIFEST2_IDENTIFIERS: self.fhashdict[t] = {} - self.compat = manifest1_compat if not from_scratch: self._read() if fetchlist_dict != None: @@ -144,26 +117,9 @@ class Manifest(object): """ Similar to getDigests(), but restricted to files of the given type. """ return self.fhashdict[ftype] - def _readDigests(self, myhashdict=None): - """ Parse old style digest files for this Manifest instance """ - if myhashdict is None: - myhashdict = {} - try: - for d in os.listdir(os.path.join(self.pkgdir, "files")): - if d.startswith("digest-"): - self._readManifest(os.path.join(self.pkgdir, "files", d), mytype="DIST", - myhashdict=myhashdict) - except (IOError, OSError), e: - if e.errno == errno.ENOENT: - pass - else: - raise - return myhashdict - def _readManifest(self, file_path, myhashdict=None, **kwargs): - """Parse a manifest or an old style digest. If myhashdict is given - then data will be added too it. Otherwise, a new dict will be created - and returned.""" + """Parse a manifest. If myhashdict is given then data will be added too it. + Otherwise, a new dict will be created and returned.""" try: fd = open(file_path, "r") if myhashdict is None: @@ -183,8 +139,6 @@ class Manifest(object): self._readManifest(self.getFullname(), myhashdict=self.fhashdict) except FileNotFound: pass - if self.compat: - self._readDigests(myhashdict=self.fhashdict) def _parseManifestLines(self, mylines): """Parse manifest lines and return a list of manifest entries.""" @@ -212,63 +166,6 @@ class Manifest(object): myhashdict[myentry_type][myentry.name].update(myentry.hashes) return myhashdict - def _writeDigests(self, force=False): - """ Create old style digest files for this Manifest instance """ - cpvlist = [os.path.join(self._pkgdir_category(), x[:-7]) for x in os.listdir(self.pkgdir) if x.endswith(".ebuild")] - rval = [] - try: - os.makedirs(os.path.join(self.pkgdir, "files")) - except OSError, oe: - if oe.errno == errno.EEXIST: - pass - else: - raise - for cpv in cpvlist: - dname = os.path.join(self.pkgdir, "files", "digest-%s" % self._catsplit(cpv)[1]) - distlist = self._getCpvDistfiles(cpv) - missing_digests = set() - for f in distlist: - if f not in self.fhashdict["DIST"] or len(self.fhashdict["DIST"][f]) == 0: - missing_digests.add(f) - if missing_digests: - # This allows us to force remove of stale digests for the - # ebuild --force digest option. - distlist = [f for f in distlist if f not in missing_digests] - update_digest = True - if not force: - try: - f = open(dname, "r") - old_data = self._parseDigests(f) - f.close() - new_data = self._getDigestData(distlist) - if not old_data and not new_data: - # SRC_URI is empty - update_digest = False - elif len(old_data) == 1 and "DIST" in old_data: - if "DIST" in new_data: - for myfile in new_data["DIST"]: - for hashname in \ - new_data["DIST"][myfile].keys(): - if hashname != "size" and hashname not in \ - portage.const.MANIFEST1_HASH_FUNCTIONS: - del new_data["DIST"][myfile][hashname] - if new_data["DIST"] == old_data["DIST"]: - update_digest = False - except (IOError, OSError), e: - if errno.ENOENT == e.errno: - pass - else: - raise - if update_digest: - mylines = self._createDigestLines1(distlist, self.fhashdict) - if mylines: - mylines = "\n".join(mylines) + "\n" - else: - mylines = "" - write_atomic(dname, mylines) - rval.append(dname) - return rval - def _getDigestData(self, distlist): """create a hash dict for a specific list of files""" myhashdict = {} @@ -280,32 +177,6 @@ class Manifest(object): myhashdict[mytype][myname].update(self.fhashdict[mytype][myname]) return myhashdict - def _createDigestLines1(self, distlist, myhashdict): - """ Create an old style digest file.""" - mylines = [] - myfiles = myhashdict["DIST"].keys() - myfiles.sort() - for f in myfiles: - if f in distlist: - myhashkeys = myhashdict["DIST"][f].keys() - myhashkeys.sort() - for h in myhashkeys: - if h not in portage.const.MANIFEST1_HASH_FUNCTIONS: - continue - myline = " ".join([h, str(myhashdict["DIST"][f][h]), f, str(myhashdict["DIST"][f]["size"])]) - mylines.append(myline) - return mylines - - def _addDigestsToManifest(self, digests, fd): - """ Add entries for old style digest files to Manifest file """ - mylines = [] - for dname in digests: - myhashes = perform_multiple_checksums(dname, portage.const.MANIFEST1_HASH_FUNCTIONS+["size"]) - for h in myhashes: - mylines.append((" ".join([h, str(myhashes[h]), os.path.join("files", os.path.basename(dname)), str(myhashes["size"])]))) - fd.write("\n".join(mylines)) - fd.write("\n") - def _createManifestEntries(self): mytypes = self.fhashdict.keys() mytypes.sort() @@ -321,36 +192,6 @@ class Manifest(object): if h not in ["size"] + portage.const.MANIFEST2_HASH_FUNCTIONS: del myentry.hashes[h] yield myentry - if self.compat and t != "DIST": - mysize = self.fhashdict[t][f]["size"] - myhashes = self.fhashdict[t][f] - for h in myhashkeys: - if h not in portage.const.MANIFEST1_HASH_FUNCTIONS: - continue - yield Manifest1Entry( - type=t, name=f, hashes={"size":mysize, h:myhashes[h]}) - - if self.compat: - cvp_list = self.fetchlist_dict.keys() - cvp_list.sort() - manifest1_hashes = set(hashname for hashname in \ - portage.const.MANIFEST1_HASH_FUNCTIONS \ - if hashname in hashfunc_map) - manifest1_hashes.add(portage.const.MANIFEST1_REQUIRED_HASH) - manifest1_hashes.add("size") - for cpv in cvp_list: - digest_path = os.path.join("files", "digest-%s" % self._catsplit(cpv)[1]) - dname = os.path.join(self.pkgdir, digest_path) - try: - myhashes = perform_multiple_checksums(dname, manifest1_hashes) - myhashkeys = myhashes.keys() - myhashkeys.sort() - for h in myhashkeys: - if h in portage.const.MANIFEST1_HASH_FUNCTIONS: - yield Manifest1Entry(type="AUX", name=digest_path, - hashes={"size":myhashes["size"], h:myhashes[h]}) - except FileNotFound: - pass def checkIntegrity(self): for t in self.fhashdict: @@ -362,8 +203,6 @@ class Manifest(object): """ Write Manifest instance to disk, optionally signing it """ self.checkIntegrity() try: - if self.compat: - self._writeDigests() myentries = list(self._createManifestEntries()) update_manifest = True if not force: @@ -451,7 +290,7 @@ class Manifest(object): distfilehashes = {} self.__init__(self.pkgdir, self.distdir, fetchlist_dict=self.fetchlist_dict, from_scratch=True, - manifest1_compat=self.compat) + manifest1_compat=False) cpvlist = [] pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() diff --git a/pym/portage/util.py b/pym/portage/util.py index 2cbcfb745..2398bad87 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -22,6 +22,11 @@ try: except ImportError: import pickle as cPickle +try: + import cStringIO as StringIO +except ImportError: + import StringIO + noiselimit = 0 def initialize_logger(level=logging.WARN): @@ -306,6 +311,15 @@ def writedict(mydict,myfilename,writekey=True): return 0 return 1 +class _tolerant_shlex(shlex.shlex): + def sourcehook(self, newfile): + try: + return shlex.shlex.sourcehook(self, newfile) + except EnvironmentError, e: + writemsg("!!! Parse error in '%s': source command failed: %s\n" % \ + (self.infile, str(e)), noiselevel=-1) + return (newfile, StringIO.StringIO()) + def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): mykeys={} try: @@ -317,10 +331,14 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): raise return None try: + if tolerant: + shlex_class = _tolerant_shlex + else: + shlex_class = shlex.shlex # The default shlex.sourcehook() implementation # only joins relative paths when the infile # attribute is properly set. - lex = shlex.shlex(f, infile=mycfg, posix=True) + lex = shlex_class(f, infile=mycfg, posix=True) lex.wordchars=string.digits+string.letters+"~!@#$%*_\:;?,./-+{}" lex.quotes="\"'" if allow_sourcing: diff --git a/pym/portage_compat_namespace.py b/pym/portage_compat_namespace.py index 9e5e691a9..b6c3da569 100644 --- a/pym/portage_compat_namespace.py +++ b/pym/portage_compat_namespace.py @@ -31,5 +31,10 @@ try: except ImportError, AttributeError: raise ImportError("No module named %s" % __oldname) -warnings.warn("DEPRECATION NOTICE: The %s module was replaced by %s" % (__oldname, __newname)) +def _formatwarning(message, category, filename, lineno): + return "%s:%s: %s: %s\n" % (filename, lineno, category.__name__, message) + +warnings.formatwarning = _formatwarning + +warnings.warn("DEPRECATION NOTICE: The %s module was replaced by %s" % (__oldname, __newname), DeprecationWarning) sys.modules[__oldname] = __realmodule