From: Fabian Groffen Date: Fri, 7 Mar 2008 20:16:47 +0000 (-0000) Subject: Merged from trunk 9428:9448 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=a59855c9866d051daad1081eb49d6d2a4cd37323;p=portage.git Merged from trunk 9428:9448 | 9429 | Bug #211949 - Fix the regex so that it won't match the = | | zmedico | character (equals sign). | | 9431 | Add support for idendification of function definitions since | | zmedico | it's needed in some cases in order to prevent some odd | | | function contents from being mistakenly identified as | | | invalid variable assignments. For example, this line from | | | _gcc-specs-directive_raw() is commonly found in | | | environment.bz2 files: $1=="*"directive":" { pspec=spec; | | | spec=""; outside=0; next } | | 9432 | Bug #211949 - Temporarily revert back to a less strict regex | | zmedico | in order to avoid false positives with multi-line variable | | | definitions that can be produced by the 'export' builtin. | | 9433 | Tighten the funct_start_re so that it doesn't match leading | | zmedico | whitespace since that's not needed. | | 9434 | Fix func_start_re so that it requires at least one | | zmedico | alphanumeric character. | | 9435 | Remove redundant CUSTOM_PROFILE_PATH from the PROFILE_PATHS | | zmedico | variable so that /etc/portage/profile/profile.bashrc will | | | only be sourced once if it exists. | | 9436 | Implement variable assignment handling in python so that we | | zmedico | can eventually make it more flexible and robust. | | 9437 | Update description to reflect the new meaning of the PATTERN | | zmedico | argument. | | 9438 | TODO: Move variable name validation and declare -r filtering | | zmedico | into filter-bash-environment.py. | | 9439 | As requested by wolf31o2, use the ROOT setting from | | zmedico | make.conf as long as it's not overridden by the calling | | | environment. | | 9440 | Handle multi-line quoted variable assignments. | | zmedico | | | 9441 | Bug #211949 - Tighten the variable name filter to exclude | | zmedico | any variables with names containing any non-alphanumeric | | | characters. | | 9442 | Fix have_end_quote() to compare the quote against the | | zmedico | correct group. | | 9443 | Pass a space-separated list of variables into | | zmedico | filter-bash-environment.py and generate the final regex on | | | the python side instead of in bash. Also, properly anchor | | | the regex so that it actually works. | | 9444 | Update the description to reflect the new meaning of | | zmedico | PATTERN. | | 9445 | Move the variable name validation regexes (for bug 211949) | | zmedico | into filter-bash-environment.py instead of passing them in | | | from bash. | | 9446 | Implement the sed-based declare -r filter in python. | | zmedico | | | 9447 | Remove egrep syntax emulation since it's not really needed. | | zmedico | | | 9448 | Use os.environ["SANDBOX_ON"] = "0" to make portageq exempt | | zmedico | from sandbox for things like writing metadata cache. Thanks | | | to ferringb for the suggestion. | svn path=/main/branches/prefix/; revision=9450 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 151ade086..1e5a4e539 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -7,7 +7,7 @@ PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-@PORTAGE_BASE@/bin}" PORTAGE_PYM_PATH="${PORTAGE_PYM_PATH:-@PORTAGE_BASE@/pym}" SANDBOX_PREDICT="${SANDBOX_PREDICT}:/proc/self/maps:/dev/console:/dev/random" -export SANDBOX_PREDICT="${SANDBOX_PREDICT}:${PORTAGE_PYM_PATH}:${PORTAGE_DEPCACHEDIR}" +export SANDBOX_PREDICT export SANDBOX_WRITE="${SANDBOX_WRITE}:/dev/shm:/dev/stdout:/dev/stderr:${PORTAGE_TMPDIR}" export SANDBOX_READ="${SANDBOX_READ}:/:/dev/shm:/dev/stdin:${PORTAGE_TMPDIR}" # Don't use sandbox's BASH_ENV for new shells because it does @@ -1438,18 +1438,16 @@ PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME" # builtin command. To avoid this problem, this function filters those # variables out and discards them. See bug #190128. filter_readonly_variables() { - local x filtered_vars var_grep + local x filtered_vars local readonly_bash_vars="DIRSTACK EUID FUNCNAME GROUPS PIPESTATUS PPID SHELLOPTS UID" local filtered_sandbox_vars="SANDBOX_ACTIVE SANDBOX_BASHRC SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB SANDBOX_LOG SANDBOX_ON" filtered_vars="${readonly_bash_vars} ${READONLY_PORTAGE_VARS} - BASH_[_[:alnum:]]* PATH - [[:digit:]][_[:alnum:]]* - [^[:space:]]*[^_[:alnum:][:space:]][^[:space:]]*" + BASH_.* PATH" if hasq --filter-sandbox $* ; then - filtered_vars="${filtered_vars} SANDBOX_[_[:alnum:]]*" + filtered_vars="${filtered_vars} SANDBOX_.*" else filtered_vars="${filtered_vars} ${filtered_sandbox_vars}" fi @@ -1463,20 +1461,8 @@ filter_readonly_variables() { ${PORTAGE_MUTABLE_FILTERED_VARS} " fi - set -f - for x in ${filtered_vars} ; do - var_grep="${var_grep}|${x}" - done - set +f - var_grep=${var_grep:1} # strip the first | - var_grep="(^|^declare[[:space:]]+-[^[:space:]]+[[:space:]]+|^export[[:space:]]+)(${var_grep})=.*" - # The sed is to remove the readonly attribute from variables such as those - # listed in READONLY_EBUILD_METADATA, since having any readonly attributes - # persisting in the saved environment can be inconvenient when it - # eventually needs to be reloaded. - "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${var_grep}" | sed -r \ - -e 's:^declare[[:space:]]+-r[[:space:]]+:declare :' \ - -e 's:^declare[[:space:]]+-([[:alnum:]]*)r([[:alnum:]]*)[[:space:]]+:declare -\1\2 :' + + "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${filtered_vars}" } # @FUNCTION: preprocess_ebuild_env diff --git a/bin/filter-bash-environment.py b/bin/filter-bash-environment.py index 90a494778..88ee40083 100755 --- a/bin/filter-bash-environment.py +++ b/bin/filter-bash-environment.py @@ -5,22 +5,62 @@ import os, re, sys -egrep_compat_map = { - "[:alnum:]" : r'\w', - "[:digit:]" : r'\d', - "[:space:]" : r'\s', -} - here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$') +func_start_re = re.compile(r'^[-\w]+\s*\(\)\s*$') +func_end_re = re.compile(r'^\}$') + +var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^export\s+)([^=\s]+)=("|\')?.*$') +close_quote_re = re.compile(r'(\\"|"|\')\s*$') +readonly_re = re.compile(r'^declare\s+-(\S*)r(\S*)\s+') -def compile_egrep_pattern(s): - for k, v in egrep_compat_map.iteritems(): - s = s.replace(k, v) - return re.compile(s) +def have_end_quote(quote, line): + """ + Check if the line has an end quote (useful for handling multi-line + quotes). This handles escaped double quotes that may occur at the + end of a line. The posix spec does not allow escaping of single + quotes inside of single quotes, so that case is not handled. + """ + close_quote_match = close_quote_re.search(line) + return close_quote_match is not None and \ + close_quote_match.group(1) == quote def filter_bash_environment(pattern, file_in, file_out): here_doc_delim = None + in_func = None + multi_line_quote = None + multi_line_quote_filter = None for line in file_in: + if multi_line_quote is not None: + if not multi_line_quote_filter: + file_out.write(line) + if have_end_quote(multi_line_quote, line): + multi_line_quote = None + multi_line_quote_filter = None + continue + if here_doc_delim is None and in_func is None: + var_assign_match = var_assign_re.match(line) + if var_assign_match is not None: + quote = var_assign_match.group(3) + filter_this = pattern.match(var_assign_match.group(2)) \ + is not None + if quote is not None and not have_end_quote(quote, line): + multi_line_quote = quote + multi_line_quote_filter = filter_this + if not filter_this: + readonly_match = readonly_re.match(line) + if readonly_match is not None: + declare_opts = "" + for i in (1, 2): + group = readonly_match.group(i) + if group is not None: + declare_opts += group + if declare_opts: + line = "declare -%s %s" % \ + (declare_opts, line[readonly_match.end():]) + else: + line = "declare " + line[readonly_match.end():] + file_out.write(line) + continue if here_doc_delim is not None: if here_doc_delim.match(line): here_doc_delim = None @@ -31,15 +71,29 @@ def filter_bash_environment(pattern, file_in, file_out): here_doc_delim = re.compile("^%s$" % here_doc.group(1)) file_out.write(line) continue - if pattern.match(line) is None: + # Note: here-documents are handled before functions since otherwise + # it would be possible for the content of a here-document to be + # mistaken as the end of a function. + if in_func: + if func_end_re.match(line) is not None: + in_func = None + file_out.write(line) + continue + in_func = func_start_re.match(line) + if in_func is not None: file_out.write(line) + continue + # This line is not recognized as part of a variable assignment, + # function definition, or here document, so just allow it to + # pass through. + file_out.write(line) if __name__ == "__main__": - description = "Filter out any lines that match a given PATTERN " + \ - "while leaving bash here-documents intact. The PATTERN should " + \ - "use python regular expression syntax but [:space:] and " + \ - "[:alnum:] character classes will be automatically translated " + \ - "for compatibility with egrep syntax." + description = "Filter out variable assignments for varable " + \ + "names matching a given PATTERN " + \ + "while leaving bash function definitions and here-documents " + \ + "intact. The PATTERN is a space separated list of variable names" + \ + " and it supports python regular expression syntax." usage = "usage: %s PATTERN" % os.path.basename(sys.argv[0]) from optparse import OptionParser parser = OptionParser(description=description, usage=usage) @@ -48,6 +102,13 @@ if __name__ == "__main__": parser.error("Missing required PATTERN argument.") file_in = sys.stdin file_out = sys.stdout + var_pattern = args[0].split() + + # Filter invalid variable names that are not supported by bash. + var_pattern.append(r'\d.*') + var_pattern.append(r'.*\W.*') + + var_pattern = "^(%s)$" % "|".join(var_pattern) filter_bash_environment( - compile_egrep_pattern(args[0]), file_in, file_out) + re.compile(var_pattern), file_in, file_out) file_out.flush() diff --git a/bin/portageq b/bin/portageq index 5ea82dbe2..920d04cad 100755 --- a/bin/portageq +++ b/bin/portageq @@ -21,6 +21,10 @@ except KeyboardInterrupt: import os +# This allows portageq to be exempt from sandbox, +# for things like updating metadata cache. +os.environ["SANDBOX_ON"] = "0" + import types #----------------------------------------------------------------------------- diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index d86fcc1d6..0dadd2d3b 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1270,13 +1270,8 @@ class config(object): self.mygcfg.pop(k, None) # Allow ROOT setting to come from make.conf if it's not overridden - # by the constructor argument (from the calling environment). As a - # special exception for a very common use case, config_root == "/" - # implies that ROOT in make.conf should be ignored. That way, the - # user can chroot into $ROOT and the ROOT setting in make.conf will - # be automatically ignored (unless config_root is other than "/"). - if config_root != "/" and \ - target_root is None and "ROOT" in self.mygcfg: + # by the constructor argument (from the calling environment). + if target_root is None and "ROOT" in self.mygcfg: target_root = self.mygcfg["ROOT"] self.configlist.append(self.mygcfg) @@ -3998,7 +3993,7 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m mysettings["ECLASSDIR"] = mysettings["PORTDIR"]+"/eclass" mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_") - mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles)+"\n"+CUSTOM_PROFILE_PATH + mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles) mysettings["P"] = mysplit[0]+"-"+mysplit[1] mysettings["PN"] = mysplit[0] mysettings["PV"] = mysplit[1]