Merged from trunk 9428:9448
authorFabian Groffen <grobian@gentoo.org>
Fri, 7 Mar 2008 20:16:47 +0000 (20:16 -0000)
committerFabian Groffen <grobian@gentoo.org>
Fri, 7 Mar 2008 20:16:47 +0000 (20:16 -0000)
   | 9429    | Bug #211949 - Fix the regex so that it won't match the =     |
   | zmedico | character (equals sign).                                     |

   | 9431    | Add support for idendification of function definitions since |
   | zmedico | it's needed in some cases in order to prevent some odd       |
   |         | function contents from being mistakenly identified as        |
   |         | invalid variable assignments. For example, this line from    |
   |         | _gcc-specs-directive_raw() is commonly found in              |
   |         | environment.bz2 files: $1=="*"directive":" { pspec=spec;     |
   |         | spec=""; outside=0; next }                                   |

   | 9432    | Bug #211949 - Temporarily revert back to a less strict regex |
   | zmedico | in order to avoid false positives with multi-line variable   |
   |         | definitions that can be produced by the 'export' builtin.    |

   | 9433    | Tighten the funct_start_re so that it doesn't match leading  |
   | zmedico | whitespace since that's not needed.                          |

   | 9434    | Fix func_start_re so that it requires at least one           |
   | zmedico | alphanumeric character.                                      |

   | 9435    | Remove redundant CUSTOM_PROFILE_PATH from the PROFILE_PATHS  |
   | zmedico | variable so that /etc/portage/profile/profile.bashrc will    |
   |         | only be sourced once if it exists.                           |

   | 9436    | Implement variable assignment handling in python so that we  |
   | zmedico | can eventually make it more flexible and robust.             |

   | 9437    | Update description to reflect the new meaning of the PATTERN |
   | zmedico | argument.                                                    |

   | 9438    | TODO: Move variable name validation and declare -r filtering |
   | zmedico | into filter-bash-environment.py.                             |

   | 9439    | As requested by wolf31o2, use the ROOT setting from          |
   | zmedico | make.conf as long as it's not overridden by the calling      |
   |         | environment.                                                 |

   | 9440    | Handle multi-line quoted variable assignments.               |
   | zmedico |                                                              |

   | 9441    | Bug #211949 - Tighten the variable name filter to exclude    |
   | zmedico | any variables with names containing any non-alphanumeric     |
   |         | characters.                                                  |

   | 9442    | Fix have_end_quote() to compare the quote against the        |
   | zmedico | correct group.                                               |

   | 9443    | Pass a space-separated list of variables into                |
   | zmedico | filter-bash-environment.py and generate the final regex on   |
   |         | the python side instead of in bash. Also, properly anchor    |
   |         | the regex so that it actually works.                         |

   | 9444    | Update the description to reflect the new meaning of         |
   | zmedico | PATTERN.                                                     |

   | 9445    | Move the variable name validation regexes (for bug 211949)   |
   | zmedico | into filter-bash-environment.py instead of passing them in   |
   |         | from bash.                                                   |

   | 9446    | Implement the sed-based declare -r filter in python.         |
   | zmedico |                                                              |

   | 9447    | Remove egrep syntax emulation since it's not really needed.  |
   | zmedico |                                                              |

   | 9448    | Use os.environ["SANDBOX_ON"] = "0" to make portageq exempt   |
   | zmedico | from sandbox for things like writing metadata cache. Thanks  |
   |         | to ferringb for the suggestion.                              |

svn path=/main/branches/prefix/; revision=9450

bin/ebuild.sh
bin/filter-bash-environment.py
bin/portageq
pym/portage/__init__.py

index 151ade086e34f7060bbbf7e16b347f0386f91b5e..1e5a4e53923a4bee082d2a8a507f98176e6d6e43 100755 (executable)
@@ -7,7 +7,7 @@ PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-@PORTAGE_BASE@/bin}"
 PORTAGE_PYM_PATH="${PORTAGE_PYM_PATH:-@PORTAGE_BASE@/pym}"
 
 SANDBOX_PREDICT="${SANDBOX_PREDICT}:/proc/self/maps:/dev/console:/dev/random"
-export SANDBOX_PREDICT="${SANDBOX_PREDICT}:${PORTAGE_PYM_PATH}:${PORTAGE_DEPCACHEDIR}"
+export SANDBOX_PREDICT
 export SANDBOX_WRITE="${SANDBOX_WRITE}:/dev/shm:/dev/stdout:/dev/stderr:${PORTAGE_TMPDIR}"
 export SANDBOX_READ="${SANDBOX_READ}:/:/dev/shm:/dev/stdin:${PORTAGE_TMPDIR}"
 # Don't use sandbox's BASH_ENV for new shells because it does
@@ -1438,18 +1438,16 @@ PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME"
 # builtin command. To avoid this problem, this function filters those
 # variables out and discards them. See bug #190128.
 filter_readonly_variables() {
-       local x filtered_vars var_grep
+       local x filtered_vars
        local readonly_bash_vars="DIRSTACK EUID FUNCNAME GROUPS
                PIPESTATUS PPID SHELLOPTS UID"
        local filtered_sandbox_vars="SANDBOX_ACTIVE SANDBOX_BASHRC
                SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB
                SANDBOX_LOG SANDBOX_ON"
        filtered_vars="${readonly_bash_vars} ${READONLY_PORTAGE_VARS}
-               BASH_[_[:alnum:]]* PATH
-               [[:digit:]][_[:alnum:]]*
-               [^[:space:]]*[^_[:alnum:][:space:]][^[:space:]]*"
+               BASH_.* PATH"
        if hasq --filter-sandbox $* ; then
-               filtered_vars="${filtered_vars} SANDBOX_[_[:alnum:]]*"
+               filtered_vars="${filtered_vars} SANDBOX_.*"
        else
                filtered_vars="${filtered_vars} ${filtered_sandbox_vars}"
        fi
@@ -1463,20 +1461,8 @@ filter_readonly_variables() {
                        ${PORTAGE_MUTABLE_FILTERED_VARS}
                "
        fi
-       set -f
-       for x in ${filtered_vars} ; do
-               var_grep="${var_grep}|${x}"
-       done
-       set +f
-       var_grep=${var_grep:1} # strip the first |
-       var_grep="(^|^declare[[:space:]]+-[^[:space:]]+[[:space:]]+|^export[[:space:]]+)(${var_grep})=.*"
-       # The sed is to remove the readonly attribute from variables such as those
-       # listed in READONLY_EBUILD_METADATA, since having any readonly attributes
-       # persisting in the saved environment can be inconvenient when it
-       # eventually needs to be reloaded.
-       "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${var_grep}" | sed -r \
-               -e 's:^declare[[:space:]]+-r[[:space:]]+:declare :' \
-               -e 's:^declare[[:space:]]+-([[:alnum:]]*)r([[:alnum:]]*)[[:space:]]+:declare -\1\2 :'
+
+       "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${filtered_vars}"
 }
 
 # @FUNCTION: preprocess_ebuild_env
index 90a494778e1501baa1ed3e373bc36bb1441ae1b2..88ee400835211254841c1da07019e9b3547ccf33 100755 (executable)
@@ -5,22 +5,62 @@
 
 import os, re, sys
 
-egrep_compat_map = {
-       "[:alnum:]" : r'\w',
-       "[:digit:]" : r'\d',
-       "[:space:]" : r'\s',
-}
-
 here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$')
+func_start_re = re.compile(r'^[-\w]+\s*\(\)\s*$')
+func_end_re = re.compile(r'^\}$')
+
+var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^export\s+)([^=\s]+)=("|\')?.*$')
+close_quote_re = re.compile(r'(\\"|"|\')\s*$')
+readonly_re = re.compile(r'^declare\s+-(\S*)r(\S*)\s+')
 
-def compile_egrep_pattern(s):
-       for k, v in egrep_compat_map.iteritems():
-               s = s.replace(k, v)
-       return re.compile(s)
+def have_end_quote(quote, line):
+       """
+       Check if the line has an end quote (useful for handling multi-line
+       quotes). This handles escaped double quotes that may occur at the
+       end of a line. The posix spec does not allow escaping of single
+       quotes inside of single quotes, so that case is not handled.
+       """
+       close_quote_match = close_quote_re.search(line)
+       return close_quote_match is not None and \
+               close_quote_match.group(1) == quote
 
 def filter_bash_environment(pattern, file_in, file_out):
        here_doc_delim = None
+       in_func = None
+       multi_line_quote = None
+       multi_line_quote_filter = None
        for line in file_in:
+               if multi_line_quote is not None:
+                       if not multi_line_quote_filter:
+                               file_out.write(line)
+                       if have_end_quote(multi_line_quote, line):
+                               multi_line_quote = None
+                               multi_line_quote_filter = None
+                       continue
+               if here_doc_delim is None and in_func is None:
+                       var_assign_match = var_assign_re.match(line)
+                       if var_assign_match is not None:
+                               quote = var_assign_match.group(3)
+                               filter_this = pattern.match(var_assign_match.group(2)) \
+                                       is not None
+                               if quote is not None and not have_end_quote(quote, line):
+                                       multi_line_quote = quote
+                                       multi_line_quote_filter = filter_this
+                               if not filter_this:
+                                       readonly_match = readonly_re.match(line)
+                                       if readonly_match is not None:
+                                               declare_opts = ""
+                                               for i in (1, 2):
+                                                       group = readonly_match.group(i)
+                                                       if group is not None:
+                                                               declare_opts += group
+                                               if declare_opts:
+                                                       line = "declare -%s %s" % \
+                                                               (declare_opts, line[readonly_match.end():])
+                                               else:
+                                                       line = "declare " + line[readonly_match.end():]
+                                       file_out.write(line)
+                               continue
                if here_doc_delim is not None:
                        if here_doc_delim.match(line):
                                here_doc_delim = None
@@ -31,15 +71,29 @@ def filter_bash_environment(pattern, file_in, file_out):
                        here_doc_delim = re.compile("^%s$" % here_doc.group(1))
                        file_out.write(line)
                        continue
-               if pattern.match(line) is None:
+               # Note: here-documents are handled before functions since otherwise
+               # it would be possible for the content of a here-document to be
+               # mistaken as the end of a function.
+               if in_func:
+                       if func_end_re.match(line) is not None:
+                               in_func = None
+                       file_out.write(line)
+                       continue
+               in_func = func_start_re.match(line)
+               if in_func is not None:
                        file_out.write(line)
+                       continue
+               # This line is not recognized as part of a variable assignment,
+               # function definition, or here document, so just allow it to
+               # pass through.
+               file_out.write(line)
 
 if __name__ == "__main__":
-       description = "Filter out any lines that match a given PATTERN " + \
-               "while leaving bash here-documents intact. The PATTERN should " + \
-               "use python regular expression syntax but [:space:] and " + \
-               "[:alnum:] character classes will be automatically translated " + \
-               "for compatibility with egrep syntax."
+       description = "Filter out variable assignments for varable " + \
+               "names matching a given PATTERN " + \
+               "while leaving bash function definitions and here-documents " + \
+               "intact. The PATTERN is a space separated list of variable names" + \
+               " and it supports python regular expression syntax."
        usage = "usage: %s PATTERN" % os.path.basename(sys.argv[0])
        from optparse import OptionParser
        parser = OptionParser(description=description, usage=usage)
@@ -48,6 +102,13 @@ if __name__ == "__main__":
                parser.error("Missing required PATTERN argument.")
        file_in = sys.stdin
        file_out = sys.stdout
+       var_pattern = args[0].split()
+
+       # Filter invalid variable names that are not supported by bash.
+       var_pattern.append(r'\d.*')
+       var_pattern.append(r'.*\W.*')
+
+       var_pattern = "^(%s)$" % "|".join(var_pattern)
        filter_bash_environment(
-               compile_egrep_pattern(args[0]), file_in, file_out)
+               re.compile(var_pattern), file_in, file_out)
        file_out.flush()
index 5ea82dbe2a9b4f4e574eaafd05b55d2f45a79008..920d04cad70f2c37a65f4c0a1d94c53ac1498cdf 100755 (executable)
@@ -21,6 +21,10 @@ except KeyboardInterrupt:
 
 import os
 
+# This allows portageq to be exempt from sandbox,
+# for things like updating metadata cache.
+os.environ["SANDBOX_ON"] = "0"
+
 import types
 
 #-----------------------------------------------------------------------------
index d86fcc1d60eaaddd82b0ec39fabeffae17fbb7eb..0dadd2d3bb39a3a0a8585687fc86dd4033e0b37b 100644 (file)
@@ -1270,13 +1270,8 @@ class config(object):
                                self.mygcfg.pop(k, None)
                        
                        # Allow ROOT setting to come from make.conf if it's not overridden
-                       # by the constructor argument (from the calling environment).  As a
-                       # special exception for a very common use case, config_root == "/"
-                       # implies that ROOT in make.conf should be ignored.  That way, the
-                       # user can chroot into $ROOT and the ROOT setting in make.conf will
-                       # be automatically ignored (unless config_root is other than "/").
-                       if config_root != "/" and \
-                               target_root is None and "ROOT" in self.mygcfg:
+                       # by the constructor argument (from the calling environment).
+                       if target_root is None and "ROOT" in self.mygcfg:
                                target_root = self.mygcfg["ROOT"]
                        
                        self.configlist.append(self.mygcfg)
@@ -3998,7 +3993,7 @@ def doebuild_environment(myebuild, mydo, myroot, mysettings, debug, use_cache, m
        mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
        mysettings["SANDBOX_LOG"] = mycpv.replace("/", "_-_")
 
-       mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles)+"\n"+CUSTOM_PROFILE_PATH
+       mysettings["PROFILE_PATHS"] = "\n".join(mysettings.profiles)
        mysettings["P"]  = mysplit[0]+"-"+mysplit[1]
        mysettings["PN"] = mysplit[0]
        mysettings["PV"] = mysplit[1]