From: stevenknight Date: Mon, 5 Jan 2004 14:06:26 +0000 (+0000) Subject: Refactor construction variable expansion to handle recursive substitution of variables. X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=3bcf4038193d91f777bf13a38ab94737cdf701f4;p=scons.git Refactor construction variable expansion to handle recursive substitution of variables. git-svn-id: http://scons.tigris.org/svn/scons/trunk@868 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 4d0ad9a7..ba14d78a 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -103,6 +103,10 @@ RELEASE 0.95 - XXX not. The default value is zipfile.ZIP_DEFLATED, which generates a compressed file. + - Refactor construction variable expansion to support recursive + expansion of variables (e.g. CCFLAGS = "$CCFLAGS -g") without going + into an infinite loop. + From Vincent Risi: - Add support for the bcc32, ilink32 and tlib Borland tools. diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index 17915324..719e2134 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -116,7 +116,8 @@ def _do_create_action(act, strfunction=_null, varlist=[]): # like a function or a CommandGenerator in that variable # instead of a string. return CommandGeneratorAction(LazyCmdGenerator(var)) - listCmds = map(lambda x: CommandAction(x), string.split(act, '\n')) + listCmds = map(lambda x: CommandAction(x), + string.split(str(act), '\n')) if len(listCmds) == 1: return listCmds[0] else: @@ -175,9 +176,7 @@ class CommandAction(ActionBase): self.cmd_list = cmd def strfunction(self, target, source, env): - cmd_list = SCons.Util.scons_subst_list(self.cmd_list, env, - SCons.Util.SUBST_CMD, - target, source) + cmd_list = env.subst_list(self.cmd_list, 0, target, source) return map(_string_from_cmd_list, cmd_list) def __call__(self, target, source, env): @@ -221,9 +220,7 @@ class CommandAction(ActionBase): else: raise SCons.Errors.UserError('Missing SPAWN construction variable.') - cmd_list = SCons.Util.scons_subst_list(self.cmd_list, env, - SCons.Util.SUBST_CMD, - target, source) + cmd_list = env.subst_list(self.cmd_list, 0, target, source) for cmd_line in cmd_list: if len(cmd_line): if print_actions: @@ -253,11 +250,10 @@ class CommandAction(ActionBase): # and will produce something reasonable for # just about everything else: ENV[key] = str(value) - + # Escape the command line for the command # interpreter we are using - map(lambda x, e=escape: x.escape(e), cmd_line) - cmd_line = map(str, cmd_line) + cmd_line = SCons.Util.escape_list(cmd_line, escape) if pipe_build: ret = pspawn( shell, escape, cmd_line[0], cmd_line, ENV, pstdout, pstderr ) diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index 53aa00ae..73961188 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -8,7 +8,7 @@ # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: - +# # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # @@ -33,10 +33,12 @@ def Func(): import os import re import StringIO +import string import sys import types import unittest import UserDict +import UserString import SCons.Action import SCons.Environment @@ -66,13 +68,13 @@ if os.environ.has_key( 'ACTPY_PIPE' ): if os.environ.has_key( 'PIPE_STDOUT_FILE' ): stdout_msg = open(os.environ['PIPE_STDOUT_FILE'], 'r').read() else: - stdout_msg = "act.py: stdout: executed act.py\\n" + stdout_msg = "act.py: stdout: executed act.py %s\\n" % string.join(sys.argv[1:]) sys.stdout.write( stdout_msg ) if os.environ.has_key( 'PIPE_STDERR_FILE' ): stderr_msg = open(os.environ['PIPE_STDERR_FILE'], 'r').read() else: - stderr_msg = "act.py: stderr: executed act.py\\n" - sys.stderr.write( stderr_msg ) + stderr_msg = "act.py: stderr: executed act.py %s\\n" % string.join(sys.argv[1:]) + sys.stderr.write( stderr_msg ) sys.exit(0) """) @@ -88,6 +90,32 @@ scons_env = SCons.Environment.Environment() # so it doesn't clutter the output. sys.stdout = StringIO.StringIO() +class CmdStringHolder(UserString.UserString): + # Copped from SCons.Util + def __init__(self, cmd, literal=None): + UserString.UserString.__init__(self, cmd) + self.literal = literal + + def is_literal(self): + return self.literal + + def escape(self, escape_func): + """Escape the string with the supplied function. The + function is expected to take an arbitrary string, then + return it with all special characters escaped and ready + for passing to the command interpreter. + + After calling this function, the next call to str() will + return the escaped string. + """ + + if self.is_literal(): + return escape_func(self.data) + elif ' ' in self.data or '\t' in self.data: + return '"%s"' % self.data + else: + return self.data + class Environment: def __init__(self, **kw): self.d = {} @@ -97,18 +125,70 @@ class Environment: self.d['ESCAPE'] = scons_env['ESCAPE'] for k, v in kw.items(): self.d[k] = v - def subst(self, s): - if not SCons.Util.is_String(s): - return s + def subst_dict(self, target, source): + dict = self.d.copy() + if not SCons.Util.is_List(target): + target = [target] + if not SCons.Util.is_List(source): + source = [source] + dict['TARGETS'] = target + dict['SOURCES'] = source try: - if s[0] == '$': - if s[1] == '{': - return self.d.get(s[2:-1], '') - else: - return self.d.get(s[1:], '') + dict['TARGET'] = target[0] except IndexError: - pass - return self.d.get(s, s) + dict['TARGET'] = '' + try: + dict['SOURCE'] = source[0] + except IndexError: + dict['SOURCE'] = '' + return dict + def subst(self, strSubst): + if not SCons.Util.is_String(strSubst): + return strSubst + try: + s0, s1 = strSubst[:2] + except (IndexError, ValueError): + return strSubst + if s0 == '$': + if s1 == '{': + return self.d.get(strSubst[2:-1], '') + else: + return self.d.get(strSubst[1:], '') + return strSubst + def subst_list(self, strSubst, raw=0, target=[], source=[]): + dict = self.subst_dict(target, source) + if SCons.Util.is_String(strSubst): + strSubst = string.split(strSubst) + elif not SCons.Util.is_List(strSubst): + return strSubst + result = [] + for s in strSubst: + if SCons.Util.is_String(s): + try: + s0, s1 = s[:2] + except (IndexError, ValueError): + result.append(s) + else: + if s0 == '$': + if s1 == '{': + s = eval(s[2:-1], {}, dict) + else: + s = dict.get(s[1:], '') + if s: + if not SCons.Util.is_List(s): + s = [s] + result.extend(s) + else: + result.append(s) + def l(obj): + try: + l = obj.is_literal + except AttributeError: + literal = None + else: + literal = l() + return CmdStringHolder(str(obj), literal) + return [map(l, result)] def __getitem__(self, item): return self.d[item] def __setitem__(self, item, value): @@ -125,7 +205,7 @@ class Environment: res = Environment() res.d = SCons.Environment.our_deepcopy(self.d) for k, v in kw.items(): - res.d[k] = v + res.d[k] = v return res def sig_dict(self): d = {} @@ -341,7 +421,7 @@ class ActionBaseTestCase(unittest.TestCase): pass else: assert 0, "Should have thrown a TypeError adding to an int." - + class CommandActionTestCase(unittest.TestCase): def test_init(self): @@ -353,7 +433,7 @@ class CommandActionTestCase(unittest.TestCase): def test_strfunction(self): """Test fetching the string representation of command Actions """ - + act = SCons.Action.CommandAction('xyzzy $TARGET $SOURCE') s = act.strfunction([], [], Environment()) assert s == ['xyzzy'], s @@ -391,7 +471,7 @@ class CommandActionTestCase(unittest.TestCase): env = self.env except AttributeError: env = Environment() - + cmd1 = r'%s %s %s xyzzy' % (python, act_py, outfile) act = SCons.Action.CommandAction(cmd1) @@ -427,17 +507,15 @@ class CommandActionTestCase(unittest.TestCase): cmd4 = r'%s %s %s ${SOURCES[:2]}' % (python, act_py, outfile) act = SCons.Action.CommandAction(cmd4) - r = act([], - source = [DummyNode('three'), - DummyNode('four'), - DummyNode('five')], - env = env.Copy()) + sources = [DummyNode('three'), DummyNode('four'), DummyNode('five')] + env2 = env.Copy() + r = act([], source = sources, env = env2) assert r == 0 c = test.read(outfile, 'r') assert c == "act.py: 'three' 'four'\n", c cmd5 = r'%s %s %s $TARGET XYZZY' % (python, act_py, outfile) - + act = SCons.Action.CommandAction(cmd5) env5 = Environment() if scons_env.has_key('ENV'): @@ -446,18 +524,18 @@ class CommandActionTestCase(unittest.TestCase): else: env5['ENV'] = {} PATH = '' - + env5['ENV']['XYZZY'] = 'xyzzy' r = act(target = DummyNode('out5'), source = [], env = env5) act = SCons.Action.CommandAction(cmd5) r = act(target = DummyNode('out5'), source = [], - env = env.Copy(ENV = {'XYZZY' : 'xyzzy', + env = env.Copy(ENV = {'XYZZY' : 'xyzzy5', 'PATH' : PATH})) assert r == 0 c = test.read(outfile, 'r') - assert c == "act.py: 'out5' 'XYZZY'\nact.py: 'xyzzy'\n", c + assert c == "act.py: 'out5' 'XYZZY'\nact.py: 'xyzzy5'\n", c class Obj: def __init__(self, str): @@ -479,24 +557,6 @@ class CommandActionTestCase(unittest.TestCase): c = test.read(outfile, 'r') assert c == "act.py: '222' '111' '333' '444'\n", c - cmd7 = '%s %s %s one\n\n%s %s %s two' % (python, act_py, outfile, - python, act_py, outfile) - expect7 = '%s %s %s one\n%s %s %s two\n' % (python, act_py, outfile, - python, act_py, outfile) - - act = SCons.Action.CommandAction(cmd7) - - global show_string - show_string = "" - def my_show(string): - global show_string - show_string = show_string + string + "\n" - act.show = my_show - - r = act([], [], env.Copy()) - assert r == 0 - assert show_string == expect7, show_string - if os.name == 'nt': # NT treats execs of directories and non-executable files # as "file not found" errors @@ -510,7 +570,7 @@ class CommandActionTestCase(unittest.TestCase): expect_nonexecutable = 126 # Test that a nonexistent command returns 127 - act = SCons.Action.CommandAction(python + "_XyZzY_") + act = SCons.Action.CommandAction(python + "_no_such_command_") r = act([], [], env.Copy(out = outfile)) assert r == expect_nonexistent, "r == %d" % r @@ -544,8 +604,8 @@ class CommandActionTestCase(unittest.TestCase): # intermixed, so count the lines separately. outlines = re.findall(act_out, pipe_out) errlines = re.findall(act_err, pipe_out) - assert len(outlines) == 8, outlines - assert len(errlines) == 8, errlines + assert len(outlines) == 6, pipe_out + repr(outlines) + assert len(errlines) == 6, pipe_out + repr(errlines) # test redirection operators def test_redirect(self, redir, stdout_msg, stderr_msg): @@ -613,20 +673,25 @@ class CommandActionTestCase(unittest.TestCase): self.data = x def __str__(self): return self.data + def escape(self, escape_func): + return escape_func(self.data) def is_literal(self): return 1 a = SCons.Action.CommandAction(["xyzzy"]) - a([], [], Environment(SPAWN = func)) - assert t.executed == [ 'xyzzy' ] + e = Environment(SPAWN = func) + a([], [], e) + assert t.executed == [ 'xyzzy' ], t.executed a = SCons.Action.CommandAction(["xyzzy"]) - a([], [], Environment(SPAWN = func, SHELL = 'fake shell')) - assert t.executed == [ 'xyzzy' ] - assert t.shell == 'fake shell' + e = Environment(SPAWN = func, SHELL = 'fake shell') + a([], [], e) + assert t.executed == [ 'xyzzy' ], t.executed + assert t.shell == 'fake shell', t.shell a = SCons.Action.CommandAction([ LiteralStr("xyzzy") ]) - a([], [], Environment(SPAWN = func, ESCAPE = escape_func)) + e = Environment(SPAWN = func, ESCAPE = escape_func) + a([], [], e) assert t.executed == [ '**xyzzy**' ], t.executed def test_get_raw_contents(self): diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index 34822a61..00a9bedd 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -45,8 +45,53 @@ import SCons.Node try: from UserString import UserString except ImportError: + # "Borrowed" from the Python 2.2 UserString module + # and modified slightly for use with SCons. class UserString: - pass + def __init__(self, seq): + if is_String(seq): + self.data = seq + elif isinstance(seq, UserString): + self.data = seq.data[:] + else: + self.data = str(seq) + def __str__(self): return str(self.data) + def __repr__(self): return repr(self.data) + def __int__(self): return int(self.data) + def __long__(self): return long(self.data) + def __float__(self): return float(self.data) + def __complex__(self): return complex(self.data) + def __hash__(self): return hash(self.data) + + def __cmp__(self, string): + if isinstance(string, UserString): + return cmp(self.data, string.data) + else: + return cmp(self.data, string) + def __contains__(self, char): + return char in self.data + + def __len__(self): return len(self.data) + def __getitem__(self, index): return self.__class__(self.data[index]) + def __getslice__(self, start, end): + start = max(start, 0); end = max(end, 0) + return self.__class__(self.data[start:end]) + + def __add__(self, other): + if isinstance(other, UserString): + return self.__class__(self.data + other.data) + elif is_String(other): + return self.__class__(self.data + other) + else: + return self.__class__(self.data + str(other)) + def __radd__(self, other): + if is_String(other): + return self.__class__(other + self.data) + else: + return self.__class__(str(other) + self.data) + def __mul__(self, n): + return self.__class__(self.data*n) + __rmul__ = __mul__ _altsep = os.altsep if _altsep is None and sys.platform == 'win32': @@ -77,6 +122,12 @@ def updrive(path): path = string.upper(drive) + rest return path +# +# Generic convert-to-string functions that abstract away whether or +# not the Python we're executing has Unicode support. The wrapper +# to_String_for_signature() will use a for_signature() method if the +# specified object has one. +# if hasattr(types, 'UnicodeType'): def to_String(s): if isinstance(s, UserString): @@ -90,6 +141,17 @@ if hasattr(types, 'UnicodeType'): else: to_String = str +def to_String_for_signature(obj): + try: + f = obj.for_signature + except: + return to_String(obj) + else: + return f() + +# Indexed by the SUBST_* constants below. +_strconv = [to_String, to_String, to_String_for_signature] + class Literal: """A wrapper for a string. If you use this object wrapped around a string, then it will be interpreted as literal. @@ -101,31 +163,46 @@ class Literal: def __str__(self): return self.lstr + def escape(self, escape_func): + return escape_func(self.lstr) + + def for_signature(self): + return self.lstr + def is_literal(self): return 1 -class SpecialAttrWrapper(Literal): +class SpecialAttrWrapper: """This is a wrapper for what we call a 'Node special attribute.' This is any of the attributes of a Node that we can reference from Environment variable substitution, such as $TARGET.abspath or - $SOURCES[1].filebase. We inherit from Literal so we can handle - special characters, plus we implement a for_signature method, - such that we can return some canonical string during signatutre + $SOURCES[1].filebase. We implement the same methods as Literal + so we can handle special characters, plus a for_signature method, + such that we can return some canonical string during signature calculation to avoid unnecessary rebuilds.""" def __init__(self, lstr, for_signature=None): """The for_signature parameter, if supplied, will be the canonical string we return from for_signature(). Else we will simply return lstr.""" - Literal.__init__(self, lstr) + self.lstr = lstr if for_signature: self.forsig = for_signature else: self.forsig = lstr + def __str__(self): + return self.lstr + + def escape(self, escape_func): + return escape_func(self.lstr) + def for_signature(self): return self.forsig + def is_literal(self): + return 1 + class CallableComposite(UserList.UserList): """A simple composite callable class that, when called, will invoke all of its contained callables with the same arguments.""" @@ -165,7 +242,7 @@ class NodeList(UserList.UserList): # If there is nothing in the list, then we have no attributes to # pass through, so raise AttributeError for everything. raise AttributeError, "NodeList has no attribute: %s" % name - + # Return a list of the attribute, gotten from every element # in the list attrList = map(lambda x, n=name: getattr(x, n), self.data) @@ -178,9 +255,6 @@ class NodeList(UserList.UserList): return CallableComposite(attrList) return self.__class__(attrList) - def is_literal(self): - return 1 - _valid_var = re.compile(r'[_a-zA-Z]\w*$') _get_env_var = re.compile(r'^\$([_a-zA-Z]\w*|{[_a-zA-Z]\w*})$') @@ -206,111 +280,28 @@ def get_environment_var(varstr): return None def quote_spaces(arg): + """Generic function for putting double quotes around any string that + has white space in it.""" if ' ' in arg or '\t' in arg: return '"%s"' % arg else: return str(arg) -# Several functions below deal with Environment variable -# substitution. Part of this process involves inserting -# a bunch of special escape sequences into the string -# so that when we are all done, we know things like -# where to split command line args, what strings to -# interpret literally, etc. A dictionary of these -# sequences follows: -# -# \0\1 signifies a division between arguments in -# a command line. -# -# \0\2 signifies a division between multiple distinct -# commands, i.e., a newline -# -# \0\3 This string should be interpreted literally. -# This code occurring anywhere in the string means -# the whole string should have all special characters -# escaped. -# -# \0\4 A literal dollar sign '$' -# -# \0\5 Placed before and after interpolated variables -# so that we do not accidentally smush two variables -# together during the recursive interpolation process. - -_cv = re.compile(r'\$([_a-zA-Z][\.\w]*|{[^}]*})') -_space_sep = re.compile(r'[\t ]+(?![^{]*})') -_newline = re.compile(r'[\r\n]+') - -def _convertArg(x, strconv=to_String): - """This function converts an individual argument. If the - argument is to be interpreted literally, with all special - characters escaped, then we insert a special code in front - of it, so that the command interpreter will know this.""" - literal = 0 - - try: - if x.is_literal(): - literal = 1 - except AttributeError: - pass - - if not literal: - # escape newlines as '\0\2', '\0\1' denotes an argument split - # Also escape double-dollar signs to mean the literal dollar sign. - return string.replace(_newline.sub('\0\2', strconv(x)), '$$', '\0\4') - else: - # Interpret non-string args as literals. - # The special \0\3 code will tell us to encase this string - # in a Literal instance when we are all done - # Also escape out any $ signs because we don't want - # to continue interpolating a literal. - return '\0\3' + string.replace(strconv(x), '$', '\0\4') - -def _convert(x, strconv = to_String): - """This function is used to convert construction variable - values or the value of strSubst to a string for interpolation. - This function follows the rules outlined in the documentaion - for scons_subst_list()""" - if x is None: - return '' - elif is_String(x): - # escape newlines as '\0\2', '\0\1' denotes an argument split - return _convertArg(_space_sep.sub('\0\1', x), strconv) - elif is_List(x): - # '\0\1' denotes an argument split - return string.join(map(lambda x, s=strconv: _convertArg(x, s), x), - '\0\1') - else: - return _convertArg(x, strconv) - -class CmdStringHolder: - """This is a special class used to hold strings generated - by scons_subst_list(). It defines a special method escape(). - When passed a function with an escape algorithm for a - particular platform, it will return the contained string - with the proper escape sequences inserted.""" - - def __init__(self, cmd): - """This constructor receives a string. The string - can contain the escape sequence \0\3. - If it does, then we will escape all special characters - in the string before passing it to the command interpreter.""" - self.data = cmd - - # Populate flatdata (the thing returned by str()) with the - # non-escaped string - self.escape(lambda x: x, lambda x: x) - - def __str__(self): - """Return the string in its current state.""" - return self.flatdata +class CmdStringHolder(UserString): + """This is a special class used to hold strings generated by + scons_subst() and scons_subst_list(). It defines a special method + escape(). When passed a function with an escape algorithm for a + particular platform, it will return the contained string with the + proper escape sequences inserted. - def __len__(self): - """Return the length of the string in its current state.""" - return len(self.flatdata) + This should really be a subclass of UserString, but that module + doesn't exist in Python 1.5.2.""" + def __init__(self, cmd, literal=None): + UserString.__init__(self, cmd) + self.literal = literal - def __getitem__(self, index): - """Return the index'th element of the string in its current state.""" - return self.flatdata[index] + def is_literal(self): + return self.literal def escape(self, escape_func, quote_func=quote_spaces): """Escape the string with the supplied function. The @@ -322,16 +313,13 @@ class CmdStringHolder: return the escaped string. """ - if string.find(self.data, '\0\3') >= 0: - self.flatdata = escape_func(string.replace(self.data, '\0\3', '')) + if self.is_literal(): + return escape_func(self.data) elif ' ' in self.data or '\t' in self.data: - self.flatdata = quote_func(self.data) + return quote_func(self.data) else: - self.flatdata = self.data + return self.data - def __cmp__(self, rhs): - return cmp(self.flatdata, str(rhs)) - class DisplayEngine: def __init__(self): self.__call__ = self.print_it @@ -348,6 +336,18 @@ class DisplayEngine: else: self.__call__ = self.dont_print +def escape_list(list, escape_func): + """Escape a list of arguments by running the specified escape_func + on every object in the list that has an escape() method.""" + def escape(obj, escape_func=escape_func): + try: + e = obj.escape + except AttributeError: + return obj + else: + return e(escape_func) + return map(escape, list) + def target_prep(target): if target and not isinstance(target, NodeList): if not is_List(target): @@ -406,178 +406,298 @@ SUBST_SIG = 2 _rm = re.compile(r'\$[()]') _remove = re.compile(r'\$\(([^\$]|\$[^\(])*?\$\)') -def _canonicalize(obj): - """Attempt to call the object's for_signature method, - which is expected to return a string suitable for use in calculating - a command line signature (i.e., it only changes when we should - rebuild the target). For instance, file Nodes will report only - their file name (with no path), so changing Repository settings - will not cause a rebuild.""" - try: - return obj.for_signature() - except AttributeError: - return to_String(obj) - # Indexed by the SUBST_* constants above. _regex_remove = [ None, _rm, _remove ] -_strconv = [ to_String, to_String, _canonicalize ] -def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None): - """ - This function serves the same purpose as scons_subst(), except - this function returns the interpolated list as a list of lines, where - each line is a list of command line arguments. In other words: - The first (outer) list is a list of lines, where the - substituted stirng has been broken along newline characters. - The inner lists are lists of command line arguments, i.e., - the argv array that should be passed to a spawn or exec - function. - - There are a few simple rules this function follows in order to - determine how to parse strSubst and construction variables into lines - and arguments: - - 1) A string is interpreted as a space delimited list of arguments. - 2) A list is interpreted as a list of arguments. This allows arguments - with spaces in them to be expressed easily. - 4) Anything that is not a list or string (e.g. a Node instance) is - interpreted as a single argument, and is converted to a string. - 3) Newline (\n) characters delimit lines. The newline parsing is done - after all the other parsing, so it is not possible for arguments - (e.g. file names) to contain embedded newline characters. - """ - - remove = _regex_remove[mode] - strconv = _strconv[mode] - - def repl(m, - target=target, - source=source, - env=env, - local_vars = subst_dict(target, source, env), - global_vars = env.Dictionary(), - strconv=strconv, - sig=(mode != SUBST_CMD)): - key = m.group(1) - if key[0] == '{': - key = key[1:-1] - try: - e = eval(key, global_vars, local_vars) - except (IndexError, NameError, TypeError): - return '\0\5' - if callable(e): - # We wait to evaluate callables until the end of everything - # else. For now, we instert a special escape sequence - # that we will look for later. - return '\0\5' + _convert(e(target=target, - source=source, - env=env, - for_signature=sig), - strconv) + '\0\5' - else: - # The \0\5 escape code keeps us from smushing two or more - # variables together during recusrive substitution, i.e. - # foo=$bar bar=baz barbaz=blat => $foo$bar->blat (bad) - return "\0\5" + _convert(e, strconv) + "\0\5" +# This regular expression splits a string into the following types of +# arguments for use by the scons_subst() and scons_subst_list() functions: +# +# "$$" +# "$(" +# "$)" +# "$variable" [must begin with alphabetic or underscore] +# "${any stuff}" +# " " [white space] +# "non-white-space" [without any dollar signs] +# "$" [single dollar sign] +# +_separate_args = re.compile(r'(\$[\$\(\)]|\$[_a-zA-Z][\.\w]*|\${[^}]*}|\s+|[^\s\$]+|\$)') - # Convert the argument to a string: - strSubst = _convert(strSubst, strconv) +# This regular expression is used to replace strings of multiple white +# space characters in the string result from the scons_subst() function. +_space_sep = re.compile(r'[\t ]+(?![^{]*})') - # Do the interpolation: - n = 1 - while n != 0: - strSubst, n = _cv.subn(repl, strSubst) - - # Convert the interpolated string to a list of lines: - listLines = string.split(strSubst, '\0\2') +def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None): + """Expand a string containing construction variable substitutions. - # Remove the patterns that match the remove argument: - if remove: - listLines = map(lambda x,re=remove: re.sub('', x), listLines) + This is the work-horse function for substitutions in file names + and the like. The companion scons_subst_list() function (below) + handles separating command lines into lists of arguments, so see + that function if that's what you're looking for. + """ + class StringSubber: + """A class to construct the results of a scons_subst() call. - # Process escaped $'s and remove placeholder \0\5's - listLines = map(lambda x: string.replace(string.replace(x, '\0\4', '$'), '\0\5', ''), listLines) + This binds a specific construction environment, mode, target and + source with two methods (substitute() and expand()) that handle + the expansion. + """ + def __init__(self, env, mode, target, source): + self.env = env + self.mode = mode + self.target = target + self.source = source + + self.gvars = env.Dictionary() + self.str = _strconv[mode] + + def expand(self, s, lvars): + """Expand a single "token" as necessary, returning an + appropriate string containing the expansion. + + This handles expanding different types of things (strings, + lists, callables) appropriately. It calls the wrapper + substitute() method to re-expand things as necessary, so that + the results of expansions of side-by-side strings still get + re-evaluated separately, not smushed together. + """ + if is_String(s): + try: + s0, s1 = s[:2] + except (IndexError, ValueError): + return s + if s0 == '$': + if s1 == '$': + return '$' + elif s1 in '()': + return s + else: + key = s[1:] + if key[0] == '{': + key = key[1:-1] + try: + s = eval(key, self.gvars, lvars) + except (IndexError, NameError, TypeError): + return '' + else: + # Before re-expanding the result, handle + # recursive expansion by copying the local + # variable dictionary and overwriting a null + # string for the value of the variable name + # we just expanded. + lv = lvars.copy() + var = string.split(key, '.')[0] + lv[var] = '' + return self.substitute(s, lv) + else: + return s + elif is_List(s): + r = [] + for l in s: + r.append(self.str(self.substitute(l, lvars))) + return string.join(r) + elif callable(s): + s = s(target=self.target, + source=self.source, + env=self.env, + for_signature=(self.mode != SUBST_CMD)) + return self.substitute(s, lvars) + elif s is None: + return '' + else: + return s + + def substitute(self, args, lvars): + """Substitute expansions in an argument or list of arguments. + + This serves as a wrapper for splitting up a string into + separate tokens. + """ + if is_String(args) and not isinstance(args, CmdStringHolder): + args = _separate_args.findall(args) + result = [] + for a in args: + result.append(self.str(self.expand(a, lvars))) + return string.join(result, '') + else: + return self.expand(args, lvars) + + ss = StringSubber(env, mode, target, source) + result = ss.substitute(strSubst, subst_dict(target, source, env)) + + if is_String(result): + # Remove $(-$) pairs and any stuff in between, + # if that's appropriate. + remove = _regex_remove[mode] + if remove: + result = remove.sub('', result) + if mode != SUBST_RAW: + # Compress strings of white space characters into + # a single space. + result = string.strip(_space_sep.sub(' ', result)) + + return result - # Finally split each line up into a list of arguments: - return map(lambda x: map(CmdStringHolder, filter(lambda y:y, string.split(x, '\0\1'))), - listLines) +def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None): + """Substitute construction variables in a string (or list or other + object) and separate the arguments into a command list. -def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None): - """Recursively interpolates dictionary variables into - the specified string, returning the expanded result. - Variables are specified by a $ prefix in the string and - begin with an initial underscore or alphabetic character - followed by any number of underscores or alphanumeric - characters. The construction variable names may be - surrounded by curly braces to separate the name from - trailing characters. + The companion scons_subst() function (above) handles basic + substitutions within strings, so see that function instead + if that's what you're looking for. """ - - # This function needs to be fast, so don't call scons_subst_list - - remove = _regex_remove[mode] - strconv = _strconv[mode] - - def repl(m, - target=target, - source=source, - env=env, - local_vars = subst_dict(target, source, env), - global_vars = env.Dictionary(), - strconv=strconv, - sig=(mode != SUBST_CMD)): - key = m.group(1) - if key[0] == '{': - key = key[1:-1] - try: - e = eval(key, global_vars, local_vars) - except (IndexError, NameError, TypeError): - return '\0\5' - if callable(e): - e = e(target=target, source=source, env=env, - for_signature = sig) - - def conv(arg, strconv=strconv): - literal = 0 - try: - if arg.is_literal(): - literal = 1 - except AttributeError: - pass - ret = strconv(arg) - if literal: - # Escape dollar signs to prevent further - # substitution on literals. - ret = string.replace(ret, '$', '\0\4') - return ret - if e is None: - s = '' - elif is_List(e): - s = string.join(map(conv, e), ' ') - else: - s = conv(e) - # Insert placeholders to avoid accidentally smushing - # separate variables together. - return "\0\5" + s + "\0\5" - - # Now, do the substitution - n = 1 - while n != 0: - # escape double dollar signs - strSubst = string.replace(strSubst, '$$', '\0\4') - strSubst,n = _cv.subn(repl, strSubst) - - # remove the remove regex - if remove: - strSubst = remove.sub('', strSubst) - - # Un-escape the string - strSubst = string.replace(string.replace(strSubst, '\0\4', '$'), - '\0\5', '') - # strip out redundant white-space - if mode != SUBST_RAW: - strSubst = string.strip(_space_sep.sub(' ', strSubst)) - return strSubst + class ListSubber(UserList.UserList): + """A class to construct the results of a scons_subst_list() call. + + Like StringSubber, this class binds a specific binds a specific + construction environment, mode, target and source with two methods + (substitute() and expand()) that handle the expansion. + + In addition, however, this class is used to track the state of + the result(s) we're gathering so we can do the appropriate thing + whenever we have to append another word to the result--start a new + line, start a new word, append to the current word, etc. We do + this by setting the "append" attribute to the right method so + that our wrapper methods only need ever call ListSubber.append(), + and the rest of the object takes care of doing the right thing + internally. + """ + def __init__(self, env, mode, target, source): + UserList.UserList.__init__(self, []) + self.env = env + self.mode = mode + self.target = target + self.source = source + + self.gvars = env.Dictionary() + + if self.mode == SUBST_RAW: + self.add_strip = lambda x, s=self: s.append(x) + else: + self.add_strip = lambda x, s=self: None + self.str = _strconv[mode] + self.in_strip = None + self.next_line() + + def expand(self, s, lvars, within_list): + """Expand a single "token" as necessary, appending the + expansion to the current result. + + This handles expanding different types of things (strings, + lists, callables) appropriately. It calls the wrapper + substitute() method to re-expand things as necessary, so that + the results of expansions of side-by-side strings still get + re-evaluated separately, not smushed together. + """ + if is_String(s): + try: + s0, s1 = s[:2] + except (IndexError, ValueError): + self.append(s) + return + if s0 == '$': + if s1 == '$': + self.append('$') + elif s1 == '(': + self.open_strip('$(') + elif s1 == ')': + self.close_strip('$)') + else: + key = s[1:] + if key[0] == '{': + key = key[1:-1] + try: + s = eval(key, self.gvars, lvars) + except (IndexError, NameError, TypeError): + return + else: + # Before re-expanding the result, handle + # recursive expansion by copying the local + # variable dictionary and overwriting a null + # string for the value of the variable name + # we just expanded. + lv = lvars.copy() + var = string.split(key, '.')[0] + lv[var] = '' + self.substitute(s, lv, 0) + self.this_word() + else: + self.append(s) + elif is_List(s): + for a in s: + self.substitute(a, lvars, 1) + self.next_word() + elif callable(s): + s = s(target=self.target, + source=self.source, + env=self.env, + for_signature=(self.mode != SUBST_CMD)) + self.substitute(s, lvars, within_list) + elif not s is None: + self.append(s) + + def substitute(self, args, lvars, within_list): + """Substitute expansions in an argument or list of arguments. + + This serves as a wrapper for splitting up a string into + separate tokens. + """ + if is_String(args) and not isinstance(args, CmdStringHolder): + args = _separate_args.findall(args) + for a in args: + if a[0] in ' \t\n\r\f\v': + if '\n' in a: + self.next_line() + elif within_list: + self.append(a) + else: + self.next_word() + else: + self.expand(a, lvars, within_list) + else: + self.expand(args, lvars, within_list) + + def next_line(self): + """Arrange for the next word to start a new line. This + is like starting a new word, except that we have to append + another line to the result.""" + UserList.UserList.append(self, []) + self.next_word() + def this_word(self): + """Arrange for the next word to append to the end of the + current last word in the result.""" + self.append = self.add_to_current_word + def next_word(self): + """Arrange for the next word to start a new word.""" + self.append = self.add_new_word + + def add_to_current_word(self, x): + if not self.in_strip or self.mode != SUBST_SIG: + self[-1][-1] = self[-1][-1] + x + def add_new_word(self, x): + if not self.in_strip or self.mode != SUBST_SIG: + try: + l = x.is_literal + except AttributeError: + literal = None + else: + literal = l() + self[-1].append(CmdStringHolder(self.str(x), literal)) + self.append = self.add_to_current_word + + def open_strip(self, x): + """Handle the "open strip" $( token.""" + self.add_strip(x) + self.in_strip = 1 + def close_strip(self, x): + """Handle the "close strip" $) token.""" + self.add_strip(x) + self.in_strip = None + + ls = ListSubber(env, mode, target, source) + ls.substitute(strSubst, subst_dict(target, source, env), 0) + + return ls.data def render_tree(root, child_func, prune=0, margin=[0], visited={}): """ @@ -657,7 +777,7 @@ def mapPaths(paths, dir, env=None): paths = [ paths ] ret = map(mapPathFunc, paths) return ret - + if hasattr(types, 'UnicodeType'): def is_String(e): @@ -673,7 +793,7 @@ class Proxy: subject. Inherit from this class to create a Proxy.""" def __init__(self, subject): self.__subject = subject - + def __getattr__(self, name): return getattr(self.__subject, name) @@ -882,7 +1002,7 @@ def AppendPath(oldpath, newpath, sep = os.pathsep): newpaths = paths + newpaths # append new paths newpaths.reverse() - + normpaths = [] paths = [] # now we add them only of they are unique diff --git a/src/engine/SCons/UtilTests.py b/src/engine/SCons/UtilTests.py index d6777086..3dbeebc4 100644 --- a/src/engine/SCons/UtilTests.py +++ b/src/engine/SCons/UtilTests.py @@ -101,9 +101,16 @@ else: class UtilTestCase(unittest.TestCase): def test_subst(self): - """Test the subst function""" + """Test the subst() function""" class MyNode(DummyNode): """Simple node work-alike with some extra stuff for testing.""" + def __init__(self, name): + DummyNode.__init__(self, name) + class Attribute: + pass + self.attribute = Attribute() + self.attribute.attr1 = 'attr$1-' + os.path.basename(name) + self.attribute.attr2 = 'attr$2-' + os.path.basename(name) def get_stuff(self, extra): return self.name + extra foo = 1 @@ -174,9 +181,9 @@ class UtilTestCase(unittest.TestCase): 'FUNC2' : lambda target, source, env, for_signature: ['x$CCC'], # Test recursion. - #'RECURSE' : 'foo $RECURSE bar', - #'RRR' : 'foo $SSS bar', - #'SSS' : '$RRR', + 'RECURSE' : 'foo $RECURSE bar', + 'RRR' : 'foo $SSS bar', + 'SSS' : '$RRR', } env = DummyEnv(loc) @@ -232,6 +239,19 @@ class UtilTestCase(unittest.TestCase): "test ${SOURCES[0:2].get_stuff('blah')}", "test foo/blah.cppblah /bar/ack.cppblah", + "test ${SOURCES.attribute.attr1}", + "test attr$1-blah.cpp attr$1-ack.cpp attr$1-ack.c", + + "test ${SOURCES.attribute.attr2}", + "test attr$2-blah.cpp attr$2-ack.cpp attr$2-ack.c", + + # Test adjacent expansions. + "foo$BAR", + "foobaz", + + "foo${BAR}", + "foobaz", + # Test that adjacent expansions don't get re-interpreted # together. The correct disambiguated expansion should be: # $XXX$HHH => ${FFF}III => GGGIII @@ -260,7 +280,7 @@ class UtilTestCase(unittest.TestCase): '$N', '', '$X', 'x', '$Y', 'x', - #'$R', '', + '$R', '', '$S', 'x y', '$LS', 'x y', '$L', 'x y', @@ -279,7 +299,6 @@ class UtilTestCase(unittest.TestCase): while cases: input, expect = cases[:2] expect = cvt(expect) - #print " " + input result = apply(scons_subst, (input, env), kwargs) if result != expect: if failed == 0: print @@ -310,15 +329,15 @@ class UtilTestCase(unittest.TestCase): "a aA b", "a aA b", - #"$RECURSE", - # "foo bar", - # "foo bar", - # "foo bar", + "$RECURSE", + "foo bar", + "foo bar", + "foo bar", - #"$RRR", - # "foo bar", - # "foo bar", - # "foo bar", + "$RRR", + "foo bar", + "foo bar", + "foo bar", ] failed = 0 @@ -358,12 +377,22 @@ class UtilTestCase(unittest.TestCase): def test_subst_list(self): """Testing the scons_subst_list() method...""" - target = [ DummyNode("./foo/bar.exe"), - DummyNode("/bar/baz with spaces.obj"), - DummyNode("../foo/baz.obj") ] - source = [ DummyNode("./foo/blah with spaces.cpp"), - DummyNode("/bar/ack.cpp"), - DummyNode("../foo/ack.c") ] + class MyNode(DummyNode): + """Simple node work-alike with some extra stuff for testing.""" + def __init__(self, name): + DummyNode.__init__(self, name) + class Attribute: + pass + self.attribute = Attribute() + self.attribute.attr1 = 'attr$1-' + os.path.basename(name) + self.attribute.attr2 = 'attr$2-' + os.path.basename(name) + + target = [ MyNode("./foo/bar.exe"), + MyNode("/bar/baz with spaces.obj"), + MyNode("../foo/baz.obj") ] + source = [ MyNode("./foo/blah with spaces.cpp"), + MyNode("/bar/ack.cpp"), + MyNode("../foo/ack.c") ] loc = { 'xxx' : None, @@ -405,6 +434,11 @@ class UtilTestCase(unittest.TestCase): 'FUNCCALL' : '${FUNC1("$AAA $FUNC2 $BBB")}', 'FUNC1' : lambda x: x, 'FUNC2' : lambda target, source, env, for_signature: ['x$CCC'], + + # Test recursion. + 'RECURSE' : 'foo $RECURSE bar', + 'RRR' : 'foo $SSS bar', + 'SSS' : '$RRR', } env = DummyEnv(loc) @@ -427,6 +461,26 @@ class UtilTestCase(unittest.TestCase): ["after"], ], + "foo$FFF", + [ + ["fooGGG"], + ], + + "foo${FFF}", + [ + ["fooGGG"], + ], + + "test ${SOURCES.attribute.attr1}", + [ + ["test", "attr$1-blah with spaces.cpp", "attr$1-ack.cpp", "attr$1-ack.c"], + ], + + "test ${SOURCES.attribute.attr2}", + [ + ["test", "attr$2-blah with spaces.cpp", "attr$2-ack.cpp", "attr$2-ack.c"], + ], + "$DO --in=$FOO --out=$BAR", [ ["do something", "--in=foo.in", "--out=bar with spaces.out"], @@ -459,7 +513,7 @@ class UtilTestCase(unittest.TestCase): # Test various combinations of strings, lists and functions. None, [[]], - #[None], [[]], + [None], [[]], '', [[]], [''], [[]], 'x', [['x']], @@ -481,17 +535,23 @@ class UtilTestCase(unittest.TestCase): ['$LS'], [['x y']], '$L', [['x', 'y']], ['$L'], [['x', 'y']], - #cs, [['cs']], - #[cs], [['cs']], - #cl, [['cl']], - #[cl], [['cl']], + cs, [['cs']], + [cs], [['cs']], + cl, [['cl']], + [cl], [['cl']], '$CS', [['cs']], ['$CS'], [['cs']], '$CL', [['cl']], ['$CL'], [['cl']], - # Test + # Test function calls within ${}. '$FUNCCALL', [['a', 'xc', 'b']], + + # Test handling of newlines in white space. + 'foo\nbar', [['foo'], ['bar']], + 'foo\n\nbar', [['foo'], ['bar']], + 'foo \n \n bar', [['foo'], ['bar']], + 'foo \nmiddle\n bar', [['foo'], ['middle'], ['bar']], ] kwargs = {'target' : target, 'source' : source} @@ -523,14 +583,68 @@ class UtilTestCase(unittest.TestCase): 'foo\nwith\nnewlines', 'bar\nwith\nnewlines', 'xyz']], cmd_list - cmd_list[0][0].escape(escape_func) - assert cmd_list[0][0] == 'abc', c - cmd_list[0][1].escape(escape_func) - assert cmd_list[0][1] == '**foo\nwith\nnewlines**', c - cmd_list[0][2].escape(escape_func) - assert cmd_list[0][2] == '**bar\nwith\nnewlines**', c - cmd_list[0][3].escape(escape_func) - assert cmd_list[0][3] == 'xyz', c + c = cmd_list[0][0].escape(escape_func) + assert c == 'abc', c + c = cmd_list[0][1].escape(escape_func) + assert c == '**foo\nwith\nnewlines**', c + c = cmd_list[0][2].escape(escape_func) + assert c == '**bar\nwith\nnewlines**', c + c = cmd_list[0][3].escape(escape_func) + assert c == 'xyz', c + + # Tests of the various SUBST_* modes of substitution. + subst_list_cases = [ + "test $xxx", + [["test"]], + [["test"]], + [["test"]], + + "test $($xxx$)", + [["test", "$($)"]], + [["test"]], + [["test"]], + + "test $( $xxx $)", + [["test", "$(", "$)"]], + [["test"]], + [["test"]], + + "$AAA ${AAA}A $BBBB $BBB", + [["a", "aA", "b"]], + [["a", "aA", "b"]], + [["a", "aA", "b"]], + + "$RECURSE", + [["foo", "bar"]], + [["foo", "bar"]], + [["foo", "bar"]], + + "$RRR", + [["foo", "bar"]], + [["foo", "bar"]], + [["foo", "bar"]], + ] + + failed = 0 + while subst_list_cases: + input, eraw, ecmd, esig = subst_list_cases[:4] + result = scons_subst_list(input, env, mode=SUBST_RAW) + if result != eraw: + if failed == 0: print + print " input %s => RAW %s did not match %s" % (repr(input), repr(result), repr(eraw)) + failed = failed + 1 + result = scons_subst_list(input, env, mode=SUBST_CMD) + if result != ecmd: + if failed == 0: print + print " input %s => CMD %s did not match %s" % (repr(input), repr(result), repr(ecmd)) + failed = failed + 1 + result = scons_subst_list(input, env, mode=SUBST_SIG) + if result != esig: + if failed == 0: print + print " input %s => SIG %s did not match %s" % (repr(input), repr(result), repr(esig)) + failed = failed + 1 + del subst_list_cases[:4] + assert failed == 0, "%d subst() mode cases failed" % failed def test_splitext(self): assert splitext('foo') == ('foo','') @@ -809,8 +923,7 @@ class UtilTestCase(unittest.TestCase): return '**' + cmd + '**' cmd_list = scons_subst_list(input_list, dummy_env) - map(lambda x, e=escape_func: x.escape(e), cmd_list[0]) - cmd_list = map(str, cmd_list[0]) + cmd_list = SCons.Util.escape_list(cmd_list[0], escape_func) assert cmd_list == ['BAZ', '**$BAR**'], cmd_list def test_SpecialAttrWrapper(self): @@ -822,13 +935,11 @@ class UtilTestCase(unittest.TestCase): return '**' + cmd + '**' cmd_list = scons_subst_list(input_list, dummy_env) - map(lambda x, e=escape_func: x.escape(e), cmd_list[0]) - cmd_list = map(str, cmd_list[0]) + cmd_list = SCons.Util.escape_list(cmd_list[0], escape_func) assert cmd_list == ['BAZ', '**$BAR**'], cmd_list cmd_list = scons_subst_list(input_list, dummy_env, mode=SUBST_SIG) - map(lambda x, e=escape_func: x.escape(e), cmd_list[0]) - cmd_list = map(str, cmd_list[0]) + cmd_list = SCons.Util.escape_list(cmd_list[0], escape_func) assert cmd_list == ['BAZ', '**BLEH**'], cmd_list def test_mapPaths(self): @@ -1057,7 +1168,7 @@ class UtilTestCase(unittest.TestCase): r = adjustixes('pre-file.xxx', 'pre-', '-suf') assert r == 'pre-file.xxx', r r = adjustixes('dir/file', 'pre-', '-suf') - assert r == 'dir/pre-file-suf', r + assert r == os.path.join('dir', 'pre-file-suf'), r if __name__ == "__main__": suite = unittest.makeSuite(UtilTestCase, 'test_')