From 68889b7e1f2eb3b27fa08483a5647384e97623fe Mon Sep 17 00:00:00 2001 From: stevenknight Date: Tue, 22 Jul 2003 18:19:40 +0000 Subject: [PATCH] Avoid hangs on POSIX systems when reading a lot of output from a piped commnd. (Christoph Wiedemann) git-svn-id: http://scons.tigris.org/svn/scons/trunk@746 fdb21ef1-2011-0410-befe-b5e4ea1792b1 --- src/CHANGES.txt | 2 + src/engine/SCons/ActionTests.py | 74 ++++++++++++++++++++---------- src/engine/SCons/Platform/posix.py | 35 +++++++++----- 3 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 4810df63..b84aa610 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -75,6 +75,8 @@ RELEASE 0.XX - XXX - Be smarter about linking: use $CC by default and $CXX only if we're linking with any C++ objects. + - Avoid SCons hanging when a piped command has a lot of output to read. + RELEASE 0.90 - Wed, 25 Jun 2003 14:24:52 -0500 diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index f980fb7a..d14edfdf 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -63,8 +63,16 @@ except: pass f.close() if os.environ.has_key( 'ACTPY_PIPE' ): - sys.stdout.write( 'act.py: stdout: executed act.py\\n' ) - sys.stderr.write( 'act.py: stderr: executed act.py\\n' ) + if os.environ.has_key( 'PIPE_STDOUT_MSG' ): + stdout_msg = os.environ['PIPE_STDOUT_MSG'] + else: + stdout_msg = "act.py: stdout: executed act.py\\n" + sys.stdout.write( stdout_msg ) + if os.environ.has_key( 'PIPE_STDERR_MSG' ): + stderr_msg = os.environ['PIPE_STDERR_MSG'] + else: + stderr_msg = "act.py: stderr: executed act.py\\n" + sys.stderr.write( stderr_msg ) sys.exit(0) """) @@ -528,39 +536,59 @@ class CommandActionTestCase(unittest.TestCase): self.test_execute() self.env['PSTDOUT'].close() pipe_out = test.read( pipe_file ) + if sys.platform == 'win32': cr = '\r' else: cr = '' - act_out = "act.py: stdout: executed act.py" - act_err = "act.py: stderr: executed act.py" - found = re.findall( "%s%s\n%s%s\n" % (act_out, cr, act_err, cr), - pipe_out ) - assert len(found) == 8, found + act_out = "act.py: stdout: executed act.py\n" + act_err = "act.py: stderr: executed act.py\n" + + # Since we are now using select(), stdout and stderr can be + # intermixed, so count the lines separately. + outlines = re.findall(act_out + cr, pipe_out) + errlines = re.findall(act_err + cr, pipe_out) + assert len(outlines) == 8, outlines + assert len(errlines) == 8, errlines # test redirection operators - def test_redirect(self, redir): + def test_redirect(self, redir, stdout_msg, stderr_msg): cmd = r'%s %s %s xyzzy %s' % (python, act_py, outfile, redir) pipe = open( pipe_file, "w" ) act = SCons.Action.CommandAction(cmd) - r = act([], [], self.env.Copy(PSTDOUT = pipe, PSTDERR = pipe)) + env = Environment( ENV = {'ACTPY_PIPE' : '1', + 'PIPE_STDOUT_MSG' : stdout_msg, + 'PIPE_STDERR_MSG' : stderr_msg}, + PIPE_BUILD = 1, + PSTDOUT = pipe, PSTDERR = pipe ) + r = act([], [], env) pipe.close() assert r == 0 return (test.read(outfile2, 'r'), test.read(pipe_file, 'r')) - - (redirected, pipe_out) = test_redirect(self,'> %s' % outfile2 ) - assert redirected == "%s\n" % act_out - assert pipe_out == "%s\n" % act_err - - (redirected, pipe_out) = test_redirect(self,'2> %s' % outfile2 ) - assert redirected == "%s\n" % act_err - assert pipe_out == "%s\n" % act_out - - (redirected, pipe_out) = test_redirect(self,'> %s 2>&1' % outfile2 ) - assert (redirected == "%s\n%s\n" % (act_out, act_err) or - redirected == "%s\n%s\n" % (act_err, act_out)) - assert pipe_out == "" - + + (redirected, pipe_out) = test_redirect(self,'> %s' % outfile2, + act_out, act_err) + assert redirected == act_out + assert pipe_out == act_err + + (redirected, pipe_out) = test_redirect(self,'2> %s' % outfile2, + act_out, act_err) + assert redirected == act_err + assert pipe_out == act_out + + (redirected, pipe_out) = test_redirect(self,'> %s 2>&1' % outfile2, + act_out, act_err) + assert (redirected == act_out + act_err or + redirected == act_err + act_out) + assert pipe_out == "" + + act_err = "Long Command Output\n"*3000 + # the size of the string should exceed the system's default block size + act_out = "" + (redirected, pipe_out) = test_redirect(self,'> %s' % outfile2, + act_out, act_err) + assert (redirected == act_out) + assert (pipe_out == act_err) def test_set_handler(self): """Test setting the command handler... diff --git a/src/engine/SCons/Platform/posix.py b/src/engine/SCons/Platform/posix.py index 3546a174..9b7e11c4 100644 --- a/src/engine/SCons/Platform/posix.py +++ b/src/engine/SCons/Platform/posix.py @@ -37,6 +37,7 @@ import os.path import popen2 import string import sys +import select import SCons.Util @@ -93,18 +94,33 @@ def fork_spawn(sh, escape, cmd, args, env): return stat | 0x80 return stat >> 8 +def process_cmd_output(cmd_stdout, cmd_stderr, stdout, stderr): + stdout_eof = stderr_eof = 0 + while not (stdout_eof and stderr_eof): + (i,o,e) = select.select([cmd_stdout, cmd_stderr], [], []) + if cmd_stdout in i: + str = cmd_stdout.read() + if len(str) == 0: + stdout_eof = 1 + elif stdout != None: + stdout.write(str) + if cmd_stderr in i: + str = cmd_stderr.read() + if len(str) == 0: + #sys.__stderr__.write( "stderr_eof=1\n" ) + stderr_eof = 1 + else: + #sys.__stderr__.write( "str(stderr) = %s\n" % str ) + stderr.write(str) + + def piped_env_spawn(sh, escape, cmd, args, env, stdout, stderr): # spawn using Popen3 combined with the env command # the command name and the command's stdout is written to stdout # the command's stderr is written to stderr s = _get_env_command( sh, escape, cmd, args, env) proc = popen2.Popen3(s, 1) - # process stdout - if stdout != None: - stdout.write(proc.fromchild.read()) - # process stderr - if stderr != None: - stderr.write(proc.childerr.read()) + process_cmd_output(proc.fromchild, proc.childerr, stdout, stderr) stat = proc.wait() if stat & 0xff: return stat | 0x80 @@ -151,12 +167,7 @@ def piped_fork_spawn(sh, escape, cmd, args, env, stdout, stderr): childErr = os.fdopen( rFdErr ) else: childErr = childOut - # process stdout - if stdout != None: - stdout.write( childOut.read() ) - # process stderr - if stderr != None: - stderr.write( childErr.read() ) + process_cmd_output(childOut, childErr, stdout, stderr) os.close( rFdOut ) if stdout != stderr: os.close( rFdErr ) -- 2.26.2