From 3a5f7502057734308d620d710307840fd944472a Mon Sep 17 00:00:00 2001 From: Julianne Swinoga Date: Thu, 9 Jan 2025 12:02:41 -0500 Subject: [PATCH 1/2] Add console keyword to run_command(), keeping stdout/stderr capture --- docs/yaml/functions/run_command.yaml | 8 +++ mesonbuild/interpreter/interpreter.py | 10 ++-- mesonbuild/interpreter/interpreterobjects.py | 54 +++++++++++++++++--- mesonbuild/interpreter/kwargs.py | 1 + mesonbuild/programs.py | 1 + test cases/common/33 run program/meson.build | 54 ++++++++++++++++++-- unittests/allplatformstests.py | 26 ++++++++++ 7 files changed, 138 insertions(+), 16 deletions(-) diff --git a/docs/yaml/functions/run_command.yaml b/docs/yaml/functions/run_command.yaml index 5803f82386bf..e13c4c0ddeb6 100644 --- a/docs/yaml/functions/run_command.yaml +++ b/docs/yaml/functions/run_command.yaml @@ -39,6 +39,14 @@ kwargs: the `.stdout()` method. If it is false, then `.stdout()` will return an empty string. + console: + type: bool + since: 1.7.0 + default: false + description: | + stdout and stderr are written to console as it is generated by the command. + Meant for commands that are resource-intensive and take a long time to finish. + env: type: env | list[str] | dict[str] since: 0.50.0 diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index 02a59e3986d5..8a63423b9e92 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -756,6 +756,7 @@ def validate_arguments(self, args, argcount, arg_types): 'run_command', KwargInfo('check', (bool, NoneType), since='0.47.0'), KwargInfo('capture', bool, default=True, since='0.47.0'), + KwargInfo('console', bool, default=False, since='1.7.0'), ENV_KW.evolve(since='0.50.0'), ) def func_run_command(self, node: mparser.BaseNode, @@ -770,7 +771,6 @@ def run_command_impl(self, kwargs: 'kwtypes.RunCommand', in_builddir: bool = False) -> RunProcess: cmd, cargs = args - capture = kwargs['capture'] env = kwargs['env'] srcdir = self.environment.get_source_dir() builddir = self.environment.get_build_dir() @@ -838,7 +838,8 @@ def run_command_impl(self, return RunProcess(cmd, expanded_args, env, srcdir, builddir, self.subdir, self.environment.get_build_command() + ['introspect'], - in_builddir=in_builddir, check=check, capture=capture) + in_builddir=in_builddir, check=check, capture=kwargs['capture'], + console=kwargs['console']) def func_option(self, nodes, args, kwargs): raise InterpreterException('Tried to call option() in build description file. All options must be in the option file.') @@ -2752,7 +2753,10 @@ def func_configure_file(self, node: mparser.BaseNode, args: T.List[TYPE_var], mlog.log('Configuring', mlog.bold(output), 'with command') cmd, *args = _cmd res = self.run_command_impl((cmd, args), - {'capture': True, 'check': True, 'env': EnvironmentVariables()}, + {'capture': True, + 'console': False, + 'check': True, + 'env': mesonlib.EnvironmentVariables()}, True) if kwargs['capture']: dst_tmp = ofile_abs + '~' diff --git a/mesonbuild/interpreter/interpreterobjects.py b/mesonbuild/interpreter/interpreterobjects.py index f4a2b4107ed3..fb89cfefd5c9 100644 --- a/mesonbuild/interpreter/interpreterobjects.py +++ b/mesonbuild/interpreter/interpreterobjects.py @@ -4,6 +4,8 @@ import subprocess import copy import textwrap +import threading +import sys from pathlib import Path, PurePath @@ -24,7 +26,7 @@ from ..interpreter.type_checking import NoneType, ENV_KW, ENV_SEPARATOR_KW, PKGCONFIG_DEFINE_KW from ..dependencies import Dependency, ExternalLibrary, InternalDependency from ..programs import ExternalProgram -from ..mesonlib import HoldableObject, listify, Popen_safe +from ..mesonlib import HoldableObject, listify import typing as T @@ -214,11 +216,13 @@ def __init__(self, mesonintrospect: T.List[str], in_builddir: bool = False, check: bool = False, - capture: bool = True) -> None: + capture: bool = True, + console: bool = False) -> None: super().__init__() if not isinstance(cmd, ExternalProgram): raise AssertionError('BUG: RunProcess must be passed an ExternalProgram') self.capture = capture + self.console = console self.returncode, self.stdout, self.stderr = self.run_command(cmd, args, env, source_dir, build_dir, subdir, mesonintrospect, in_builddir, check) self.methods.update({'returncode': self.returncode_method, 'stdout': self.stdout_method, @@ -248,18 +252,52 @@ def run_command(self, child_env = os.environ.copy() child_env.update(menv) child_env = env.get_env(child_env) - stdout = subprocess.PIPE if self.capture else subprocess.DEVNULL + + def proc_output_thread(pipe: T.IO, capture_list: T.List, io_obj: T.TextIO) -> None: + while True: + line = pipe.readline() + if not line: + break + if self.console: + io_obj.write(line.decode('utf-8', errors='replace')) + io_obj.flush() + if self.capture: + capture_list.append(line) + pipe.close() + + stdin: T.Union[T.TextIO, int] = sys.stdin if self.console else subprocess.DEVNULL + stdout = subprocess.PIPE + stderr = subprocess.PIPE + mlog.debug('Running command:', mesonlib.join_args(command_array)) try: - p, o, e = Popen_safe(command_array, stdout=stdout, env=child_env, cwd=cwd) + p = subprocess.Popen(command_array, stdin=stdin, stdout=stdout, stderr=stderr, env=child_env, cwd=cwd) + + o_list: T.List = [] + e_list: T.List = [] + stdout_thread = threading.Thread( + target=proc_output_thread, + kwargs={'pipe': p.stdout, 'capture_list': o_list, 'io_obj': sys.stdout}, daemon=True) + stderr_thread = threading.Thread( + target=proc_output_thread, + kwargs={'pipe': p.stderr, 'capture_list': e_list, 'io_obj': sys.stderr}, daemon=True) + for t in (stdout_thread, stderr_thread): + t.start() + + p.wait() + for t in (stdout_thread, stderr_thread): + t.join() + + o = b''.join(o_list).decode('utf-8', errors='replace') + e = b''.join(e_list).decode('utf-8', errors='replace') + if self.capture: mlog.debug('--- stdout ---') mlog.debug(o) + mlog.debug('--- stderr ---') + mlog.debug(e) else: - o = '' - mlog.debug('--- stdout disabled ---') - mlog.debug('--- stderr ---') - mlog.debug(e) + mlog.debug('--- capture output disabled ---') mlog.debug('') if check and p.returncode != 0: diff --git a/mesonbuild/interpreter/kwargs.py b/mesonbuild/interpreter/kwargs.py index 87f121e90b0f..8e728c832ca7 100644 --- a/mesonbuild/interpreter/kwargs.py +++ b/mesonbuild/interpreter/kwargs.py @@ -248,6 +248,7 @@ class RunCommand(TypedDict): check: bool capture: T.Optional[bool] + console: T.Optional[bool] env: EnvironmentVariables diff --git a/mesonbuild/programs.py b/mesonbuild/programs.py index d01440cce193..c36d0ace3f8e 100644 --- a/mesonbuild/programs.py +++ b/mesonbuild/programs.py @@ -110,6 +110,7 @@ def get_version(self, interpreter: T.Optional['Interpreter'] = None) -> str: if interpreter: res = interpreter.run_command_impl((self, [self.version_arg]), {'capture': True, + 'console': False, 'check': True, 'env': mesonlib.EnvironmentVariables()}, True) diff --git a/test cases/common/33 run program/meson.build b/test cases/common/33 run program/meson.build index 2257d93c7fc1..1c8f47b6e9a6 100644 --- a/test cases/common/33 run program/meson.build +++ b/test cases/common/33 run program/meson.build @@ -57,13 +57,57 @@ endif py3 = import('python3').find_python() -ret = run_command(py3, '-c', 'print("some output")', check: false) +# No runtime output expected, not captured +ret = run_command( + py3, + '-c', + 'import sys; print("stdout|capture:false,console:false"); print("stderr|capture:false,console:false", file=sys.stderr)', + check: true, + capture: false, + console: false, +) assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr()) -assert(ret.stdout() == 'some output\n', 'failed to run python3') - -ret = run_command(py3, '-c', 'print("some output")', check: false, capture: false) +assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout())) +assert(ret.stderr() == '', 'stderr is "@0@" instead of empty'.format(ret.stderr())) + +# Expect runtime output, not captured +ret = run_command( + py3, + '-c', + 'import sys; print("stdout|capture:false,console:true"); print("stderr|capture:false,console:true", file=sys.stderr)', + check: true, + capture: false, + console: true, +) assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr()) assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout())) +assert(ret.stderr() == '', 'stderr is "@0@" instead of empty'.format(ret.stderr())) + +# No runtime output expected, but captured +ret = run_command( + py3, + '-c', + 'import sys; print("stdout|capture:true,console:false"); print("stderr|capture:true,console:false", file=sys.stderr)', + check: true, + capture: true, + console: false, +) +assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr()) +assert(ret.stdout() == 'stdout|capture:true,console:false\n', 'failed to capture stdout:' + ret.stdout()) +assert(ret.stderr() == 'stderr|capture:true,console:false\n', 'failed to capture stderr:' + ret.stderr()) + +# Expect runtime output, and captured +ret = run_command( + py3, + '-c', + 'import sys; print("stdout|capture:true,console:true"); print("stderr|capture:true,console:true", file=sys.stderr)', + check: true, + capture: true, + console: true, +) +assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr()) +assert(ret.stdout() == 'stdout|capture:true,console:true\n', 'failed to capture stdout:' + ret.stdout()) +assert(ret.stderr() == 'stderr|capture:true,console:true\n', 'failed to capture stderr:' + ret.stderr()) c_env = environment() c_env.append('CUSTOM_ENV_VAR', 'FOOBAR') @@ -73,7 +117,7 @@ assert(ret.stdout() == 'FOOBAR\n', 'stdout is "@0@" instead of FOOBAR'.format(re dd = find_program('dd', required : false) if dd.found() - ret = run_command(dd, 'if=/dev/urandom', 'bs=10', 'count=1', check: false, capture: false) + ret = run_command(dd, 'if=/dev/urandom', 'bs=10', 'count=1', 'of=/dev/null', check: false, capture: false) assert(ret.returncode() == 0, 'failed to run dd: ' + ret.stderr()) assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout())) endif diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index 96576b0ee888..36afe7d31954 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -5084,3 +5084,29 @@ def test_rsp_support(self): 'link', 'lld-link', 'mwldarm', 'mwldeppc', 'optlink', 'xilink', } self.assertEqual(cc.linker.get_accepts_rsp(), has_rsp) + + def test_run_command_output(self): + ''' + Test that run_command will output to stdout/stderr if `check: false`. + ''' + testdir = os.path.join(self.common_test_dir, '33 run program') + + # inprocess=False uses BasePlatformTests::_run, which by default + # redirects all stderr to stdout, so we look for the expected stderr + # in the merged stdout. + # inprocess=True captures stderr and stdout separately, but doesn't + # return stderr (only printing it on failure) so unless we change the + # function signature we can't get at the stderr output + out = self.init(testdir, inprocess=False) + # No output at all + assert('stdout|capture:false,console:false' not in out) + assert('stderr|capture:false,console:false' not in out) + # Output not captured, but printed + assert('stdout|capture:false,console:true' in out) + assert('stderr|capture:false,console:true' in out) + # Output captured, but not printed + assert('stdout|capture:true,console:false' not in out) + assert('stderr|capture:true,console:false' not in out) + # Output captured and printed + assert('stdout|capture:true,console:true' in out) + assert('stderr|capture:true,console:true' in out) From 7ef69b0058500a64309a1dcaf67295a855790712 Mon Sep 17 00:00:00 2001 From: Julianne Swinoga Date: Thu, 9 Jan 2025 16:59:00 -0500 Subject: [PATCH 2/2] fixup! Add console keyword to run_command(), keeping stdout/stderr capture Line ending replace, to fix Windows CI --- mesonbuild/interpreter/interpreterobjects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/interpreter/interpreterobjects.py b/mesonbuild/interpreter/interpreterobjects.py index fb89cfefd5c9..f45401e6bdb7 100644 --- a/mesonbuild/interpreter/interpreterobjects.py +++ b/mesonbuild/interpreter/interpreterobjects.py @@ -288,8 +288,8 @@ def proc_output_thread(pipe: T.IO, capture_list: T.List, io_obj: T.TextIO) -> No for t in (stdout_thread, stderr_thread): t.join() - o = b''.join(o_list).decode('utf-8', errors='replace') - e = b''.join(e_list).decode('utf-8', errors='replace') + o = b''.join(o_list).decode('utf-8', errors='replace').replace('\r\n', '\n') + e = b''.join(e_list).decode('utf-8', errors='replace').replace('\r\n', '\n') if self.capture: mlog.debug('--- stdout ---')