Add console keyword to run_command()

docs/yaml/functions/run_command.yaml

@@ -39,6 +39,14 @@ kwargs:
       the `.stdout()` method. If it is false, then `.stdout()` will return an
       empty string.
 
+  console:
+    type: bool
+    since: 1.7.0
+    default: false
+    description: |
+      stdout and stderr are written to console as it is generated by the command.
+      Meant for commands that are resource-intensive and take a long time to finish.
+
   env:
     type: env | list[str] | dict[str]
     since: 0.50.0

mesonbuild/interpreter/interpreter.py

@@ -756,6 +756,7 @@ def validate_arguments(self, args, argcount, arg_types):
         'run_command',
         KwargInfo('check', (bool, NoneType), since='0.47.0'),
         KwargInfo('capture', bool, default=True, since='0.47.0'),
+        KwargInfo('console', bool, default=False, since='1.7.0'),
         ENV_KW.evolve(since='0.50.0'),
     )
     def func_run_command(self, node: mparser.BaseNode,
@@ -770,7 +771,6 @@ def run_command_impl(self,
                          kwargs: 'kwtypes.RunCommand',
                          in_builddir: bool = False) -> RunProcess:
         cmd, cargs = args
-        capture = kwargs['capture']
         env = kwargs['env']
         srcdir = self.environment.get_source_dir()
         builddir = self.environment.get_build_dir()
@@ -838,7 +838,8 @@ def run_command_impl(self,
 
         return RunProcess(cmd, expanded_args, env, srcdir, builddir, self.subdir,
                           self.environment.get_build_command() + ['introspect'],
-                          in_builddir=in_builddir, check=check, capture=capture)
+                          in_builddir=in_builddir, check=check, capture=kwargs['capture'],
+                          console=kwargs['console'])
 
     def func_option(self, nodes, args, kwargs):
         raise InterpreterException('Tried to call option() in build description file. All options must be in the option file.')
@@ -2752,7 +2753,10 @@ def func_configure_file(self, node: mparser.BaseNode, args: T.List[TYPE_var],
             mlog.log('Configuring', mlog.bold(output), 'with command')
             cmd, *args = _cmd
             res = self.run_command_impl((cmd, args),
-                                        {'capture': True, 'check': True, 'env': EnvironmentVariables()},
+                                        {'capture': True,
+                                         'console': False,
+                                         'check': True,
+                                         'env': mesonlib.EnvironmentVariables()},
                                         True)
             if kwargs['capture']:
                 dst_tmp = ofile_abs + '~'

mesonbuild/interpreter/interpreterobjects.py

@@ -4,6 +4,8 @@
 import subprocess
 import copy
 import textwrap
+import threading
+import sys
 
 from pathlib import Path, PurePath
 
@@ -24,7 +26,7 @@
 from ..interpreter.type_checking import NoneType, ENV_KW, ENV_SEPARATOR_KW, PKGCONFIG_DEFINE_KW
 from ..dependencies import Dependency, ExternalLibrary, InternalDependency
 from ..programs import ExternalProgram
-from ..mesonlib import HoldableObject, listify, Popen_safe
+from ..mesonlib import HoldableObject, listify
 
 import typing as T
 
@@ -214,11 +216,13 @@ def __init__(self,
                  mesonintrospect: T.List[str],
                  in_builddir: bool = False,
                  check: bool = False,
-                 capture: bool = True) -> None:
+                 capture: bool = True,
+                 console: bool = False) -> None:
         super().__init__()
         if not isinstance(cmd, ExternalProgram):
             raise AssertionError('BUG: RunProcess must be passed an ExternalProgram')
         self.capture = capture
+        self.console = console
         self.returncode, self.stdout, self.stderr = self.run_command(cmd, args, env, source_dir, build_dir, subdir, mesonintrospect, in_builddir, check)
         self.methods.update({'returncode': self.returncode_method,
                              'stdout': self.stdout_method,
@@ -248,18 +252,52 @@ def run_command(self,
         child_env = os.environ.copy()
         child_env.update(menv)
         child_env = env.get_env(child_env)
-        stdout = subprocess.PIPE if self.capture else subprocess.DEVNULL
+
+        def proc_output_thread(pipe: T.IO, capture_list: T.List, io_obj: T.TextIO) -> None:
+            while True:
+                line = pipe.readline()
+                if not line:
+                    break
+                if self.console:
+                    io_obj.write(line.decode('utf-8', errors='replace'))
+                    io_obj.flush()
+                if self.capture:
+                    capture_list.append(line)
+            pipe.close()
+
+        stdin: T.Union[T.TextIO, int] = sys.stdin if self.console else subprocess.DEVNULL
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+
         mlog.debug('Running command:', mesonlib.join_args(command_array))
         try:
-            p, o, e = Popen_safe(command_array, stdout=stdout, env=child_env, cwd=cwd)
+            p = subprocess.Popen(command_array, stdin=stdin, stdout=stdout, stderr=stderr, env=child_env, cwd=cwd)
+
+            o_list: T.List = []
+            e_list: T.List = []
+            stdout_thread = threading.Thread(
+                target=proc_output_thread,
+                kwargs={'pipe': p.stdout, 'capture_list': o_list, 'io_obj': sys.stdout}, daemon=True)
+            stderr_thread = threading.Thread(
+                target=proc_output_thread,
+                kwargs={'pipe': p.stderr, 'capture_list': e_list, 'io_obj': sys.stderr}, daemon=True)
+            for t in (stdout_thread, stderr_thread):
+                t.start()
+
+            p.wait()
+            for t in (stdout_thread, stderr_thread):
+                t.join()
+
+            o = b''.join(o_list).decode('utf-8', errors='replace').replace('\r\n', '\n')
+            e = b''.join(e_list).decode('utf-8', errors='replace').replace('\r\n', '\n')
+
             if self.capture:
                 mlog.debug('--- stdout ---')
                 mlog.debug(o)
+                mlog.debug('--- stderr ---')
+                mlog.debug(e)
             else:
-                o = ''
-                mlog.debug('--- stdout disabled ---')
-            mlog.debug('--- stderr ---')
-            mlog.debug(e)
+                mlog.debug('--- capture output disabled ---')
             mlog.debug('')
 
             if check and p.returncode != 0:

mesonbuild/interpreter/kwargs.py

@@ -248,6 +248,7 @@ class RunCommand(TypedDict):
 
     check: bool
     capture: T.Optional[bool]
+    console: T.Optional[bool]
     env: EnvironmentVariables
 
 

mesonbuild/programs.py

@@ -110,6 +110,7 @@ def get_version(self, interpreter: T.Optional['Interpreter'] = None) -> str:
             if interpreter:
                 res = interpreter.run_command_impl((self, [self.version_arg]),
                                                    {'capture': True,
+                                                    'console': False,
                                                     'check': True,
                                                     'env': mesonlib.EnvironmentVariables()},
                                                    True)

test cases/common/33 run program/meson.build

@@ -57,13 +57,57 @@ endif
 
 py3 = import('python3').find_python()
 
-ret = run_command(py3, '-c', 'print("some output")', check: false)
+# No runtime output expected, not captured
+ret = run_command(
+    py3,
+    '-c',
+    'import sys; print("stdout|capture:false,console:false"); print("stderr|capture:false,console:false", file=sys.stderr)',
+    check: true,
+    capture: false,
+    console: false,
+)
 assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr())
-assert(ret.stdout() == 'some output\n', 'failed to run python3')
-
-ret = run_command(py3, '-c', 'print("some output")', check: false, capture: false)
+assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout()))
+assert(ret.stderr() == '', 'stderr is "@0@" instead of empty'.format(ret.stderr()))
+
+# Expect runtime output, not captured
+ret = run_command(
+    py3,
+    '-c',
+    'import sys; print("stdout|capture:false,console:true"); print("stderr|capture:false,console:true", file=sys.stderr)',
+    check: true,
+    capture: false,
+    console: true,
+)
 assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr())
 assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout()))
+assert(ret.stderr() == '', 'stderr is "@0@" instead of empty'.format(ret.stderr()))
+
+# No runtime output expected, but captured
+ret = run_command(
+    py3,
+    '-c',
+    'import sys; print("stdout|capture:true,console:false"); print("stderr|capture:true,console:false", file=sys.stderr)',
+    check: true,
+    capture: true,
+    console: false,
+)
+assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr())
+assert(ret.stdout() == 'stdout|capture:true,console:false\n', 'failed to capture stdout:' + ret.stdout())
+assert(ret.stderr() == 'stderr|capture:true,console:false\n', 'failed to capture stderr:' + ret.stderr())
+
+# Expect runtime output, and captured
+ret = run_command(
+    py3,
+    '-c',
+    'import sys; print("stdout|capture:true,console:true"); print("stderr|capture:true,console:true", file=sys.stderr)',
+    check: true,
+    capture: true,
+    console: true,
+)
+assert(ret.returncode() == 0, 'failed to run python3: ' + ret.stderr())
+assert(ret.stdout() == 'stdout|capture:true,console:true\n', 'failed to capture stdout:' + ret.stdout())
+assert(ret.stderr() == 'stderr|capture:true,console:true\n', 'failed to capture stderr:' + ret.stderr())
 
 c_env = environment()
 c_env.append('CUSTOM_ENV_VAR', 'FOOBAR')
@@ -73,7 +117,7 @@ assert(ret.stdout() == 'FOOBAR\n', 'stdout is "@0@" instead of FOOBAR'.format(re
 
 dd = find_program('dd', required : false)
 if dd.found()
-  ret = run_command(dd, 'if=/dev/urandom', 'bs=10', 'count=1', check: false, capture: false)
+  ret = run_command(dd, 'if=/dev/urandom', 'bs=10', 'count=1', 'of=/dev/null', check: false, capture: false)
   assert(ret.returncode() == 0, 'failed to run dd: ' + ret.stderr())
   assert(ret.stdout() == '', 'stdout is "@0@" instead of empty'.format(ret.stdout()))
 endif

unittests/allplatformstests.py

@@ -5084,3 +5084,29 @@ def test_rsp_support(self):
             'link', 'lld-link', 'mwldarm', 'mwldeppc', 'optlink', 'xilink',
         }
         self.assertEqual(cc.linker.get_accepts_rsp(), has_rsp)
+
+    def test_run_command_output(self):
+        '''
+        Test that run_command will output to stdout/stderr if `check: false`.
+        '''
+        testdir = os.path.join(self.common_test_dir, '33 run program')
+
+        # inprocess=False uses BasePlatformTests::_run, which by default
+        # redirects all stderr to stdout, so we look for the expected stderr
+        # in the merged stdout.
+        # inprocess=True captures stderr and stdout separately, but doesn't
+        # return stderr (only printing it on failure) so unless we change the
+        # function signature we can't get at the stderr output
+        out = self.init(testdir, inprocess=False)
+        # No output at all
+        assert('stdout|capture:false,console:false' not in out)
+        assert('stderr|capture:false,console:false' not in out)
+        # Output not captured, but printed
+        assert('stdout|capture:false,console:true' in out)
+        assert('stderr|capture:false,console:true' in out)
+        # Output captured, but not printed
+        assert('stdout|capture:true,console:false' not in out)
+        assert('stderr|capture:true,console:false' not in out)
+        # Output captured and printed
+        assert('stdout|capture:true,console:true' in out)
+        assert('stderr|capture:true,console:true' in out)