From 2e0a73817d2816390c54bd2cd67ce87202be7deb Mon Sep 17 00:00:00 2001 From: PerseoGI Date: Fri, 27 Dec 2024 13:29:54 +0100 Subject: [PATCH] Refactor docker runner, updated logging --- conan/api/conan_api.py | 3 +- conan/api/output.py | 7 +- conan/internal/runner/docker.py | 237 +++++++++++++++++--------------- 3 files changed, 130 insertions(+), 117 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 0b280e5ed58..3060ff57091 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -1,4 +1,5 @@ import sys +from typing import Optional from conan.api.output import init_colorama from conan.api.subapi.cache import CacheAPI @@ -26,7 +27,7 @@ class ConanAPI: - def __init__(self, cache_folder=None): + def __init__(self, cache_folder: Optional[str]=None): version = sys.version_info if version.major == 2 or version.minor < 6: diff --git a/conan/api/output.py b/conan/api/output.py index cce692af638..fba7ca915cb 100644 --- a/conan/api/output.py +++ b/conan/api/output.py @@ -186,7 +186,7 @@ def rewrite_line(self, line): self.stream.flush() self._color = tmp_color - def _write_message(self, msg, fg=None, bg=None): + def _write_message(self, msg, fg=None, bg=None, newline=True): if isinstance(msg, dict): # For traces we can receive a dict already, we try to transform then into more natural # text @@ -206,8 +206,11 @@ def _write_message(self, msg, fg=None, bg=None): else: ret += "{}".format(msg) + if newline: + ret = "%s\n" % ret + with self.lock: - self.stream.write("{}\n".format(ret)) + self.stream.write(ret) self.stream.flush() def trace(self, msg): diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index ad23e646b96..1357eb1d040 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -1,68 +1,24 @@ -from collections import namedtuple +from argparse import Namespace import os +import sys import json import platform import shutil +from typing import Optional, NamedTuple import yaml +from conan.api.conan_api import ConanAPI from conan.api.model import ListPattern from conan.api.output import Color, ConanOutput -from conan.api.conan_api import ConfigAPI from conan.cli import make_abs_path from conan.internal.runner import RunnerException from conan.errors import ConanException +from conans.model.profile import Profile from conans.model.version import Version - - -def config_parser(file_path): - Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) - Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes', 'network']) - Conf = namedtuple('Conf', ['image', 'build', 'run']) - if file_path: - def _instans_or_error(value, obj): - if value and (not isinstance(value, obj)): - raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}") - return value - with open(file_path, 'r') as f: - runnerfile = yaml.safe_load(f) - return Conf( - image=_instans_or_error(runnerfile.get('image'), str), - build=Build( - dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), - build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), - build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), - cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), - ), - run=Run( - name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), - environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), - user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), - privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), - cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), - security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list), - volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), - network=_instans_or_error(runnerfile.get('run', {}).get('network'), str), - ) - ) - else: - return Conf( - image=None, - build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), - run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None, - security_opt=None, volumes=None, network=None) - ) - - -def _docker_info(msg, error=False): - fg=Color.BRIGHT_MAGENTA - if error: - fg=Color.BRIGHT_RED - ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) - ConanOutput().status(f'| {msg} |', fg=fg) - ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) +from pathlib import Path class DockerRunner: - def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): + def __init__(self, conan_api: ConanAPI, command: str, host_profile: Profile, build_profile: Profile, args: Namespace, raw_args: list[str]): import docker import docker.api.build try: @@ -89,13 +45,16 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar self.conan_api = conan_api self.build_profile = build_profile self.args = args - self.abs_host_path = make_abs_path(args.path) + abs_path = make_abs_path(args.path) + if abs_path is None: + raise ConanException("Could not determine absolute path") + self.abs_host_path = Path(abs_path) if args.format: raise ConanException("format argument is forbidden if running in a docker runner") # Container config # https://containers.dev/implementors/json_reference/ - self.configfile = config_parser(host_profile.runner.get('configfile')) + self.configfile = self.config_parser(host_profile.runner.get('configfile')) self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile self.docker_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context self.image = host_profile.runner.get('image') or self.configfile.image @@ -108,79 +67,89 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar self.container = None # Runner config> - self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') + self.abs_runner_home_path = self.abs_host_path / '.conanrunner' self.docker_user_name = self.configfile.run.user or 'root' self.abs_docker_path = os.path.join(f'/{self.docker_user_name}/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) + self.logger = DockerOutput(self.name) - def run(self): + def run(self) -> None: """ - run conan inside a Docker continer + Run conan inside a Docker container """ - if self.dockerfile: - _docker_info(f'Building the Docker image: {self.image}') - self.build_image() - volumes, environment = self.create_runner_environment() - error = False - try: - if self.docker_client.containers.list(all=True, filters={'name': self.name}): - _docker_info('Starting the docker container') - self.container = self.docker_client.containers.get(self.name) - self.container.start() - else: - if self.configfile.run.environment: - environment.update(self.configfile.run.environment) - if self.configfile.run.volumes: - volumes.update(self.configfile.run.volumes) - _docker_info('Creating the docker container') - self.container = self.docker_client.containers.run( - self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - name=self.name, - volumes=volumes, - environment=environment, - user=self.configfile.run.user, - privileged=self.configfile.run.privileged, - cap_add=self.configfile.run.cap_add, - security_opt=self.configfile.run.security_opt, - detach=True, - auto_remove=False, - network=self.configfile.run.network) - _docker_info(f'Container {self.name} running') - except Exception as e: - raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' - f'\n\n{str(e)}') + self.build_image() + self.start_container() try: self.init_container() self.run_command(self.command) self.update_local_cache() - except ConanException as e: - error = True - raise e except RunnerException as e: - error = True raise ConanException(f'"{e.command}" inside docker fail' f'\n\nLast command output: {str(e.stdout_log)}') finally: if self.container: - error_prefix = 'ERROR: ' if error else '' - _docker_info(f'{error_prefix}Stopping container', error) + error = sys.exc_info()[0] is not None # Check if error has been raised + log = self.logger.error if error else self.logger.status + log('Stopping container') self.container.stop() if self.remove: - _docker_info(f'{error_prefix}Removing container', error) + log('Removing container') self.container.remove() - def build_image(self): + Build = NamedTuple('Build', [('dockerfile', Optional[str]), ('build_context', Optional[str]), ('build_args', Optional[dict]), ('cache_from', Optional[list])]) + Run = NamedTuple('Run', [('name', Optional[str]), ('environment', Optional[dict]), ('user', Optional[str]), ('privileged', Optional[bool]), ('cap_add', Optional[list]), ('security_opt', Optional[list]), ('volumes', Optional[dict]), ('network', Optional[str])]) + Conf = NamedTuple('Conf', [('image', Optional[str]), ('build', Build), ('run', Run)]) + + @staticmethod + def config_parser(file_path: str) -> Conf: + if file_path: + def _instans_or_error(value, obj): + if value and (not isinstance(value, obj)): + raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}") + return value + with open(file_path, 'r') as f: + runnerfile = yaml.safe_load(f) + return DockerRunner.Conf( + image=_instans_or_error(runnerfile.get('image'), str), + build=DockerRunner.Build( + dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), + build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), + build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), + cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), + ), + run=DockerRunner.Run( + name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), + environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), + user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), + privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), + cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), + security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list), + volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), + network=_instans_or_error(runnerfile.get('run', {}).get('network'), str), + ) + ) + else: + return DockerRunner.Conf( + image=None, + build=DockerRunner.Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), + run=DockerRunner.Run(name=None, environment=None, user=None, privileged=None, cap_add=None, + security_opt=None, volumes=None, network=None) + ) + + def build_image(self) -> None: + if not self.dockerfile: + return + self.logger.status(f'Building the Docker image: {self.image}') dockerfile_file_path = self.dockerfile if os.path.isdir(self.dockerfile): dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') with open(dockerfile_file_path) as f: build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path) - ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") - ConanOutput().highlight(f"Docker build context: '{build_path}'\n") + self.logger.highlight(f"Dockerfile path: '{dockerfile_file_path}'") + self.logger.highlight(f"Docker build context: '{build_path}'\n") docker_build_logs = self.docker_api.build( path=build_path, dockerfile=f.read(), @@ -189,16 +158,47 @@ def build_image(self): cache_from=self.configfile.build.cache_from, ) for chunk in docker_build_logs: - for line in chunk.decode("utf-8").split('\r\n'): - if line: - stream = json.loads(line).get('stream') - if stream: - ConanOutput().status(stream.strip()) + for line in chunk.decode("utf-8").split('\r\n'): + if line: + stream = json.loads(line).get('stream') + if stream: + ConanOutput().status(stream.strip()) + + def start_container(self) -> None: + volumes, environment = self.create_runner_environment() + try: + if self.docker_client.containers.list(all=True, filters={'name': self.name}): + self.logger.status('Starting the docker container', fg=Color.BRIGHT_MAGENTA) + self.container = self.docker_client.containers.get(self.name) + self.container.start() + else: + if self.configfile.run.environment: + environment.update(self.configfile.run.environment) + if self.configfile.run.volumes: + volumes.update(self.configfile.run.volumes) + self.logger.status('Creating the docker container', fg=Color.BRIGHT_MAGENTA) + self.container = self.docker_client.containers.run( + self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + user=self.configfile.run.user, + privileged=self.configfile.run.privileged, + cap_add=self.configfile.run.cap_add, + security_opt=self.configfile.run.security_opt, + detach=True, + auto_remove=False, + network=self.configfile.run.network) + self.logger.status(f'Container {self.name} running', fg=Color.BRIGHT_MAGENTA) + except Exception as e: + raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' + f'\n\n{str(e)}') - def run_command(self, command, workdir=None, log=True): + def run_command(self, command: str, workdir: Optional[str] = None, log: bool = True) -> tuple[str, str]: workdir = workdir or self.abs_docker_path if log: - _docker_info(f'Running in container: "{command}"') + self.logger.status(f'Running in container: "{command}"') exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", workdir=workdir, tty=True) exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,) stderr_log, stdout_log = '', '' @@ -224,7 +224,7 @@ def run_command(self, command, workdir=None, log=True): raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) return stdout_log, stderr_log - def create_runner_environment(self): + def create_runner_environment(self) -> tuple[dict, dict]: shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} @@ -246,17 +246,16 @@ def create_runner_environment(self): if self.cache == 'copy': tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') - _docker_info(f'Save host cache in: {tgz_path}') + self.logger.status(f'Save host cache in: {tgz_path}') self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) return volumes, environment - def init_container(self): + def init_container(self) -> None: min_conan_version = '2.1' stdout, _ = self.run_command('conan --version', log=True) docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color if Version(docker_conan_version) <= Version(min_conan_version): - ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) - raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') + raise ConanException(f'conan version inside the container must be greater than {min_conan_version}') if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) @@ -266,10 +265,20 @@ def init_container(self): if self.cache in ['copy']: self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"') - def update_local_cache(self): + def update_local_cache(self) -> None: if self.cache != 'shared': self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') - _docker_info(f'Restore host cache from: {tgz_path}') - package_list = self.conan_api.cache.restore(tgz_path) + self.logger.status(f'Restore host cache from: {tgz_path}') + self.conan_api.cache.restore(tgz_path) + +class DockerOutput(ConanOutput): + def __init__(self, image: str): + super().__init__() + self.image = image + + def _write_message(self, msg, fg=None, bg=None, newline=True): + super()._write_message(f"===> Docker Runner ({self.image}): ", Color.BLACK, + Color.BRIGHT_YELLOW, newline=False) + super()._write_message(msg, fg, bg, newline)