Skip to content

Commit

Permalink
Refactor docker runner, updated logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@perseoGI
perseoGI committed Dec 31, 2024
1 parent e9e4bb1 commit 2e0a738
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 117 deletions.
3 changes: 2 additions & 1 deletion conan/api/conan_api.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import sys
from typing import Optional

from conan.api.output import init_colorama
from conan.api.subapi.cache import CacheAPI
Expand Down Expand Up @@ -26,7 +27,7 @@


class ConanAPI:
def __init__(self, cache_folder=None):
def __init__(self, cache_folder: Optional[str]=None):

version = sys.version_info
if version.major == 2 or version.minor < 6:
Expand Down
7 changes: 5 additions & 2 deletions conan/api/output.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ def rewrite_line(self, line):
self.stream.flush()
self._color = tmp_color

def _write_message(self, msg, fg=None, bg=None):
def _write_message(self, msg, fg=None, bg=None, newline=True):
if isinstance(msg, dict):
# For traces we can receive a dict already, we try to transform then into more natural
# text
Expand All @@ -206,8 +206,11 @@ def _write_message(self, msg, fg=None, bg=None):
else:
ret += "{}".format(msg)

if newline:
ret = "%s\n" % ret

with self.lock:
self.stream.write("{}\n".format(ret))
self.stream.write(ret)
self.stream.flush()

def trace(self, msg):
Expand Down
237 changes: 123 additions & 114 deletions conan/internal/runner/docker.py
View file
Original file line number Diff line number Diff line change
@@ -1,68 +1,24 @@
from collections import namedtuple
from argparse import Namespace
import os
import sys
import json
import platform
import shutil
from typing import Optional, NamedTuple
import yaml
from conan.api.conan_api import ConanAPI
from conan.api.model import ListPattern
from conan.api.output import Color, ConanOutput
from conan.api.conan_api import ConfigAPI
from conan.cli import make_abs_path
from conan.internal.runner import RunnerException
from conan.errors import ConanException
from conans.model.profile import Profile
from conans.model.version import Version


def config_parser(file_path):
Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from'])
Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes', 'network'])
Conf = namedtuple('Conf', ['image', 'build', 'run'])
if file_path:
def _instans_or_error(value, obj):
if value and (not isinstance(value, obj)):
raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}")
return value
with open(file_path, 'r') as f:
runnerfile = yaml.safe_load(f)
return Conf(
image=_instans_or_error(runnerfile.get('image'), str),
build=Build(
dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str),
build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str),
build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict),
cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list),
),
run=Run(
name=_instans_or_error(runnerfile.get('run', {}).get('name'), str),
environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict),
user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str),
privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool),
cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list),
security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list),
volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict),
network=_instans_or_error(runnerfile.get('run', {}).get('network'), str),
)
)
else:
return Conf(
image=None,
build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None),
run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None,
security_opt=None, volumes=None, network=None)
)


def _docker_info(msg, error=False):
fg=Color.BRIGHT_MAGENTA
if error:
fg=Color.BRIGHT_RED
ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg)
ConanOutput().status(f'| {msg} |', fg=fg)
ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg)
from pathlib import Path


class DockerRunner:
def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args):
def __init__(self, conan_api: ConanAPI, command: str, host_profile: Profile, build_profile: Profile, args: Namespace, raw_args: list[str]):
import docker
import docker.api.build
try:
Expand All @@ -89,13 +45,16 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
self.conan_api = conan_api
self.build_profile = build_profile
self.args = args
self.abs_host_path = make_abs_path(args.path)
abs_path = make_abs_path(args.path)
if abs_path is None:
raise ConanException("Could not determine absolute path")
self.abs_host_path = Path(abs_path)
if args.format:
raise ConanException("format argument is forbidden if running in a docker runner")

# Container config
# https://containers.dev/implementors/json_reference/
self.configfile = config_parser(host_profile.runner.get('configfile'))
self.configfile = self.config_parser(host_profile.runner.get('configfile'))
self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile
self.docker_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context
self.image = host_profile.runner.get('image') or self.configfile.image
Expand All @@ -108,79 +67,89 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
self.container = None

# Runner config>
self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner')
self.abs_runner_home_path = self.abs_host_path / '.conanrunner'
self.docker_user_name = self.configfile.run.user or 'root'
self.abs_docker_path = os.path.join(f'/{self.docker_user_name}/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/")

# Update conan command and some paths to run inside the container
raw_args[raw_args.index(args.path)] = self.abs_docker_path
self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json'])
self.logger = DockerOutput(self.name)

def run(self):
def run(self) -> None:
"""
run conan inside a Docker continer
Run conan inside a Docker container
"""
if self.dockerfile:
_docker_info(f'Building the Docker image: {self.image}')
self.build_image()
volumes, environment = self.create_runner_environment()
error = False
try:
if self.docker_client.containers.list(all=True, filters={'name': self.name}):
_docker_info('Starting the docker container')
self.container = self.docker_client.containers.get(self.name)
self.container.start()
else:
if self.configfile.run.environment:
environment.update(self.configfile.run.environment)
if self.configfile.run.volumes:
volumes.update(self.configfile.run.volumes)
_docker_info('Creating the docker container')
self.container = self.docker_client.containers.run(
self.image,
"/bin/bash -c 'while true; do sleep 30; done;'",
name=self.name,
volumes=volumes,
environment=environment,
user=self.configfile.run.user,
privileged=self.configfile.run.privileged,
cap_add=self.configfile.run.cap_add,
security_opt=self.configfile.run.security_opt,
detach=True,
auto_remove=False,
network=self.configfile.run.network)
_docker_info(f'Container {self.name} running')
except Exception as e:
raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"'
f'\n\n{str(e)}')
self.build_image()
self.start_container()
try:
self.init_container()
self.run_command(self.command)
self.update_local_cache()
except ConanException as e:
error = True
raise e
except RunnerException as e:
error = True
raise ConanException(f'"{e.command}" inside docker fail'
f'\n\nLast command output: {str(e.stdout_log)}')
finally:
if self.container:
error_prefix = 'ERROR: ' if error else ''
_docker_info(f'{error_prefix}Stopping container', error)
error = sys.exc_info()[0] is not None # Check if error has been raised
log = self.logger.error if error else self.logger.status
log('Stopping container')
self.container.stop()
if self.remove:
_docker_info(f'{error_prefix}Removing container', error)
log('Removing container')
self.container.remove()

def build_image(self):
Build = NamedTuple('Build', [('dockerfile', Optional[str]), ('build_context', Optional[str]), ('build_args', Optional[dict]), ('cache_from', Optional[list])])
Run = NamedTuple('Run', [('name', Optional[str]), ('environment', Optional[dict]), ('user', Optional[str]), ('privileged', Optional[bool]), ('cap_add', Optional[list]), ('security_opt', Optional[list]), ('volumes', Optional[dict]), ('network', Optional[str])])
Conf = NamedTuple('Conf', [('image', Optional[str]), ('build', Build), ('run', Run)])

@staticmethod
def config_parser(file_path: str) -> Conf:
if file_path:
def _instans_or_error(value, obj):
if value and (not isinstance(value, obj)):
raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}")
return value
with open(file_path, 'r') as f:
runnerfile = yaml.safe_load(f)
return DockerRunner.Conf(
image=_instans_or_error(runnerfile.get('image'), str),
build=DockerRunner.Build(
dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str),
build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str),
build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict),
cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list),
),
run=DockerRunner.Run(
name=_instans_or_error(runnerfile.get('run', {}).get('name'), str),
environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict),
user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str),
privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool),
cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list),
security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list),
volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict),
network=_instans_or_error(runnerfile.get('run', {}).get('network'), str),
)
)
else:
return DockerRunner.Conf(
image=None,
build=DockerRunner.Build(dockerfile=None, build_context=None, build_args=None, cache_from=None),
run=DockerRunner.Run(name=None, environment=None, user=None, privileged=None, cap_add=None,
security_opt=None, volumes=None, network=None)
)

def build_image(self) -> None:
if not self.dockerfile:
return
self.logger.status(f'Building the Docker image: {self.image}')
dockerfile_file_path = self.dockerfile
if os.path.isdir(self.dockerfile):
dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile')
with open(dockerfile_file_path) as f:
build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path)
ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'")
ConanOutput().highlight(f"Docker build context: '{build_path}'\n")
self.logger.highlight(f"Dockerfile path: '{dockerfile_file_path}'")
self.logger.highlight(f"Docker build context: '{build_path}'\n")
docker_build_logs = self.docker_api.build(
path=build_path,
dockerfile=f.read(),
Expand All @@ -189,16 +158,47 @@ def build_image(self):
cache_from=self.configfile.build.cache_from,
)
for chunk in docker_build_logs:
for line in chunk.decode("utf-8").split('\r\n'):
if line:
stream = json.loads(line).get('stream')
if stream:
ConanOutput().status(stream.strip())
for line in chunk.decode("utf-8").split('\r\n'):
if line:
stream = json.loads(line).get('stream')
if stream:
ConanOutput().status(stream.strip())

def start_container(self) -> None:
volumes, environment = self.create_runner_environment()
try:
if self.docker_client.containers.list(all=True, filters={'name': self.name}):
self.logger.status('Starting the docker container', fg=Color.BRIGHT_MAGENTA)
self.container = self.docker_client.containers.get(self.name)
self.container.start()
else:
if self.configfile.run.environment:
environment.update(self.configfile.run.environment)
if self.configfile.run.volumes:
volumes.update(self.configfile.run.volumes)
self.logger.status('Creating the docker container', fg=Color.BRIGHT_MAGENTA)
self.container = self.docker_client.containers.run(
self.image,
"/bin/bash -c 'while true; do sleep 30; done;'",
name=self.name,
volumes=volumes,
environment=environment,
user=self.configfile.run.user,
privileged=self.configfile.run.privileged,
cap_add=self.configfile.run.cap_add,
security_opt=self.configfile.run.security_opt,
detach=True,
auto_remove=False,
network=self.configfile.run.network)
self.logger.status(f'Container {self.name} running', fg=Color.BRIGHT_MAGENTA)
except Exception as e:
raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"'
f'\n\n{str(e)}')

def run_command(self, command, workdir=None, log=True):
def run_command(self, command: str, workdir: Optional[str] = None, log: bool = True) -> tuple[str, str]:
workdir = workdir or self.abs_docker_path
if log:
_docker_info(f'Running in container: "{command}"')
self.logger.status(f'Running in container: "{command}"')
exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", workdir=workdir, tty=True)
exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,)
stderr_log, stdout_log = '', ''
Expand All @@ -224,7 +224,7 @@ def run_command(self, command, workdir=None, log=True):
raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log)
return stdout_log, stderr_log

def create_runner_environment(self):
def create_runner_environment(self) -> tuple[dict, dict]:
shutil.rmtree(self.abs_runner_home_path, ignore_errors=True)
volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}}
environment = {'CONAN_RUNNER_ENVIRONMENT': '1'}
Expand All @@ -246,17 +246,16 @@ def create_runner_environment(self):

if self.cache == 'copy':
tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz')
_docker_info(f'Save host cache in: {tgz_path}')
self.logger.status(f'Save host cache in: {tgz_path}')
self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path)
return volumes, environment

def init_container(self):
def init_container(self) -> None:
min_conan_version = '2.1'
stdout, _ = self.run_command('conan --version', log=True)
docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color
if Version(docker_conan_version) <= Version(min_conan_version):
ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED)
raise ConanException( f'conan version inside the container must be greater than {min_conan_version}')
raise ConanException(f'conan version inside the container must be greater than {min_conan_version}')
if self.cache != 'shared':
self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False)
self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False)
Expand All @@ -266,10 +265,20 @@ def init_container(self):
if self.cache in ['copy']:
self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"')

def update_local_cache(self):
def update_local_cache(self) -> None:
if self.cache != 'shared':
self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False)
self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz')
tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz')
_docker_info(f'Restore host cache from: {tgz_path}')
package_list = self.conan_api.cache.restore(tgz_path)
self.logger.status(f'Restore host cache from: {tgz_path}')
self.conan_api.cache.restore(tgz_path)

class DockerOutput(ConanOutput):
def __init__(self, image: str):
super().__init__()
self.image = image

def _write_message(self, msg, fg=None, bg=None, newline=True):
super()._write_message(f"===> Docker Runner ({self.image}): ", Color.BLACK,
Color.BRIGHT_YELLOW, newline=False)
super()._write_message(msg, fg, bg, newline)

0 comments on commit 2e0a738

Please sign in to comment.