From 19b99578262cb834194730bd7787f0d60db629bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20Tekin=20=C3=96z?= Date: Tue, 19 Nov 2024 12:32:57 +0300 Subject: [PATCH] Add registry mirrors, preload snapd and core20 (#799) --------- Co-authored-by: Lucian Petrut --- .github/workflows/integration.yaml | 1 + .github/workflows/nightly-test.yaml | 1 + k8s/scripts/inspect.sh | 14 ++ .../integration/templates/registry/hosts.toml | 2 + .../templates/registry/registry-config.yaml | 22 +++ .../templates/registry/registry.service | 15 ++ tests/integration/tests/conftest.py | 57 ++++++ tests/integration/tests/test_util/config.py | 29 +++ tests/integration/tests/test_util/registry.py | 178 ++++++++++++++++++ tests/integration/tests/test_util/util.py | 8 +- 10 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 tests/integration/templates/registry/hosts.toml create mode 100644 tests/integration/templates/registry/registry-config.yaml create mode 100644 tests/integration/templates/registry/registry.service create mode 100644 tests/integration/tests/test_util/registry.py diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index bfcc903a0..2aefbb939 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -109,6 +109,7 @@ jobs: TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic" # Upgrading from 1.30 is not supported. TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31" + TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}]' run: | cd tests/integration && sg lxd -c 'tox -e integration' - name: Prepare inspection reports diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml index d2fd49972..3e7fd8972 100644 --- a/.github/workflows/nightly-test.yaml +++ b/.github/workflows/nightly-test.yaml @@ -48,6 +48,7 @@ jobs: # Upgrading from 1.30 is not supported. TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31" TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict" + TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}]' run: | export PATH="/home/runner/.local/bin:$PATH" cd tests/integration && sg lxd -c 'tox -vve integration' diff --git a/k8s/scripts/inspect.sh b/k8s/scripts/inspect.sh index a4ae51cc5..9f559451c 100755 --- a/k8s/scripts/inspect.sh +++ b/k8s/scripts/inspect.sh @@ -111,6 +111,17 @@ function collect_service_diagnostics { journalctl -n 100000 -u "snap.$service" &>"$INSPECT_DUMP/$service/journal.log" } +function collect_registry_mirror_logs { + local mirror_units=`systemctl list-unit-files --state=enabled | grep "registry-" | awk '{print $1}'` + if [ -n "$mirror_units" ]; then + mkdir -p "$INSPECT_DUMP/mirrors" + + for mirror_unit in $mirror_units; do + journalctl -n 100000 -u "$mirror_unit" &>"$INSPECT_DUMP/mirrors/$mirror_unit.log" + done + fi +} + function collect_network_diagnostics { log_info "Copy network diagnostics to the final report tarball" ip a &>"$INSPECT_DUMP/ip-a.log" || true @@ -182,6 +193,9 @@ else check_expected_services "${worker_services[@]}" fi +printf -- 'Collecting registry mirror logs\n' +collect_registry_mirror_logs + printf -- 'Collecting service arguments\n' collect_args diff --git a/tests/integration/templates/registry/hosts.toml b/tests/integration/templates/registry/hosts.toml new file mode 100644 index 000000000..416c9a642 --- /dev/null +++ b/tests/integration/templates/registry/hosts.toml @@ -0,0 +1,2 @@ +[host."http://$IP:$PORT"] +capabilities = ["pull", "resolve"] diff --git a/tests/integration/templates/registry/registry-config.yaml b/tests/integration/templates/registry/registry-config.yaml new file mode 100644 index 000000000..28610ffbb --- /dev/null +++ b/tests/integration/templates/registry/registry-config.yaml @@ -0,0 +1,22 @@ +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry/$NAME +http: + addr: :$PORT + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +proxy: + remoteurl: $REMOTE + username: $USERNAME + password: $PASSWORD diff --git a/tests/integration/templates/registry/registry.service b/tests/integration/templates/registry/registry.service new file mode 100644 index 000000000..83de843a6 --- /dev/null +++ b/tests/integration/templates/registry/registry.service @@ -0,0 +1,15 @@ + [Unit] + Description=registry-$NAME + Documentation=https://github.com/distribution/distribution + + [Service] + Type=simple + Restart=always + RestartSec=5s + LimitNOFILE=40000 + TimeoutStartSec=0 + + ExecStart=/bin/registry serve /etc/distribution/$NAME.yaml + + [Install] + WantedBy=multi-user.target diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py index 04efacd98..7333c2167 100644 --- a/tests/integration/tests/conftest.py +++ b/tests/integration/tests/conftest.py @@ -9,9 +9,14 @@ import pytest from test_util import config, harness, util from test_util.etcd import EtcdCluster +from test_util.registry import Registry LOG = logging.getLogger(__name__) +# The following snaps will be downloaded once per test run and preloaded +# into the harness instances to reduce the number of downloads. +PRELOADED_SNAPS = ["snapd", "core20"] + def _harness_clean(h: harness.Harness): "Clean up created instances within the test harness." @@ -79,6 +84,35 @@ def h() -> harness.Harness: _harness_clean(h) +@pytest.fixture(scope="session") +def registry(h: harness.Harness) -> Optional[Registry]: + if config.USE_LOCAL_MIRROR: + yield Registry(h) + else: + # local image mirror disabled, avoid initializing the + # registry mirror instance. + yield None + + +@pytest.fixture(scope="session", autouse=True) +def snapd_preload() -> None: + if not config.PRELOAD_SNAPS: + LOG.info("Snap preloading disabled, skipping...") + return + + LOG.info(f"Downloading snaps for preloading: {PRELOADED_SNAPS}") + for snap in PRELOADED_SNAPS: + util.run( + [ + "snap", + "download", + snap, + f"--basename={snap}", + "--target-directory=/tmp", + ] + ) + + def pytest_configure(config): config.addinivalue_line( "markers", @@ -141,6 +175,7 @@ def network_type(request) -> Union[str, None]: @pytest.fixture(scope="function") def instances( h: harness.Harness, + registry: Registry, node_count: int, tmp_path: Path, disable_k8s_bootstrapping: bool, @@ -163,9 +198,31 @@ def instances( # Create instances and setup the k8s snap in each. instance = h.new_instance(network_type=network_type) instances.append(instance) + + if config.PRELOAD_SNAPS: + for preloaded_snap in PRELOADED_SNAPS: + ack_file = f"{preloaded_snap}.assert" + remote_path = (tmp_path / ack_file).as_posix() + instance.send_file( + source=f"/tmp/{ack_file}", + destination=remote_path, + ) + instance.exec(["snap", "ack", remote_path]) + + snap_file = f"{preloaded_snap}.snap" + remote_path = (tmp_path / snap_file).as_posix() + instance.send_file( + source=f"/tmp/{snap_file}", + destination=remote_path, + ) + instance.exec(["snap", "install", remote_path]) + if not no_setup: util.setup_k8s_snap(instance, tmp_path, snap) + if config.USE_LOCAL_MIRROR: + registry.apply_configuration(instance) + if not disable_k8s_bootstrapping and not no_setup: first_node, *_ = instances diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py index 40e375c23..4ba31b337 100644 --- a/tests/integration/tests/test_util/config.py +++ b/tests/integration/tests/test_util/config.py @@ -1,6 +1,7 @@ # # Copyright 2024 Canonical, Ltd. # +import json import os from pathlib import Path @@ -21,6 +22,18 @@ # ETCD_VERSION is the version of etcd to use. ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.4.34" +# REGISTRY_DIR contains all templates required to setup an registry mirror. +REGISTRY_DIR = MANIFESTS_DIR / "registry" + +# REGISTRY_URL is the url from which the registry binary should be downloaded. +REGISTRY_URL = ( + os.getenv("REGISTRY_URL") + or "https://github.com/distribution/distribution/releases/download" +) + +# REGISTRY_VERSION is the version of registry to use. +REGISTRY_VERSION = os.getenv("REGISTRY_VERSION") or "v2.8.3" + # FLAVOR is the flavor of the snap to use. FLAVOR = os.getenv("TEST_FLAVOR") or "" @@ -129,3 +142,19 @@ STRICT_INTERFACE_CHANNELS = ( os.environ.get("TEST_STRICT_INTERFACE_CHANNELS", "").strip().split() ) + +# Cache and preload certain snaps such as snapd and core20 to avoid fetching them +# for every test instance. Note that k8s-snap is currently based on core20. +PRELOAD_SNAPS = (os.getenv("TEST_PRELOAD_SNAPS") or "1") == "1" + +# Setup a local image mirror to reduce the number of image pulls. The mirror +# will be configured to run in a session scoped harness instance (e.g. LXD container) +USE_LOCAL_MIRROR = (os.getenv("TEST_USE_LOCAL_MIRROR") or "1") == "1" + +DEFAULT_MIRROR_LIST = [ + {"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io"}, + {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io"}, +] + +# Local mirror configuration. +MIRROR_LIST = json.loads(os.getenv("TEST_MIRROR_LIST", "{}")) or DEFAULT_MIRROR_LIST diff --git a/tests/integration/tests/test_util/registry.py b/tests/integration/tests/test_util/registry.py new file mode 100644 index 000000000..6f2bd0e52 --- /dev/null +++ b/tests/integration/tests/test_util/registry.py @@ -0,0 +1,178 @@ +# +# Copyright 2024 Canonical, Ltd. +# +import logging +from string import Template +from typing import List, Optional + +from test_util import config +from test_util.harness import Harness, Instance +from test_util.util import get_default_ip + +LOG = logging.getLogger(__name__) + + +class Mirror: + def __init__( + self, + name: str, + port: int, + remote: str, + username: Optional[str] = None, + password: Optional[str] = None, + ): + """ + Initialize the Mirror object. + + Args: + name (str): The name of the mirror. + port (int): The port of the mirror. + remote (str): The remote URL of the upstream registry. + username (str, optional): Authentication username. + password (str, optional): Authentication password. + """ + self.name = name + self.port = port + self.remote = remote + self.username = username + self.password = password + + +class Registry: + + def __init__(self, h: Harness): + """ + Initialize the Registry object. + + Args: + h (Harness): The test harness object. + """ + self.registry_url = config.REGISTRY_URL + self.registry_version = config.REGISTRY_VERSION + self.instance: Instance = None + self.harness: Harness = h + self._mirrors: List[Mirror] = self.get_configured_mirrors() + self.instance = self.harness.new_instance() + + arch = self.instance.arch + self.instance.exec( + [ + "curl", + "-L", + f"{self.registry_url}/{self.registry_version}/registry_{self.registry_version[1:]}_linux_{arch}.tar.gz", + "-o", + f"/tmp/registry_{self.registry_version}_linux_{arch}.tar.gz", + ] + ) + + self.instance.exec( + [ + "tar", + "xzvf", + f"/tmp/registry_{self.registry_version}_linux_{arch}.tar.gz", + "-C", + "/bin/", + "registry", + ], + ) + + self._ip = get_default_ip(self.instance) + + self.add_mirrors() + + def get_configured_mirrors(self) -> List[Mirror]: + mirrors: List[Mirror] = [] + for mirror_dict in config.MIRROR_LIST: + for field in ["name", "port", "remote"]: + if field not in mirror_dict: + raise Exception( + f"Invalid 'TEST_MIRROR_LIST' configuration. Missing field: {field}" + ) + + mirror = Mirror( + mirror_dict["name"], + mirror_dict["port"], + mirror_dict["remote"], + mirror_dict.get("username"), + mirror_dict.get("password"), + ) + mirrors.append(mirror) + return mirrors + + def add_mirrors(self): + for mirror in self._mirrors: + self.add_mirror(mirror) + + def add_mirror(self, mirror: Mirror): + substitutes = { + "NAME": mirror.name, + "PORT": mirror.port, + "REMOTE": mirror.remote, + "USERNAME": mirror.username or "", + "PASSWORD": mirror.password or "", + } + + self.instance.exec(["mkdir", "-p", "/etc/distribution"]) + self.instance.exec(["mkdir", "-p", f"/var/lib/registry/{mirror.name}"]) + + with open( + config.REGISTRY_DIR / "registry-config.yaml", "r" + ) as registry_template: + src = Template(registry_template.read()) + self.instance.exec( + ["dd", f"of=/etc/distribution/{mirror.name}.yaml"], + sensitive_kwargs=True, + input=str.encode(src.substitute(substitutes)), + ) + + with open(config.REGISTRY_DIR / "registry.service", "r") as registry_template: + src = Template(registry_template.read()) + self.instance.exec( + ["dd", f"of=/etc/systemd/system/registry-{mirror.name}.service"], + sensitive_kwargs=True, + input=str.encode(src.substitute(substitutes)), + ) + + self.instance.exec(["systemctl", "daemon-reload"]) + self.instance.exec(["systemctl", "enable", f"registry-{mirror.name}.service"]) + self.instance.exec(["systemctl", "start", f"registry-{mirror.name}.service"]) + + @property + def mirrors(self) -> List[Mirror]: + """ + Get the list of mirrors in the registry. + + Returns: + List[Mirror]: The list of mirrors. + """ + return self._mirrors + + @property + def ip(self) -> str: + """ + Get the IP address of the registry. + + Returns: + str: The IP address of the registry. + """ + return self._ip + + # Configure the specified instance to use this registry mirror. + def apply_configuration(self, instance): + for mirror in self.mirrors: + substitutes = { + "IP": self.ip, + "PORT": mirror.port, + } + + instance.exec(["mkdir", "-p", f"/etc/containerd/hosts.d/{mirror.name}"]) + + with open(config.REGISTRY_DIR / "hosts.toml", "r") as registry_template: + src = Template(registry_template.read()) + instance.exec( + [ + "dd", + f"of=/etc/containerd/hosts.d/{mirror.name}/hosts.toml", + ], + input=str.encode(src.substitute(substitutes)), + ) diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 5ae53f2e2..1c3106178 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -33,7 +33,13 @@ def run(command: list, **kwargs) -> subprocess.CompletedProcess: """Log and run command.""" kwargs.setdefault("check", True) - LOG.debug("Execute command %s (kwargs=%s)", shlex.join(command), kwargs) + sensitive_command = kwargs.pop("sensitive_command", False) + sensitive_kwargs = kwargs.pop("sensitive_kwargs", sensitive_command) + + logged_command = shlex.join(command) if not sensitive_command else "" + logged_kwargs = kwargs if not sensitive_kwargs else "" + + LOG.debug("Execute command %s (kwargs=%s)", logged_command, logged_kwargs) return subprocess.run(command, **kwargs)