Add registry mirrors, preload snapd and core20 (#799)

---------

Co-authored-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>

.github/workflows/integration.yaml

@@ -109,6 +109,7 @@ jobs:
           TEST_VERSION_UPGRADE_CHANNELS: "recent 6 classic"
           # Upgrading from 1.30 is not supported.
           TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31"
+          TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}]'
         run: |
           cd tests/integration && sg lxd -c 'tox -e integration'
       - name: Prepare inspection reports

.github/workflows/nightly-test.yaml

@@ -48,6 +48,7 @@ jobs:
           # Upgrading from 1.30 is not supported.
           TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31"
           TEST_STRICT_INTERFACE_CHANNELS: "recent 6 strict"
+          TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}]'
         run: |
           export PATH="/home/runner/.local/bin:$PATH"
           cd tests/integration && sg lxd -c 'tox -vve integration'

k8s/scripts/inspect.sh

@@ -111,6 +111,17 @@ function collect_service_diagnostics {
   journalctl -n 100000 -u "snap.$service" &>"$INSPECT_DUMP/$service/journal.log"
 }
 
+function collect_registry_mirror_logs {
+  local mirror_units=`systemctl list-unit-files --state=enabled | grep "registry-" | awk '{print $1}'`
+  if [ -n "$mirror_units" ]; then
+    mkdir -p "$INSPECT_DUMP/mirrors"
+
+    for mirror_unit in $mirror_units; do
+      journalctl -n 100000 -u "$mirror_unit" &>"$INSPECT_DUMP/mirrors/$mirror_unit.log"
+    done
+  fi
+}
+
 function collect_network_diagnostics {
   log_info "Copy network diagnostics to the final report tarball"
   ip a &>"$INSPECT_DUMP/ip-a.log" || true
@@ -182,6 +193,9 @@ else
   check_expected_services "${worker_services[@]}"
 fi
 
+printf -- 'Collecting registry mirror logs\n'
+collect_registry_mirror_logs
+
 printf -- 'Collecting service arguments\n'
 collect_args
 

tests/integration/templates/registry/hosts.toml

@@ -0,0 +1,2 @@
+[host."http://$IP:$PORT"]
+capabilities = ["pull", "resolve"]

tests/integration/templates/registry/registry-config.yaml

@@ -0,0 +1,22 @@
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  cache:
+    blobdescriptor: inmemory
+  filesystem:
+    rootdirectory: /var/lib/registry/$NAME
+http:
+  addr: :$PORT
+  headers:
+    X-Content-Type-Options: [nosniff]
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3
+proxy:
+  remoteurl: $REMOTE
+  username: $USERNAME
+  password: $PASSWORD

tests/integration/templates/registry/registry.service

@@ -0,0 +1,15 @@
+    [Unit]
+    Description=registry-$NAME
+    Documentation=https://github.com/distribution/distribution
+
+    [Service]
+    Type=simple
+    Restart=always
+    RestartSec=5s
+    LimitNOFILE=40000
+    TimeoutStartSec=0
+
+    ExecStart=/bin/registry serve /etc/distribution/$NAME.yaml
+
+    [Install]
+    WantedBy=multi-user.target

tests/integration/tests/conftest.py

@@ -9,9 +9,14 @@
 import pytest
 from test_util import config, harness, util
 from test_util.etcd import EtcdCluster
+from test_util.registry import Registry
 
 LOG = logging.getLogger(__name__)
 
+# The following snaps will be downloaded once per test run and preloaded
+# into the harness instances to reduce the number of downloads.
+PRELOADED_SNAPS = ["snapd", "core20"]
+
 
 def _harness_clean(h: harness.Harness):
     "Clean up created instances within the test harness."
@@ -79,6 +84,35 @@ def h() -> harness.Harness:
     _harness_clean(h)
 
 
+@pytest.fixture(scope="session")
+def registry(h: harness.Harness) -> Optional[Registry]:
+    if config.USE_LOCAL_MIRROR:
+        yield Registry(h)
+    else:
+        # local image mirror disabled, avoid initializing the
+        # registry mirror instance.
+        yield None
+
+
+@pytest.fixture(scope="session", autouse=True)
+def snapd_preload() -> None:
+    if not config.PRELOAD_SNAPS:
+        LOG.info("Snap preloading disabled, skipping...")
+        return
+
+    LOG.info(f"Downloading snaps for preloading: {PRELOADED_SNAPS}")
+    for snap in PRELOADED_SNAPS:
+        util.run(
+            [
+                "snap",
+                "download",
+                snap,
+                f"--basename={snap}",
+                "--target-directory=/tmp",
+            ]
+        )
+
+
 def pytest_configure(config):
     config.addinivalue_line(
         "markers",
@@ -141,6 +175,7 @@ def network_type(request) -> Union[str, None]:
 @pytest.fixture(scope="function")
 def instances(
     h: harness.Harness,
+    registry: Registry,
     node_count: int,
     tmp_path: Path,
     disable_k8s_bootstrapping: bool,
@@ -163,9 +198,31 @@ def instances(
         # Create <node_count> instances and setup the k8s snap in each.
         instance = h.new_instance(network_type=network_type)
         instances.append(instance)
+
+        if config.PRELOAD_SNAPS:
+            for preloaded_snap in PRELOADED_SNAPS:
+                ack_file = f"{preloaded_snap}.assert"
+                remote_path = (tmp_path / ack_file).as_posix()
+                instance.send_file(
+                    source=f"/tmp/{ack_file}",
+                    destination=remote_path,
+                )
+                instance.exec(["snap", "ack", remote_path])
+
+                snap_file = f"{preloaded_snap}.snap"
+                remote_path = (tmp_path / snap_file).as_posix()
+                instance.send_file(
+                    source=f"/tmp/{snap_file}",
+                    destination=remote_path,
+                )
+                instance.exec(["snap", "install", remote_path])
+
         if not no_setup:
             util.setup_k8s_snap(instance, tmp_path, snap)
 
+            if config.USE_LOCAL_MIRROR:
+                registry.apply_configuration(instance)
+
     if not disable_k8s_bootstrapping and not no_setup:
         first_node, *_ = instances
 

tests/integration/tests/test_util/config.py

@@ -1,6 +1,7 @@
 #
 # Copyright 2024 Canonical, Ltd.
 #
+import json
 import os
 from pathlib import Path
 
@@ -21,6 +22,18 @@
 # ETCD_VERSION is the version of etcd to use.
 ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.4.34"
 
+# REGISTRY_DIR contains all templates required to setup an registry mirror.
+REGISTRY_DIR = MANIFESTS_DIR / "registry"
+
+# REGISTRY_URL is the url from which the registry binary should be downloaded.
+REGISTRY_URL = (
+    os.getenv("REGISTRY_URL")
+    or "https://github.com/distribution/distribution/releases/download"
+)
+
+# REGISTRY_VERSION is the version of registry to use.
+REGISTRY_VERSION = os.getenv("REGISTRY_VERSION") or "v2.8.3"
+
 # FLAVOR is the flavor of the snap to use.
 FLAVOR = os.getenv("TEST_FLAVOR") or ""
 
@@ -129,3 +142,19 @@
 STRICT_INTERFACE_CHANNELS = (
     os.environ.get("TEST_STRICT_INTERFACE_CHANNELS", "").strip().split()
 )
+
+# Cache and preload certain snaps such as snapd and core20 to avoid fetching them
+# for every test instance. Note that k8s-snap is currently based on core20.
+PRELOAD_SNAPS = (os.getenv("TEST_PRELOAD_SNAPS") or "1") == "1"
+
+# Setup a local image mirror to reduce the number of image pulls. The mirror
+# will be configured to run in a session scoped harness instance (e.g. LXD container)
+USE_LOCAL_MIRROR = (os.getenv("TEST_USE_LOCAL_MIRROR") or "1") == "1"
+
+DEFAULT_MIRROR_LIST = [
+    {"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io"},
+    {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io"},
+]
+
+# Local mirror configuration.
+MIRROR_LIST = json.loads(os.getenv("TEST_MIRROR_LIST", "{}")) or DEFAULT_MIRROR_LIST