Fix nightly tests (#876)

* Fix nightly job

The runner labels used by the nightly job are not correctly
evaluated.

We'll fix this by switching to the newly added one-string node
labels.

* Apply Docker iptables workaround

* Fix test_node_cleanup_new_containerd_path test

The containerd base dir setting is currently ignored when joining
nodes. The reason is that yaml parser ignores a struct field which
is marked as private. Making it public fixes the issue.

At the same time, the test expects the default containerd folders
to be missing when configured to use a different base dir.
However, the tests are currently placing the registry settings
in /etc/containerd. We also get an empty /run/containerd folder.
For now, we'll update the test to ignore these folders.

There's also an os.path.join call that doesn't produce the expected
result if the last element is an absolute path, we'll need to remove
a slash when constructing the paths.

* Fix linter errors

* Run all tests for backports or when there are test changes

We're adding a few checks to the integration tests workflow. We'll
use the "weekly" tag in case of backports (targeting the release-*
branches) and whenever there are test changes.

* tests: Add containerd cfg dir marker

One of the tests uses a custom containerd base dir. We'll add a
marker so that the registry mirror settings can be applied to that
containerd folder.

* Add yaml parser note

.github/workflows/integration.yaml

@@ -107,7 +107,17 @@ jobs:
           TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31"
           TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]'
         run: |
-          cd tests/integration && sg lxd -c 'tox -e integration -- --tags pull_request'
+          tags="pull_request"
+          # Run all tests if there are test changes. In case of a PR, we'll
+          # get a merge commit that includes all changes.
+          if git diff HEAD HEAD~1 --name-only | grep "tests/"; then
+            tags="up_to_weekly"
+          fi
+          # Run all tests on backports.
+          if echo ${{ github.base_ref }} | grep "release-"; then
+            tags="up_to_weekly"
+          fi
+          cd tests/integration && sg lxd -c "tox -e integration -- --tags $tags"
       - name: Prepare inspection reports
         if: failure()
         run: |

.github/workflows/nightly-test.yaml

@@ -17,7 +17,7 @@ jobs:
         release: ["latest/edge"]
       fail-fast: false # TODO: remove once arm64 works
 
-    runs-on: ${{ matrix.arch == 'arm64' && ["self-hosted", "Linux", "ARM64", "jammy", "large"] || ["self-hosted", "Linux", "AMD64", "jammy", "large"] }}
+    runs-on: ${{ matrix.arch == 'arm64' && 'self-hosted-linux-arm64-jammy-large' || 'self-hosted-linux-amd64-jammy-large' }}
 
     steps:
       - name: Checking out repo
@@ -30,6 +30,8 @@ jobs:
           sudo lxd init --auto
           sudo usermod --append --groups lxd $USER
           sg lxd -c 'lxc version'
+          sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT
+          sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
       - name: Create build directory
         run: mkdir -p build
       - name: Install ${{ matrix.release }} k8s snap

src/k8s/pkg/k8sd/api/cluster_join.go

@@ -37,16 +37,18 @@ func (e *Endpoints) postClusterJoin(s state.State, r *http.Request) response.Res
 
 	joinConfig := struct {
 		// We only care about this field from the entire join config.
-		containerdBaseDir string `yaml:"containerd-base-dir,omitempty"`
+		// Note that the field must be public (uppercase) in order for the yaml
+		// parser to handle it.
+		ContainerdBaseDir string `yaml:"containerd-base-dir,omitempty"`
 	}{}
 
 	if err := yaml.Unmarshal([]byte(req.Config), &joinConfig); err != nil {
 		return response.BadRequest(fmt.Errorf("failed to parse request config: %w", err))
 	}
 
-	if joinConfig.containerdBaseDir != "" {
+	if joinConfig.ContainerdBaseDir != "" {
 		// append k8s-containerd to the given base dir, so we don't flood it with our own folders.
-		e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.containerdBaseDir, "k8s-containerd"))
+		e.provider.Snap().SetContainerdBaseDir(filepath.Join(joinConfig.ContainerdBaseDir, "k8s-containerd"))
 	}
 
 	config := map[string]string{}

tests/integration/tests/conftest.py

@@ -139,6 +139,7 @@ def pytest_configure(config):
         "bootstrap_config: Provide a custom bootstrap config to the bootstrapping node.\n"
         "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that.\n"
         "no_setup: No setup steps (pushing snap, bootstrapping etc.) are performed on any node for this test.\n"
+        "containerd_cfgdir: The instance containerd config directory, defaults to /etc/containerd."
         "network_type: Specify network type to use for the infrastructure (IPv4, Dualstack or IPv6).\n"
         "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n"
         "node_count: Mark a test to specify how many instance nodes need to be created\n"
@@ -192,6 +193,15 @@ def network_type(request) -> Union[str, None]:
     return network_type
 
 
+@pytest.fixture(scope="function")
+def containerd_cfgdir(request) -> str:
+    marker = request.node.get_closest_marker("containerd_cfgdir")
+    if not marker:
+        return "/etc/containerd"
+    cfgdir, *_ = marker.args
+    return cfgdir
+
+
 @pytest.fixture(scope="function")
 def instances(
     h: harness.Harness,
@@ -200,6 +210,7 @@ def instances(
     tmp_path: Path,
     disable_k8s_bootstrapping: bool,
     no_setup: bool,
+    containerd_cfgdir: str,
     bootstrap_config: Union[str, None],
     request,
     network_type: str,
@@ -241,7 +252,7 @@ def instances(
             util.setup_k8s_snap(instance, tmp_path, snap)
 
             if config.USE_LOCAL_MIRROR:
-                registry.apply_configuration(instance)
+                registry.apply_configuration(instance, containerd_cfgdir)
 
     if not disable_k8s_bootstrapping and not no_setup:
         first_node, *_ = instances

tests/integration/tests/test_cleanup.py

@@ -42,6 +42,7 @@ def test_node_cleanup(instances: List[harness.Instance], tmp_path):
 
 @pytest.mark.node_count(2)
 @pytest.mark.disable_k8s_bootstrapping()
+@pytest.mark.containerd_cfgdir("/home/ubuntu/etc/containerd")
 @pytest.mark.tags(tags.NIGHTLY)
 def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]):
     main = instances[0]
@@ -64,15 +65,26 @@ def test_node_cleanup_new_containerd_path(instances: List[harness.Instance]):
 
     boostrap_config = yaml.safe_load(containerd_path_bootstrap_config)
     new_containerd_paths = [
-        os.path.join(boostrap_config["containerd-base-dir"], "k8s-containerd", p)
+        os.path.join(
+            boostrap_config["containerd-base-dir"], "k8s-containerd", p.lstrip("/")
+        )
         for p in CONTAINERD_PATHS
     ]
+
+    # /run/containerd gets created but isn't actually used (requires further
+    # investigation).
+    exp_missing_paths = [
+        "/etc/containerd",
+        "/run/containerd/containerd.sock",
+        "/var/lib/containerd",
+    ]
+
     for instance in instances:
         # Check that the containerd-related folders are not in the default locations.
         process = instance.exec(
-            ["ls", *CONTAINERD_PATHS], capture_output=True, text=True, check=False
+            ["ls", *exp_missing_paths], capture_output=True, text=True, check=False
         )
-        for path in CONTAINERD_PATHS:
+        for path in exp_missing_paths:
             assert (
                 f"cannot access '{path}': No such file or directory" in process.stderr
             )

tests/integration/tests/test_util/registry.py

@@ -2,6 +2,7 @@
 # Copyright 2024 Canonical, Ltd.
 #
 import logging
+import os
 from pathlib import Path
 from string import Template
 from typing import List, Optional
@@ -165,21 +166,22 @@ def ip(self) -> str:
         return self._ip
 
     # Configure the specified instance to use this registry mirror.
-    def apply_configuration(self, instance):
+    def apply_configuration(self, instance, containerd_basedir="/etc/containerd"):
         for mirror in self.mirrors:
             substitutes = {
                 "IP": self.ip,
                 "PORT": mirror.port,
             }
 
-            instance.exec(["mkdir", "-p", f"/etc/containerd/hosts.d/{mirror.name}"])
+            mirror_dir = os.path.join(containerd_basedir, "hosts.d", mirror.name)
+            instance.exec(["mkdir", "-p", mirror_dir])
 
             with open(config.REGISTRY_DIR / "hosts.toml", "r") as registry_template:
                 src = Template(registry_template.read())
                 instance.exec(
                     [
                         "dd",
-                        f"of=/etc/containerd/hosts.d/{mirror.name}/hosts.toml",
+                        f"of={mirror_dir}/hosts.toml",
                     ],
                     input=str.encode(src.substitute(substitutes)),
                 )