Update agent charms

agent/charms/testflinger-agent-charm/config.yaml

@@ -1,18 +1,10 @@
 options:
-  testflinger-agent-repo:
+  testflinger-repo:
     type: string
-    description: git repo for testflinger-agent
-    default: "https://github.com/canonical/testflinger-agent"
-  testflinger-agent-branch:
+    description: git repo for testflinger
+    default: "https://github.com/canonical/testflinger"
+  testflinger-branch:
     type: string
-    description: git branch for testflinger-agent
-    default: "main"
-  device-agent-repo:
-    type: string
-    description: git repo for device-agent
-    default: "https://github.com/canonical/snappy-device-agents"
-  device-agent-branch:
-    type: string
-    description: git branch for device-agent
+    description: git branch for testflinger
     default: "main"
 

agent/charms/testflinger-agent-charm/src/charm.py

@@ -41,16 +41,14 @@ class TestflingerAgentCharm(CharmBase):
 
     def __init__(self, *args):
         super().__init__(*args)
-        self.framework.observe(self.on.install, self._on_install)
-        self.framework.observe(self.on.config_changed, self._on_config_changed)
-        self.framework.observe(self.on.start, self._on_start)
-        self.framework.observe(self.on.remove, self._on_remove)
-        self.framework.observe(self.on.update_action, self._on_update_action)
+        self.framework.observe(self.on.install, self.on_install)
+        self.framework.observe(self.on.config_changed, self.on_config_changed)
+        self.framework.observe(self.on.start, self.on_start)
+        self.framework.observe(self.on.remove, self.on_remove)
+        self.framework.observe(self.on.update_action, self.on_update_action)
         self._stored.set_default(
-            testflinger_agent_repo="",
-            testflinger_agent_branch="",
-            device_agent_repo="",
-            device_agent_branch="",
+            testflinger_repo="",
+            testflinger_branch="",
             unit_path=(
                 f"/etc/systemd/system/testflinger-agent-{self.app.name}"
                 ".service"
@@ -59,7 +57,7 @@ def __init__(self, *args):
             venv_path=f"/srv/testflinger-agent/{self.app.name}/env",
         )
 
-    def _on_install(self, _):
+    def on_install(self, _):
         """Install hook"""
         self.unit.status = MaintenanceStatus("Installing dependencies")
         # Ensure we have a fresh agent dir to start with
@@ -68,7 +66,7 @@ def _on_install(self, _):
         os.makedirs("/home/ubuntu/testflinger", exist_ok=True)
         shutil.chown("/home/ubuntu/testflinger", "ubuntu", "ubuntu")
 
-        self._install_apt_packages(
+        self.install_apt_packages(
             [
                 "python3-pip",
                 "python3-virtualenv",
@@ -79,12 +77,12 @@ def _on_install(self, _):
             ]
         )
         # Create the virtualenv
-        self._run_with_logged_errors(
+        self.run_with_logged_errors(
             ["python3", "-m", "virtualenv", f"{self._stored.venv_path}"],
         )
-        self._render_systemd_unit()
+        self.render_systemd_unit()
 
-    def _run_with_logged_errors(self, cmd):
+    def run_with_logged_errors(self, cmd):
         """Run a command, log output if errors, return proc just in case"""
         proc = subprocess.run(
             cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, text=True
@@ -93,106 +91,114 @@ def _run_with_logged_errors(self, cmd):
             logger.error(proc.stdout)
         return proc
 
-    def _write_file(self, location, contents):
+    def write_file(self, location, contents):
         # Sanity check to make sure we're actually about to write something
         if not contents:
             return
         with open(location, "w", encoding="utf-8", errors="ignore") as out:
             out.write(contents)
 
-    def _on_start(self, _):
+    def on_start(self, _):
         """Start the service"""
         service_name = f"testflinger-agent-{self.app.name}"
         systemd.service_restart(service_name)
         self.unit.status = ActiveStatus()
 
-    def _on_remove(self, _):
+    def on_remove(self, _):
         """Stop the service"""
         service_name = f"testflinger-agent-{self.app.name}"
         systemd.service_stop(service_name)
         # remove the old systemd unit file and agent directory
         try:
             os.unlink(self._stored.unit_path)
         except FileNotFoundError:
-            logger.error("No systemd unit file found when removing: %s",
-                         self._stored.unit_path)
+            logger.error(
+                "No systemd unit file found when removing: %s",
+                self._stored.unit_path,
+            )
         systemd.daemon_reload()
         shutil.rmtree(self._stored.agent_path, ignore_errors=True)
 
-    def _check_update_repos_needed(self):
+    def check_update_repos_needed(self):
         """
         Determine if any config settings change which require
         an update to the git repos
         """
-        update_repos = False
-        repo = self.config.get("testflinger-agent-repo")
-        if repo != self._stored.testflinger_agent_repo:
-            self._stored.testflinger_agent_repo = repo
-            update_repos = True
-        branch = self.config.get("testflinger-agent-branch")
-        if branch != self._stored.testflinger_agent_branch:
-            self._stored.testflinger_agent_branch = branch
-            update_repos = True
-        repo = self.config.get("device-agent-repo")
-        if repo != self._stored.device_agent_repo:
-            self._stored.device_agent_repo = repo
-            update_repos = True
-        branch = self.config.get("device-agent-branch")
-        if branch != self._stored.device_agent_branch:
-            self._stored.device_agent_branch = branch
-            update_repos = True
-        if update_repos:
-            self._update_repos()
+        update_needed = False
+        repo = self.config.get("testflinger-repo")
+        if repo != self._stored.testflinger_repo:
+            self._stored.testflinger_repo = repo
+            update_needed = True
+        branch = self.config.get("testflinger-branch")
+        if branch != self._stored.testflinger_branch:
+            self._stored.testflinger_branch = branch
+            update_needed = True
+        if update_needed:
+            self.update_repos()
 
-    def _update_repos(self):
+    def update_repos(self):
         """Recreate the git repos and reinstall everything needed"""
-        tf_agent_dir = f"{self._stored.agent_path}/testflinger-agent"
-        device_agent_dir = f"{self._stored.agent_path}/snappy-device-agents"
-        shutil.rmtree(tf_agent_dir, ignore_errors=True)
-        shutil.rmtree(device_agent_dir, ignore_errors=True)
-        Repo.clone_from(
-            self._stored.testflinger_agent_repo,
-            tf_agent_dir,
-            multi_options=[f"-b {self._stored.testflinger_agent_branch}"],
-        )
-        self._run_with_logged_errors(
-            [f"{self._stored.venv_path}/bin/pip3", "install", "-I",
-             tf_agent_dir]
+        self.cleanup_agent_dirs()
+        repo_path = f"{self._stored.agent_path}/testflinger"
+        repo = Repo.clone_from(
+            url=self._stored.testflinger_repo,
+            to_path=repo_path,
+            no_checkout=True,
+            depth=1,
         )
-        Repo.clone_from(
-            self._stored.device_agent_repo,
-            device_agent_dir,
-            multi_options=[f"-b {self._stored.device_agent_branch}"],
-        )
-        self._run_with_logged_errors(
-            [f"{self._stored.venv_path}/bin/pip3", "install", "-I",
-             device_agent_dir]
+        # do a sparse checkout of only agent and device-connectors
+        repo.git.checkout(
+            f"origin/{self._stored.testflinger_branch}",
+            "--",
+            "agent",
+            "device-connectors",
         )
+        # Install the agent and device-connectors
+        for dir in ("agent", "device-connectors"):
+            self.run_with_logged_errors(
+                [
+                    f"{self._stored.venv_path}/bin/pip3",
+                    "install",
+                    "-I",
+                    f"{repo_path}/{dir}",
+                ]
+            )
 
-    def _signal_restart_agent(self):
+    def cleanup_agent_dirs(self):
+        """Remove old agent dirs before checking out again"""
+        dirs_to_remove = ("testflinger",)
+        # Temporarily skip removing the following two things so that we
+        # don't accidentally remove it from a job in progress. Add these
+        # back after this version has deployed everywhere.
+        #    "testflinger-agent",
+        #    "snappy-device-agents",
+        for dir in dirs_to_remove:
+            shutil.rmtree(
+                f"{self._stored.agent_path}/{dir}", ignore_errors=True
+            )
+
+    def signal_restart_agent(self):
         """Signal testflinger-agent to restart when it's not busy"""
         restart_file = PosixPath(
-                f"/tmp/TESTFLINGER-DEVICE-RESTART-{self.app.name}")
+            f"/tmp/TESTFLINGER-DEVICE-RESTART-{self.app.name}"
+        )
         if restart_file.exists():
             return
         restart_file.open(mode="w").close()
         shutil.chown(restart_file, "ubuntu", "ubuntu")
 
-    def _write_config_files(self):
+    def write_config_files(self):
         """Overwrite the config files if they were changed"""
         tf_agent_config_path = (
-            f"{self._stored.agent_path}/testflinger-agent/"
-            "testflinger-agent.conf"
-        )
-        tf_agent_config = self._read_resource("testflinger_agent_configfile")
-        self._write_file(tf_agent_config_path, tf_agent_config)
-        device_config_path = (
-            f"{self._stored.agent_path}/" "snappy-device-agents/default.yaml"
+            f"{self._stored.agent_path}/testflinger-agent.conf"
         )
-        device_config = self._read_resource("device_configfile")
-        self._write_file(device_config_path, device_config)
+        tf_agent_config = self.read_resource("testflinger_agent_configfile")
+        self.write_file(tf_agent_config_path, tf_agent_config)
+        device_config_path = f"{self._stored.agent_path}/default.yaml"
+        device_config = self.read_resource("device_configfile")
+        self.write_file(device_config_path, device_config)
 
-    def _render_systemd_unit(self):
+    def render_systemd_unit(self):
         """Render the systemd unit for Gunicorn to a file"""
         # Open the template systemd unit file
         with open(
@@ -218,14 +224,15 @@ def _render_systemd_unit(self):
         # Reload systemd units
         systemd.daemon_reload()
 
-    def _on_config_changed(self, _):
+    def on_config_changed(self, _):
         self.unit.status = MaintenanceStatus("Handling config_changed hook")
-        self._check_update_repos_needed()
-        self._write_config_files()
-        self._signal_restart_agent()
+        self.check_update_repos_needed()
+        self.write_config_files()
+        self.render_systemd_unit()
+        self.signal_restart_agent()
         self.unit.status = ActiveStatus()
 
-    def _install_apt_packages(self, packages: list):
+    def install_apt_packages(self, packages: list):
         """Simple wrapper around 'apt-get install -y"""
         try:
             apt.update()
@@ -239,29 +246,29 @@ def _install_apt_packages(self, packages: list):
             logger.error("could not install package")
             self.unit.status = BlockedStatus("Failed to install packages")
 
-    def _read_resource(self, resource):
+    def read_resource(self, resource):
         """Read the specified resource and return the contents"""
         try:
             resource_file = self.model.resources.fetch(resource)
         except ModelError:
             # resource doesn't exist yet, return empty string
             return ""
         if (
-            not isinstance(resource_file, PosixPath) or not
-            resource_file.exists()
+            not isinstance(resource_file, PosixPath)
+            or not resource_file.exists()
         ):
             # Return empty string if it's invalid
             return ""
         with open(resource_file, encoding="utf-8", errors="ignore") as res:
             contents = res.read()
         return contents
 
-    def _on_update_action(self, event):
+    def on_update_action(self, event):
         """Force an update of git trees and config files"""
         self.unit.status = MaintenanceStatus("Handling update action")
-        self._update_repos()
-        self._write_config_files()
-        self._signal_restart_agent()
+        self.update_repos()
+        self.write_config_files()
+        self.signal_restart_agent()
         self.unit.status = ActiveStatus()
 
 

agent/charms/testflinger-agent-charm/templates/testflinger-agent.service.j2

@@ -6,7 +6,7 @@ After=network.target
 User=ubuntu
 Group=ubuntu
 WorkingDirectory={{ project_root }}
-ExecStart=/bin/sh -c ". env/bin/activate && PYTHONIOENCODING=utf-8 testflinger-agent -c testflinger-agent/testflinger-agent.conf"
+ExecStart=/bin/sh -c ". env/bin/activate && PYTHONIOENCODING=utf-8 testflinger-agent -c testflinger-agent.conf"
 Restart=always
 
 [Install]