diff --git a/agent/charms/testflinger-agent-charm/config.yaml b/agent/charms/testflinger-agent-charm/config.yaml index c73126c2..557ff74b 100644 --- a/agent/charms/testflinger-agent-charm/config.yaml +++ b/agent/charms/testflinger-agent-charm/config.yaml @@ -1,18 +1,10 @@ options: - testflinger-agent-repo: + testflinger-repo: type: string - description: git repo for testflinger-agent - default: "https://github.com/canonical/testflinger-agent" - testflinger-agent-branch: + description: git repo for testflinger + default: "https://github.com/canonical/testflinger" + testflinger-branch: type: string - description: git branch for testflinger-agent - default: "main" - device-agent-repo: - type: string - description: git repo for device-agent - default: "https://github.com/canonical/snappy-device-agents" - device-agent-branch: - type: string - description: git branch for device-agent + description: git branch for testflinger default: "main" diff --git a/agent/charms/testflinger-agent-charm/src/charm.py b/agent/charms/testflinger-agent-charm/src/charm.py index 06954831..7fb222f1 100755 --- a/agent/charms/testflinger-agent-charm/src/charm.py +++ b/agent/charms/testflinger-agent-charm/src/charm.py @@ -41,16 +41,14 @@ class TestflingerAgentCharm(CharmBase): def __init__(self, *args): super().__init__(*args) - self.framework.observe(self.on.install, self._on_install) - self.framework.observe(self.on.config_changed, self._on_config_changed) - self.framework.observe(self.on.start, self._on_start) - self.framework.observe(self.on.remove, self._on_remove) - self.framework.observe(self.on.update_action, self._on_update_action) + self.framework.observe(self.on.install, self.on_install) + self.framework.observe(self.on.config_changed, self.on_config_changed) + self.framework.observe(self.on.start, self.on_start) + self.framework.observe(self.on.remove, self.on_remove) + self.framework.observe(self.on.update_action, self.on_update_action) self._stored.set_default( - testflinger_agent_repo="", - testflinger_agent_branch="", - device_agent_repo="", - device_agent_branch="", + testflinger_repo="", + testflinger_branch="", unit_path=( f"/etc/systemd/system/testflinger-agent-{self.app.name}" ".service" @@ -59,7 +57,7 @@ def __init__(self, *args): venv_path=f"/srv/testflinger-agent/{self.app.name}/env", ) - def _on_install(self, _): + def on_install(self, _): """Install hook""" self.unit.status = MaintenanceStatus("Installing dependencies") # Ensure we have a fresh agent dir to start with @@ -68,7 +66,7 @@ def _on_install(self, _): os.makedirs("/home/ubuntu/testflinger", exist_ok=True) shutil.chown("/home/ubuntu/testflinger", "ubuntu", "ubuntu") - self._install_apt_packages( + self.install_apt_packages( [ "python3-pip", "python3-virtualenv", @@ -79,12 +77,12 @@ def _on_install(self, _): ] ) # Create the virtualenv - self._run_with_logged_errors( + self.run_with_logged_errors( ["python3", "-m", "virtualenv", f"{self._stored.venv_path}"], ) - self._render_systemd_unit() + self.render_systemd_unit() - def _run_with_logged_errors(self, cmd): + def run_with_logged_errors(self, cmd): """Run a command, log output if errors, return proc just in case""" proc = subprocess.run( cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, text=True @@ -93,20 +91,20 @@ def _run_with_logged_errors(self, cmd): logger.error(proc.stdout) return proc - def _write_file(self, location, contents): + def write_file(self, location, contents): # Sanity check to make sure we're actually about to write something if not contents: return with open(location, "w", encoding="utf-8", errors="ignore") as out: out.write(contents) - def _on_start(self, _): + def on_start(self, _): """Start the service""" service_name = f"testflinger-agent-{self.app.name}" systemd.service_restart(service_name) self.unit.status = ActiveStatus() - def _on_remove(self, _): + def on_remove(self, _): """Stop the service""" service_name = f"testflinger-agent-{self.app.name}" systemd.service_stop(service_name) @@ -114,85 +112,93 @@ def _on_remove(self, _): try: os.unlink(self._stored.unit_path) except FileNotFoundError: - logger.error("No systemd unit file found when removing: %s", - self._stored.unit_path) + logger.error( + "No systemd unit file found when removing: %s", + self._stored.unit_path, + ) systemd.daemon_reload() shutil.rmtree(self._stored.agent_path, ignore_errors=True) - def _check_update_repos_needed(self): + def check_update_repos_needed(self): """ Determine if any config settings change which require an update to the git repos """ - update_repos = False - repo = self.config.get("testflinger-agent-repo") - if repo != self._stored.testflinger_agent_repo: - self._stored.testflinger_agent_repo = repo - update_repos = True - branch = self.config.get("testflinger-agent-branch") - if branch != self._stored.testflinger_agent_branch: - self._stored.testflinger_agent_branch = branch - update_repos = True - repo = self.config.get("device-agent-repo") - if repo != self._stored.device_agent_repo: - self._stored.device_agent_repo = repo - update_repos = True - branch = self.config.get("device-agent-branch") - if branch != self._stored.device_agent_branch: - self._stored.device_agent_branch = branch - update_repos = True - if update_repos: - self._update_repos() + update_needed = False + repo = self.config.get("testflinger-repo") + if repo != self._stored.testflinger_repo: + self._stored.testflinger_repo = repo + update_needed = True + branch = self.config.get("testflinger-branch") + if branch != self._stored.testflinger_branch: + self._stored.testflinger_branch = branch + update_needed = True + if update_needed: + self.update_repos() - def _update_repos(self): + def update_repos(self): """Recreate the git repos and reinstall everything needed""" - tf_agent_dir = f"{self._stored.agent_path}/testflinger-agent" - device_agent_dir = f"{self._stored.agent_path}/snappy-device-agents" - shutil.rmtree(tf_agent_dir, ignore_errors=True) - shutil.rmtree(device_agent_dir, ignore_errors=True) - Repo.clone_from( - self._stored.testflinger_agent_repo, - tf_agent_dir, - multi_options=[f"-b {self._stored.testflinger_agent_branch}"], - ) - self._run_with_logged_errors( - [f"{self._stored.venv_path}/bin/pip3", "install", "-I", - tf_agent_dir] + self.cleanup_agent_dirs() + repo_path = f"{self._stored.agent_path}/testflinger" + repo = Repo.clone_from( + url=self._stored.testflinger_repo, + to_path=repo_path, + no_checkout=True, + depth=1, ) - Repo.clone_from( - self._stored.device_agent_repo, - device_agent_dir, - multi_options=[f"-b {self._stored.device_agent_branch}"], - ) - self._run_with_logged_errors( - [f"{self._stored.venv_path}/bin/pip3", "install", "-I", - device_agent_dir] + # do a sparse checkout of only agent and device-connectors + repo.git.checkout( + f"origin/{self._stored.testflinger_branch}", + "--", + "agent", + "device-connectors", ) + # Install the agent and device-connectors + for dir in ("agent", "device-connectors"): + self.run_with_logged_errors( + [ + f"{self._stored.venv_path}/bin/pip3", + "install", + "-I", + f"{repo_path}/{dir}", + ] + ) - def _signal_restart_agent(self): + def cleanup_agent_dirs(self): + """Remove old agent dirs before checking out again""" + dirs_to_remove = ("testflinger",) + # Temporarily skip removing the following two things so that we + # don't accidentally remove it from a job in progress. Add these + # back after this version has deployed everywhere. + # "testflinger-agent", + # "snappy-device-agents", + for dir in dirs_to_remove: + shutil.rmtree( + f"{self._stored.agent_path}/{dir}", ignore_errors=True + ) + + def signal_restart_agent(self): """Signal testflinger-agent to restart when it's not busy""" restart_file = PosixPath( - f"/tmp/TESTFLINGER-DEVICE-RESTART-{self.app.name}") + f"/tmp/TESTFLINGER-DEVICE-RESTART-{self.app.name}" + ) if restart_file.exists(): return restart_file.open(mode="w").close() shutil.chown(restart_file, "ubuntu", "ubuntu") - def _write_config_files(self): + def write_config_files(self): """Overwrite the config files if they were changed""" tf_agent_config_path = ( - f"{self._stored.agent_path}/testflinger-agent/" - "testflinger-agent.conf" - ) - tf_agent_config = self._read_resource("testflinger_agent_configfile") - self._write_file(tf_agent_config_path, tf_agent_config) - device_config_path = ( - f"{self._stored.agent_path}/" "snappy-device-agents/default.yaml" + f"{self._stored.agent_path}/testflinger-agent.conf" ) - device_config = self._read_resource("device_configfile") - self._write_file(device_config_path, device_config) + tf_agent_config = self.read_resource("testflinger_agent_configfile") + self.write_file(tf_agent_config_path, tf_agent_config) + device_config_path = f"{self._stored.agent_path}/default.yaml" + device_config = self.read_resource("device_configfile") + self.write_file(device_config_path, device_config) - def _render_systemd_unit(self): + def render_systemd_unit(self): """Render the systemd unit for Gunicorn to a file""" # Open the template systemd unit file with open( @@ -218,14 +224,15 @@ def _render_systemd_unit(self): # Reload systemd units systemd.daemon_reload() - def _on_config_changed(self, _): + def on_config_changed(self, _): self.unit.status = MaintenanceStatus("Handling config_changed hook") - self._check_update_repos_needed() - self._write_config_files() - self._signal_restart_agent() + self.check_update_repos_needed() + self.write_config_files() + self.render_systemd_unit() + self.signal_restart_agent() self.unit.status = ActiveStatus() - def _install_apt_packages(self, packages: list): + def install_apt_packages(self, packages: list): """Simple wrapper around 'apt-get install -y""" try: apt.update() @@ -239,7 +246,7 @@ def _install_apt_packages(self, packages: list): logger.error("could not install package") self.unit.status = BlockedStatus("Failed to install packages") - def _read_resource(self, resource): + def read_resource(self, resource): """Read the specified resource and return the contents""" try: resource_file = self.model.resources.fetch(resource) @@ -247,8 +254,8 @@ def _read_resource(self, resource): # resource doesn't exist yet, return empty string return "" if ( - not isinstance(resource_file, PosixPath) or not - resource_file.exists() + not isinstance(resource_file, PosixPath) + or not resource_file.exists() ): # Return empty string if it's invalid return "" @@ -256,12 +263,12 @@ def _read_resource(self, resource): contents = res.read() return contents - def _on_update_action(self, event): + def on_update_action(self, event): """Force an update of git trees and config files""" self.unit.status = MaintenanceStatus("Handling update action") - self._update_repos() - self._write_config_files() - self._signal_restart_agent() + self.update_repos() + self.write_config_files() + self.signal_restart_agent() self.unit.status = ActiveStatus() diff --git a/agent/charms/testflinger-agent-charm/templates/testflinger-agent.service.j2 b/agent/charms/testflinger-agent-charm/templates/testflinger-agent.service.j2 index 2d6e69e6..8598f1a0 100644 --- a/agent/charms/testflinger-agent-charm/templates/testflinger-agent.service.j2 +++ b/agent/charms/testflinger-agent-charm/templates/testflinger-agent.service.j2 @@ -6,7 +6,7 @@ After=network.target User=ubuntu Group=ubuntu WorkingDirectory={{ project_root }} -ExecStart=/bin/sh -c ". env/bin/activate && PYTHONIOENCODING=utf-8 testflinger-agent -c testflinger-agent/testflinger-agent.conf" +ExecStart=/bin/sh -c ". env/bin/activate && PYTHONIOENCODING=utf-8 testflinger-agent -c testflinger-agent.conf" Restart=always [Install] diff --git a/agent/charms/testflinger-agent-host-charm/src/charm.py b/agent/charms/testflinger-agent-host-charm/src/charm.py index c53e746c..1e1f067c 100755 --- a/agent/charms/testflinger-agent-host-charm/src/charm.py +++ b/agent/charms/testflinger-agent-host-charm/src/charm.py @@ -30,29 +30,29 @@ class TestflingerAgentHostCharm(CharmBase): def __init__(self, *args): super().__init__(*args) - self.framework.observe(self.on.install, self._on_install) - self.framework.observe(self.on.start, self._on_start) - self.framework.observe(self.on.config_changed, self._on_config_changed) + self.framework.observe(self.on.install, self.on_install) + self.framework.observe(self.on.start, self.on_start) + self.framework.observe(self.on.config_changed, self.on_config_changed) self._stored.set_default( ssh_priv="", ssh_pub="", ) - def _on_install(self, _): + def on_install(self, _): """Install hook""" self.unit.status = MaintenanceStatus("Installing dependencies") - self._install_apt_packages( + self.install_apt_packages( ["python3-pip", "python3-virtualenv", "docker.io"] ) # maas cli comes from maas snap now - self._run_with_logged_errors(["snap", "install", "maas"]) - self._setup_docker() + self.run_with_logged_errors(["snap", "install", "maas"]) + self.setup_docker() - def _setup_docker(self): - self._run_with_logged_errors(["groupadd", "docker"]) - self._run_with_logged_errors(["gpasswd", "-a", "ubuntu", "docker"]) + def setup_docker(self): + self.run_with_logged_errors(["groupadd", "docker"]) + self.run_with_logged_errors(["gpasswd", "-a", "ubuntu", "docker"]) - def _run_with_logged_errors(self, cmd): + def run_with_logged_errors(self, cmd): """Run a command, log output if errors, return proc just in case""" proc = subprocess.run( cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, text=True @@ -61,33 +61,33 @@ def _run_with_logged_errors(self, cmd): logger.error(proc.stdout) return proc - def _write_file(self, location, contents): + def write_file(self, location, contents): # Sanity check to make sure we're actually about to write something if not contents: return with open(location, "w", encoding="utf-8", errors="ignore") as out: out.write(contents) - def _copy_ssh_keys(self): - priv_key = self._read_resource("ssh_priv_key") + def copy_ssh_keys(self): + priv_key = self.read_resource("ssh_priv_key") if self._stored.ssh_priv != priv_key: self._stored.ssh_priv = priv_key - self._write_file("/home/ubuntu/.ssh/id_rsa", priv_key) - pub_key = self._read_resource("ssh_pub_key") + self.write_file("/home/ubuntu/.ssh/id_rsa", priv_key) + pub_key = self.read_resource("ssh_pub_key") if self._stored.ssh_pub != pub_key: self._stored.ssh_pub = pub_key - self._write_file("/home/ubuntu/.ssh/id_rsa.pub", pub_key) + self.write_file("/home/ubuntu/.ssh/id_rsa.pub", pub_key) - def _on_start(self, _): + def on_start(self, _): """Start the service""" self.unit.status = ActiveStatus() - def _on_config_changed(self, _): + def on_config_changed(self, _): self.unit.status = MaintenanceStatus("Handling config_changed hook") - self._copy_ssh_keys() + self.copy_ssh_keys() self.unit.status = ActiveStatus() - def _install_apt_packages(self, packages: list): + def install_apt_packages(self, packages: list): """Simple wrapper around 'apt-get install -y""" try: apt.update() @@ -101,7 +101,7 @@ def _install_apt_packages(self, packages: list): logger.error("could not install package") self.unit.status = BlockedStatus("Failed to install packages") - def _read_resource(self, resource): + def read_resource(self, resource): """Read the specified resource and return the contents""" try: resource_file = self.model.resources.fetch(resource) @@ -109,8 +109,8 @@ def _read_resource(self, resource): # resource doesn't exist yet, return empty string return "" if ( - not isinstance(resource_file, PosixPath) or not - resource_file.exists() + not isinstance(resource_file, PosixPath) + or not resource_file.exists() ): # Return empty string if it's invalid return ""