diff --git a/.github/workflows/tox.yml b/.github/workflows/agent-tox.yml similarity index 75% rename from .github/workflows/tox.yml rename to .github/workflows/agent-tox.yml index c9011a8b..c085338a 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/agent-tox.yml @@ -1,13 +1,20 @@ -name: Run unit tests +name: "[agent] Run unit tests" on: push: branches: [ main ] + paths: + - agent/** pull_request: branches: [ main ] + paths: + - agent/** jobs: build: + defaults: + run: + working-directory: agent runs-on: ubuntu-latest strategy: matrix: diff --git a/.github/workflows/cli-tox.yml b/.github/workflows/cli-tox.yml new file mode 100644 index 00000000..bef84f2c --- /dev/null +++ b/.github/workflows/cli-tox.yml @@ -0,0 +1,32 @@ +name: "[cli] Run unit tests" + +on: + push: + branches: [ main ] + paths: + - cli/** + pull_request: + branches: [ main ] + paths: + - cli/** + +jobs: + build: + defaults: + run: + working-directory: cli + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.10"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install tox + run: pip install tox + - name: Run tests + run: | + tox diff --git a/.github/workflows/device-tox.yml b/.github/workflows/device-tox.yml new file mode 100644 index 00000000..1a681070 --- /dev/null +++ b/.github/workflows/device-tox.yml @@ -0,0 +1,32 @@ +name: "[device-connectors] Run unit tests" + +on: + push: + branches: [ main ] + paths: + - device-connectors/** + pull_request: + branches: [ main ] + paths: + - device-connectors/** + +jobs: + build: + defaults: + run: + working-directory: device-connectors + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.10"] + steps: + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install tox + run: pip install tox + - name: Run tests + run: | + tox diff --git a/.github/workflows/charm_check_libs.yml b/.github/workflows/server-charm-check-libs.yml similarity index 88% rename from .github/workflows/charm_check_libs.yml rename to .github/workflows/server-charm-check-libs.yml index 1a62c972..5d6d33e7 100644 --- a/.github/workflows/charm_check_libs.yml +++ b/.github/workflows/server-charm-check-libs.yml @@ -4,6 +4,8 @@ on: pull_request: branches: - main + paths: + - server/** jobs: build: @@ -17,6 +19,6 @@ jobs: - name: Check libraries uses: canonical/charming-actions/check-libraries@2.4.0 with: - charm-path: charm + charm-path: server/charm credentials: "${{ secrets.CHARMHUB_TOKEN }}" github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/charm_release_edge.yml b/.github/workflows/server-charm-release-edge.yml similarity index 89% rename from .github/workflows/charm_release_edge.yml rename to .github/workflows/server-charm-release-edge.yml index af15a086..3b1ccd2b 100644 --- a/.github/workflows/charm_release_edge.yml +++ b/.github/workflows/server-charm-release-edge.yml @@ -4,6 +4,8 @@ on: push: branches: - main + paths: + - server/** jobs: build: @@ -17,7 +19,7 @@ jobs: - name: Upload charm to charmhub uses: canonical/charming-actions/upload-charm@2.4.0 with: - charm-path: charm + charm-path: server/charm credentials: "${{ secrets.CHARMHUB_TOKEN }}" github-token: "${{ secrets.GITHUB_TOKEN }}" upload-image: "true" diff --git a/.github/workflows/publish_oci_image.yml b/.github/workflows/server-publish-oci-image.yml similarity index 95% rename from .github/workflows/publish_oci_image.yml rename to .github/workflows/server-publish-oci-image.yml index bf9935e5..25d1f7b8 100644 --- a/.github/workflows/publish_oci_image.yml +++ b/.github/workflows/server-publish-oci-image.yml @@ -3,6 +3,8 @@ on: push: branches: ["main"] tags: ["v*.*.*"] + paths: + - server/** env: REGISTRY: ghcr.io @@ -37,7 +39,7 @@ jobs: - name: Build and push backend Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: . + context: ./server file: Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/server-tox.yml b/.github/workflows/server-tox.yml new file mode 100644 index 00000000..37ca7cdd --- /dev/null +++ b/.github/workflows/server-tox.yml @@ -0,0 +1,32 @@ +name: "[server] Run unit tests" + +on: + push: + branches: [ main ] + paths: + - server/** + pull_request: + branches: [ main ] + paths: + - server/** + +jobs: + build: + defaults: + run: + working-directory: server + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install tox + run: pip install tox + - name: Run tests + run: | + tox diff --git a/agent/.gitignore b/agent/.gitignore new file mode 100644 index 00000000..da344c16 --- /dev/null +++ b/agent/.gitignore @@ -0,0 +1,12 @@ +*~ +.coverage +.vscode/ +build/ +env/ +__pycache__/ +*.py[cod] +*$py.class +*.conf +*.egg* +*.bak +.swp diff --git a/agent/.pmr-merge-hook b/agent/.pmr-merge-hook new file mode 100755 index 00000000..394df162 --- /dev/null +++ b/agent/.pmr-merge-hook @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +rm -rf tfenv +virtualenv -q -p python3 tfenv +. tfenv/bin/activate +./setup.py test diff --git a/agent/README.rst b/agent/README.rst new file mode 100644 index 00000000..b2d6ac93 --- /dev/null +++ b/agent/README.rst @@ -0,0 +1,192 @@ +================= +Testflinger Agent +================= + +Testflinger agent waits for job requests on a configured queue, then processes +them. The Testflinger Server submits those jobs, and once the job is complete, +the agent can submit outcome data with limited results back to the server. + +Overview +-------- + +Testflinger-agent connects to the Testflinger microservice to request and +service requests for tests. + +Installation +------------ + +To create a virtual environment and install testflinger-agent: + +.. code-block:: console + + $ virtualenv env + $ . env/bin/activate + $ ./setup install + +Testing +------- + +To run the unit tests, first install (see above) then: + +.. code-block:: console + + $ ./setup test + +Configuration +------------- + +Configuration is loaded from a yaml configuration file called +testflinger-agent.conf by default. You can specify a different file +to use for config data using the -c option. + +The following configuration options are supported: + +- **agent_id**: + + - Unique identifier for this agent + +- **polling_interval**: + + - Time to sleep between polling for new tests (default: 10s) + +- **server_address**: + + - Host/IP and port of the testflinger server + +- **execution_basedir**: + + - Base directory to use for running jobs (default: /tmp/testflinger/run) + +- **logging_basedir**: + + - Base directory to use for agent logging (default: /tmp/testflinger/logs) + +- **results_basedir**: + + - Base directory to use for temporary storage of test results to be transmitted to the server (default: /tmp/testflinger/results) + +- **logging_level**: + + - Python loglevel name to use for logging (default: INFO) + +- **logging_quiet**: + + - Only log to the logfile, and not to the console (default: False) + +- **job_queues**: + + - List of queues that can be serviced by this device + +- **advertised_queues**: + + - List of public queue names that should be reported to the server to report to users + +- **advertised_images**: + + - List of images to associate with a queue name so that they can be referenced by name when using testflinger reserve + +- **global_timeout**: + + - Maximum global timeout (in seconds) a job is allowed to specify for this device agent. The job will timeout during the provision or test phase if it takes longer than the requested global_timeout to run. (Default 4 hours) + +- **output_timeout**: + + - Maximum output timeout (in seconds) a job is allowed to specify for this device agent. The job will timeout if there has been no output in the test phase for longer than the requested output_timeout. (Default 15 min.) + +- **setup_command**: + + - Command to run for the setup phase + +- **provision_command**: + + - Command to run for the provision phase + +- **allocate_command**: + + - Command to run for the allocate phase + +- **test_command**: + + - Command to run for the testing phase + +- **reserve_command**: + + - Command to run for the reserve phase + +- **cleanup_command**: + + - Command to run for the cleanup phase + +Test Phases +----------- +The test will go through several phases depending on the configuration of the +test job and the configuration testflinger agent itself. If a _command +is not set in the testflinger-agent.conf (see above), then that phase will +be skipped. Even if the phase_command is configured, there are some phases +that are not mandatory, and will be skipped if the job does not contain data +for it, such as the provision, test, allocate, and reserve phases. + +The following test phases are currently supported: + +- **setup**: + + - This phase is run first, and is used to setup the environment for the + test. The test job has no input for this phase and it is completely up to + the device owner to include commands that may need to run here. + +- **provision**: + + - This phase is run after the setup phase, and is used to provision the + device by installing (if possible) the image requested in the test job. + If the provision_data section is missing from the job, this phase will + not run. + +- **test**: + + - This phase is run after the provision phase, and is used to run the + test_cmds defined in the test_data section of the job. If the test_data + section is missing from the job, this will not run. + +- **allocate**: + + - This phase is normally only used by multi-device jobs and is used to + lock the agent into an allocated state to be externally controlled by + another job. During this phase, it will gather device_ip information + and push that information to the results data on the testflinger server + under the running job's job_id. Once that data is pushed successfully + to the server, it will transition the job to a **allocated** state, which + is just a signal that the parent job can make use of that data. The + **allocated** state is just a *job* state though, and not a phase that + needs a separate command configured on the agent. + Normally, the allocate_data section will be missing from the test job, + and this phase will be skipped. + +- **reserve**: + + - This phase is used for reserving a system for manual control. This + will push the requested ssh key specified in the job data to the + device once it's provisioned and ready for use, then publish output + to the polling log with information on how to reach the device over + ssh. If the reserve_data section is missing from the job, then this + phase will be skipped. + +- **cleanup**: + + - This phase is run after the reserve phase, and is used to cleanup the + device after the test. The test job has no input for this phase and + it is completely up to the device owner to include commands + that may need to run here. + +Usage +----- + +When running testflinger, your output will be automatically accumulated +for each stage (setup, provision, test, cleanup) and sent to the testflinger +server, along with an exit status for each stage. If any stage encounters a +non-zero exit code, no further stages will be executed, but the outcome will +still be sent. + +If you have additional artifacts that you would like to save along with +the output, you can create a 'artifacts' directory from your test command. +Any files in the artifacts directory under your test execution directory +will automatically be compressed (tar.gz) and sent to the testflinger server. diff --git a/agent/pyproject.toml b/agent/pyproject.toml new file mode 100644 index 00000000..3f51fac8 --- /dev/null +++ b/agent/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = [ + "setuptools", + "setuptools-scm", +] +build-backend = "setuptools.build_meta" + +[project] +name = "testflinger-agent" +description = "Testflinger agent" +readme = "README.rst" +dependencies = [ + "PyYAML", + "requests", + "voluptuous", + "influxdb", +] +dynamic = ["version"] + +[project.scripts] +testflinger-agent = "testflinger_agent.cmd:main" + +[tool.black] +line-length = 79 diff --git a/renovate.json b/agent/renovate.json similarity index 100% rename from renovate.json rename to agent/renovate.json diff --git a/agent/setup.cfg b/agent/setup.cfg new file mode 100644 index 00000000..31ad82b6 --- /dev/null +++ b/agent/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = pytest diff --git a/agent/setup.py b/agent/setup.py new file mode 100755 index 00000000..a3456e80 --- /dev/null +++ b/agent/setup.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# Copyright (C) 2016-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from setuptools import setup + +setup() diff --git a/agent/testflinger-agent.conf.example b/agent/testflinger-agent.conf.example new file mode 100644 index 00000000..5534121f --- /dev/null +++ b/agent/testflinger-agent.conf.example @@ -0,0 +1,50 @@ +# Unique identifier for this agent +# agent_id: agent-007 + +# Time to sleep between polling for new tests (default: 10s) +# polling_interval: 10 + +# Host/IP and port of the testflinger server +# server_address: 127.0.0.1:8000 + +# Base directory to use for running jobs +# (default: /tmp/testflinger/run) +# execution_basedir: /tmp/testflinger/run + +# Base directory to use for agent logging +# (default: /tmp/testflinger/logs) +# logging_basedir: /tmp/testflinger/logs + +# Base directory to use for queuing results for retransmit +# (default: /tmp/testflinger/results) +# results_basedir: /tmp/testflinger/results + +# Python loglevel name to use for logging (default: INFO) +# logging_level: DEBUG + +# Only log to the logfile, and not to the console (default: False) +# logging_quiet: True + +# List of queues that can be serviced by this device +# job_queues: +# - myqueue +# - anotherqueue + +# List of advertised queues to show users when they ask +# advertised_queues: +# myqueue: A brief description of myqueue + +# List of advertised images and the provision_data for using them +# advertised_images: +# myqueue: +# - latest: "url: http://path/to/latest.img.xz" +# - stable: "url: http://path/to/stable.img.xz" + +# Command to run for the setup phase +# setup_command: echo setup phase && run-setup-tasks.sh + +# Command to run for the provision phase +# provision_command: echo provision phase && provision-system.sh + +# Command to run for the testing phase +# test_command: echo test phase && run-test.sh diff --git a/agent/testflinger_agent/__init__.py b/agent/testflinger_agent/__init__.py new file mode 100644 index 00000000..77c4eac0 --- /dev/null +++ b/agent/testflinger_agent/__init__.py @@ -0,0 +1,200 @@ +# Copyright (C) 2016-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import logging +import os +import time +import yaml +import requests +from requests.adapters import HTTPAdapter, Retry +from urllib3.exceptions import HTTPError +from urllib.parse import urljoin +from collections import deque +from threading import Timer + +from testflinger_agent import schema +from testflinger_agent.agent import TestflingerAgent +from testflinger_agent.client import TestflingerClient +from logging.handlers import TimedRotatingFileHandler + +logger = logging.getLogger(__name__) + + +class ReqBufferTimer(Timer): + """Requests buffer flush""" + + def run(self): + """Loop timer""" + while not self.finished.wait(self.interval): + self.function(*self.args, **self.kwargs) + + +class ReqBufferHandler(logging.Handler): + """Requests logging handler""" + + def __init__(self, agent, server): + super().__init__() + if not server.lower().startswith("http"): + server = "http://" + server + uri = urljoin(server, "/v1/agents/data/") + self.url = urljoin(uri, agent) + self.qdepth = 100 # messages + self.reqbuffer = deque([], maxlen=self.qdepth) + self.reqbuff_timer = None + self.reqbuff_interval = 10.0 # seconds + self._start_reqbuff_timer() + # reuse socket + self.session = self._requests_retry() + + def _requests_retry(self, retries=3): + """Retry api server""" + session = requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=0.3, + status_forcelist=(500, 502, 503, 504), + allowed_methods=False, # allow retry on all methods + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + def _start_reqbuff_timer(self): + """Periodically check and send buffer""" + self.reqbuff_timer = ReqBufferTimer(self.reqbuff_interval, self.flush) + # terminate timer on exit + self.reqbuff_timer.daemon = True + self.reqbuff_timer.start() + + def emit(self, record): + """Write logging events to buffer""" + if len(self.reqbuffer) >= self.qdepth: + self.reqbuffer.popleft() + + self.reqbuffer.append(record) + + def flush(self): + """Flush and post buffer""" + # list conversion for atomic iteration + records = [record.getMessage() for record in list(self.reqbuffer)] + + try: + self.session.post( + url=self.url, json=self.format(records), timeout=5 + ) + except (requests.RequestException, HTTPError) as error: + logger.debug(error) + + return # preserve buffer + + self.reqbuffer.clear() + + def close(self): + """Cleanup on handler close""" + self.reqbuff_timer.cancel() + + +class ReqBufferFormatter(logging.Formatter): + """Format logging messages""" + + def format(self, records): + return {"log": records} + + +def start_agent(): + args = parse_args() + config = load_config(args.config) + configure_logging(config) + check_interval = config.get("polling_interval") + client = TestflingerClient(config) + agent = TestflingerAgent(client) + while True: + offline_file = agent.check_offline() + if offline_file: + logger.error( + "Agent %s is offline, not processing jobs! " + "Remove %s to resume processing" + % (config.get("agent_id"), offline_file) + ) + while agent.check_offline(): + time.sleep(check_interval) + logger.info("Checking jobs") + agent.process_jobs() + logger.info("Sleeping for {}".format(check_interval)) + time.sleep(check_interval) + + +def load_config(configfile): + with open(configfile) as f: + config = yaml.safe_load(f) + config = schema.validate(config) + return config + + +def configure_logging(config): + # Create these at the beginning so we fail early if there are + # permission problems + os.makedirs(config.get("logging_basedir"), exist_ok=True) + os.makedirs(config.get("results_basedir"), exist_ok=True) + log_level = logging.getLevelName(config.get("logging_level")) + # This should help if they specify something invalid + if not isinstance(log_level, int): + log_level = logging.INFO + logfmt = logging.Formatter( + fmt=( + "[%(asctime)s] %(levelname)+7.7s: " + "(%(filename)s:%(lineno)d)| %(message)s" + ), + datefmt="%y-%m-%d %H:%M:%S", + ) + log_path = os.path.join( + config.get("logging_basedir"), "testflinger-agent.log" + ) + file_log = TimedRotatingFileHandler( + log_path, when="midnight", interval=1, backupCount=6 + ) + file_log.setFormatter(logfmt) + logger.addHandler(file_log) + # requests logging + # inherit from logger __name__ + """ DEBUG: Temporarily disable sending agent logs to the server + req_logger = logging.getLogger() + request_formatter = ReqBufferFormatter() + request_handler = ReqBufferHandler( + config.get("agent_id"), config.get("server_address") + ) + request_handler.setFormatter(request_formatter) + req_logger.addHandler(request_handler) + req_logger.setLevel(log_level) + """ + if not config.get("logging_quiet"): + console_log = logging.StreamHandler() + console_log.setFormatter(logfmt) + logger.addHandler(console_log) + logger.setLevel(log_level) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Testflinger Agent") + parser.add_argument( + "--config", + "-c", + default="testflinger-agent.conf", + help="Testflinger agent config file", + ) + return parser.parse_args() diff --git a/agent/testflinger_agent/agent.py b/agent/testflinger_agent/agent.py new file mode 100644 index 00000000..a9f049b6 --- /dev/null +++ b/agent/testflinger_agent/agent.py @@ -0,0 +1,213 @@ +# Copyright (C) 2017-2020 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +import json +import logging +import os +import shutil + +from testflinger_agent.job import TestflingerJob +from testflinger_agent.errors import TFServerError + +logger = logging.getLogger(__name__) + + +class TestflingerAgent: + def __init__(self, client): + self.client = client + self.set_agent_state("waiting") + self._post_initial_agent_data() + + def _post_initial_agent_data(self): + """Post the initial agent data to the server once on agent startup""" + + location = self.client.config.get("location", "") + advertised_queues = self._post_advertised_queues() + self._post_advertised_images() + + if advertised_queues or location: + self.client.post_agent_data( + {"queues": advertised_queues, "location": location} + ) + + def _post_advertised_queues(self): + """ + Get the advertised queues from the config and send them to the server + + :return: Dictionary of advertised queues + """ + advertised_queues = self.client.config.get("advertised_queues", {}) + if advertised_queues: + self.client.post_queues(advertised_queues) + return advertised_queues + + def _post_advertised_images(self): + """ + Get the advertised images from the config and post them to the server + """ + advertised_images = self.client.config.get("advertised_images") + if advertised_images: + self.client.post_images(advertised_images) + + def set_agent_state(self, state): + """Send the agent state to the server""" + self.client.post_agent_data({"state": state}) + self.client.post_influx(state) + + def get_offline_files(self): + # Return possible restart filenames with and without dashes + # i.e. support both: + # TESTFLINGER-DEVICE-OFFLINE-devname-001 + # TESTFLINGER-DEVICE-OFFLINE-devname001 + agent = self.client.config.get("agent_id") + files = [ + "/tmp/TESTFLINGER-DEVICE-OFFLINE-{}".format(agent), + "/tmp/TESTFLINGER-DEVICE-OFFLINE-{}".format( + agent.replace("-", "") + ), + ] + return files + + def get_restart_files(self): + # Return possible restart filenames with and without dashes + # i.e. support both: + # TESTFLINGER-DEVICE-RESTART-devname-001 + # TESTFLINGER-DEVICE-RESTART-devname001 + agent = self.client.config.get("agent_id") + files = [ + "/tmp/TESTFLINGER-DEVICE-RESTART-{}".format(agent), + "/tmp/TESTFLINGER-DEVICE-RESTART-{}".format( + agent.replace("-", "") + ), + ] + return files + + def check_offline(self): + possible_files = self.get_offline_files() + for offline_file in possible_files: + if os.path.exists(offline_file): + self.set_agent_state("offline") + return offline_file + self.set_agent_state("waiting") + return "" + + def check_restart(self): + possible_files = self.get_restart_files() + for restart_file in possible_files: + if os.path.exists(restart_file): + try: + os.unlink(restart_file) + logger.info("Restarting agent") + self.set_agent_state("offline") + raise SystemExit("Restart Requested") + except OSError: + logger.error( + "Restart requested, but unable to remove marker file!" + ) + break + + def mark_device_offline(self): + # Create the offline file, this should work even if it exists + open(self.get_offline_files()[0], "w").close() + + def process_jobs(self): + """Coordinate checking for new jobs and handling them if they exists""" + TEST_PHASES = ["setup", "provision", "test", "allocate", "reserve"] + + # First, see if we have any old results that we couldn't send last time + self.retry_old_results() + + self.check_restart() + + job_data = self.client.check_jobs() + while job_data: + try: + job = TestflingerJob(job_data, self.client) + logger.info("Starting job %s", job.job_id) + rundir = os.path.join( + self.client.config.get("execution_basedir"), job.job_id + ) + os.makedirs(rundir) + self.client.post_agent_data({"job_id": job.job_id}) + # Dump the job data to testflinger.json in our execution dir + with open(os.path.join(rundir, "testflinger.json"), "w") as f: + json.dump(job_data, f) + # Create json outcome file where phases will store their output + with open( + os.path.join(rundir, "testflinger-outcome.json"), "w" + ) as f: + json.dump({}, f) + + for phase in TEST_PHASES: + # First make sure the job hasn't been cancelled + if self.client.check_job_state(job.job_id) == "cancelled": + logger.info("Job cancellation was requested, exiting.") + break + self.client.post_job_state(job.job_id, phase) + self.set_agent_state(phase) + + exitcode = job.run_test_phase(phase, rundir) + + self.client.post_influx(phase, exitcode) + + # exit code 46 is our indication that recovery failed! + # In this case, we need to mark the device offline + if exitcode == 46: + self.mark_device_offline() + if phase != "test" and exitcode: + logger.debug("Phase %s failed, aborting job" % phase) + break + except Exception as e: + logger.exception(e) + finally: + # Always run the cleanup, even if the job was cancelled + job.run_test_phase("cleanup", rundir) + # clear job id + self.client.post_agent_data({"job_id": ""}) + + try: + self.client.transmit_job_outcome(rundir) + except Exception as e: + # TFServerError will happen if we get other-than-good status + # Other errors can happen too for things like connection + # problems + logger.exception(e) + results_basedir = self.client.config.get("results_basedir") + shutil.move(rundir, results_basedir) + self.set_agent_state("waiting") + + self.check_restart() + if self.check_offline(): + # Don't get a new job if we are now marked offline + break + job_data = self.client.check_jobs() + + def retry_old_results(self): + """Retry sending results that we previously failed to send""" + + results_dir = self.client.config.get("results_basedir") + # List all the directories in 'results_basedir', where we store the + # results that we couldn't transmit before + old_results = [ + os.path.join(results_dir, d) + for d in os.listdir(results_dir) + if os.path.isdir(os.path.join(results_dir, d)) + ] + for result in old_results: + try: + logger.info("Attempting to send result: %s" % result) + self.client.transmit_job_outcome(result) + except TFServerError: + # Problems still, better luck next time? + pass diff --git a/agent/testflinger_agent/client.py b/agent/testflinger_agent/client.py new file mode 100644 index 00000000..df01db0c --- /dev/null +++ b/agent/testflinger_agent/client.py @@ -0,0 +1,338 @@ +# Copyright (C) 2016-2022 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import json +import os +import requests +import shutil +import tempfile +import time + +from urllib.parse import urljoin +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +from requests.exceptions import RequestException, ConnectionError +from influxdb import InfluxDBClient +from influxdb.exceptions import InfluxDBClientError + +from testflinger_agent.errors import TFServerError + +logger = logging.getLogger(__name__) + + +class TestflingerClient: + def __init__(self, config): + self.config = config + self.server = self.config.get( + "server_address", "https://testflinger.canonical.com" + ) + if not self.server.lower().startswith("http"): + self.server = "http://" + self.server + self.session = self._requests_retry(retries=5) + self.influx_agent_db = "agent_jobs" + self.influx_client = self._configure_influx() + + def _requests_retry(self, retries=3): + session = requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=0.3, + status_forcelist=(500, 502, 503, 504), + allowed_methods=False, # allow retry on all methods + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + + def _configure_influx(self): + """Configure InfluxDB client using environment variables. + + :return: influxdb object or None + """ + host = os.environ.get("INFLUX_HOST") + if not host: + logger.error("InfluxDB host undefined") + return None + port = int(os.environ.get("INFLUX_PORT", 8086)) + user = os.environ.get("INFLUX_USER", "") + password = os.environ.get("INFLUX_PW", "") + + influx_client = InfluxDBClient( + host, port, user, password, self.influx_agent_db + ) + + # ensure we can connect to influxdb + try: + influx_client.create_database(self.influx_agent_db) + except ConnectionError as exc: + logger.error(exc) + else: + return influx_client + + def check_jobs(self): + """Check for new jobs for on the Testflinger server + + :return: Dict with job data, or None if no job found + """ + try: + job_uri = urljoin(self.server, "/v1/job") + queue_list = self.config.get("job_queues") + logger.debug("Requesting a job") + job_request = self.session.get( + job_uri, params={"queue": queue_list}, timeout=30 + ) + if job_request.content: + return job_request.json() + else: + return None + except RequestException as exc: + logger.error(exc) + # Wait a little extra before trying again + time.sleep(60) + + def check_job_state(self, job_id): + job_data = self.get_result(job_id) + if job_data: + return job_data.get("job_state") + + def repost_job(self, job_data): + """ "Resubmit the job to the testflinger server with the same id + + :param job_id: + id for the job on which we want to post results + """ + job_uri = urljoin(self.server, "/v1/job") + job_id = job_data.get("job_id") + logger.info("Resubmitting job: %s", job_id) + job_output = """ + There was an unrecoverable error while running this stage. Your job + will attempt to be automatically resubmitted back to the queue. + Resubmitting job: {}\n""".format( + job_id + ) + self.post_live_output(job_id, job_output) + try: + job_request = self.session.post(job_uri, json=job_data) + except RequestException as exc: + logger.error(exc) + raise TFServerError("other exception") from exc + if not job_request: + logger.error( + "Unable to re-post job to: %s (error: %s)" + % (job_uri, job_request.status_code) + ) + raise TFServerError(job_request.status_code) + + def post_job_state(self, job_id, phase): + """Update the job_state on the testflinger server""" + try: + self.post_result(job_id, {"job_state": phase}) + except TFServerError: + pass + + def post_result(self, job_id, data): + """Post data to the testflinger server result for this job + + :param job_id: + id for the job on which we want to post results + :param data: + dict with data to be posted in json + """ + result_uri = urljoin(self.server, "/v1/result/") + result_uri = urljoin(result_uri, job_id) + try: + job_request = self.session.post(result_uri, json=data, timeout=30) + except RequestException as exc: + logger.error(exc) + raise TFServerError("other exception") from exc + if not job_request: + logger.error( + "Unable to post results to: %s (error: %s)" + % (result_uri, job_request.status_code) + ) + raise TFServerError(job_request.status_code) + + def get_result(self, job_id): + """Get current results data to the testflinger server for this job + + :param job_id: + id for the job on which we want to post results + :param data: + dict with data to be posted in json or an empty dict if + there was an error + """ + result_uri = urljoin(self.server, "/v1/result/") + result_uri = urljoin(result_uri, job_id) + try: + job_request = self.session.get(result_uri, timeout=30) + except RequestException as exc: + logger.error(exc) + return {} + if not job_request: + logger.error( + "Unable to get results from: %s (error: %s)" + % (result_uri, job_request.status_code) + ) + return {} + if job_request.content: + return job_request.json() + else: + return {} + + def transmit_job_outcome(self, rundir): + """Post job outcome json data to the testflinger server + + :param rundir: + Execution dir where the results can be found + """ + with open(os.path.join(rundir, "testflinger.json")) as f: + job_data = json.load(f) + job_id = job_data.get("job_id") + # If we find an 'artifacts' dir under rundir, archive it, and transmit + # it to the Testflinger server + artifacts_dir = os.path.join(rundir, "artifacts") + if os.path.isdir(artifacts_dir): + with tempfile.TemporaryDirectory() as tmpdir: + artifact_file = os.path.join(tmpdir, "artifacts") + shutil.make_archive( + artifact_file, + format="gztar", + root_dir=rundir, + base_dir="artifacts", + ) + # Create uri for API: /v1/result/ + artifact_uri = urljoin( + self.server, "/v1/result/{}/artifact".format(job_id) + ) + with open(artifact_file + ".tar.gz", "rb") as tarball: + file_upload = { + "file": ("file", tarball, "application/x-gzip") + } + artifact_request = self.session.post( + artifact_uri, files=file_upload, timeout=600 + ) + if not artifact_request: + logger.error( + "Unable to post results to: %s (error: %s)" + % (artifact_uri, artifact_request.status_code) + ) + raise TFServerError(artifact_request.status_code) + else: + shutil.rmtree(artifacts_dir) + # Do not retransmit outcome if it's already been done and removed + outcome_file = os.path.join(rundir, "testflinger-outcome.json") + if os.path.isfile(outcome_file): + logger.info("Submitting job outcome for job: %s" % job_id) + with open(outcome_file) as f: + data = json.load(f) + data["job_state"] = "complete" + self.post_result(job_id, data) + # Remove the outcome file so we don't retransmit + os.unlink(outcome_file) + shutil.rmtree(rundir) + + def post_live_output(self, job_id, data): + """Post output data to the testflinger server for this job + + :param job_id: + id for the job on which we want to post results + :param data: + string with latest output data + """ + output_uri = urljoin( + self.server, "/v1/result/{}/output".format(job_id) + ) + try: + job_request = self.session.post( + output_uri, data=data.encode("utf-8"), timeout=60 + ) + except RequestException as exc: + logger.error(exc) + return False + return bool(job_request) + + def post_queues(self, data): + """Post the list of advertised queues to testflinger server + + :param data: + dict of queue name and descriptions to send to the server + """ + queues_uri = urljoin(self.server, "/v1/agents/queues") + try: + self.session.post(queues_uri, json=data, timeout=30) + except RequestException as exc: + logger.error(exc) + + def post_images(self, data): + """Post the list of advertised images to testflinger server + + :param data: + dict of queues containing dicts of imgae names and provision data + """ + images_uri = urljoin(self.server, "/v1/agents/images") + try: + self.session.post(images_uri, json=data, timeout=30) + except RequestException as exc: + logger.error(exc) + + def post_agent_data(self, data): + """Post the relevant data points to testflinger server + + :param data: + dict of various agent data points to send to the api server + """ + agent_data_uri = urljoin(self.server, "/v1/agents/data/") + agent_data_url = urljoin(agent_data_uri, self.config.get("agent_id")) + try: + self.session.post(agent_data_url, json=data, timeout=30) + except RequestException as exc: + logger.error(exc) + + def post_influx(self, phase, result=None): + """Post the relevant data points to testflinger server + + :param data: + dict of various agent data points to send to the api server + """ + if not self.influx_client: + return + + fields = {"phase": phase} + + if result is not None: + fields["result"] = result + + data = [ + { + "measurement": "phase result", + "tags": { + "agent": self.config.get("agent_id"), + }, + "fields": fields, + "time": time.time_ns(), + } + ] + + try: + self.influx_client.write_points( + data, + database=self.influx_agent_db, + protocol="json", + ) + except InfluxDBClientError as exc: + logger.error(exc) diff --git a/agent/testflinger_agent/cmd.py b/agent/testflinger_agent/cmd.py new file mode 100644 index 00000000..f507cb3c --- /dev/null +++ b/agent/testflinger_agent/cmd.py @@ -0,0 +1,32 @@ +# Copyright (C) 2016-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Entrypoint for the testflinger-agent command""" + +import logging +import sys + +from testflinger_agent import start_agent + +logger = logging.getLogger(__name__) + + +def main(): + """main() entrypoint for the testflinger-agent command""" + try: + start_agent() + except KeyboardInterrupt: + logger.info("Caught interrupt, exiting!") + sys.exit(0) + except Exception as exc: # pylint: disable=broad-except + logger.exception(exc) diff --git a/agent/testflinger_agent/errors.py b/agent/testflinger_agent/errors.py new file mode 100644 index 00000000..17010491 --- /dev/null +++ b/agent/testflinger_agent/errors.py @@ -0,0 +1,22 @@ +# Copyright (C) 2016 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class TFServerError(Exception): + def __init__(self, m): + self.code = m + self.message = "HTTP Status: {}".format(m) + + def __str__(self): + return self.message diff --git a/agent/testflinger_agent/job.py b/agent/testflinger_agent/job.py new file mode 100644 index 00000000..77bdc4f6 --- /dev/null +++ b/agent/testflinger_agent/job.py @@ -0,0 +1,335 @@ +# Copyright (C) 2017 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +import fcntl +import json +import logging +import os +import signal +import sys +import subprocess +import time + +from testflinger_agent.errors import TFServerError + +logger = logging.getLogger(__name__) + + +class TestflingerJob: + def __init__(self, job_data, client): + """ + :param job_data: + Dictionary containing data for the test job_data + :param client: + Testflinger client object for communicating with the server + """ + self.client = client + self.job_data = job_data + self.job_id = job_data.get("job_id") + self.phase = "unknown" + + def run_test_phase(self, phase, rundir): + """Run the specified test phase in rundir + + :param phase: + Name of the test phase (setup, provision, test, ...) + :param rundir: + Directory in which to run the command defined for the phase + :return: + Returncode from the command that was executed, 0 will be returned + if there was no command to run + """ + self.phase = phase + cmd = self.client.config.get(phase + "_command") + node = self.client.config.get("agent_id") + if not cmd: + logger.info("No %s_command configured, skipping...", phase) + return 0 + if phase == "provision" and not self.job_data.get("provision_data"): + logger.info("No provision_data defined in job data, skipping...") + return 0 + if phase == "test" and not self.job_data.get("test_data"): + logger.info("No test_data defined in job data, skipping...") + return 0 + if phase == "allocate" and not self.job_data.get("allocate_data"): + return 0 + if phase == "reserve" and not self.job_data.get("reserve_data"): + return 0 + results_file = os.path.join(rundir, "testflinger-outcome.json") + output_log = os.path.join(rundir, phase + ".log") + serial_log = os.path.join(rundir, phase + "-serial.log") + logger.info("Running %s_command: %s", phase, cmd) + # Set the exitcode to some failed status in case we get interrupted + exitcode = 99 + + for line in self.banner( + "Starting testflinger {} phase on {}".format(phase, node) + ): + self.run_with_log("echo '{}'".format(line), output_log, rundir) + try: + exitcode = self.run_with_log(cmd, output_log, rundir) + except Exception as e: + logger.exception(e) + finally: + self._update_phase_results( + results_file, phase, exitcode, output_log, serial_log + ) + if phase == "allocate": + self.allocate_phase(rundir) + return exitcode + + def _update_phase_results( + self, results_file, phase, exitcode, output_log, serial_log + ): + """Update the results file with the results of the specified phase + + :param results_file: + Path to the results file + :param phase: + Name of the phase + :param exitcode: + Exitcode from the device agent + :param output_log: + Path to the output log file + :param serial_log: + Path to the serial log file + """ + with open(results_file, "r+") as results: + outcome_data = json.load(results) + if os.path.exists(output_log): + with open(output_log, "r+", encoding="utf-8") as logfile: + self._set_truncate(logfile) + outcome_data[phase + "_output"] = logfile.read() + if os.path.exists(serial_log): + with open(serial_log, "r+", encoding="utf-8") as logfile: + self._set_truncate(logfile) + outcome_data[phase + "_serial"] = logfile.read() + outcome_data[phase + "_status"] = exitcode + results.seek(0) + json.dump(outcome_data, results) + + def allocate_phase(self, rundir): + """ + Read the json dict from "device-info.json" and send it to the server + so that the multi-device agent can find the IP addresses of all + subordinate jobs + """ + device_info_file = os.path.join(rundir, "device-info.json") + with open(device_info_file, "r") as f: + device_info = json.load(f) + + # The allocated state MUST be reflected on the server or the multi- + # device job can't continue + while True: + try: + self.client.post_result(self.job_id, device_info) + break + except TFServerError: + logger.warning("Failed to post device_info, retrying...") + time.sleep(60) + + self.client.post_job_state(self.job_id, "allocated") + + self.wait_for_completion() + + def wait_for_completion(self): + """Monitor the parent job and exit when it completes""" + + while True: + try: + this_job_state = self.client.check_job_state(self.job_id) + if this_job_state in ("complete", "completed", "cancelled"): + logger.info("This job completed, exiting...") + break + + parent_job_id = self.job_data.get("parent_job_id") + if not parent_job_id: + logger.warning("No parent job ID found while allocated") + continue + parent_job_state = self.client.check_job_state( + self.job_data.get("parent_job_id") + ) + if parent_job_state in ("complete", "completed", "cancelled"): + logger.info("Parent job completed, exiting...") + break + except TFServerError: + logger.warning("Failed to get allocated job status, retrying") + time.sleep(60) + + def _set_truncate(self, f, size=1024 * 1024): + """Set up an open file so that we don't read more than a specified + size. We want to read from the end of the file rather than the + beginning. Write a warning at the end of the file if it was too big. + + :param f: + The file object, which should be opened for read/write + :param size: + Maximum number of bytes we want to allow from reading the file + """ + end = f.seek(0, 2) + if end > size: + f.write("\nWARNING: File has been truncated due to length!") + f.seek(end - size, 0) + else: + f.seek(0, 0) + + def run_with_log(self, cmd, logfile, cwd=None): + """Execute command in a subprocess and log the output + + :param cmd: + Command to run + :param logfile: + Filename to save the output in + :param cwd: + Path to run the command from + :return: + returncode from the process + """ + env = os.environ.copy() + # Make sure there all values we add are strings + env.update( + {k: v for k, v in self.client.config.items() if isinstance(v, str)} + ) + global_timeout = self.get_global_timeout() + output_timeout = self.get_output_timeout() + start_time = time.time() + with open(logfile, "a", encoding="utf-8") as f: + live_output_buffer = "" + buffer_timeout = time.time() + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + cwd=cwd, + env=env, + ) + + def cleanup(signum, frame): + process.kill() + + signal.signal(signal.SIGTERM, cleanup) + set_nonblock(process.stdout.fileno()) + + while True: + line = process.stdout.read() + if not line and process.poll() is not None: + # Process exited + break + + if line: + # Write the latest output to the log file, stdout, and + # the live output buffer + buf = line.decode(sys.stdout.encoding, errors="replace") + sys.stdout.write(buf) + live_output_buffer += buf + f.write(buf) + f.flush() + else: + if ( + self.phase == "test" + and time.time() - buffer_timeout > output_timeout + ): + buf = ( + "\nERROR: Output timeout reached! " + "({}s)\n".format(output_timeout) + ) + live_output_buffer += buf + f.write(buf) + process.kill() + break + + # Check if it's time to send the output buffer to the server + if live_output_buffer and time.time() - buffer_timeout > 10: + if self.client.post_live_output( + self.job_id, live_output_buffer + ): + live_output_buffer = "" + buffer_timeout = time.time() + + # Check global timeout + if ( + self.phase != "reserve" + and time.time() - start_time > global_timeout + ): + buf = "\nERROR: Global timeout reached! ({}s)\n".format( + global_timeout + ) + live_output_buffer += buf + f.write(buf) + process.kill() + break + + # Check if job was canceled + if ( + self.client.check_job_state(self.job_id) == "cancelled" + and self.phase != "provision" + ): + logger.info("Job cancellation was requested, exiting.") + process.kill() + break + + if live_output_buffer: + self.client.post_live_output(self.job_id, live_output_buffer) + + try: + status = process.wait(10) # process.returncode + except TimeoutError: + status = 99 # Default in case something goes wrong + return status + + def get_global_timeout(self): + """Get the global timeout for the test run in seconds""" + # Default timeout is 4 hours + default_timeout = 4 * 60 * 60 + + # Don't exceed the maximum timeout configured for the device! + return min( + self.job_data.get("global_timeout", default_timeout), + self.client.config.get("global_timeout", default_timeout), + ) + + def get_output_timeout(self): + """Get the output timeout for the test run in seconds""" + # Default timeout is 15 minutes + default_timeout = 15 * 60 + + # Don't exceed the maximum timeout configured for the device! + return min( + self.job_data.get("output_timeout", default_timeout), + self.client.config.get("output_timeout", default_timeout), + ) + + def banner(self, line): + """Yield text lines to print a banner around a sting + + :param line: + Line of text to print a banner around + """ + yield "*" * (len(line) + 4) + yield "* {} *".format(line) + yield "*" * (len(line) + 4) + + +def set_nonblock(fd): + """Set the specified fd to nonblocking output + + :param fd: + File descriptor that should be set to nonblocking mode + """ + + # XXX: This is only used in one place right now, may want to consider + # moving it if it gets wider use in the future + fl = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) diff --git a/agent/testflinger_agent/schema.py b/agent/testflinger_agent/schema.py new file mode 100644 index 00000000..f429686f --- /dev/null +++ b/agent/testflinger_agent/schema.py @@ -0,0 +1,55 @@ +# Copyright (C) 2016-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Schema validation for testflinger-agent config files""" + +import voluptuous + +SCHEMA_V1 = { + voluptuous.Required("agent_id"): str, + voluptuous.Required("polling_interval", default=10): int, + voluptuous.Required("server_address"): str, + voluptuous.Required( + "execution_basedir", default="/tmp/testflinger/run" + ): str, + voluptuous.Required( + "logging_basedir", default="/tmp/testflinger/logs" + ): str, + voluptuous.Required( + "results_basedir", default="/tmp/testflinger/results" + ): str, + voluptuous.Required("logging_level", default="INFO"): str, + voluptuous.Required("logging_quiet", default=False): bool, + voluptuous.Required("job_queues"): list, + voluptuous.Required("setup_command", default=""): str, + voluptuous.Required("provision_command", default=""): str, + voluptuous.Required("test_command", default=""): str, + voluptuous.Required("allocate_command", default=""): str, + voluptuous.Required("reserve_command", default=""): str, + voluptuous.Required("cleanup_command", default=""): str, + voluptuous.Optional("provision_type"): str, + voluptuous.Optional("global_timeout"): int, + voluptuous.Optional("output_timeout"): int, + voluptuous.Optional("advertised_queues"): dict, + voluptuous.Optional("advertised_images"): dict, +} + + +def validate(data): + """Validate data according to known schemas + + :param data: + Data to validate + """ + schema_v1 = voluptuous.Schema(SCHEMA_V1) + return schema_v1(data) diff --git a/tests/__init__.py b/agent/testflinger_agent/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to agent/testflinger_agent/tests/__init__.py diff --git a/agent/testflinger_agent/tests/test_agent.py b/agent/testflinger_agent/tests/test_agent.py new file mode 100644 index 00000000..591f9a38 --- /dev/null +++ b/agent/testflinger_agent/tests/test_agent.py @@ -0,0 +1,205 @@ +import json +import os +import shutil +import tempfile +import uuid +import requests_mock as rmock +import pytest + +from mock import patch + +import testflinger_agent +from testflinger_agent.errors import TFServerError +from testflinger_agent.client import TestflingerClient as _TestflingerClient +from testflinger_agent.agent import TestflingerAgent as _TestflingerAgent + + +class TestClient: + @pytest.fixture + def agent(self): + self.tmpdir = tempfile.mkdtemp() + self.config = { + "agent_id": "test01", + "polling_interval": "2", + "server_address": "127.0.0.1:8000", + "job_queues": ["test"], + "execution_basedir": self.tmpdir, + "logging_basedir": self.tmpdir, + "results_basedir": os.path.join(self.tmpdir, "results"), + "test_string": "ThisIsATest", + } + testflinger_agent.configure_logging(self.config) + client = _TestflingerClient(self.config) + yield _TestflingerAgent(client) + # Inside tests, we patch rmtree so that we can check files after the + # run, so we need to clean up the tmpdirs here + shutil.rmtree(self.tmpdir) + + def test_check_and_run_setup(self, agent, requests_mock): + self.config["setup_command"] = "echo setup1" + fake_job_data = {"job_id": str(uuid.uuid1()), "job_queue": "test"} + requests_mock.get( + rmock.ANY, [{"text": json.dumps(fake_job_data)}, {"text": "{}"}] + ) + requests_mock.post(rmock.ANY, status_code=200) + with patch("shutil.rmtree"): + agent.process_jobs() + setuplog = open( + os.path.join(self.tmpdir, fake_job_data.get("job_id"), "setup.log") + ).read() + assert "setup1" == setuplog.splitlines()[-1].strip() + + def test_check_and_run_provision(self, agent, requests_mock): + self.config["provision_command"] = "echo provision1" + fake_job_data = { + "job_id": str(uuid.uuid1()), + "job_queue": "test", + "provision_data": {"url": "foo"}, + } + requests_mock.get( + rmock.ANY, [{"text": json.dumps(fake_job_data)}, {"text": "{}"}] + ) + requests_mock.post(rmock.ANY, status_code=200) + with patch("shutil.rmtree"): + agent.process_jobs() + provisionlog = open( + os.path.join( + self.tmpdir, fake_job_data.get("job_id"), "provision.log" + ) + ).read() + assert "provision1" == provisionlog.splitlines()[-1].strip() + + def test_check_and_run_test(self, agent, requests_mock): + self.config["test_command"] = "echo test1" + fake_job_data = { + "job_id": str(uuid.uuid1()), + "job_queue": "test", + "test_data": {"test_cmds": "foo"}, + } + requests_mock.get( + rmock.ANY, [{"text": json.dumps(fake_job_data)}, {"text": "{}"}] + ) + requests_mock.post(rmock.ANY, status_code=200) + with patch("shutil.rmtree"): + agent.process_jobs() + testlog = open( + os.path.join(self.tmpdir, fake_job_data.get("job_id"), "test.log") + ).read() + assert "test1" == testlog.splitlines()[-1].strip() + + def test_config_vars_in_env(self, agent, requests_mock): + self.config["test_command"] = "echo test_string is $test_string" + fake_job_data = { + "job_id": str(uuid.uuid1()), + "job_queue": "test", + "test_data": {"test_cmds": "foo"}, + } + requests_mock.get( + rmock.ANY, [{"text": json.dumps(fake_job_data)}, {"text": "{}"}] + ) + requests_mock.post(rmock.ANY, status_code=200) + with patch("shutil.rmtree"): + agent.process_jobs() + testlog = open( + os.path.join(self.tmpdir, fake_job_data.get("job_id"), "test.log") + ).read() + assert "ThisIsATest" in testlog + + def test_phase_failed(self, agent, requests_mock): + # Make sure we stop running after a failed phase + self.config["provision_command"] = "/bin/false" + self.config["test_command"] = "echo test1" + fake_job_data = { + "job_id": str(uuid.uuid1()), + "job_queue": "test", + "provision_data": {"url": "foo"}, + "test_data": {"test_cmds": "foo"}, + } + requests_mock.get( + rmock.ANY, [{"text": json.dumps(fake_job_data)}, {"text": "{}"}] + ) + requests_mock.post(rmock.ANY, status_code=200) + with patch("shutil.rmtree"), patch("os.unlink"): + agent.process_jobs() + outcome_file = os.path.join( + os.path.join( + self.tmpdir, + fake_job_data.get("job_id"), + "testflinger-outcome.json", + ) + ) + with open(outcome_file) as f: + outcome_data = json.load(f) + assert outcome_data.get("provision_status") == 1 + assert outcome_data.get("test_status") is None + + def test_retry_transmit(self, agent, requests_mock): + # Make sure we retry sending test results + self.config["provision_command"] = "/bin/false" + self.config["test_command"] = "echo test1" + fake_job_data = {"job_id": str(uuid.uuid1()), "job_queue": "test"} + # Send an extra empty data since we will be calling get 3 times + requests_mock.get( + rmock.ANY, + [ + {"text": json.dumps(fake_job_data)}, + {"text": "{}"}, + {"text": "{}"}, + ], + ) + requests_mock.post(rmock.ANY, status_code=200) + with patch.object( + testflinger_agent.client.TestflingerClient, "transmit_job_outcome" + ) as mock_transmit_job_outcome: + # Make sure we fail the first time when transmitting the results + mock_transmit_job_outcome.side_effect = [TFServerError(404), ""] + agent.process_jobs() + first_dir = os.path.join( + self.config.get("execution_basedir"), + fake_job_data.get("job_id"), + ) + mock_transmit_job_outcome.assert_called_with(first_dir) + # Try processing jobs again, now it should be in results_basedir + agent.process_jobs() + retry_dir = os.path.join( + self.config.get("results_basedir"), fake_job_data.get("job_id") + ) + mock_transmit_job_outcome.assert_called_with(retry_dir) + + def test_recovery_failed(self, agent, requests_mock): + # Make sure we stop processing jobs after a device recovery error + OFFLINE_FILE = "/tmp/TESTFLINGER-DEVICE-OFFLINE-test001" + if os.path.exists(OFFLINE_FILE): + os.unlink(OFFLINE_FILE) + self.config["agent_id"] = "test001" + self.config["provision_command"] = "exit 46" + self.config["test_command"] = "echo test1" + job_id = str(uuid.uuid1()) + fake_job_data = { + "job_id": job_id, + "job_queue": "test", + "provision_data": {"url": "foo"}, + "test_data": {"test_cmds": "foo"}, + } + # In this case we are making sure that the repost job request + # gets good status + with rmock.Mocker() as m: + m.get( + "http://127.0.0.1:8000/v1/job?queue=test", json=fake_job_data + ) + m.get("http://127.0.0.1:8000/v1/result/" + job_id, text="{}") + m.post("http://127.0.0.1:8000/v1/result/" + job_id, text="{}") + m.post( + "http://127.0.0.1:8000/v1/result/" + job_id + "/output", + text="{}", + ) + m.post( + "http://127.0.0.1:8000/v1/agents/data/" + + self.config.get("agent_id"), + text="OK", + ) + + agent.process_jobs() + assert agent.check_offline() + if os.path.exists(OFFLINE_FILE): + os.unlink(OFFLINE_FILE) diff --git a/agent/testflinger_agent/tests/test_client.py b/agent/testflinger_agent/tests/test_client.py new file mode 100644 index 00000000..e7fd3a67 --- /dev/null +++ b/agent/testflinger_agent/tests/test_client.py @@ -0,0 +1,40 @@ +# Copyright (C) 2016 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pytest +import uuid + +import requests_mock as rmock + +from testflinger_agent.client import TestflingerClient as _TestflingerClient + + +class TestClient: + @pytest.fixture + def client(self): + yield _TestflingerClient({"server_address": "127.0.0.1:8000"}) + + def test_check_jobs_empty(self, client, requests_mock): + requests_mock.get(rmock.ANY, status_code=200) + job_data = client.check_jobs() + assert job_data is None + + def test_check_jobs_with_job(self, client, requests_mock): + fake_job_data = { + "job_id": str(uuid.uuid1()), + "job_queue": "test_queue", + } + requests_mock.get(rmock.ANY, json=fake_job_data) + job_data = client.check_jobs() + assert job_data == fake_job_data diff --git a/agent/testflinger_agent/tests/test_config.py b/agent/testflinger_agent/tests/test_config.py new file mode 100644 index 00000000..d25cd88d --- /dev/null +++ b/agent/testflinger_agent/tests/test_config.py @@ -0,0 +1,56 @@ +# Copyright (C) 2016 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import tempfile +import testflinger_agent +import voluptuous + +from unittest import TestCase + +GOOD_CONFIG = """ +agent_id: test01 +polling_interval: 10 +server_address: 127.0.0.1:8000 +job_queues: + - test +""" + +BAD_CONFIG = """ +badkey: foo +""" + + +class ConfigTest(TestCase): + def setUp(self): + with tempfile.NamedTemporaryFile(delete=False) as config: + self.configfile = config.name + + def tearDown(self): + os.unlink(self.configfile) + + def test_config_good(self): + with open(self.configfile, "w") as config: + config.write(GOOD_CONFIG) + config = testflinger_agent.load_config(self.configfile) + self.assertEqual("test01", config.get("agent_id")) + + def test_config_bad(self): + with open(self.configfile, "w") as config: + config.write(BAD_CONFIG) + self.assertRaises( + voluptuous.error.MultipleInvalid, + testflinger_agent.load_config, + self.configfile, + ) diff --git a/agent/testflinger_agent/tests/test_job.py b/agent/testflinger_agent/tests/test_job.py new file mode 100644 index 00000000..627b3c96 --- /dev/null +++ b/agent/testflinger_agent/tests/test_job.py @@ -0,0 +1,164 @@ +import os +import pytest +import shutil +import tempfile + +import requests_mock as rmock + +import testflinger_agent +from testflinger_agent.client import TestflingerClient as _TestflingerClient +from testflinger_agent.job import TestflingerJob as _TestflingerJob + + +class TestJob: + @pytest.fixture + def client(self): + self.tmpdir = tempfile.mkdtemp() + self.config = { + "agent_id": "test01", + "polling_interval": "2", + "server_address": "127.0.0.1:8000", + "job_queues": ["test"], + "execution_basedir": self.tmpdir, + "logging_basedir": self.tmpdir, + "results_basedir": os.path.join(self.tmpdir, "results"), + } + testflinger_agent.configure_logging(self.config) + yield _TestflingerClient(self.config) + shutil.rmtree(self.tmpdir) + + def test_skip_missing_provision_data(self, client): + """ + Test that provision phase is skipped when provision_data is + absent + """ + self.config["provision_command"] = "/bin/true" + fake_job_data = {"global_timeout": 1} + job = _TestflingerJob(fake_job_data, client) + job.run_test_phase("provision", None) + logfile = os.path.join(self.tmpdir, "testflinger-agent.log") + with open(logfile) as log: + log_output = log.read() + assert "No provision_data defined in job data" in log_output + + def test_skip_empty_provision_data(self, client): + """ + Test that provision phase is skipped when provision_data is + present but empty + """ + self.config["provision_command"] = "/bin/true" + fake_job_data = {"global_timeout": 1, "provision_data": ""} + job = _TestflingerJob(fake_job_data, client) + job.run_test_phase("provision", None) + logfile = os.path.join(self.tmpdir, "testflinger-agent.log") + with open(logfile) as log: + log_output = log.read() + assert "No provision_data defined in job data" in log_output + + def test_job_global_timeout(self, client, requests_mock): + """Test that timeout from job_data is respected""" + timeout_str = "\nERROR: Global timeout reached! (1s)\n" + logfile = os.path.join(self.tmpdir, "testlog") + fake_job_data = {"global_timeout": 1} + requests_mock.post(rmock.ANY, status_code=200) + requests_mock.get(rmock.ANY, status_code=200) + job = _TestflingerJob(fake_job_data, client) + job.phase = "test" + job.run_with_log("sleep 3", logfile) + with open(logfile) as log: + log_data = log.read() + assert timeout_str == log_data + + def test_config_global_timeout(self, client, requests_mock): + """Test that timeout from device config is preferred""" + timeout_str = "\nERROR: Global timeout reached! (1s)\n" + logfile = os.path.join(self.tmpdir, "testlog") + self.config["global_timeout"] = 1 + fake_job_data = {"global_timeout": 3} + requests_mock.post(rmock.ANY, status_code=200) + requests_mock.get(rmock.ANY, status_code=200) + job = _TestflingerJob(fake_job_data, client) + job.phase = "test" + job.run_with_log("sleep 3", logfile) + with open(logfile) as log: + log_data = log.read() + assert timeout_str == log_data + + def test_job_output_timeout(self, client, requests_mock): + """Test that output timeout from job_data is respected""" + timeout_str = "\nERROR: Output timeout reached! (1s)\n" + logfile = os.path.join(self.tmpdir, "testlog") + fake_job_data = {"output_timeout": 1} + requests_mock.post(rmock.ANY, status_code=200) + requests_mock.get(rmock.ANY, status_code=200) + job = _TestflingerJob(fake_job_data, client) + job.phase = "test" + # unfortunately, we need to sleep for longer that 10 seconds here + # or else we fall under the polling time + job.run_with_log("sleep 12", logfile) + with open(logfile) as log: + log_data = log.read() + assert timeout_str == log_data + + def test_config_output_timeout(self, client, requests_mock): + """Test that output timeout from device config is preferred""" + timeout_str = "\nERROR: Output timeout reached! (1s)\n" + logfile = os.path.join(self.tmpdir, "testlog") + self.config["output_timeout"] = 1 + fake_job_data = {"output_timeout": 30} + requests_mock.post(rmock.ANY, status_code=200) + requests_mock.get(rmock.ANY, status_code=200) + job = _TestflingerJob(fake_job_data, client) + job.phase = "test" + # unfortunately, we need to sleep for longer that 10 seconds here + # or else we fall under the polling time + job.run_with_log("sleep 12", logfile) + with open(logfile) as log: + log_data = log.read() + assert timeout_str == log_data + + def test_no_output_timeout_in_provision(self, client, requests_mock): + """Test that output timeout is ignored when not in test phase""" + timeout_str = "complete\n" + logfile = os.path.join(self.tmpdir, "testlog") + fake_job_data = {"output_timeout": 1} + requests_mock.post(rmock.ANY, status_code=200) + requests_mock.get(rmock.ANY, status_code=200) + job = _TestflingerJob(fake_job_data, client) + job.phase = "provision" + # unfortunately, we need to sleep for longer that 10 seconds here + # or else we fall under the polling time + job.run_with_log("sleep 12 && echo complete", logfile) + with open(logfile) as log: + log_data = log.read() + assert timeout_str == log_data + + def test_set_truncate(self, client): + """Test the _set_truncate method of TestflingerJob""" + job = _TestflingerJob({}, client) + with tempfile.TemporaryFile(mode="r+") as f: + # First check that a small file doesn't get truncated + f.write("x" * 100) + job._set_truncate(f, size=100) + contents = f.read() + assert len(contents) == 100 + assert "WARNING" not in contents + + # Now check that a larger file does get truncated + f.write("x" * 100) + job._set_truncate(f, size=100) + contents = f.read() + # It won't be exactly 100 bytes, because a warning is added + assert len(contents) < 150 + assert "WARNING" in contents + + @pytest.mark.timeout(1) + def test_wait_for_completion(self, client): + """Test that wait_for_completion works""" + + # Make sure we return "completed" for the parent job state + client.check_job_state = lambda _: "completed" + + job = _TestflingerJob({"parent_job_id": "999"}, client) + job.wait_for_completion() + # No assertions needed, just make sure we don't timeout diff --git a/agent/tox.ini b/agent/tox.ini new file mode 100644 index 00000000..cd335062 --- /dev/null +++ b/agent/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = py +skipsdist = true + +[testenv] +setenv = + HOME = {envtmpdir} +deps = + black + flake8 + mock + pytest + pylint + pytest-mock + pytest-cov + pytest-timeout + requests-mock +commands = + {envbindir}/python setup.py develop + {envbindir}/python -m black --check setup.py testflinger_agent + {envbindir}/python -m flake8 setup.py testflinger_agent + {envbindir}/python -m pytest --doctest-modules --cov=testflinger_agent diff --git a/cli/.gitignore b/cli/.gitignore new file mode 100644 index 00000000..ca62b051 --- /dev/null +++ b/cli/.gitignore @@ -0,0 +1,6 @@ +env/ +dist/ +build/ +testflinger_cli.egg-info/ +**/__pycache__/ +.coverage diff --git a/cli/.pylintrc b/cli/.pylintrc new file mode 100644 index 00000000..691ed1ec --- /dev/null +++ b/cli/.pylintrc @@ -0,0 +1,7 @@ +[MASTER] +good-names = k, v + +[MESSAGES CONTROL] +# We currently have some older systems running this which +# don't support f-strings yet +disable = C0209 diff --git a/cli/README.rst b/cli/README.rst new file mode 100644 index 00000000..520cd4f9 --- /dev/null +++ b/cli/README.rst @@ -0,0 +1,73 @@ +=============== +Testflinger CLI +=============== + +Overview +-------- + +The testflinger-cli tool is used for interacting with testflinger +server. It can be used for things like submitting jobs, checking +the status of them, and getting results. + +Installation +------------ + +You can either run testflinger-cli from a checkout of the code, or +install it like any other python project. + +To run it from a checkout, please make sure to first install python3-click +and python3-requests + +To install it in a virtual environment: + +.. code-block:: console + + $ virtualenv -p python3 env + $ . env/bin/activate + $ pip install . + + +Usage +----- + +After installing testflinger-cli, you can get help by just running +'testflinger-cli' on its own, or by using the '--help' parameter. + +To specify a different server to use, you can use the '--server' +parameter, otherwise it will default to the one running on +http://testflinger.canonical.com +You may also set the environment variable 'TESTFLINGER_SERVER' to +the URI of your server, and it will prefer that over the default +or the string specified by --server. + +To submit a new test job, first create a yaml or json file containing +the job definition. Then run: +.. code-block:: console + + $ testflinger-cli submit mytest.json + +If successful, this will return the job_id of the test job you submitted. +You can check on the status of that job by running: +.. code-block:: console + + $ testflinger-cli status + +To watch the output from the job as it runs, you can use the 'poll' +subcommand. This will display output in 10s chunks and exit when the +job is completed. +.. code-block:: console + + $ testflinger-cli poll + +To get the full json results from the job when it is done running, you can +use the 'results' subcommand: +.. code-block:: console + + $ testflinger-cli results + +Finally, to download the artifact tarball from the job, you can use the +'artifact' subcommand: +.. code-block:: console + + $ testflinger-cli artifact [--filename ] + diff --git a/cli/pyproject.toml b/cli/pyproject.toml new file mode 100644 index 00000000..1b2ba197 --- /dev/null +++ b/cli/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = [ + "setuptools", + "setuptools-scm", +] +build-backend = "setuptools.build_meta" + +[project] +name = "testflinger-cli" +description = "Testflinger CLI" +readme = "README.rst" +dependencies = [ + "PyYAML", + "requests", + "xdg<5.2", +] +dynamic = ["version"] + +[project.scripts] +testflinger-cli = "testflinger_cli:cli" +testflinger = "testflinger_cli:cli" + +[tool.black] +line-length = 79 \ No newline at end of file diff --git a/cli/renovate.json b/cli/renovate.json new file mode 100644 index 00000000..39a2b6e9 --- /dev/null +++ b/cli/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} diff --git a/cli/setup.py b/cli/setup.py new file mode 100755 index 00000000..77b2fc18 --- /dev/null +++ b/cli/setup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from setuptools import setup + +setup() diff --git a/cli/snapcraft.yaml b/cli/snapcraft.yaml new file mode 100644 index 00000000..df4c0e2c --- /dev/null +++ b/cli/snapcraft.yaml @@ -0,0 +1,36 @@ +name: testflinger-cli +summary: testflinger-cli +description: | + The testflinger-cli tool is used for interacting with the testflinger + server for submitting test jobs, checking status, getting results, and + streaming output. +confinement: strict +base: core20 +adopt-info: testflinger-cli + +architectures: + - build-on: amd64 + - build-on: arm64 + - build-on: armhf + +apps: + testflinger-cli: + command: bin/testflinger-cli.wrapper + plugs: + - home + - network + +parts: + launcher: + plugin: dump + source: . + organize: + testflinger-cli.wrapper: bin/testflinger-cli.wrapper + testflinger-cli: + plugin: python + source: . + override-pull: | + set -e + snapcraftctl pull + snapcraftctl set-version "$(date +%Y%m%d)" + snapcraftctl set-grade "stable" diff --git a/cli/testflinger-cli b/cli/testflinger-cli new file mode 100755 index 00000000..f3a0644c --- /dev/null +++ b/cli/testflinger-cli @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# Copyright (C) 2017-2019 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +from testflinger_cli import cli + + +if __name__ == "__main__": + cli() diff --git a/cli/testflinger-cli.wrapper b/cli/testflinger-cli.wrapper new file mode 100755 index 00000000..92788620 --- /dev/null +++ b/cli/testflinger-cli.wrapper @@ -0,0 +1,4 @@ +#!/bin/sh +export LC_ALL=C.UTF-8 +export LANG=C.UTF-8 +exec testflinger-cli $@ diff --git a/cli/testflinger_cli/__init__.py b/cli/testflinger_cli/__init__.py new file mode 100644 index 00000000..241c8983 --- /dev/null +++ b/cli/testflinger_cli/__init__.py @@ -0,0 +1,647 @@ +# Copyright (C) 2017-2022 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +TestflingerCli module +""" + + +import inspect +import json +import logging +import os +import sys +import time +from argparse import ArgumentParser +from datetime import datetime +import yaml + +from testflinger_cli import client, config, history + +logger = logging.getLogger(__name__) + +# Make it easier to run from a checkout +basedir = os.path.abspath(os.path.join(__file__, "..")) +if os.path.exists(os.path.join(basedir, "setup.py")): + sys.path.insert(0, basedir) + + +def cli(): + """Generate the TestflingerCli instance and run it""" + try: + tfcli = TestflingerCli() + configure_logging() + tfcli.run() + except KeyboardInterrupt as exc: + raise SystemExit from exc + + +def configure_logging(): + """Configure default logging""" + logging.basicConfig( + level=logging.WARNING, + format=( + "%(levelname)s: %(asctime)s %(filename)s:%(lineno)d -- %(message)s" + ), + datefmt="%Y-%m-%d %H:%M:%S", + ) + + +def _get_image(images): + image = "" + flex_url = "" + if images and images[list(images.keys())[0]].startswith("url:"): + # If this device can take URLs, offer to let the user enter one + # instead of just using the known images + flex_url = "or URL for a valid image starting with http(s)://... " + while not image or image == "?": + image = input( + "\nEnter the name of the image you want to use " + + flex_url + + "('?' to list) " + ) + if image == "?": + if not images: + print( + "WARNING: There are no images defined for this " + "device. You may also provide the URL to an image " + "that can be booted with this device though." + ) + continue + for image_id in sorted(images.keys()): + print(" " + image_id) + continue + if image.startswith(("http://", "https://")): + return image + if image not in images.keys(): + print( + "ERROR: '{}' is not in the list of known images for " + "that queue, please select another.".format(image) + ) + image = "" + return image + + +def _get_ssh_keys(): + ssh_keys = "" + while not ssh_keys.strip(): + ssh_keys = input( + "\nEnter the ssh key(s) you wish to use: " + "(ex: lp:userid, gh:userid) " + ) + key_list = [ssh_key.strip() for ssh_key in ssh_keys.split(",")] + for ssh_key in key_list: + if not ssh_key.startswith("lp:") and not ssh_key.startswith("gh:"): + ssh_keys = "" + print("Please enter keys in the form lp:userid or gh:userid") + return key_list + + +class TestflingerCli: + """Class for handling the Testflinger CLI""" + + def __init__(self): + self.get_args() + self.config = config.TestflingerCliConfig(self.args.configfile) + server = ( + self.args.server + or self.config.get("server") + or os.environ.get("TESTFLINGER_SERVER") + or "https://testflinger.canonical.com" + ) + # Allow config subcommand without worrying about server or client + if ( + hasattr(self.args, "func") + and self.args.func == self.configure # pylint: disable=W0143 + ): + return + if not server.startswith(("http://", "https://")): + raise SystemExit( + 'Server must start with "http://" or "https://" ' + '- currently set to: "{}"'.format(server) + ) + self.client = client.Client(server) + self.history = history.TestflingerCliHistory() + + def run(self): + """Run the subcommand specified in command line arguments""" + if hasattr(self.args, "func"): + raise SystemExit(self.args.func()) + print(self.help) + + def get_args(self): + """Handle command line arguments""" + parser = ArgumentParser() + parser.add_argument( + "-c", + "--configfile", + default=None, + help="Configuration file to use", + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="Enable debug logging" + ) + parser.add_argument( + "--server", default=None, help="Testflinger server to use" + ) + sub = parser.add_subparsers() + arg_artifacts = sub.add_parser( + "artifacts", + help="Download a tarball of artifacts saved for a specified job", + ) + arg_artifacts.set_defaults(func=self.artifacts) + arg_artifacts.add_argument("--filename", default="artifacts.tgz") + arg_artifacts.add_argument("job_id") + arg_cancel = sub.add_parser( + "cancel", help="Tell the server to cancel a specified JOB_ID" + ) + arg_cancel.set_defaults(func=self.cancel) + arg_cancel.add_argument("job_id") + arg_config = sub.add_parser( + "config", help="Get or set configuration options" + ) + arg_config.set_defaults(func=self.configure) + arg_config.add_argument("setting", nargs="?", help="setting=value") + arg_jobs = sub.add_parser( + "jobs", help="List the previously started test jobs" + ) + arg_jobs.set_defaults(func=self.jobs) + arg_jobs.add_argument( + "--status", + "-s", + action="store_true", + help="Include job status (may add delay)", + ) + arg_list_queues = sub.add_parser( + "list-queues", + help="List the advertised queues on the Testflinger server", + ) + arg_list_queues.set_defaults(func=self.list_queues) + arg_poll = sub.add_parser( + "poll", help="Poll for output from a job until it is completed" + ) + arg_poll.set_defaults(func=self.poll) + arg_poll.add_argument( + "--oneshot", + "-o", + action="store_true", + help="Get latest output and exit immediately", + ) + arg_poll.add_argument("job_id") + arg_reserve = sub.add_parser( + "reserve", help="Install and reserve a system" + ) + arg_reserve.set_defaults(func=self.reserve) + arg_reserve.add_argument( + "--queue", "-q", help="Name of the queue to use" + ) + arg_reserve.add_argument( + "--image", "-i", help="Name of the image to use for provisioning" + ) + arg_reserve.add_argument( + "--key", + "-k", + nargs="*", + help=( + "Ssh key(s) to use for reservation " + "(ex: -k lp:userid -k gh:userid)" + ), + ) + arg_results = sub.add_parser( + "results", help="Get results JSON for a completed JOB_ID" + ) + arg_results.set_defaults(func=self.results) + arg_results.add_argument("job_id") + arg_show = sub.add_parser( + "show", help="Show the requested job JSON for a specified JOB_ID" + ) + arg_show.set_defaults(func=self.show) + arg_show.add_argument("job_id") + arg_status = sub.add_parser( + "status", help="Show the status of a specified JOB_ID" + ) + arg_status.set_defaults(func=self.status) + arg_status.add_argument("job_id") + arg_submit = sub.add_parser( + "submit", help="Submit a new test job to the server" + ) + arg_submit.set_defaults(func=self.submit) + arg_submit.add_argument("--poll", "-p", action="store_true") + arg_submit.add_argument("--quiet", "-q", action="store_true") + arg_submit.add_argument("filename") + + self.args = parser.parse_args() + self.help = parser.format_help() + + def status(self): + """Show the status of a specified JOB_ID""" + job_state = self.get_job_state(self.args.job_id) + if job_state != "unknown": + self.history.update(self.args.job_id, job_state) + print(job_state) + + def cancel(self, job_id=None): + """Tell the server to cancel a specified JOB_ID""" + if not job_id: + try: + job_id = self.args.job_id + except AttributeError as exc: + raise SystemExit("No job id specified to cancel.") from exc + try: + self.client.put(f"/v1/job/{job_id}/action", {"action": "cancel"}) + self.history.update(job_id, "cancelled") + except client.HTTPError as exc: + if exc.status == 400: + raise SystemExit( + "Invalid job ID specified or the job is already " + "completed/cancelled." + ) from exc + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + raise + + def configure(self): + """Print or set configuration values""" + if self.args.setting: + setting = self.args.setting.split("=") + if len(setting) == 2: + self.config.set(*setting) + return + if len(setting) == 1: + print( + "{} = {}".format(setting[0], self.config.get(setting[0])) + ) + return + print("Current Configuration") + print("---------------------") + for k, v in self.config.data.items(): + print("{} = {}".format(k, v)) + print() + + def submit(self): + """Submit a new test job to the server""" + if self.args.filename == "-": + data = sys.stdin.read() + else: + try: + with open( + self.args.filename, encoding="utf-8", errors="ignore" + ) as job_file: + data = job_file.read() + except FileNotFoundError as exc: + raise SystemExit( + "File not found: {}".format(self.args.filename) + ) from exc + job_id = self.submit_job_data(data) + queue = yaml.safe_load(data).get("job_queue") + self.history.new(job_id, queue) + if self.args.quiet: + print(job_id) + else: + print("Job submitted successfully!") + print("job_id: {}".format(job_id)) + if self.args.poll: + self.do_poll(job_id) + + def submit_job_data(self, data): + """Submit data that was generated or read from a file as a test job""" + try: + job_id = self.client.submit_job(data) + except client.HTTPError as exc: + if exc.status == 400: + raise SystemExit( + "The job you submitted contained bad data or " + "bad formatting, or did not specify a " + "job_queue." + ) from exc + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + # This shouldn't happen, so let's get more information + raise SystemExit( + "Unexpected error status from testflinger " + "server: {}".format(exc.status) + ) from exc + return job_id + + def show(self): + """Show the requested job JSON for a specified JOB_ID""" + try: + results = self.client.show_job(self.args.job_id) + except client.HTTPError as exc: + if exc.status == 204: + raise SystemExit("No data found for that job id.") from exc + if exc.status == 400: + raise SystemExit( + "Invalid job id specified. Check the job id " + "to be sure it is correct" + ) from exc + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + # This shouldn't happen, so let's get more information + raise SystemExit( + "Unexpected error status from testflinger " + "server: {}".format(exc.status) + ) from exc + print(json.dumps(results, sort_keys=True, indent=4)) + + def results(self): + """Get results JSON for a completed JOB_ID""" + try: + results = self.client.get_results(self.args.job_id) + except client.HTTPError as exc: + if exc.status == 204: + raise SystemExit("No results found for that job id.") from exc + if exc.status == 400: + raise SystemExit( + "Invalid job id specified. Check the job id " + "to be sure it is correct" + ) from exc + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + # This shouldn't happen, so let's get more information + raise SystemExit( + "Unexpected error status from testflinger " + "server: {}".format(exc.status) + ) from exc + + print(json.dumps(results, sort_keys=True, indent=4)) + + def artifacts(self): + """Download a tarball of artifacts saved for a specified job""" + print("Downloading artifacts tarball...") + try: + self.client.get_artifact(self.args.job_id, self.args.filename) + except client.HTTPError as exc: + if exc.status == 204: + raise SystemExit( + "No artifacts tarball found for that job id." + ) from exc + if exc.status == 400: + raise SystemExit( + "Invalid job id specified. Check the job id " + "to be sure it is correct" + ) from exc + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + # This shouldn't happen, so let's get more information + raise SystemExit( + "Unexpected error status from testflinger " + "server: {}".format(exc.status) + ) from exc + print("Artifacts downloaded to {}".format(self.args.filename)) + + def poll(self): + """Poll for output from a job until it is completed""" + if self.args.oneshot: + # This could get an IOError for connection errors or timeouts + # Raise it since it's not running continuously in this mode + output = self.get_latest_output(self.args.job_id) + if output: + print(output, end="", flush=True) + sys.exit(0) + self.do_poll(self.args.job_id) + + def do_poll(self, job_id): + """Poll for output from a running job and print it while it runs + + :param str job_id: Job ID + """ + job_state = self.get_job_state(job_id) + self.history.update(job_id, job_state) + prev_queue_pos = None + if job_state == "waiting": + print("This job is waiting on a node to become available.") + while True: + try: + job_state = self.get_job_state(job_id) + self.history.update(job_id, job_state) + if job_state in ("cancelled", "complete", "completed"): + break + if job_state == "waiting": + queue_pos = self.client.get_job_position(job_id) + if int(queue_pos) != prev_queue_pos: + prev_queue_pos = int(queue_pos) + print("Jobs ahead in queue: {}".format(queue_pos)) + time.sleep(10) + output = "" + output = self.get_latest_output(job_id) + if output: + print(output, end="", flush=True) + except (IOError, client.HTTPError): + # Ignore/retry or debug any connection errors or timeouts + if self.args.debug: + logging.exception("Error polling for job output") + except KeyboardInterrupt: + choice = input( + "\nCancel job {} before exiting " + "(y)es/(N)o/(c)ontinue? ".format(job_id) + ) + if choice: + choice = choice[0].lower() + if choice == "c": + continue + if choice == "y": + self.cancel(job_id) + # Both y and n will allow the external handler deal with it + raise + + print(job_state) + + def jobs(self): + """List the previously started test jobs""" + if self.args.status: + # Getting job state may be slow, only include if requested + status_text = "Status" + else: + status_text = "" + print( + "{:36} {:9} {} {}".format( + "Job ID", status_text, "Submission Time", "Queue" + ) + ) + print("-" * 79) + for job_id, jobdata in self.history.history.items(): + if self.args.status: + job_state = jobdata.get("job_state") + if job_state not in ("cancelled", "complete", "completed"): + job_state = self.get_job_state(job_id) + self.history.update(job_id, job_state) + else: + job_state = "" + print( + "{} {:9} {} {}".format( + job_id, + job_state, + datetime.fromtimestamp( + jobdata.get("submission_time") + ).strftime("%a %b %d %H:%M"), + jobdata.get("queue"), + ) + ) + print() + + def list_queues(self): + """List the advertised queues on the current Testflinger server""" + try: + queues = self.client.get_queues() + except client.HTTPError as exc: + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + print("Advertised queues on this server:") + for name, description in sorted(queues.items()): + print(" {} - {}".format(name, description)) + + def reserve(self): + """Install and reserve a system""" + try: + queues = self.client.get_queues() + except OSError: + logger.warning("Unable to get a list of queues from the server!") + queues = {} + queue = self.args.queue or self._get_queue(queues) + if queue not in queues.keys(): + print( + "WARNING: '{}' is not in the list of known " + "queues".format(queue) + ) + try: + images = self.client.get_images(queue) + except OSError: + logger.warning("Unable to get a list of images from the server!") + images = {} + image = self.args.image or _get_image(images) + if ( + not image.startswith(("http://", "https://")) + and image not in images.keys() + ): + raise SystemExit( + "ERROR: '{}' is not in the list of known " + "images for that queue, please select " + "another.".format(image) + ) + if image.startswith(("http://", "https://")): + image = "url: " + image + else: + image = images[image] + ssh_keys = self.args.key or _get_ssh_keys() + for ssh_key in ssh_keys: + if not ssh_key.startswith("lp:") and not ssh_key.startswith("gh:"): + raise SystemExit( + "Please enter keys in the form lp:userid or gh:userid" + ) + template = inspect.cleandoc( + """job_queue: {queue} + provision_data: + {image} + reserve_data: + ssh_keys:""" + ) + for ssh_key in ssh_keys: + template += "\n - {}".format(ssh_key) + job_data = template.format(queue=queue, image=image) + print("\nThe following yaml will be submitted:") + print(job_data) + answer = input("Proceed? (Y/n) ") + if answer in ("Y", "y", ""): + job_id = self.submit_job_data(job_data) + print("Job submitted successfully!") + print("job_id: {}".format(job_id)) + self.do_poll(job_id) + + def _get_queue(self, queues): + queue = "" + while not queue or queue == "?": + queue = input("\nWhich queue do you want to use? ('?' to list) ") + if not queue: + continue + if queue == "?": + print("\nAdvertised queues on this server:") + for name, description in sorted(queues.items()): + print(" {} - {}".format(name, description)) + queue = self._get_queue(queues) + if queue not in queues.keys(): + print( + "WARNING: '{}' is not in the list of known " + "queues".format(queue) + ) + answer = input("Do you still want to use it? (y/N) ") + if answer.lower() != "y": + queue = "" + return queue + + def get_latest_output(self, job_id): + """Get the latest output from a running job + + :param str job_id: Job ID + :return str: New output from the running job + """ + output = "" + try: + output = self.client.get_output(job_id) + except client.HTTPError as exc: + if exc.status == 204: + # We are still waiting for the job to start + pass + return output + + def get_job_state(self, job_id): + """Return the job state for the specified job_id + + :param str job_id: Job ID + :raises SystemExit: Exit with HTTP error code + :return str : Job state + """ + try: + return self.client.get_status(job_id) + except client.HTTPError as exc: + if exc.status == 204: + raise SystemExit( + "No data found for that job id. Check the " + "job id to be sure it is correct" + ) from exc + if exc.status == 400: + raise SystemExit( + "Invalid job id specified. Check the job id " + "to be sure it is correct" + ) from exc + if exc.status == 404: + raise SystemExit( + "Received 404 error from server. Are you " + "sure this is a testflinger server?" + ) from exc + except (IOError, ValueError): + # For other types of network errors, or JSONDecodeError if we got + # a bad return from get_status() + logger.warning("Unable to retrieve job state.") + return "unknown" diff --git a/cli/testflinger_cli/client.py b/cli/testflinger_cli/client.py new file mode 100644 index 00000000..8096f4ef --- /dev/null +++ b/cli/testflinger_cli/client.py @@ -0,0 +1,207 @@ +# Copyright (C) 2017-2022 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Testflinger client module +""" + +import json +import logging +import sys +import urllib.parse +import requests +import yaml + + +logger = logging.getLogger(__name__) + + +class HTTPError(Exception): + """Exception class for HTTP error codes""" + + def __init__(self, status): + super().__init__(status) + self.status = status + + +class Client: + """Testflinger connection client""" + + def __init__(self, server): + self.server = server + + def get(self, uri_frag, timeout=15): + """Submit a GET request to the server + :param uri_frag: + endpoint for the GET request + :return: + String containing the response from the server + """ + uri = urllib.parse.urljoin(self.server, uri_frag) + try: + req = requests.get(uri, timeout=timeout) + except requests.exceptions.ConnectionError: + logger.error("Unable to communicate with specified server.") + raise + except IOError: + # This should catch all other timeout cases + logger.error( + "Timeout while trying to communicate with the server." + ) + raise + if req.status_code != 200: + raise HTTPError(req.status_code) + return req.text + + def put(self, uri_frag, data, timeout=15): + """Submit a POST request to the server + :param uri_frag: + endpoint for the POST request + :return: + String containing the response from the server + """ + uri = urllib.parse.urljoin(self.server, uri_frag) + try: + req = requests.post(uri, json=data, timeout=timeout) + except requests.exceptions.ConnectTimeout: + logger.error("Timout while trying to communicate with the server.") + sys.exit(1) + except requests.exceptions.ConnectionError: + logger.error("Unable to communicate with specified server.") + sys.exit(1) + if req.status_code != 200: + raise HTTPError(req.status_code) + return req.text + + def get_status(self, job_id): + """Get the status of a test job + + :param job_id: + ID for the test job + :return: + String containing the job_state for the specified ID + (waiting, setup, provision, test, reserved, released, + cancelled, completed) + """ + endpoint = "/v1/result/{}".format(job_id) + data = json.loads(self.get(endpoint)) + return data.get("job_state") + + def post_job_state(self, job_id, state): + """Post the status of a test job + + :param job_id: + ID for the test job + :param state: + Job state to set for the specified job + """ + endpoint = "/v1/result/{}".format(job_id) + data = {"job_state": state} + self.put(endpoint, data) + + def submit_job(self, job_data): + """Submit a test job to the testflinger server + + :param job_data: + String containing json or yaml data for the job to submit + :return: + ID for the test job + """ + endpoint = "/v1/job" + data = yaml.safe_load(job_data) + response = self.put(endpoint, data) + return json.loads(response).get("job_id") + + def show_job(self, job_id): + """Show the JSON job definition for the specified ID + + :param job_id: + ID for the test job + :return: + JSON job definition for the specified ID + """ + endpoint = "/v1/job/{}".format(job_id) + return json.loads(self.get(endpoint)) + + def get_results(self, job_id): + """Get results for a specified test job + + :param job_id: + ID for the test job + :return: + Dict containing the results returned from the server + """ + endpoint = "/v1/result/{}".format(job_id) + return json.loads(self.get(endpoint)) + + def get_artifact(self, job_id, path): + """Get results for a specified test job + + :param job_id: + ID for the test job + :param path: + Path and filename for the artifact file + """ + endpoint = "/v1/result/{}/artifact".format(job_id) + uri = urllib.parse.urljoin(self.server, endpoint) + req = requests.get(uri, timeout=15, stream=True) + if req.status_code != 200: + raise HTTPError(req.status_code) + with open(path, "wb") as artifact: + for chunk in req.raw.stream(4096, decode_content=False): + if chunk: + artifact.write(chunk) + + def get_output(self, job_id): + """Get the latest output for a specified test job + + :param job_id: + ID for the test job + :return: + String containing the latest output from the job + """ + endpoint = "/v1/result/{}/output".format(job_id) + return self.get(endpoint) + + def get_job_position(self, job_id): + """Get the status of a test job + + :param job_id: + ID for the test job + :return: + String containing the queue position for the specified ID + i.e. how many jobs are ahead of it in the queue + """ + endpoint = "/v1/job/{}/position".format(job_id) + return self.get(endpoint) + + def get_queues(self): + """Get the advertised queues from the testflinger server""" + endpoint = "/v1/agents/queues" + data = self.get(endpoint) + try: + return json.loads(data) + except ValueError: + return {} + + def get_images(self, queue): + """Get the advertised images from the testflinger server""" + endpoint = "/v1/agents/images/" + queue + data = self.get(endpoint) + try: + return json.loads(data) + except ValueError: + return {} diff --git a/cli/testflinger_cli/config.py b/cli/testflinger_cli/config.py new file mode 100644 index 00000000..3e12140e --- /dev/null +++ b/cli/testflinger_cli/config.py @@ -0,0 +1,60 @@ +# Copyright (C) 2020-2022 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Testflinger config module +""" + +import configparser +import os +from collections import OrderedDict +import xdg + + +class TestflingerCliConfig: + """TestflingerCliConfig class load values from files, env, and params""" + + def __init__(self, configfile=None): + config = configparser.ConfigParser() + if not configfile: + os.makedirs(xdg.XDG_CONFIG_HOME, exist_ok=True) + configfile = os.path.join( + xdg.XDG_CONFIG_HOME, "testflinger-cli.conf" + ) + config.read(configfile) + # Default empty config in case there's no config file + self.data = OrderedDict() + if "testflinger-cli" in config.sections(): + self.data = OrderedDict(config["testflinger-cli"]) + self.configfile = configfile + + def get(self, key): + """Get config item""" + return self.data.get(key) + + def set(self, key, value): + """Set config item""" + self.data[key] = value + self._save() + + def _save(self): + """Save config back to the config file""" + config = configparser.ConfigParser() + config.read_dict({"testflinger-cli": self.data}) + with open( + self.configfile, "w", encoding="utf-8", errors="ignore" + ) as config_file: + config.write(config_file) diff --git a/cli/testflinger_cli/history.py b/cli/testflinger_cli/history.py new file mode 100644 index 00000000..27c2d117 --- /dev/null +++ b/cli/testflinger_cli/history.py @@ -0,0 +1,83 @@ +# Copyright (C) 2020-2022 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Testflinger history module +""" + +import json +import logging +import os +from collections import OrderedDict +from datetime import datetime +import xdg + + +logger = logging.getLogger(__name__) + + +class TestflingerCliHistory: + """History class used for storing job history on a device""" + + def __init__(self): + os.makedirs(xdg.XDG_DATA_HOME, exist_ok=True) + self.historyfile = os.path.join( + xdg.XDG_DATA_HOME, "testflinger-cli-history.json" + ) + self.load() + + def new(self, job_id, queue): + """Add a new job to the history""" + submission_time = datetime.now().timestamp() + self.history[job_id] = { + "queue": queue, + "submission_time": submission_time, + "job_state": "unknown", + } + + # limit job history to last 10 jobs + if len(self.history) > 10: + self.history.popitem(last=False) + self.save() + + def load(self): + """Load the history file""" + if not hasattr(self, "history"): + self.history = OrderedDict() + if os.path.exists(self.historyfile): + with open( + self.historyfile, encoding="utf-8", errors="ignore" + ) as history_file: + try: + self.history.update(json.load(history_file)) + except (OSError, ValueError): + # If there's any error loading the history, ignore it + logger.error( + "Error loading history file from %s", self.historyfile + ) + + def save(self): + """Save the history out to the history file""" + with open( + self.historyfile, "w", encoding="utf-8", errors="ignore" + ) as history_file: + json.dump(self.history, history_file, indent=2) + + def update(self, job_id, state): + """Update job state in the history file""" + if job_id in self.history: + self.history[job_id]["job_state"] = state + self.save() diff --git a/src/api/__init__.py b/cli/testflinger_cli/tests/__init__.py similarity index 100% rename from src/api/__init__.py rename to cli/testflinger_cli/tests/__init__.py diff --git a/cli/testflinger_cli/tests/test_cli.py b/cli/testflinger_cli/tests/test_cli.py new file mode 100644 index 00000000..137f876c --- /dev/null +++ b/cli/testflinger_cli/tests/test_cli.py @@ -0,0 +1,109 @@ +# Copyright (C) 2022 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Unit tests for testflinger-cli +""" + +import json +import sys +import uuid +import pytest + +import testflinger_cli +from testflinger_cli.client import HTTPError + + +URL = "https://testflinger.canonical.com" + + +def test_status(capsys, requests_mock): + """Status should report job_state data""" + jobid = str(uuid.uuid1()) + fake_return = {"job_state": "completed"} + requests_mock.get(URL + "/v1/result/" + jobid, json=fake_return) + sys.argv = ["", "status", jobid] + tfcli = testflinger_cli.TestflingerCli() + tfcli.status() + std = capsys.readouterr() + assert std.out == "completed\n" + + +def test_cancel_503(requests_mock): + """Cancel should fail loudly if cancel action returns 503""" + jobid = str(uuid.uuid1()) + requests_mock.post( + URL + "/v1/job/" + jobid + "/action", + status_code=503, + ) + sys.argv = ["", "cancel", jobid] + tfcli = testflinger_cli.TestflingerCli() + with pytest.raises(HTTPError) as err: + tfcli.cancel() + assert err.value.status == 503 + + +def test_cancel(requests_mock): + """Cancel should fail if /v1/job//action URL returns 400 code""" + jobid = str(uuid.uuid1()) + requests_mock.post( + URL + "/v1/job/" + jobid + "/action", + status_code=400, + ) + sys.argv = ["", "cancel", jobid] + tfcli = testflinger_cli.TestflingerCli() + with pytest.raises(SystemExit) as err: + tfcli.cancel() + assert "already completed/cancelled" in err.value.args[0] + + +def test_submit(capsys, tmp_path, requests_mock): + """Make sure jobid is read back from submitted job""" + jobid = str(uuid.uuid1()) + fake_data = {"queue": "fake", "provision_data": {"distro": "fake"}} + testfile = tmp_path / "test.json" + testfile.write_text(json.dumps(fake_data)) + fake_return = {"job_id": jobid} + requests_mock.post(URL + "/v1/job", json=fake_return) + sys.argv = ["", "submit", str(testfile)] + tfcli = testflinger_cli.TestflingerCli() + tfcli.submit() + std = capsys.readouterr() + assert jobid in std.out + + +def test_show(capsys, requests_mock): + """Exercise show command""" + jobid = str(uuid.uuid1()) + fake_return = {"job_state": "completed"} + requests_mock.get(URL + "/v1/job/" + jobid, json=fake_return) + sys.argv = ["", "show", jobid] + tfcli = testflinger_cli.TestflingerCli() + tfcli.show() + std = capsys.readouterr() + assert "completed" in std.out + + +def test_results(capsys, requests_mock): + """results should report job_state data""" + jobid = str(uuid.uuid1()) + fake_return = {"job_state": "completed"} + requests_mock.get(URL + "/v1/result/" + jobid, json=fake_return) + sys.argv = ["", "results", jobid] + tfcli = testflinger_cli.TestflingerCli() + tfcli.results() + std = capsys.readouterr() + assert "completed" in std.out diff --git a/cli/tox.ini b/cli/tox.ini new file mode 100644 index 00000000..2ba286ad --- /dev/null +++ b/cli/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = py +skipsdist = true + +[testenv] +setenv = + HOME = {envtmpdir} +deps = + black + flake8 + mock + pytest + pylint + pytest-mock + pytest-cov + requests-mock +commands = + {envbindir}/pip install . + {envbindir}/python -m black --check setup.py testflinger-cli testflinger_cli + {envbindir}/python -m flake8 setup.py testflinger_cli + {envbindir}/python -m pylint testflinger_cli + {envbindir}/python -m pytest --doctest-modules --cov=. diff --git a/device-connectors/.gitignore b/device-connectors/.gitignore new file mode 100644 index 00000000..8de9a2f1 --- /dev/null +++ b/device-connectors/.gitignore @@ -0,0 +1,11 @@ +__pycache__ +*.pyc +*.egg* +*.swp +env +venv +build +dist +.tox +.coverage +.vscode diff --git a/device-connectors/.pmr-merge-hook b/device-connectors/.pmr-merge-hook new file mode 100755 index 00000000..828e4a23 --- /dev/null +++ b/device-connectors/.pmr-merge-hook @@ -0,0 +1,10 @@ +#!/bin/sh +# This hook is executed for all incoming merge requests +set -e + +rm -rf env +virtualenv -p python3 env +. env/bin/activate +pip install -r test_requirements.txt +./setup.py flake8 +rm -rf env diff --git a/device-connectors/AUTHORS b/device-connectors/AUTHORS new file mode 100644 index 00000000..19e3ab33 --- /dev/null +++ b/device-connectors/AUTHORS @@ -0,0 +1 @@ +Paul Larson diff --git a/COPYING b/device-connectors/COPYING similarity index 100% rename from COPYING rename to device-connectors/COPYING diff --git a/device-connectors/README-pip-cache.rst b/device-connectors/README-pip-cache.rst new file mode 100644 index 00000000..85820d1f --- /dev/null +++ b/device-connectors/README-pip-cache.rst @@ -0,0 +1,14 @@ +Generating pip-cache +#################### + +The pip-cache can be used for distributing this with all dependencies. +To regenerate or update the cache contents, use the following process:: + + $ mkdir pip-cache + $ cd pip-cache + $ virtualenv -p python3 ve + $ . ve/bin/activate + $ pip install --download . -r requirements.txt + +This will download the missing wheels or tarballs, and put them in +pip-cache for redistribution. diff --git a/device-connectors/README.rst b/device-connectors/README.rst new file mode 100644 index 00000000..728307c2 --- /dev/null +++ b/device-connectors/README.rst @@ -0,0 +1,36 @@ +Testflinger Device Connectors +#################### + +Device connectors scripts for provisioning and running tests on Testflinger +devices + +Supported Devices +================= + +The following device connector types are currently supported, however most of them +require a very specific environment in order to work properly. That's part of +the reason why they are broken out into a separate project. Nothing here is +really required to run testflinger, only to support these devices in our +environment. Alternative device connectors could be written in order to support +testing on other types of devices. + +- cm3 - Raspberry PI CM3 with a sidecar device and tools to support putting it in otg mode to flash an image +- dragonboard - dragonboard with a stable image on usb and test images are flashed to a wiped sd with a dual boot process +- maas2 - Metal as a Service (MaaS) systems, which support additional features such as disk layouts. Images provisioned must be imported first! +- multi - multi-device connector used for provisioning jobs that span multiple devices at once +- muxpi - muxpi/sdwire provisioned devices that utilize a device that can write to an sd the boot it on the DUT +- netboot - minimal netboot initramfs process for a specific device that couldn't be provisioned with MaaS +- noprovision - devices which need to run tests, but can't be provisioned (yet) +- oemrecovery - anything (such as core fde images) that can't be provisioned but can run a set of commands to recover back to the initial state +- oemscript - uses a script that supports some oem images and allows injection of an iso to the recovery partition to install that image + + +Exit Status +=========== + +Device connectors will exit with a value of ''46'' if something goes wrong during +device recovery. This can be used as an indication that the device is unusable +for some reason, and can't be recovere using automated recovery mechanisms. +The system calling the device connector may want to take further action, such +as alerting someone that it needs manual recovery, or to stop attempting to +run tests on it until it's fixed. diff --git a/device-connectors/pyproject.toml b/device-connectors/pyproject.toml new file mode 100644 index 00000000..a0f64200 --- /dev/null +++ b/device-connectors/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = [ + "wheel", + "setuptools", + "setuptools-scm", +] +build-backend = "setuptools.build_meta" + +[project] +name = "testflinger-device-connectors" +version = "0.0.2" +description = "Testflinger device connectors" +license = {text = "GPLv3"} +readme = "README.rst" +requires-python = ">=3.8" +dependencies = [ + "PyYAML>=3.11", + "requests", +] + +[project.scripts] +snappy-device-agent = "testflinger_device_connectors.cmd:main" +testflinger-device-connector = "testflinger_device_connectors.cmd:main" + +[tool.setuptools.package-data] +testflinger_device_connectors = ["data/**"] + +[tool.black] +line-length = 79 diff --git a/device-connectors/renovate.json b/device-connectors/renovate.json new file mode 100644 index 00000000..39a2b6e9 --- /dev/null +++ b/device-connectors/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} diff --git a/device-connectors/setup.cfg b/device-connectors/setup.cfg new file mode 100644 index 00000000..31ad82b6 --- /dev/null +++ b/device-connectors/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = pytest diff --git a/device-connectors/setup.py b/device-connectors/setup.py new file mode 100755 index 00000000..8104ab0f --- /dev/null +++ b/device-connectors/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# Copyright (C) 2015-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from setuptools import setup + +setup() diff --git a/device-connectors/src/testflinger_device_connectors/__init__.py b/device-connectors/src/testflinger_device_connectors/__init__.py new file mode 100644 index 00000000..a57ca4af --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/__init__.py @@ -0,0 +1,489 @@ +# Copyright (C) 2015 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""General functions used by device connectors""" + +import bz2 +import gzip +import json +import logging +import lzma +import os +import shutil +import socket +import string +import subprocess +import sys +import time +import urllib.request + +IMAGEFILE = "install.img" + +logger = logging.getLogger() + + +class CmdTimeoutError(Exception): + """Exception for timeout running running commands""" + + +def get_test_opportunity(job_data="testflinger.json"): + """ + Read the json test opportunity data from testflinger.json. + + :param job_data: + Filename and path of the json data if not the default + :return test_opportunity: + Dictionary of values read from the json file + """ + with open(job_data, encoding="utf-8") as job_data_json: + test_opportunity = json.load(job_data_json) + return test_opportunity + + +def filetype(filename): + """Attempt to determine the compression type of a specified file""" + magic_headers = { + b"\x1f\x8b\x08": "gz", + b"\x42\x5a\x68": "bz2", + b"\xfd\x37\x7a\x58\x5a\x00": "xz", + b"\x51\x46\x49\xfb": "qcow2", + } + with open(filename, "rb") as checkfile: + filehead = checkfile.read(1024) + ftype = "unknown" + for k, val in magic_headers.items(): + if filehead.startswith(k): + ftype = val + break + return ftype + + +def download(url, filename=None): + """ + Download the at the specified URL + + :param url: + URL of the file to download + :param filename: + Filename to save the file as, defaults to the basename from the url + :return filename: + Filename of the downloaded core image + """ + logger.info("Downloading file from %s", url) + if filename is None: + filename = os.path.basename(url) + urllib.request.urlretrieve(url, filename) + return filename + + +def delayretry(func, args, max_retries=3, delay=0): + """ + Retry the called function with a delay inserted between attempts + + :param func: + Function to retry + :param args: + List of args to pass to func() + :param max_retries: + Maximum number of times to retry + :delay: + Time (in seconds) to delay between attempts + """ + for retry_count in range(max_retries): + try: + ret = func(*args) + except Exception: # pylint: disable=broad-except + time.sleep(delay) + if retry_count == max_retries - 1: + raise + continue + return ret + + +def get_test_username(job_data="testflinger.json", default="ubuntu"): + """ + If the test_data specifies a default username, use it. Otherwise + allow the provisioning method pick a default, or use ubuntu as a safe bet + + :return username: + Returns the test image username + """ + testflinger_data = get_test_opportunity(job_data) + try: + user = testflinger_data["test_data"]["test_username"] + except KeyError: + user = default + return user + + +def get_test_password(job_data="testflinger.json", default="ubuntu"): + """ + If the test_data specifies a default password, use it. Otherwise + allow the provisioning method pick a default, or use ubuntu as a safe bet + + :return password: + Returns the test image password + """ + testflinger_data = get_test_opportunity(job_data) + try: + password = testflinger_data["test_data"]["test_password"] + except KeyError: + password = default + return password + + +def get_image(job_data="testflinger.json"): + """ + Read the json data for a test opportunity from SPI and retrieve or + create the requested image. + + :return compressed_filename: + Returns the filename of the compressed image, or empty string if + there was an error + """ + testflinger_data = get_test_opportunity(job_data) + provision_data = testflinger_data.get("provision_data") + if "url" not in provision_data: + logger.error('provision_data needs to contain "url" for the image') + return "" + url = testflinger_data["provision_data"]["url"] + try: + image = download(url, IMAGEFILE) + except OSError: + logger.exception('Error getting "%s":', url) + return "" + return compress_file(image) + + +def get_local_ip_addr(): + """ + Return our default IP address for another system to connect to + + :return ipaddr: + Returns the ip address of this system + """ + # Use SOCK_DGRAM since we don't need to send any data and to avoid timeout + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.connect(("10.0.0.0", 0)) + ipaddr = sock.getsockname()[0] + return ipaddr + + +def serve_file(queue, filename): + """ + Wait for a connection, then send the specified file one time + + :param queue: + multiprocessing queue used to send the port number back + :param filename: + The file to transmit + """ + server = socket.socket() + server.bind(("0.0.0.0", 0)) + port = server.getsockname()[1] + queue.put(port) + server.listen(1) + (client, _) = server.accept() + with open(filename, mode="rb") as imagefile: + while True: + data = imagefile.read(16 * 1024 * 1024) + if not data: + break + client.send(data) + client.close() + server.close() + + +def compress_file(filename): + """ + Gzip the specified file, return the filename of the compressed image + + :param filename: + The file to compress + :return compressed_filename: + The filename of the compressed file + """ + compressed_filename = f"{filename}.xz" + try: + # Remove the compressed_filename if it exists, just in case + os.unlink(compressed_filename) + except FileNotFoundError: + pass + if filetype(filename) == "xz": + # just hard link it so we can unlink later without special handling + os.rename(filename, compressed_filename) + elif filetype(filename) == "gz": + with lzma.open(compressed_filename, "wb") as compressed_image: + with gzip.GzipFile(filename, "rb") as old_compressed: + shutil.copyfileobj(old_compressed, compressed_image) + elif filetype(filename) == "bz2": + with lzma.open(compressed_filename, "wb") as compressed_image: + with bz2.BZ2File(filename, "rb") as old_compressed: + shutil.copyfileobj(old_compressed, compressed_image) + elif filetype(filename) == "qcow2": + raw_filename = f"{filename}.raw" + try: + # Remove the original file, unless we already did + os.unlink(raw_filename) + except FileNotFoundError: + pass + cmd = ["qemu-img", "convert", "-O", "raw", filename, raw_filename] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as exc: + logger.error("Image Conversion Output:\n %s", exc.output) + raise + with open(raw_filename, "rb") as uncompressed_image: + with lzma.open(compressed_filename, "wb") as compressed_image: + shutil.copyfileobj(uncompressed_image, compressed_image) + os.unlink(raw_filename) + else: + # filetype is 'unknown' so assumed to be raw image + with open(filename, "rb") as uncompressed_image: + with lzma.open(compressed_filename, "wb") as compressed_image: + shutil.copyfileobj(uncompressed_image, compressed_image) + try: + # Remove the original file, unless we already did + os.unlink(filename) + except FileNotFoundError: + pass + return compressed_filename + + +def configure_logging(config): + """Setup logging""" + + class AgentFilter( + logging.Filter + ): # pylint: disable=too-few-public-methods + """Add agent_name to log records""" + + def __init__(self, agent_name): + super().__init__() + self.agent_name = agent_name + + def filter(self, record): + record.agent_name = self.agent_name + return True + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(agent_name)s %(levelname)s: " + "DEVICE CONNECTOR: " + "%(message)s", + ) + agent_name = config.get("agent_name", "") + logger.addFilter(AgentFilter(agent_name)) + + +def logmsg(level, msg, *args): + """ + Front end to logging that splits messages into 4096 byte chunks + + :param level: + log level + :param msg: + log message + :param args: + args for filling message variables + """ + + if args: + msg = msg % args + logger.log(level, msg[:4096]) + if len(msg) > 4096: + logmsg(level, msg[4096:]) + + +def runcmd(cmd, env=None, timeout=None): + """ + Run a command and stream the output to stdout + + :param cmd: + Command to run + :param env: + Environment to pass to Popen + :param timeout: + Seconds after which we should timeout + :return returncode: + Return value from running the command + """ + + # Sanitize the environment, eliminate null values or Popen may choke + if not env: + env = {} + env = {x: y for x, y in env.items() if y} + + if timeout: + deadline = time.time() + timeout + else: + deadline = None + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + env=env, + ) as process: + while process.poll() is None: + if deadline and time.time() > deadline: + process.terminate() + raise CmdTimeoutError + line = process.stdout.readline() + if line: + sys.stdout.write(line.decode(errors="replace")) + line = process.stdout.read() + if line: + sys.stdout.write(line.decode(errors="replace")) + return process.returncode + + +def run_test_cmds(cmds, config=None, env=None): + """ + Run the test commands provided + This is just a frontend to determine the type of cmds we + were passed and do the right thing with it + + :param cmds: + Commands to run as a string or list of strings + :param config: + Config data for the device which can be used for filling templates + :param env: + Environment to pass when running the commands + """ + + if not env: + env = os.environ.copy() + config_env = config.get("env", {}) + env.update(config_env) + if isinstance(cmds, list): + return _run_test_cmds_list(cmds, config, env) + if isinstance(cmds, str): + return _run_test_cmds_str(cmds, config, env) + logmsg(logging.ERROR, "test_cmds field must be a list or string") + return 1 + + +def _process_cmds_template_vars(cmds, config=None): + """ + Fill in templated values for test command string. Ignore any values + in braces for which we don't have a config item. + + :param cmds: + Commands to run as a list of strings + :param config: + Config data for the device which can be used for filling templates + """ + + logmsg( + logging.WARNING, + "DEPRECATED - Detected use of double-braces in test_cmds", + ) + + class IgnoreUnknownFormatter(string.Formatter): + """Try to allow both double and single curly braces""" + + def vformat(self, format_string, args, kwargs): + tokens = [] + for literal, field_name, spec, conv in self.parse(format_string): + # replace double braces if parse removed them + literal = literal.replace("{", "{{").replace("}", "}}") + # if the field is {}, just add escaped empty braces + if field_name == "": + tokens.extend([literal, "{{}}"]) + continue + # if field name was None, we just add the literal token + if field_name is None: + tokens.extend([literal]) + continue + # if conf and spec are not defined, set to '' + conv = "!" + conv if conv else "" + spec = ":" + spec if spec else "" + # only consider field before index + field = field_name.split("[")[0].split(".")[0] + # If this field is one we've defined, fill template value + if field in kwargs: + tokens.extend([literal, "{", field_name, conv, spec, "}"]) + else: + # If not, the use escaped braces to pass it through + tokens.extend( + [literal, "{{", field_name, conv, spec, "}}"] + ) + format_string = "".join(tokens) + return string.Formatter.vformat(self, format_string, args, kwargs) + + # Ensure config is a dict + if not isinstance(config, dict): + config = {} + formatter = IgnoreUnknownFormatter() + return formatter.format(cmds, **config) + + +def _run_test_cmds_list(cmds, config=None, env=None): + """ + Run the test commands provided + + :param cmds: + Commands to run as a list of strings + :param config: + Config data for the device which can be used for filling templates + :param env: + Environment to pass when running the commands + :return returncode: + Return 0 if everything succeeded, or exit code from failed command + """ + + if not env: + env = {} + for cmd in cmds: + # Settings from the device yaml configfile like device_ip can be + # formatted in test commands like "foo {device_ip}" + if "{{" in cmd: + cmd = _process_cmds_template_vars(cmd, config) + + logmsg(logging.INFO, "Running: %s", cmd) + result = runcmd(cmd, env) + if result: + logmsg(logging.WARNING, "Command failed, rc=%d", result) + return result + + +def _run_test_cmds_str(cmds, config=None, env=None): + """ + Run the test commands provided + + :param cmds: + Commands to run as a string + :param config: + Config data for the device which can be used for filling templates + :param env: + Environment to pass when running the commands + :return returncode: + Return the value of the return code from the script + """ + + if not env: + env = {} + # If cmds doesn't specify an interpreter, pick a safe default + if not cmds.startswith("#!"): + cmds = "#!/bin/bash\n" + cmds + + if "{{" in cmds: + cmds = _process_cmds_template_vars(cmds, config) + with open("tf_cmd_script", mode="w", encoding="utf-8") as tf_cmd_script: + tf_cmd_script.write(cmds) + os.chmod("tf_cmd_script", 0o775) + result = runcmd("./tf_cmd_script", env) + if result: + logmsg(logging.WARNING, "Tests failed, rc=%d", result) + return result diff --git a/device-connectors/src/testflinger_device_connectors/cmd.py b/device-connectors/src/testflinger_device_connectors/cmd.py new file mode 100755 index 00000000..f3614203 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/cmd.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Main testflinger-device-connectors command module +""" + + +import argparse +import logging + +from testflinger_device_connectors.devices import load_devices + +logger = logging.getLogger() + + +def main(): + """main command function for testflinger-device-connectors""" + devices = load_devices() + parser = argparse.ArgumentParser() + + # First add a subcommand for each supported device type + dev_parser = parser.add_subparsers() + for dev_name, dev_class in devices: + dev_subparser = dev_parser.add_parser(dev_name) + dev_module = dev_class() + # Next add the subcommands that can be used and the methods they run + cmd_subparser = dev_subparser.add_subparsers() + for cmd, func in ( + ("provision", dev_module.provision), + ("runtest", dev_module.runtest), + ("allocate", dev_module.allocate), + ("reserve", dev_module.reserve), + ("cleanup", dev_module.cleanup), + ): + cmd_parser = cmd_subparser.add_parser(cmd) + cmd_parser.add_argument( + "-c", + "--config", + required=True, + help="Config file for this device", + ) + cmd_parser.add_argument( + "job_data", help="Testflinger json data file" + ) + cmd_parser.set_defaults(func=func) + args = parser.parse_args() + raise SystemExit(args.func(args)) diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/ce-oem-iot/user-data b/device-connectors/src/testflinger_device_connectors/data/muxpi/ce-oem-iot/user-data new file mode 100644 index 00000000..ef4d0d40 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/ce-oem-iot/user-data @@ -0,0 +1,14 @@ +#cloud-config +ssh_pwauth: True +users: + - name: ubuntu + gecos: ubuntu + uid: 1000 + shell: /bin/bash + lock_passwd: False + groups: [ adm, dialout, cdrom, floppy, sudo, audio, dip, video, plugdev, lxd, netdev, render ] + plain_text_passwd: 'ubuntu' +chpasswd: + list: | + ubuntu:ubuntu + expire: False diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/classic/meta-data b/device-connectors/src/testflinger_device_connectors/data/muxpi/classic/meta-data new file mode 100644 index 00000000..e19f3471 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/classic/meta-data @@ -0,0 +1 @@ +instance_id: cloud-image diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/classic/user-data b/device-connectors/src/testflinger_device_connectors/data/muxpi/classic/user-data new file mode 100644 index 00000000..39335263 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/classic/user-data @@ -0,0 +1,7 @@ +#cloud-config +password: ubuntu +chpasswd: + list: + - ubuntu:ubuntu + expire: False +ssh_pwauth: True diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/README b/device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/README new file mode 100644 index 00000000..0bca4482 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/README @@ -0,0 +1,2 @@ +This recovery-from-iso.sh comes from: +https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-scripts/tree/recovery-from-iso.sh diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/recovery-from-iso.sh b/device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/recovery-from-iso.sh new file mode 100755 index 00000000..d20543d5 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/recovery-from-iso.sh @@ -0,0 +1,566 @@ +#!/bin/bash +set -ex + +jenkins_job_for_iso="" +jenkins_job_build_no="lastSuccessfulBuild" +script_on_target_machine="inject_recovery_from_iso.sh" +additional_grub_for_ubuntu_recovery="99_ubuntu_recovery" +user_on_target="ubuntu" +SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +SSH="ssh $SSH_OPTS" +SCP="scp $SSH_OPTS" +#TAR="tar -C $temp_folder" +temp_folder="$(mktemp -d -p "$PWD")" +GIT="git -C $temp_folder" +ubuntu_release="" +enable_sb="no" + +enable_secureboot() { + if [ "${ubr}" != "yes" ] && [ "$enable_sb" = "yes" ]; then + ssh -o StrictHostKeyChecking=no "$user_on_target"@"$target_ip" sudo sb_fixup + ssh -o StrictHostKeyChecking=no "$user_on_target"@"$target_ip" sudo reboot + fi +} + +clear_all() { + rm -rf "$temp_folder" + # remove Ubiquity in the end to match factory and Stock Ubuntu image behavior. + # and it also workaround some debsum error from ubiquity. + ssh -o StrictHostKeyChecking=no "$user_on_target"@"$target_ip" sudo apt-get -o DPkg::Lock::Timeout=-1 purge -y ubiquity +} +trap clear_all EXIT +# shellcheck disable=SC2046 +eval set -- $(getopt -o "su:c:j:b:t:h" -l "local-iso:,sync,url:,jenkins-credential:,jenkins-job:,jenkins-job-build-no:,oem-share-url:,oem-share-credential:,target-ip:,ubr,enable-secureboot,inject-ssh-key:,help" -- "$@") + +usage() { + set +x +cat << EOF +Usage: + # This triggers sync job, downloads the image from oem-share, upload the + # image to target DUT, and starts recovery. + $(basename "$0") \\ + -s -u http://10.102.135.50:8080 \\ + -c JENKINS_USERNAME:JENKINS_CREDENTIAL \\ + -j dell-bto-jammy-jellyfish -b 17 \\ + --oem-share-url https://oem-share.canonical.com/share/lyoncore/jenkins/job \\ + --oem-share-credential OEM_SHARE_USERNAME:OEM_SHARE_PASSWORD \\ + -t 192.168.101.68 + + # This downloads the image from Jenkins, upload the image to target DUT, + # and starts recovery. + $(basename "$0") \\ + -u 10.101.46.50 \\ + -j dell-bto-jammy-jellyfish -b 17 \\ + -t 192.168.101.68 + + # This upload the image from local to target DUT, and starts recovery. + $(basename "$0") \\ + --local-iso ./dell-bto-jammy-jellyfish-X10-20220519-17.iso \\ + -t 192.168.101.68 + + # This upload the image from local to target DUT, and starts recovery. The + # image is using ubuntu-recovery. + $(basename "$0") \\ + --local-iso ./pc-stella-cmit-focal-amd64-X00-20210618-1563.iso \\ + --ubr -t 192.168.101.68 + +Limition: + It will failed when target recovery partition size smaller than target iso + file. + +The assumption of using this tool: + - An root account 'ubuntu' on target machine. + - The root account 'ubuntu' can execute command with root permission with + \`sudo\` without password. + - Host executing this tool can access target machine without password over ssh. + +OPTIONS: + --local-iso + Use local + + -s | --sync + Trigger sync job \`infrastructure-swift-client\` in Jenkins in --url, + then download image from --oem-share-url. + + -u | --url + URL of jenkins server. + + -c | --jenkins-credential + Jenkins credential in the form of username:password, used with --sync. + + -j | --jenkins-job + Get iso from jenkins-job. + + -b | --jenkins-job-build-no + The build number of the Jenkins job assigned by --jenkins-job. + + --oem-share-url + URL of oem-share, used with --sync. + + --oem-share-credential + Credential in the form of username:password of lyoncore, used with --sync. + + -t | --target-ip + The IP address of target machine. It will be used for ssh accessing. + Please put your ssh key on target machine. This tool no yet support + keyphase for ssh. + + --enable-secureboot + Enable Secure Boot. When this option is on, the script will not install + file that prevents turning on Secure Boot after installation. Only + effective with dell-recovery images that enables Secure Boot on + Somerville platforms. + + --ubr + DUT which using ubuntu recovery (volatile-task). + + --inject-ssh-key + Path to ssh key to inject into the target machine. + + -h | --help + Print this message +EOF + set -x +exit 1 +} + +download_preseed() { + echo " == download_preseed == " + if [ "${ubr}" == "yes" ]; then + if [ "$enable_sb" = "yes" ]; then + echo "error: --enable-secureboot does not apply to ubuntu-recovery images" + exit 1 + fi + # TODO: sync togother + # replace $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-no-secureboot --depth 1 + # Why need it? + # reokace $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-skip-storage-selecting --depth 1 + mkdir "$temp_folder/preseed/" + echo "# Ubuntu Recovery configuration preseed + +ubiquity ubuntu-oobe/user-interface string dynamic +ubiquity ubuntu-recovery/recovery_partition_filesystem string 0c +ubiquity ubuntu-recovery/active_partition string 1 +ubiquity ubuntu-recovery/dual_boot_layout string primary +ubiquity ubuntu-recovery/disk_layout string gpt +ubiquity ubuntu-recovery/swap string dynamic +ubiquity ubuntu-recovery/dual_boot boolean false +ubiquity ubiquity/reboot boolean true +ubiquity ubiquity/poweroff boolean false +ubiquity ubuntu-recovery/recovery_hotkey/partition_label string PQSERVICE +ubiquity ubuntu-recovery/recovery_type string dev +" | tee ubuntu-recovery.cfg + mv ubuntu-recovery.cfg "$temp_folder/preseed" + $SCP "$user_on_target"@"$target_ip":/cdrom/preseed/project.cfg ./ + sed -i 's%ubiquity/reboot boolean false%ubiquity/reboot boolean true%' ./project.cfg + sed -i 's%ubiquity/poweroff boolean true%ubiquity/poweroff boolean false%' ./project.cfg + mv project.cfg "$temp_folder/preseed" + + mkdir -p "$temp_folder/oem-fix-set-local-repo/scripts/chroot-scripts/fish/" + mkdir -p "$temp_folder/oem-fix-set-local-repo/scripts/chroot-scripts/os-post/" + cat < "$temp_folder/oem-fix-set-local-repo/scripts/chroot-scripts/fish/00-setup-local-repo" +#!/bin/bash -ex +# setup local repo +mkdir /tmp/cdrom_debs +apt-ftparchive packages /cdrom/debs > /tmp/cdrom_debs/Packages +echo 'deb [ trusted=yes ] file:/. /tmp/cdrom_debs/' >> /etc/apt/sources.list.d/$(basename "$0")_$$.list +sudo apt-get update +EOF + + cat < "$temp_folder/oem-fix-set-local-repo/scripts/chroot-scripts/os-post/99-remove-local-repo" +#!/bin/bash -ex +# remove local repo +rm -f /etc/apt/sources.list.d/$(basename "$0")_$$.list +sudo apt update +EOF + else + # get checkbox pkgs and prepare-checkbox + # get pkgs to skip OOBE + if [ "$enable_sb" = "yes" ]; then + $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-install-sbhelper --depth 1 + else + $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-no-secureboot --depth 1 + fi + $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-skip-storage-selecting --depth 1 + fi + + # install packages related to skip oobe + skip_oobe_branch="master" + if [ -n "$ubuntu_release" ]; then + # set ubuntu_release to jammy or focal, depending on detected release + skip_oobe_branch="$ubuntu_release" + fi + $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-skip-oobe --depth 1 -b "$skip_oobe_branch" + # get pkgs for ssh key and skip disk checking. + $GIT clone https://git.launchpad.net/~oem-solutions-engineers/pc-enablement/+git/oem-fix-misc-cnl-misc-for-automation --depth 1 misc_for_automation + + if [ "${ubr}" == "yes" ]; then + mkdir -p "$temp_folder"/preseed + cat < "$temp_folder/preseed/$additional_grub_for_ubuntu_recovery" +#!/bin/bash -e +source /usr/lib/grub/grub-mkconfig_lib +cat < "$temp_folder/preseed/set_env_for_ubuntu_recovery" +#!/bin/bash -ex +# replace the grub entry which ubuntu_recovery expected +recover_p=\$(lsblk -l | grep efi | cut -d ' ' -f 1 | sed 's/.$/2'/) +UUID_OF_RECOVERY_PARTITION=\$(ls -l /dev/disk/by-uuid/ | grep \$recover_p | awk '{print \$9}') +echo partition = \$UUID_OF_RECOVERY_PARTITION +sed -i "s/UUID_OF_RECOVERY_PARTITION/\$UUID_OF_RECOVERY_PARTITION/" push_preseed/preseed/$additional_grub_for_ubuntu_recovery +sudo rm -f /etc/grub.d/99_dell_recovery || true +chmod 766 push_preseed/preseed/$additional_grub_for_ubuntu_recovery +sudo cp push_preseed/preseed/$additional_grub_for_ubuntu_recovery /etc/grub.d/ + +# Force changing the recovery partition label to PQSERVICE for ubuntu-recovery +sudo fatlabel /dev/\$recover_p PQSERVICE +EOF + fi + + return 0 +} +push_preseed() { + echo " == download_preseed == " + $SSH "$user_on_target"@"$target_ip" rm -rf push_preseed + $SSH "$user_on_target"@"$target_ip" mkdir -p push_preseed + $SSH "$user_on_target"@"$target_ip" touch push_preseed/SUCCSS_push_preseed + $SSH "$user_on_target"@"$target_ip" sudo rm -f /cdrom/SUCCSS_push_preseed + + if [ "${ubr}" == "yes" ]; then + $SCP -r "$temp_folder/preseed" "$user_on_target"@"$target_ip":~/push_preseed || $SSH "$user_on_target"@"$target_ip" sudo rm -f push_preseed/SUCCSS_push_preseed + folders=( + "oem-fix-set-local-repo" + ) + else + folders=( + "oem-fix-misc-cnl-skip-storage-selecting" + ) + if [ "$enable_sb" = "yes" ]; then + folders+=("oem-fix-misc-cnl-install-sbhelper") + else + folders+=("oem-fix-misc-cnl-no-secureboot") + fi + fi + + folders+=("misc_for_automation" "oem-fix-misc-cnl-skip-oobe") + + for folder in "${folders[@]}"; do + tar -C "$temp_folder/$folder" -zcvf "$temp_folder/$folder".tar.gz . + $SCP "$temp_folder/$folder".tar.gz "$user_on_target"@"$target_ip":~ + $SSH "$user_on_target"@"$target_ip" tar -C push_preseed -zxvf "$folder".tar.gz || $SSH "$user_on_target"@"$target_ip" sudo rm -f push_preseed/SUCCSS_push_preseed + done + + $SSH "$user_on_target"@"$target_ip" sudo cp -r push_preseed/* /cdrom/ + return 0 +} +inject_preseed() { + echo " == inject_preseed == " + $SSH "$user_on_target"@"$target_ip" rm -rf /tmp/SUCCSS_inject_preseed + download_preseed && \ + push_preseed + $SCP "$user_on_target"@"$target_ip":/cdrom/SUCCSS_push_preseed "$temp_folder" || usage + + if [ "${ubr}" == "yes" ]; then + $SSH "$user_on_target"@"$target_ip" bash \$HOME/push_preseed/preseed/set_env_for_ubuntu_recovery || usage + fi + $SSH "$user_on_target"@"$target_ip" touch /tmp/SUCCSS_inject_preseed +} + +download_image() { + img_path=$1 + img_name=$2 + user=$3 + + MAX_RETRIES=10 + local retries=0 + + echo "downloading $img_name from $img_path" + curl_cmd=(curl --retry 3 --fail --show-error) + if [ -n "$user" ]; then + curl_cmd+=(--user "$user") + fi + + pushd "$temp_folder" + + while [ "$retries" -lt "$MAX_RETRIES" ]; do + ((retries+=1)) || true # arithmetics, see https://www.shellcheck.net/wiki/SC2219 + echo "Downloading checksum and image, tries $retries/$MAX_RETRIES" + "${curl_cmd[@]}" -O "$img_path/$img_name".md5sum || true + "${curl_cmd[@]}" -O "$img_path/$img_name" || true + if md5sum -c "$img_name".md5sum; then + break + fi + sleep 10; continue + done + + if [ "$retries" -ge "$MAX_RETRIES" ]; then + echo "error: max retries reached" + exit 1 + fi + + local_iso="$PWD/$img_name" + + popd +} + +download_from_jenkins() { + path="ftp://$jenkins_url/jenkins_host/jobs/$jenkins_job_for_iso/builds/$jenkins_job_build_no/archive/out" + img_name=$(wget -q "$path/" -O - | grep -o 'href=.*iso"' | awk -F/ '{print $NF}' | tr -d \") + download_image "$path" "$img_name" +} + +sync_to_swift() { + if [ -z "$jenkins_url" ] ; then + echo "error: --url not set" + exit 1 + elif [ -z "$jenkins_credential" ]; then + echo "error: --jenkins-credential not set" + exit 1 + elif [ -z "$jenkins_job_for_iso" ]; then + echo "error: --jenkins-job not set" + exit 1 + elif [ -z "$jenkins_job_build_no" ]; then + echo "error: --jenkins-job-build-no not set" + exit 1 + elif [ -z "$oem_share_url" ]; then + echo "error: --oem-share-url not set" + exit 1 + elif [ -z "$oem_share_credential" ]; then + echo "error: --oem-share-credential not set" + exit 1 + fi + + jenkins_job_name="infrastructure-swift-client" + jenkins_job_url="$jenkins_url/job/$jenkins_job_name/buildWithParameters" + curl_cmd=(curl --retry 3 --max-time 10 -sS) + headers_path="$temp_folder/build_request_headers" + + echo "sending build request" + "${curl_cmd[@]}" --user "$jenkins_credential" -X POST -D "$headers_path" "$jenkins_job_url" \ + --data option=sync \ + --data "jenkins_job=$jenkins_job_for_iso" \ + --data "build_no=$jenkins_job_build_no" + + echo "getting job id from queue" + queue_url=$(grep '^Location: ' "$headers_path" | awk '{print $2}' | tr -d '\r') + duration=0 + timeout=600 + url= + until [ -n "$timeout" ] && [[ $duration -ge $timeout ]]; do + url=$("${curl_cmd[@]}" --user "$jenkins_credential" "${queue_url}api/json" | jq -r '.executable | .url') + if [ "$url" != "null" ]; then + break + fi + sleep 5 + duration=$((duration+5)) + done + if [ "$url" = "null" ]; then + echo "error: sync job was not created in time" + exit 1 + fi + + echo "polling build status" + duration=0 + timeout=1800 + until [ -n "$timeout" ] && [[ $duration -ge $timeout ]]; do + result=$("${curl_cmd[@]}" --user "$jenkins_credential" "${url}api/json" | jq -r .result) + if [ "$result" = "SUCCESS" ]; then + break + fi + if [ "$result" = "FAILURE" ]; then + echo "error: sync job failed" + exit 1 + fi + sleep 30 + duration=$((duration+30)) + done + if [ "$result" != "SUCCESS" ]; then + echo "error: sync job has not been done in time" + exit 1 + fi + + oem_share_path="$oem_share_url/$jenkins_job_for_iso/$jenkins_job_build_no" + img_name=$(curl -sS --user "$oem_share_credential" "$oem_share_path/" | grep -o 'href=.*iso"' | tr -d \") + img_name=${img_name#"href="} + download_image "$oem_share_path" "$img_name" "$oem_share_credential" +} + +download_iso() { + if [ "$enable_sync_to_swift" = true ]; then + sync_to_swift + else + download_from_jenkins + fi +} + +inject_recovery_iso() { + if [ -z "$local_iso" ]; then + download_iso + fi + + img_name="$(basename "$local_iso")" + if [ -z "${img_name##*stella*}" ] || + [ -z "${img_name##*sutton*}" ]; then + ubr="yes" + fi + if [ -z "${img_name##*jammy*}" ]; then + ubuntu_release="jammy" + elif [ -z "${img_name##*focal*}" ]; then + ubuntu_release="focal" + fi + rsync_opts="--exclude=efi --delete --temp-dir=/var/tmp/rsync" + $SCP "$local_iso" "$user_on_target"@"$target_ip":~/ +cat < "$temp_folder/$script_on_target_machine" +#!/bin/bash +set -ex +sudo umount /cdrom /mnt || true +sudo mount -o loop $img_name /mnt && \ +recover_p=\$(lsblk -l | grep efi | cut -d ' ' -f 1 | sed 's/.$/2'/) && \ +sudo mount /dev/\$recover_p /cdrom && \ +df | grep "cdrom\|mnt" | awk '{print \$2" "\$6}' | sort | tail -n1 | grep -q cdrom && \ +sudo mkdir -p /var/tmp/rsync && \ +sudo rsync -alv /mnt/ /cdrom/ $rsync_opts && \ +sudo cp /mnt/.disk/ubuntu_dist_channel /cdrom/.disk/ && \ +touch /tmp/SUCCSS_inject_recovery_iso +EOF + $SCP "$temp_folder"/"$script_on_target_machine" "$user_on_target"@"$target_ip":~/ + $SSH "$user_on_target"@"$target_ip" chmod +x "\$HOME/$script_on_target_machine" + $SSH "$user_on_target"@"$target_ip" "\$HOME/$script_on_target_machine" + $SCP "$user_on_target"@"$target_ip":/tmp/SUCCSS_inject_recovery_iso "$temp_folder" || usage +} +prepare() { + echo "prepare" + inject_recovery_iso + inject_preseed +} + +inject_ssh_key() { + while(:); do + echo "Attempting to inject ssh key" + if [ "$(sshpass -p u ssh-copy-id $SSH_OPTS -f -i "$ssh_key" "$user_on_target@$target_ip")" ] ; then + break + fi + sleep 180 + done +} + +poll_recovery_status() { + while(:); do + if [ "$($SSH "$user_on_target"@"$target_ip" systemctl is-active ubiquity)" = "inactive" ] ; then + break + fi + sleep 180 + done +} + +do_recovery() { + if [ "${ubr}" == "yes" ]; then + echo GRUB_DEFAULT='"ubuntu-recovery restore"' | $SSH "$user_on_target"@"$target_ip" -T "sudo tee -a /etc/default/grub.d/automatic-oem-config.cfg" + echo GRUB_TIMEOUT_STYLE=menu | $SSH "$user_on_target"@"$target_ip" -T "sudo tee -a /etc/default/grub.d/automatic-oem-config.cfg" + echo GRUB_TIMEOUT=5 | $SSH "$user_on_target"@"$target_ip" -T "sudo tee -a /etc/default/grub.d/automatic-oem-config.cfg" + $SSH "$user_on_target"@"$target_ip" sudo update-grub + $SSH "$user_on_target"@"$target_ip" sudo reboot & + else + $SSH "$user_on_target"@"$target_ip" sudo dell-restore-system -y & + fi + sleep 300 # sleep to make sure the target system has been rebooted to recovery mode. + if [ -n "$ssh_key" ]; then + inject_ssh_key + fi + poll_recovery_status +} + +main() { + while [ $# -gt 0 ] + do + case "$1" in + --local-iso) + shift + local_iso="$1" + ;; + -s | --sync) + enable_sync_to_swift=true + ;; + -u | --url) + shift + jenkins_url="$1" + ;; + -c | --jenkins-credential) + shift + jenkins_credential="$1" + ;; + -j | --jenkins-job) + shift + jenkins_job_for_iso="$1" + ;; + -b | --jenkins-job-build-no) + shift + jenkins_job_build_no="$1" + ;; + --oem-share-url) + shift + oem_share_url="$1" + ;; + --oem-share-credential) + shift + oem_share_credential="$1" + ;; + -t | --target-ip) + shift + target_ip="$1" + ;; + --ubr) + ubr="yes" + ;; + --enable-secureboot) + enable_sb="yes" + ;; + --inject-ssh-key) + shift + ssh_key="$1" + ;; + -h | --help) + usage 0 + exit 0 + ;; + --) + ;; + *) + echo "Not recognize $1" + usage + exit 1 + ;; + esac + shift + done + prepare + do_recovery + clear_all + enable_secureboot +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + main "$@" +fi diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/pi-desktop/oem-config.service b/device-connectors/src/testflinger_device_connectors/data/muxpi/pi-desktop/oem-config.service new file mode 100644 index 00000000..758caa5a --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/pi-desktop/oem-config.service @@ -0,0 +1,26 @@ +[Unit] +Description=End-user configuration after initial OEM installation +ConditionFileIsExecutable=/usr/sbin/oem-config-firstboot +ConditionPathExists=/dev/tty1 + +# We never want to run the oem-config job in the live environment (as is the +# case in some custom configurations) or in recovery mode. +ConditionKernelCommandLine=!boot=casper +ConditionKernelCommandLine=!single +ConditionKernelCommandLine=!rescue +ConditionKernelCommandLine=!emergency + +[Service] +Type=oneshot +StandardInput=tty +StandardOutput=tty +StandardError=tty +TTYPath=/dev/tty1 +TTYReset=yes +TTYVHangup=yes +ExecStart=/bin/sh -ec '\ + while ! debconf-set-selections /preseed.cfg; do sleep 30;done; \ + exec oem-config-firstboot --automatic' + +[Install] +WantedBy=oem-config.target diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/pi-desktop/preseed.cfg b/device-connectors/src/testflinger_device_connectors/data/muxpi/pi-desktop/preseed.cfg new file mode 100644 index 00000000..45b8ed4a --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/pi-desktop/preseed.cfg @@ -0,0 +1,105 @@ +#### Contents of the preconfiguration file (for groovy) +### Localization +# Preseeding only locale sets language, country and locale. +d-i localechooser/languagelist select en +d-i debian-installer/locale string en_US.UTF-8 + +# The values can also be preseeded individually for greater flexibility. +#d-i debian-installer/language string en +#d-i debian-installer/country string NL +#d-i debian-installer/locale string en_GB.UTF-8 +# Optionally specify additional locales to be generated. +#d-i localechooser/supported-locales multiselect en_US.UTF-8, nl_NL.UTF-8 + +# Keyboard selection. +# Disable automatic (interactive) keymap detection. +d-i console-setup/ask_detect boolean false +d-i keyboard-configuration/xkb-keymap select us +# To select a variant of the selected layout: +#d-i keyboard-configuration/xkb-keymap select us(dvorak) +# d-i keyboard-configuration/toggle select No toggling + +# netcfg will choose an interface that has link if possible. This makes it +# skip displaying a list if there is more than one interface. +d-i netcfg/choose_interface select auto +# Any hostname and domain names assigned from dhcp take precedence over +# values set here. However, setting the values still prevents the questions +# from being shown, even if values come from dhcp. +d-i netcfg/get_hostname string unassigned-hostname +d-i netcfg/get_domain string unassigned-domain +# Disable that annoying WEP key dialog. +d-i netcfg/wireless_wep string +# The wacky dhcp hostname that some ISPs use as a password of sorts. +#d-i netcfg/dhcp_hostname string radish + +### Mirror settings +# If you select ftp, the mirror/country string does not need to be set. +#d-i mirror/protocol string ftp +d-i mirror/country string manual +d-i mirror/http/hostname string archive.ubuntu.com +d-i mirror/http/directory string /ubuntu +d-i mirror/http/proxy string + + +# Set to true if you want to encrypt the first user's home directory. +d-i user-setup/encrypt-home boolean false + +### Clock and time zone setup +# Controls whether or not the hardware clock is set to UTC. +d-i clock-setup/utc boolean true + +# You may set this to any valid setting for $TZ; see the contents of +# /usr/share/zoneinfo/ for valid values. +d-i time/zone string US/Eastern + +# Controls whether to use NTP to set the clock during the install +d-i clock-setup/ntp boolean true +# NTP server to use. The default is almost always fine here. +#d-i clock-setup/ntp-server string ntp.example.com + +### Package selection +tasksel tasksel/first multiselect ubuntu-desktop +#tasksel tasksel/first multiselect lamp-server, print-server +#tasksel tasksel/first multiselect kubuntu-desktop + +# Avoid that last message about the install being complete. +d-i finish-install/reboot_in_progress note + +d-i netcfg/get_hostname string ubuntu +d-i mirror/http/hostname string archive.ubuntu.com +d-i passwd/auto-login boolean true +d-i passwd/user-fullname string Ubuntu User +d-i passwd/username string ubuntu +d-i passwd/user-password password ubuntu +d-i passwd/user-password-again password ubuntu +d-i user-setup/allow-password-weak boolean true +d-i debian-installer/allow_unauthenticated boolean true +d-i preseed/late_command string \ + systemctl start NetworkManager; \ + apt-get update; \ + apt-get install -y --force-yes ssh; \ + apt-get clean + +d-i console-setup/ask_detect boolean false +d-i console-setup/layoutcode string us +d-i debian-installer/locale string en_US +d-i keyboard-configuration/ask_detect boolean false +d-i keyboard-configuration/layoutcode string us +d-i keyboard-configuration/xkb-keymap select us +ubiquity countrychooser/shortlist select US +ubiquity languagechooser/language-name select English +ubiquity localechooser/supported-locales multiselect en_US.UTF-8 + +ubiquity ubiquity/summary note +ubiquity ubiquity/reboot boolean true +ubiquity ubiquity/poweroff boolean true +ubiquity ubiquity/success_command string \ + systemctl start NetworkManager; \ + apt-get update; \ + apt-get install -y --force-yes ssh; \ + apt-get clean + +# Enable extras.ubuntu.com. +d-i apt-setup/extras boolean true +# Install the Ubuntu desktop. +tasksel tasksel/first multiselect ubuntu-desktop diff --git a/device-connectors/src/testflinger_device_connectors/data/muxpi/uc20/99_nocloud.cfg b/device-connectors/src/testflinger_device_connectors/data/muxpi/uc20/99_nocloud.cfg new file mode 100644 index 00000000..2ea88244 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/muxpi/uc20/99_nocloud.cfg @@ -0,0 +1,14 @@ +#cloud-config +datasource_list: [ NoCloud, None ] +datasource: + NoCloud: + user-data: | + #cloud-config + password: ubuntu + chpasswd: + list: + - ubuntu:ubuntu + expire: False + ssh_pwauth: True + meta-data: | + instance_id: cloud-image diff --git a/device-connectors/src/testflinger_device_connectors/data/pi-desktop/oem-config.service b/device-connectors/src/testflinger_device_connectors/data/pi-desktop/oem-config.service new file mode 100644 index 00000000..758caa5a --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/pi-desktop/oem-config.service @@ -0,0 +1,26 @@ +[Unit] +Description=End-user configuration after initial OEM installation +ConditionFileIsExecutable=/usr/sbin/oem-config-firstboot +ConditionPathExists=/dev/tty1 + +# We never want to run the oem-config job in the live environment (as is the +# case in some custom configurations) or in recovery mode. +ConditionKernelCommandLine=!boot=casper +ConditionKernelCommandLine=!single +ConditionKernelCommandLine=!rescue +ConditionKernelCommandLine=!emergency + +[Service] +Type=oneshot +StandardInput=tty +StandardOutput=tty +StandardError=tty +TTYPath=/dev/tty1 +TTYReset=yes +TTYVHangup=yes +ExecStart=/bin/sh -ec '\ + while ! debconf-set-selections /preseed.cfg; do sleep 30;done; \ + exec oem-config-firstboot --automatic' + +[Install] +WantedBy=oem-config.target diff --git a/device-connectors/src/testflinger_device_connectors/data/pi-desktop/preseed.cfg b/device-connectors/src/testflinger_device_connectors/data/pi-desktop/preseed.cfg new file mode 100644 index 00000000..45b8ed4a --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/data/pi-desktop/preseed.cfg @@ -0,0 +1,105 @@ +#### Contents of the preconfiguration file (for groovy) +### Localization +# Preseeding only locale sets language, country and locale. +d-i localechooser/languagelist select en +d-i debian-installer/locale string en_US.UTF-8 + +# The values can also be preseeded individually for greater flexibility. +#d-i debian-installer/language string en +#d-i debian-installer/country string NL +#d-i debian-installer/locale string en_GB.UTF-8 +# Optionally specify additional locales to be generated. +#d-i localechooser/supported-locales multiselect en_US.UTF-8, nl_NL.UTF-8 + +# Keyboard selection. +# Disable automatic (interactive) keymap detection. +d-i console-setup/ask_detect boolean false +d-i keyboard-configuration/xkb-keymap select us +# To select a variant of the selected layout: +#d-i keyboard-configuration/xkb-keymap select us(dvorak) +# d-i keyboard-configuration/toggle select No toggling + +# netcfg will choose an interface that has link if possible. This makes it +# skip displaying a list if there is more than one interface. +d-i netcfg/choose_interface select auto +# Any hostname and domain names assigned from dhcp take precedence over +# values set here. However, setting the values still prevents the questions +# from being shown, even if values come from dhcp. +d-i netcfg/get_hostname string unassigned-hostname +d-i netcfg/get_domain string unassigned-domain +# Disable that annoying WEP key dialog. +d-i netcfg/wireless_wep string +# The wacky dhcp hostname that some ISPs use as a password of sorts. +#d-i netcfg/dhcp_hostname string radish + +### Mirror settings +# If you select ftp, the mirror/country string does not need to be set. +#d-i mirror/protocol string ftp +d-i mirror/country string manual +d-i mirror/http/hostname string archive.ubuntu.com +d-i mirror/http/directory string /ubuntu +d-i mirror/http/proxy string + + +# Set to true if you want to encrypt the first user's home directory. +d-i user-setup/encrypt-home boolean false + +### Clock and time zone setup +# Controls whether or not the hardware clock is set to UTC. +d-i clock-setup/utc boolean true + +# You may set this to any valid setting for $TZ; see the contents of +# /usr/share/zoneinfo/ for valid values. +d-i time/zone string US/Eastern + +# Controls whether to use NTP to set the clock during the install +d-i clock-setup/ntp boolean true +# NTP server to use. The default is almost always fine here. +#d-i clock-setup/ntp-server string ntp.example.com + +### Package selection +tasksel tasksel/first multiselect ubuntu-desktop +#tasksel tasksel/first multiselect lamp-server, print-server +#tasksel tasksel/first multiselect kubuntu-desktop + +# Avoid that last message about the install being complete. +d-i finish-install/reboot_in_progress note + +d-i netcfg/get_hostname string ubuntu +d-i mirror/http/hostname string archive.ubuntu.com +d-i passwd/auto-login boolean true +d-i passwd/user-fullname string Ubuntu User +d-i passwd/username string ubuntu +d-i passwd/user-password password ubuntu +d-i passwd/user-password-again password ubuntu +d-i user-setup/allow-password-weak boolean true +d-i debian-installer/allow_unauthenticated boolean true +d-i preseed/late_command string \ + systemctl start NetworkManager; \ + apt-get update; \ + apt-get install -y --force-yes ssh; \ + apt-get clean + +d-i console-setup/ask_detect boolean false +d-i console-setup/layoutcode string us +d-i debian-installer/locale string en_US +d-i keyboard-configuration/ask_detect boolean false +d-i keyboard-configuration/layoutcode string us +d-i keyboard-configuration/xkb-keymap select us +ubiquity countrychooser/shortlist select US +ubiquity languagechooser/language-name select English +ubiquity localechooser/supported-locales multiselect en_US.UTF-8 + +ubiquity ubiquity/summary note +ubiquity ubiquity/reboot boolean true +ubiquity ubiquity/poweroff boolean true +ubiquity ubiquity/success_command string \ + systemctl start NetworkManager; \ + apt-get update; \ + apt-get install -y --force-yes ssh; \ + apt-get clean + +# Enable extras.ubuntu.com. +d-i apt-setup/extras boolean true +# Install the Ubuntu desktop. +tasksel tasksel/first multiselect ubuntu-desktop diff --git a/device-connectors/src/testflinger_device_connectors/devices/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/__init__.py new file mode 100644 index 00000000..4d799c46 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/__init__.py @@ -0,0 +1,293 @@ +# Copyright (C) 2015 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +import imp +import json +import logging +import multiprocessing +import os +import select +import socket +import subprocess +import time +from datetime import datetime, timedelta + +import yaml + +import testflinger_device_connectors + + +class ProvisioningError(Exception): + pass + + +class RecoveryError(Exception): + pass + + +def SerialLogger(host=None, port=None, filename=None): + """ + Factory to generate real or fake SerialLogger object based on params + """ + if host and port and filename: + return RealSerialLogger(host, port, filename) + return StubSerialLogger(host, port, filename) + + +class StubSerialLogger: + """Fake SerialLogger when we don't have Serial Logger data defined""" + + def __init__(self, host, port, filename): + pass + + def start(self): + pass + + def stop(self): + pass + + +class RealSerialLogger: + """Real SerialLogger for when we have a serial logging service""" + + def __init__(self, host, port, filename): + """Set up a subprocess to connect to an ip and collect serial logs""" + if not (host and port and filename): + self.stub = True + self.stub = False + self.host = host + self.port = int(port) + self.filename = filename + + def start(self): + """Start the serial logger connection""" + + def reconnector(): + """Reconnect when needed""" + while True: + try: + self._log_serial() + except Exception: + pass + # Keep trying if we can't connect, but sleep between attempts + testflinger_device_connectors.logmsg( + logging.ERROR, "Error connecting to serial logging server" + ) + time.sleep(30) + + self.proc = multiprocessing.Process(target=reconnector, daemon=True) + self.proc.start() + + def _log_serial(self): + """Log data to the serial data to the output file""" + with open(self.filename, "a+") as f: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((self.host, self.port)) + while True: + read_sockets, _, _ = select.select([s], [], []) + for sock in read_sockets: + data = sock.recv(4096) + if data: + f.write( + data.decode(encoding="utf-8", errors="ignore") + ) + f.flush() + else: + testflinger_device_connectors.logmsg( + logging.ERROR, "Serial Log connection closed" + ) + return + + def stop(self): + """Stop the serial logger""" + self.proc.terminate() + + +class DefaultDevice: + def runtest(self, args): + """Default method for processing test commands""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + testflinger_device_connectors.logmsg(logging.INFO, "BEGIN testrun") + + test_opportunity = testflinger_device_connectors.get_test_opportunity( + args.job_data + ) + test_cmds = test_opportunity.get("test_data").get("test_cmds") + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + serial_proc = SerialLogger(serial_host, serial_port, "test-serial.log") + serial_proc.start() + try: + exitcode = testflinger_device_connectors.run_test_cmds( + test_cmds, config + ) + except Exception as e: + raise e + finally: + serial_proc.stop() + testflinger_device_connectors.logmsg(logging.INFO, "END testrun") + return exitcode + + def allocate(self, args): + """Default method for allocating devices for multi-agent jobs""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + device_ip = config["device_ip"] + device_info = {"device_info": {"device_ip": device_ip}} + print(device_info) + with open("device-info.json", "w", encoding="utf-8") as devinfo_file: + devinfo_file.write(json.dumps(device_info)) + + def reserve(self, args): + """Default method for reserving systems""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + testflinger_device_connectors.logmsg(logging.INFO, "BEGIN reservation") + job_data = testflinger_device_connectors.get_test_opportunity( + args.job_data + ) + try: + test_username = job_data["test_data"]["test_username"] + except KeyError: + test_username = "ubuntu" + device_ip = config["device_ip"] + reserve_data = job_data["reserve_data"] + ssh_keys = reserve_data.get("ssh_keys", []) + for key in ssh_keys: + try: + os.unlink("key.pub") + except FileNotFoundError: + pass + cmd = ["ssh-import-id", "-o", "key.pub", key] + proc = subprocess.run(cmd) + if proc.returncode != 0: + testflinger_device_connectors.logmsg( + logging.ERROR, + "Unable to import ssh key from: {}".format(key), + ) + continue + cmd = [ + "ssh-copy-id", + "-f", + "-i", + "key.pub", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, device_ip), + ] + for retry in range(10): + # Retry ssh key copy just in case it's rebooting + try: + proc = subprocess.run(cmd, timeout=30) + if proc.returncode == 0: + break + except subprocess.TimeoutExpired: + # Log an error for timeout or any other problem + pass + testflinger_device_connectors.logmsg( + logging.ERROR, + "Error copying ssh key to device for: {}".format(key), + ) + if retry != 9: + testflinger_device_connectors.logmsg( + logging.INFO, "Retrying..." + ) + time.sleep(60) + else: + testflinger_device_connectors.logmsg( + logging.ERROR, "Failed to copy ssh key: {}".format(key) + ) + # default reservation timeout is 1 hour + timeout = int(reserve_data.get("timeout", "3600")) + # If max_reserve_timeout isn't specified, default to 6 hours + max_reserve_timeout = int( + config.get("max_reserve_timeout", 6 * 60 * 60) + ) + if timeout > max_reserve_timeout: + timeout = max_reserve_timeout + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + print("*** TESTFLINGER SYSTEM RESERVED ***") + print("You can now connect to {}@{}".format(test_username, device_ip)) + if serial_host and serial_port: + print( + "Serial access is available via: telnet {} {}".format( + serial_host, serial_port + ) + ) + now = datetime.utcnow().isoformat() + expire_time = ( + datetime.utcnow() + timedelta(seconds=timeout) + ).isoformat() + print("Current time: [{}]".format(now)) + print("Reservation expires at: [{}]".format(expire_time)) + print( + "Reservation will automatically timeout in {} " + "seconds".format(timeout) + ) + job_id = job_data.get("job_id", "") + print( + "To end the reservation sooner use: testflinger-cli " + "cancel {}".format(job_id) + ) + time.sleep(int(timeout)) + + def cleanup(self, _): + """Default method for cleaning up devices""" + pass + + +def catch(exception, returnval=0): + """Decorator for catching Exceptions and returning values instead + + This is useful because for certain things, like RecoveryError, we + need to give the calling process a hint that we failed for that + reason, so it can act accordingly, by disabling the device for example + """ + + def _wrapper(func): + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except exception: + return returnval + + return wrapper + + return _wrapper + + +def load_devices(): + devices = [] + device_path = os.path.dirname(os.path.realpath(__file__)) + devs = [ + os.path.join(device_path, device) + for device in os.listdir(device_path) + if os.path.isdir(os.path.join(device_path, device)) + ] + for device in devs: + if "__pycache__" in device: + continue + module = imp.load_source("module", os.path.join(device, "__init__.py")) + devices.append((module.device_name, module.DeviceConnector)) + return tuple(devices) + + +if __name__ == "__main__": + load_devices() diff --git a/device-connectors/src/testflinger_device_connectors/devices/cm3/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/cm3/__init__.py new file mode 100644 index 00000000..d8bad8f1 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/cm3/__init__.py @@ -0,0 +1,59 @@ +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu Raspberry PI CM3 support code.""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + SerialLogger, + catch, +) +from testflinger_device_connectors.devices.cm3.cm3 import CM3 + +device_name = "cm3" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = CM3(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + serial_proc = SerialLogger( + serial_host, serial_port, "provision-serial.log" + ) + serial_proc.start() + try: + device.provision() + except Exception as e: + raise e + finally: + serial_proc.stop() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/cm3/cm3.py b/device-connectors/src/testflinger_device_connectors/devices/cm3/cm3.py new file mode 100644 index 00000000..d24fcab6 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/cm3/cm3.py @@ -0,0 +1,276 @@ +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu Raspberry PI cm3 support code.""" + +import json +import logging +import os +import subprocess +import time +from contextlib import contextmanager + +import yaml + +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class CM3: + + """Device Connector for CM3.""" + + IMAGE_PATH_IDS = { + "etc": "ubuntu", + "system-data": "core", + "snaps": "core20", + } + + def __init__(self, config, job_data): + with open(config) as configfile: + self.config = yaml.safe_load(configfile) + with open(job_data) as j: + self.job_data = json.load(j) + + def _run_control(self, cmd, timeout=60): + """ + Run a command on the control host over ssh + + :param cmd: + Command to run + :param timeout: + Timeout (default 60) + :returns: + Return output from the command, if any + """ + control_host = self.config.get("control_host") + control_user = self.config.get("control_user", "ubuntu") + ssh_cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(control_user, control_host), + cmd, + ] + try: + output = subprocess.check_output( + ssh_cmd, stderr=subprocess.STDOUT, timeout=timeout + ) + except subprocess.CalledProcessError as e: + raise ProvisioningError(e.output) + return output + + def provision(self): + try: + url = self.job_data["provision_data"]["url"] + except KeyError: + raise ProvisioningError( + 'You must specify a "url" value in ' + 'the "provision_data" section of ' + "your job_data" + ) + # Remove /dev/sda if somehow it's a normal file + try: + self._run_control("test -f /dev/sda") + # paranoid, but be really certain we're not running locally + self._run_control("sudo rm -f /dev/sda") + except Exception: + pass + self._run_control("sudo pi3gpio set high 16") + time.sleep(5) + self.hardreset() + logger.info("Flashing image") + out = self._run_control( + "sudo cm3-installer {}".format(url), timeout=1800 + ) + logger.info(out) + image_type, image_dev = self.get_image_type() + with self.remote_mount(image_dev): + logger.info("Creating Test User") + self.create_user(image_type) + self._run_control("sudo sync") + time.sleep(5) + out = self._run_control("sudo udisksctl power-off -b /dev/sda ") + logger.info(out) + time.sleep(5) + self._run_control("sudo pi3gpio set low 16") + time.sleep(5) + self.hardreset() + if self.check_test_image_booted(): + return + agent_name = self.config.get("agent_name") + logger.error( + "Device %s unreachable after provisioning, deployment " "failed!", + agent_name, + ) + raise ProvisioningError("Provisioning failed!") + + @contextmanager + def remote_mount(self, remote_device, mount_point="/mnt"): + self._run_control( + "sudo mount /dev/{} {}".format(remote_device, mount_point) + ) + try: + yield mount_point + finally: + self._run_control("sudo umount {}".format(mount_point)) + + def get_image_type(self): + """ + Figure out which kind of image is on the configured block device + + :returns: + tuple of image type and device as strings + """ + dev = self.config["test_device"] + lsblk_data = self._run_control("lsblk -J {}".format(dev)) + lsblk_json = json.loads(lsblk_data.decode()) + dev_list = [ + x.get("name") + for x in lsblk_json["blockdevices"][0]["children"] + if x.get("name") + ] + for dev in dev_list: + try: + with self.remote_mount(dev): + dirs = self._run_control("ls /mnt") + for path, img_type in self.IMAGE_PATH_IDS.items(): + if path in dirs.decode().split(): + return img_type, dev + except Exception: + # If unmountable or any other error, go on to the next one + continue + # We have no idea what kind of image this is + return "unknown", dev + + def check_test_image_booted(self): + logger.info("Checking if test image booted.") + started = time.time() + # Retry for a while since we might still be rebooting + test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + test_password = self.job_data.get("test_data", {}).get( + "test_password", "ubuntu" + ) + while time.time() - started < 600: + try: + time.sleep(10) + cmd = [ + "sshpass", + "-p", + test_password, + "ssh-copy-id", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, self.config["device_ip"]), + ] + subprocess.check_output( + cmd, stderr=subprocess.STDOUT, timeout=60 + ) + return True + except Exception: + pass + # If we get here, then we didn't boot in time + raise ProvisioningError("Failed to boot test image!") + + def create_user(self, image_type): + """Create user account for default ubuntu user""" + metadata = "instance_id: cloud-image" + userdata = ( + "#cloud-config\n" + "password: ubuntu\n" + "chpasswd:\n" + " list:\n" + " - ubuntu:ubuntu\n" + " expire: False\n" + "ssh_pwauth: True" + ) + # For core20: + uc20_ci_data = ( + "#cloud-config\n" + "datasource_list: [ NoCloud, None ]\n" + "datasource:\n" + " NoCloud:\n" + " user-data: |\n" + " #cloud-config\n" + " password: ubuntu\n" + " chpasswd:\n" + " list:\n" + " - ubuntu:ubuntu\n" + " expire: False\n" + " ssh_pwauth: True\n" + " meta-data: |\n" + " instance_id: cloud-image" + ) + + base = "/mnt" + if image_type == "core": + base = "/mnt/system-data" + try: + if image_type == "core20": + ci_path = os.path.join(base, "data/etc/cloud/cloud.cfg.d") + self._run_control("sudo mkdir -p {}".format(ci_path)) + write_cmd = "sudo bash -c \"echo '{}' > /{}/{}\"" + self._run_control( + write_cmd.format(uc20_ci_data, ci_path, "99_nocloud.cfg") + ) + else: + # For core or ubuntu classic images + ci_path = os.path.join(base, "var/lib/cloud/seed/nocloud-net") + self._run_control("sudo mkdir -p {}".format(ci_path)) + write_cmd = "sudo bash -c \"echo '{}' > /{}/{}\"" + self._run_control( + write_cmd.format(metadata, ci_path, "meta-data") + ) + self._run_control( + write_cmd.format(userdata, ci_path, "user-data") + ) + if image_type == "ubuntu": + # This needs to be removed on classic for rpi, else + # cloud-init won't find the user-data we give it + rm_cmd = "sudo rm -f {}".format( + os.path.join( + base, "etc/cloud/cloud.cfg.d/99-fake?cloud.cfg" + ) + ) + self._run_control(rm_cmd) + except Exception: + raise ProvisioningError("Error creating user files") + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config["reboot_script"]: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except Exception: + raise RecoveryError("timeout reaching control host!") diff --git a/device-connectors/src/testflinger_device_connectors/devices/dell_oemscript/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/dell_oemscript/__init__.py new file mode 100644 index 00000000..e44874ad --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/dell_oemscript/__init__.py @@ -0,0 +1,52 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Ubuntu OEM Recovery provisioning for Dell OEM devices +Use this for systems that can use the oem recovery-from-iso.sh script +for provisioning, but require the --ubr flag in order to use the +"ubuntu recovery" method. +""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + catch, +) +from .dell_oemscript import DellOemScript + +device_name = "dell_oemscript" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning Dell OEM devices with an oem image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = DellOemScript(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + device.provision() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/dell_oemscript/dell_oemscript.py b/device-connectors/src/testflinger_device_connectors/devices/dell_oemscript/dell_oemscript.py new file mode 100644 index 00000000..614409ff --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/dell_oemscript/dell_oemscript.py @@ -0,0 +1,28 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu OEM Script provisioning for Dell OEM devices +this for systems that can use the oem recovery-from-iso.sh script +for provisioning, but require the --ubr flag in order to use the +"ubuntu recovery" method. +""" + +import logging +from testflinger_device_connectors.devices.oemscript.oemscript import OemScript + +logger = logging.getLogger() + + +class DellOemScript(OemScript): + """Device Agent for Dell OEM devices.""" diff --git a/device-connectors/src/testflinger_device_connectors/devices/dragonboard/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/dragonboard/__init__.py new file mode 100644 index 00000000..05a24d1a --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/dragonboard/__init__.py @@ -0,0 +1,60 @@ +# Copyright (C) 2016-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Dragonboard support code.""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + SerialLogger, + catch, +) +from testflinger_device_connectors.devices.dragonboard.dragonboard import ( + Dragonboard, +) + +device_name = "dragonboard" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = Dragonboard(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Booting Master Image") + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + serial_proc = SerialLogger( + serial_host, serial_port, "provision-serial.log" + ) + serial_proc.start() + try: + device.provision() + except Exception as e: + raise e + finally: + serial_proc.stop() diff --git a/device-connectors/src/testflinger_device_connectors/devices/dragonboard/dragonboard.py b/device-connectors/src/testflinger_device_connectors/devices/dragonboard/dragonboard.py new file mode 100644 index 00000000..899630d2 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/dragonboard/dragonboard.py @@ -0,0 +1,422 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Dragonboard support code.""" + +import json +import logging +import multiprocessing +import os +import subprocess +import time + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class Dragonboard: + + """Testflinger Device Connector for Dragonboard.""" + + def __init__(self, config, job_data): + with open(config) as configfile: + self.config = yaml.safe_load(configfile) + with open(job_data) as j: + self.job_data = json.load(j) + self.test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + self.test_password = self.job_data.get("test_data", {}).get( + "test_password", "ubuntu" + ) + + def _run_control(self, cmd, timeout=60): + """ + Run a command on the control host over ssh + + :param cmd: + Command to run + :param timeout: + Timeout (default 60) + :returns: + Return output from the command, if any + """ + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "linaro@{}".format(self.config["device_ip"]), + cmd, + ] + output = subprocess.check_output( + cmd, stderr=subprocess.STDOUT, timeout=timeout + ) + return output + + def setboot(self, mode): + """ + Set the boot mode of the device. + + :param mode: + One of 'master' or 'test' + :raises KeyError: + if script keys are missing from the config file + :raises ProvisioningError: + If the command times out or anything else fails. + + This method sets the boot method to the specified value. + """ + if mode == "master": + setboot_script = self.config["select_master_script"] + elif mode == "test": + setboot_script = self.config["select_test_script"] + else: + raise KeyError + for cmd in setboot_script: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=60) + except subprocess.TimeoutExpired: + raise ProvisioningError("timeout reaching control host!") + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config["reboot_script"]: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except subprocess.TimeoutExpired: + raise RecoveryError("timeout reaching control host!") + + def copy_ssh_id(self): + """ + Copy the ssh key to the device. + """ + cmd = [ + "sshpass", + "-p", + self.test_password, + "ssh-copy-id", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(self.test_username, self.config["device_ip"]), + ] + try: + subprocess.check_call(cmd) + except subprocess.SubprocessError: + pass + + def ensure_test_image(self): + """ + Actively switch the device to boot the test image. + + :raises ProvisioningError: + If the command times out or anything else fails. + """ + logger.info("Booting the test image") + self.setboot("test") + try: + self._run_control("sudo /sbin/reboot") + except subprocess.SubprocessError: + # Keep trying even if this command fails + pass + time.sleep(60) + + started = time.time() + # Retry for a while since we might still be rebooting + test_image_booted = False + while time.time() - started < 600: + self.copy_ssh_id() + test_image_booted = self.is_test_image_booted() + if test_image_booted: + break + # Check again if we are in the master image + if not test_image_booted: + raise ProvisioningError("Failed to boot test image!") + + def is_test_image_booted(self): + """ + Check if the master image is booted. + + :returns: + True if the test image is currently booted, False otherwise. + """ + logger.info("Checking if test image booted.") + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "snap -h", + ] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=60) + except subprocess.SubprocessError: + return False + # If we get here, the above command proved we are in the test image + return True + + def is_master_image_booted(self): + """ + Check if the master image is booted. + + :returns: + True if the master image is currently booted, False otherwise. + + .. note:: + The master image is used for writing a new image to local media + """ + # FIXME: come up with a better way of checking this + logger.info("Checking if master image booted.") + try: + output = self._run_control("cat /etc/issue") + except subprocess.SubprocessError: + logger.info("Error checking device state. Forcing reboot...") + return False + if "Debian GNU" in str(output): + return True + return False + + def ensure_master_image(self): + """ + Actively switch the device to boot the test image. + + :raises RecoveryError: + If the command times out or anything else fails. + """ + logger.info("Making sure the master image is booted") + + # most likely, we are still in a test image, check that first + test_booted = self.is_test_image_booted() + + if test_booted: + # We are not in the master image, so just hard reset + self.setboot("master") + self.hardreset() + + started = time.time() + while time.time() - started < 300: + time.sleep(10) + master_booted = self.is_master_image_booted() + if master_booted: + return + # Check again if we are in the master image + if not master_booted: + raise RecoveryError("Could not reboot to master!") + + master_booted = self.is_master_image_booted() + if not master_booted: + logging.warn( + "Device is in an unknown state, attempting to recover" + ) + self.hardreset() + started = time.time() + while time.time() - started < 300: + time.sleep(10) + if self.is_master_image_booted(): + return + elif self.is_test_image_booted(): + # device was stuck, but booted to the test image + # So rerun ourselves to get to the master image + return self.ensure_master_image() + # timeout reached, this could be a dead device + raise RecoveryError( + "Device is in an unknown state, may require manual recovery!" + ) + # If we get here, the master image was already booted, so just return + + def flash_test_image(self, server_ip, server_port): + """ + Flash the image at :image_url to the sd card. + + :param server_ip: + IP address of the image server. The image will be downloaded and + uncompressed over the SD card. + :param server_port: + TCP port to connect to on server_ip for downloading the image + :raises ProvisioningError: + If the command times out or anything else fails. + """ + # First unmount, just in case + try: + self._run_control( + "sudo umount {}*".format(self.config["test_device"]), + timeout=30, + ) + except subprocess.SubprocessError: + # We might not be mounted, so expect this to fail sometimes + pass + cmd = "nc {} {}| unxz| sudo dd of={} bs=16M".format( + server_ip, server_port, self.config["test_device"] + ) + logger.info("Running: %s", cmd) + try: + # XXX: I hope 30 min is enough? but maybe not! + self._run_control(cmd, timeout=1800) + except subprocess.TimeoutExpired: + raise ProvisioningError("timeout reached while flashing image!") + try: + self._run_control("sync") + except subprocess.SubprocessError: + # Nothing should go wrong here, but let's sleep if it does + logger.warn("Something went wrong with the sync, sleeping...") + time.sleep(30) + try: + self._run_control( + "sudo hdparm -z {}".format(self.config["test_device"]), + timeout=30, + ) + except subprocess.CalledProcessError as exc: + raise ProvisioningError( + "Unable to run hdparm to rescan" + "partitions: {}".format(exc.output) + ) + + def mount_writable_partition(self): + # Mount the writable partition + try: + self._run_control( + "sudo mount {} /mnt".format( + self.config["snappy_writable_partition"] + ) + ) + except subprocess.CalledProcessError as exc: + err = ( + "Error mounting writable partition on test image {}. " + "Check device configuration\n" + "output: {}".format( + self.config["snappy_writable_partition"], exc.output + ) + ) + raise ProvisioningError(err) + + def create_user(self): + """Create user account for default ubuntu user""" + self.mount_writable_partition() + metadata = "instance_id: cloud-image" + userdata = ( + "#cloud-config\n" + "password: ubuntu\n" + "chpasswd:\n" + " list:\n" + " - ubuntu:ubuntu\n" + " expire: False\n" + "ssh_pwauth: True" + ) + with open("meta-data", "w") as mdata: + mdata.write(metadata) + with open("user-data", "w") as udata: + udata.write(userdata) + try: + output = self._run_control("ls /mnt") + if "system-data" in str(output): + base = "/mnt/system-data" + else: + base = "/mnt" + cloud_path = os.path.join(base, "var/lib/cloud/seed/nocloud-net") + self._run_control("sudo mkdir -p {}".format(cloud_path)) + write_cmd = "sudo bash -c \"echo '{}' > /{}/{}\"" + self._run_control( + write_cmd.format(metadata, cloud_path, "meta-data") + ) + self._run_control( + write_cmd.format(userdata, cloud_path, "user-data") + ) + except subprocess.CalledProcessError as exc: + raise ProvisioningError( + "Error creating user files: {}".format(exc.output) + ) + + def setup_sudo(self): + sudo_data = "ubuntu ALL=(ALL) NOPASSWD:ALL" + sudo_path = "/mnt/system-data/etc/sudoers.d/ubuntu" + self._run_control( + "sudo mkdir -p {}".format(os.path.dirname(sudo_path)) + ) + self._run_control( + "sudo bash -c \"echo '{}' > {}\"".format(sudo_data, sudo_path) + ) + + def wipe_test_device(self): + """Safety check - wipe the test drive if things go wrong + + This way if we reboot the sytem after a failed provision, it goes + back to the control boot image which we could use to provision + something else. + """ + try: + test_device = self.config["test_device"] + logger.error("Failed to write image, cleaning up...") + self._run_control("sudo sgdisk -o {}".format(test_device)) + except subprocess.SubprocessError: + # This is an attempt to salvage a bad run, further tracebacks + # would just add to the noise + pass + + def provision(self): + """Provision the device""" + url = self.job_data["provision_data"].get("url") + self.copy_ssh_id() + self.ensure_master_image() + testflinger_device_connectors.download(url, "install.img") + image_file = testflinger_device_connectors.compress_file("install.img") + server_ip = testflinger_device_connectors.get_local_ip_addr() + serve_q = multiprocessing.Queue() + file_server = multiprocessing.Process( + target=testflinger_device_connectors.serve_file, + args=( + serve_q, + image_file, + ), + ) + file_server.start() + server_port = serve_q.get() + logger.info("Flashing Test Image") + try: + self.flash_test_image(server_ip, server_port) + file_server.terminate() + logger.info("Creating Test User") + self.create_user() + self.setup_sudo() + logger.info("Booting Test Image") + self.ensure_test_image() + except (ValueError, subprocess.SubprocessError): + # wipe out whatever we installed if things go badly + self.wipe_test_device() + raise + # Brief delay to ensure first boot tasks are complete + time.sleep(60) + logger.info("END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/lenovo_oemscript/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/lenovo_oemscript/__init__.py new file mode 100644 index 00000000..2716962c --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/lenovo_oemscript/__init__.py @@ -0,0 +1,52 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Ubuntu OEM Recovery Provisioning for Lenovo OEM devices +Use this for systems that can use the oem recovery-from-iso.sh script +for provisioning, but require the --ubr flag in order to use the +"ubuntu recovery" method. +""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + catch, +) +from .lenovo_oemscript import LenovoOemScript + +device_name = "lenovo_oemscript" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning Lenovo OEM devices with an oem image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = LenovoOemScript(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + device.provision() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/lenovo_oemscript/lenovo_oemscript.py b/device-connectors/src/testflinger_device_connectors/devices/lenovo_oemscript/lenovo_oemscript.py new file mode 100644 index 00000000..c57316f3 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/lenovo_oemscript/lenovo_oemscript.py @@ -0,0 +1,31 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu OEM Script provisioning for Lenovo OEM devices +this for systems that can use the oem recovery-from-iso.sh script +for provisioning, but require the --ubr flag in order to use the +"ubuntu recovery" method. +""" + +import logging +from testflinger_device_connectors.devices.oemscript.oemscript import OemScript + +logger = logging.getLogger() + + +class LenovoOemScript(OemScript): + """Device Agent for Lenovo OEM devices.""" + + # Extra arguments to pass to the OEM script + extra_script_args = ["--ubr"] diff --git a/device-connectors/src/testflinger_device_connectors/devices/maas2/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/maas2/__init__.py new file mode 100644 index 00000000..4cf6bd17 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/maas2/__init__.py @@ -0,0 +1,63 @@ +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu MaaS 2.x CLI support code.""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + ProvisioningError, + RecoveryError, + SerialLogger, + catch, +) +from testflinger_device_connectors.devices.maas2.maas2 import Maas2 + +device_name = "maas2" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = Maas2(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + serial_proc = SerialLogger( + serial_host, serial_port, "provision-serial.log" + ) + serial_proc.start() + try: + device.provision() + except ProvisioningError as e: + logmsg(logging.ERROR, "Provisioning failed: {}".format(str(e))) + return 1 + except Exception as e: + raise e + finally: + serial_proc.stop() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/maas2/doc/maas_storage.rst b/device-connectors/src/testflinger_device_connectors/devices/maas2/doc/maas_storage.rst new file mode 100644 index 00000000..d398c2c5 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/maas2/doc/maas_storage.rst @@ -0,0 +1,140 @@ +================= +Preamble +================= +This extension of the Testflinger Maas Testflinger Device Connector to handle a variety of node storage layout configurations. This configuration will be passed to Testflinger via the node job configuration yaml file, as part of the SUT provision data (example below). This functionality is containted in the discreet Python module (‘maas-storage.py’) that sits alongside the Maas Testflinger Device Connector, to be imported and called when this device connector is instantiated, if a storage layout configuration is supplied. + +These storage layout configurations are to be passed along to MAAS, via the CLI API, when the device connector is created as part of its provision phase. While initial scope and use of this module will be limited to SQA’s testing requirements, the availability of this module implies additional consumers can specify disk layout configurations as part of their Testflinger job definitions. + +Of note: the initial scope of storage to be supported will be limited to flat layouts and simple partitions; RAID, LVM or bcache configurations are currently unnsupported by this module. This functionality will be added in the future as the need arises. + +================= +Job Configuration +================= +The storage configuration is traditionally supplied as a node bucket config, so we can duplicate how this is laid out in the SUT job configuration at the end of this document. + +As below, the storage configuration is defined under ‘disks’ key in the yaml file. It is composed of a list of storage configuration entries, which are dictionaries with at least these two fields, ‘type’ and ‘id’: +- **type**: the type of configuration entry. Currently this is one of: + - *disk* - a physical disk on the system + - *partition* - a partition on the system + - *format* - instructions to format a volume + - *mount* - instructions to mount a formatted partition +- id: a label used to refer to this storage configuration entry. Other configuration entries will use this id to refer to this configuration item, or the component it configured. + +================= +Storage Types +================= +Each type of storage configuration entry has additional fields of: +- **Disk**: all storage configuration is ultimately applied to disks. These are referenced by ‘disk’ storage configuration entries. + - *disk* - The key of a disk in the machine's hardware configuration. By default, this is an integer value like ‘0’ or ‘1.’ + - *ptable* - Type of partition table to use for the disk (‘gpt’ or ‘msdos’). + - *name* - Optional. If specified, the name to use for the disk, as opposed to the default one which is usually something like 'sda'. This will be used in ‘/dev/disk/by-dname/’ for the disk and its partitions. So if you make the disk name 'rootdisk', it will show up at . This can be used to give predictable, meaningful names to disks, which can be referenced in juju config, etc. + - *boot* - Optional. If specified, this disk will be set as boot disk in MAAS. + - The disk's ‘id’ will be used to refer to this disk in other entries, such as ‘partition.’ +- **Partition**: A partition of a disk is represented by a ‘partition’ entry. + - *device* - The ‘id’ of the ‘disk’ entry this partition should be created on. + - *number* - Partition number. This determines the order of the partitions on the disk. + - *size* - The minimum required size of the partition, in bytes, or in larger units, given by suffixes (K, M, G, T) + - The partition's ‘id’ will be used to refer to this partition in other entries, such as ‘format.’ + - *alloc_pct* - Percent (as an int) of the parent disk this partition should consume. This is optional, if this is not given, then the ‘size’ value will be the created partition size. If multiple partitions exist on a parent disk, the total alloc_pct between them cannot exceed 100. +- **Format**: Instructions to format a volume are represented by a ‘format’ entry. + - *volume* - the ‘id’ of the entry to be formatted. This can be a disk or partition entry. + - *fstype* - The file system type to format the volume with. See MAAS docs for options. + - *label* - The label to use for the file system. + - The format's ‘id’ will be used to refer to this formatted volume in ‘mount’ entries. +- **Mount:** Instructions to mount a formatted volume are represented by a ‘mount’ entry. + - *device* - The ‘id’ of the ‘format’ entry this mount refers to. + - *path* - The path to mount the formatted volume at, e.g. ‘/boot.’ + +================= +Storage Configuration Instantiation +================= +- The existing storage configuration on the SUT is first cleared in order to start with a clean slate. +- We will then fetch the SUT block device config via the Maas API in order to verify and choose the appropriate physical disks which exist on the system. These disks must align with the configuration parameters (size, number of disks) presented in the config to proceed. +- Disk selection should be performed with the following criteria: + - In instances where all disks meet the space requirements, we can numerically assign the lowest physical disk ID (in Maas block-devices) to the first config disk. Subsequent disks will be assigned in numerical order. + - In instances where the total of a config disk’s partition map (determined by adding all configuration partitions on that disk) will only fit on certain node disks, these disks will only be selected for the parent configuration disk of said partition map. + - Disk selection will be done in numerical order as above within any smaller pool of disks that meet configuration partitioning criteria. + - Node provisioning will fail if configuration partition maps exist that will not adequately fit on any disk, or if the pool of appropriate disks is exhausted prior to accommodating all configuration partition maps. + - However, dynamic allocation of partition sizes using the alloc_pct field will enable a much more flexible allocation of partitions to parent disks, and one only needs to be able to provide the minimum partition size in order to select the most appropriate disk. +- After disk selection takes place, all configuration elements of each storage type will be grouped together for batch processing. This order is determined by the dependency each type has on the other. The types and the order in which they will be processed will be: [‘disk’, ‘partition’, ‘format’, ‘mount’]. + - As additional storage types are supported in the future, this order will need to remain consistent with any parent-child relationship that exists between storage types. +- The storage configuration will then be written to the node disks in this order. + - If a boot partition exists in the configuration, the parent disk will be flagged as a boot disk via the Maas API. The boot partition will then be created on this disk, including an EFI mount if desired. +- After the storage configuration is completed and written to the node’s physical disks, node provisioning will proceed to OS installation, in addition to any other provisioning steps outside of the node’s storage subsystem. + +================= +Job Definition Reference +================= +.. code-block:: yaml + :caption: job.yaml + :linenos: + + disks: + - id: disk0 + disk: 0 + type: disk + ptable: gpt + - id: disk0-part1 + device: disk0 + type: partition + number: 1 + size: 2G + alloc_pct: 80 + - id: disk0-part1-format + type: format + volume: disk0-part1 + fstype: ext4 + label: nova-ephemeral + - id: disk1-part1-mount + device: disk1-part1-format + path: / + type: mount + - id: disk1 + disk: 1 + type: disk + ptable: gpt + - id: disk1-part1 + device: disk1 + type: partition + number: 1 + size: 500M + alloc_pct: 10 + - id: disk1-part1-format + type: format + volume: disk1-part1 + fstype: fat32 + label: efi + - id: disk1-part1-mount + device: disk1-part1-format + path: /boot/efi + type: mount + - id: disk1-part2 + device: disk1 + type: partition + number: 2 + size: 1G + alloc_pct: 20 + - id: disk1-part2-format + volume: disk1-part2 + type: format + fstype: ext4 + label: boot + - id: disk1-part2-mount + device: disk1-part2-format + path: /boot + type: mount + - id: disk1-part3 + device: disk1 + type: partition + number: 3 + size: 10G + alloc_pct: 60 + - id: disk1-part3-format + volume: disk1-part3 + type: format + fstype: ext4 + label: ceph + - id: disk1-part3-mount + device: disk1-part3-format + path: /data + type: mount diff --git a/device-connectors/src/testflinger_device_connectors/devices/maas2/maas2.py b/device-connectors/src/testflinger_device_connectors/devices/maas2/maas2.py new file mode 100644 index 00000000..06944c1c --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/maas2/maas2.py @@ -0,0 +1,404 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu MaaS 2.x CLI support code.""" + +import base64 +import json +import logging +import subprocess +import time +from collections import OrderedDict + +import yaml + +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) +from testflinger_device_connectors.devices.maas2.maas_storage import ( + MaasStorage, + MaasStorageError, +) + + +logger = logging.getLogger() + + +class Maas2: + + """Device Connector for Maas2.""" + + def __init__(self, config, job_data): + with open(config) as configfile: + self.config = yaml.safe_load(configfile) + with open(job_data) as j: + self.job_data = json.load(j) + self.maas_user = self.config.get("maas_user") + self.node_id = self.config.get("node_id") + self.agent_name = self.config.get("agent_name") + self.timeout_min = int(self.config.get("timeout_min", 60)) + self.maas_storage = MaasStorage(self.maas_user, self.node_id) + + def _logger_debug(self, message): + logger.debug("MAAS: {}".format(message)) + + def _logger_info(self, message): + logger.info("MAAS: {}".format(message)) + + def _logger_warning(self, message): + logger.warning("MAAS: {}".format(message)) + + def _logger_error(self, message): + logger.error("MAAS: {}".format(message)) + + def _logger_critical(self, message): + logger.critical("MAAS: {}".format(message)) + + def recover(self): + self._logger_info("Releasing node {}".format(self.agent_name)) + self.node_release() + + def provision(self): + if self.config.get("reset_efi"): + self.reset_efi() + # Check if this is a device where we need to clear the tpm (dawson) + if self.config.get("clear_tpm"): + self.clear_tpm() + provision_data = self.job_data.get("provision_data") + # Default to a safe LTS if no distro is specified + distro = provision_data.get("distro", "xenial") + kernel = provision_data.get("kernel") + user_data = provision_data.get("user_data") + storage_data = provision_data.get("disks") + + self.deploy_node(distro, kernel, user_data, storage_data) + + def _install_efitools_snap(self): + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "sudo snap install efi-tools-ijohnson --devmode --edge", + ] + subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "sudo snap alias efi-tools-ijohnson.efibootmgr efibootmgr", + ] + subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + + def _get_efi_data(self): + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "sudo efibootmgr -v", + ] + p = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + # If it fails the first time, try installing efitools snap + if p.returncode: + self._install_efitools_snap() + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "sudo efibootmgr -v", + ] + p = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if p.returncode: + return None + # Use OrderedDict because often the NIC entries in EFI are in a good + # order with ipv4 ones coming first + efi_data = OrderedDict() + for line in p.stdout.decode().splitlines(): + k, v = line.split(" ", maxsplit=1) + efi_data[k] = v + return efi_data + + def _set_efi_data(self, boot_order): + # Set the boot order to the comma separated string of entries + self._logger_info("Setting boot order to {}".format(boot_order)) + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "sudo efibootmgr -o {}".format(boot_order), + ] + p = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + if p.returncode: + self._logger_error( + 'Failed to set efi boot order to "{}":\n' + "{}".format(boot_order, p.stdout.decode()) + ) + + def reset_efi(self): + # Try to reset the boot order so that NICs boot first + self._logger_info("Fixing EFI boot order before provisioning") + efi_data = self._get_efi_data() + if not efi_data: + return + bootlist = efi_data.get("BootOrder:").split(",") + new_boot_order = [] + for k, v in efi_data.items(): + if ("IPv4" in v) and "Boot" in k: + new_boot_order.append(k[4:8]) + for entry in bootlist: + if entry not in new_boot_order: + new_boot_order.append(entry) + self._set_efi_data(",".join(new_boot_order)) + + def clear_tpm(self): + self._logger_info("Clearing the TPM before provisioning") + # First see if we can run the command on the current install + if self._run_tpm_clear_cmd(): + return + # If not, then deploy bionic and try again + self.deploy_node() + if not self._run_tpm_clear_cmd(): + raise ProvisioningError("Failed to clear TPM") + + def _run_tpm_clear_cmd(self): + # Run the command to clear the tpm over ssh + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "echo 5 | sudo tee /sys/class/tpm/tpm0/ppi/request", + ] + proc = subprocess.run(cmd, timeout=30, check=False) + if proc.returncode: + return False + + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "cat /sys/class/tpm/tpm0/ppi/request", + ] + proc = subprocess.run( + cmd, timeout=30, capture_output=True, check=False + ) + if proc.returncode: + return False + + # If we now see "5" in that file, then clearing tpm succeeded + if proc.stdout.decode("utf-8").strip() == "5": + return True + return False + + def deploy_node( + self, distro="bionic", kernel=None, user_data=None, storage_data=None + ): + # Deploy the node in maas, default to bionic if nothing is specified + self.recover() + status = self.node_status() + # do not process an empty dataset + if storage_data is not None: + try: + self.maas_storage.configure_node_storage(storage_data) + except MaasStorageError as error: + self._logger_error( + f"Unable to configure node storage: {error}" + ) + raise ProvisioningError from error + else: + def_storage_data = self.config.get("default_disks") + self._logger_debug(f"Using def storage data: {def_storage_data}") + if not def_storage_data: + self._logger_warning( + "'default_disks' and/or 'disks' unspecified; " + "skipping storage layout configuration" + ) + else: + # reset to the default layout + try: + self.maas_storage.configure_node_storage( + def_storage_data, reset=True + ) + except MaasStorageError as error: + self._logger_error( + "Unable to reset node storage to " + f"default_disk layout: {error}" + ) + raise ProvisioningError from error + + self._logger_info("Acquiring node") + cmd = [ + "maas", + self.maas_user, + "machines", + "allocate", + "system_id={}".format(self.node_id), + ] + # Do not use runcmd for this - we need the output, not the end user + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + if proc.returncode: + self._logger_error(f"maas error running: {' '.join(cmd)}") + raise ProvisioningError(proc.stdout.decode()) + self._logger_info( + "Starting node {} " + "with distro {}".format(self.agent_name, distro) + ) + cmd = [ + "maas", + self.maas_user, + "machine", + "deploy", + self.node_id, + "distro_series={}".format(distro), + ] + if kernel: + cmd.append("hwe_kernel={}".format(kernel)) + if user_data: + data = base64.b64encode(user_data.encode()).decode() + cmd.append("user_data={}".format(data)) + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + if proc.returncode: + self._logger_error(f"maas-cli error running: {' '.join(cmd)}") + raise ProvisioningError(proc.stdout.decode()) + + # Make sure the device is available before returning + minutes_spent = 0 + self._logger_info( + "Timeout value: {} minutes.".format(self.timeout_min) + ) + while minutes_spent < self.timeout_min: + time.sleep(60) + minutes_spent += 1 + self._logger_info( + "{} minutes passed " "since deployment.".format(minutes_spent) + ) + status = self.node_status() + + if status == "Failed deployment": + self._logger_error("MaaS reports Failed Deployment") + exception_msg = ( + "Provisioning failed because " + + "MaaS got unexpected or " + + "deployment failure status signal." + ) + raise ProvisioningError(exception_msg) + + if status == "Deployed": + if self.check_test_image_booted(): + self._logger_info("Deployed and booted.") + return + + self._logger_error( + 'Device {} still in "{}" state, deployment ' + "failed!".format(self.agent_name, status) + ) + self._logger_error(proc.stdout.decode()) + exception_msg = ( + "Provisioning failed because deployment timeout. " + + "Deploying for more than " + + "{} minutes.".format(self.timeout_min) + ) + raise ProvisioningError(exception_msg) + + def check_test_image_booted(self): + self._logger_info("Checking if test image booted.") + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "ubuntu@{}".format(self.config["device_ip"]), + "/bin/true", + ] + try: + subprocess.run( + cmd, stderr=subprocess.STDOUT, timeout=60, check=True + ) + except subprocess.SubprocessError: + return False + # If we get here, then the above command proved we are booted + return True + + def node_status(self): + """Return status of the node according to maas: + + Ready: Node is unused + Allocated: Node is allocated + Deploying: Deployment in progress + Deployed: Node is provisioned and ready for use + """ + cmd = ["maas", self.maas_user, "machine", "read", self.node_id] + # Do not use runcmd for this - we need the output, not the end user + proc = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False + ) + if proc.returncode: + self._logger_error(f"maas error running: {' '.join(cmd)}") + raise ProvisioningError(proc.stdout.decode()) + data = json.loads(proc.stdout.decode()) + return data.get("status_name") + + def node_release(self): + """Release the node to make it available again""" + cmd = ["maas", self.maas_user, "machine", "release", self.node_id] + subprocess.run(cmd, check=False) + # Make sure the device is available before returning + for _ in range(0, 10): + time.sleep(5) + status = self.node_status() + if status == "Ready": + return + self._logger_error( + 'Device {} still in "{}" state, could not ' + "recover!".format(self.agent_name, status) + ) + raise RecoveryError("Device recovery failed!") diff --git a/device-connectors/src/testflinger_device_connectors/devices/maas2/maas_storage.py b/device-connectors/src/testflinger_device_connectors/devices/maas2/maas_storage.py new file mode 100644 index 00000000..072ace22 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/maas2/maas_storage.py @@ -0,0 +1,571 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu MaaS 2.x storage provisioning code.""" + +import logging +import subprocess +import collections +import json +import math + + +logger = logging.getLogger() + + +class MaasStorageError(Exception): + pass + + +class MaasStorage: + """Maas device connector storage module.""" + + def __init__(self, maas_user, node_id): + self.maas_user = maas_user + self.node_id = node_id + self.device_list = None + self.init_data = None + self.node_info = self._node_read() + self.block_ids = {} + self.partition_sizes = {} + + def _logger_debug(self, message): + logger.debug("MAAS: {}".format(message)) + + def _logger_info(self, message): + logger.info("MAAS: {}".format(message)) + + def _node_read(self): + """Read node block-devices. + + :return: the node's block device information + """ + cmd = ["maas", self.maas_user, "block-devices", "read", self.node_id] + return self.call_cmd(cmd, output_json=True) + + @staticmethod + def call_cmd(cmd, output_json=False): + """Run a command and return the output. + + :param cmd: command to run + :param output_json: output the result as JSON + :return: subprocess stdout + :raises MaasStorageError: on subprocess non-zero return code + """ + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + + if proc.returncode != 0: + raise MaasStorageError(proc.stdout.decode()) + + if proc.stdout: + output = proc.stdout.decode() + + if output_json: + return json.loads(output) + + return output + + @staticmethod + def convert_size_to_bytes(size_str): + """Convert given sizes to bytes; case insensitive. + + :param size_str: the size string to convert + :return: the size in bytes + :raises MaasStorageError: on invalid size unit/type + """ + size_str = size_str.upper() + size_pow = {"T": 4, "G": 3, "M": 2, "K": 1, "B": 0} + + try: + return round( + float("".join(char for char in size_str if char.isdigit())) + ) * ( + 1000 + ** size_pow[ + "".join(char for char in size_str if not char.isdigit()) + ] + ) + except KeyError: + try: + # attempt to convert the size string to an integer + return int(size_str) + except ValueError: + raise MaasStorageError( + "Sizes must end in T, G, M, K, B, or be an integer " + "when no unit is provided." + ) + + def configure_node_storage(self, storage_data, reset=False): + """Configure the node's storage layout, from provisioning data.""" + self.device_list = storage_data + + if not reset: + self._logger_info("Configuring node storage") + # map top level parent disk to every device + self.assign_parent_disk() + # tally partition requirements for each disk + self.gather_partitions() + # find appropriate block devices for each partition + self.parse_block_devices() + # map block ids to top level parents + self.map_block_ids() + # calculate partition sizes + self.create_partition_sizes() + + # group devices by type + devs_by_type = self.group_by_type() + + # clear existing storage on node + self._logger_info("Clearing existing storage configuration") + self.clear_storage_config() + # apply configured storage to node + self._logger_info("Applying storage layout") + self.process_by_dev_type(devs_by_type) + + def clear_storage_config(self): + """Clear the node's exisitng storage configuration.""" + for block_dev in self.node_info: + if block_dev["type"] == "virtual": + continue + for partition in block_dev["partitions"]: + self.call_cmd( + [ + "maas", + self.maas_user, + "partition", + "delete", + self.node_id, + str(block_dev["id"]), + str(str(partition["id"])), + ] + ) + if block_dev["filesystem"] is not None: + if block_dev["filesystem"]["mount_point"] is not None: + self.call_cmd( + [ + "maas", + self.maas_user, + "block-device", + "unmount", + self.node_id, + str(block_dev["id"]), + ] + ) + self.call_cmd( + [ + "maas", + self.maas_user, + "block-device", + "unformat", + self.node_id, + str(block_dev["id"]), + ] + ) + + def assign_parent_disk(self): + """Transverse device hierarchy to determine each device's parent + disk.""" + dev_dict = {dev["id"]: dev for dev in self.device_list} + for dev in self.device_list: + parent_id = dev.get("device") or dev.get("volume") + + if dev["type"] == "disk": + # keep 'parent_disk' key for consistency + dev["parent_disk"] = dev["id"] + + while parent_id and parent_id in dev_dict: + parent = dev_dict[parent_id] + parent_id = parent.get("device") or parent.get("volume") + if parent["type"] == "disk": + dev["parent_disk"] = parent["id"] + + def gather_partitions(self): + """Tally partition size requirements for block-device selection.""" + partitions = collections.defaultdict(list) + + for dev in self.device_list: + if dev["type"] == "partition": + # convert size to bytes before appending to the list + partitions[dev["parent_disk"]].append( + self.convert_size_to_bytes(dev["size"]) + ) + + # summing up sizes of each disk's partitions + self.partition_sizes = { + devid: sum(partitions) for devid, partitions in partitions.items() + } + + def _select_block_dev(self, partition_id, partition_size): + """Find a suitable block device for the given partition. + + :param partition_id: the id of the partition + :param partition_size: the size of the partition + :return: the id of a suitable block device if found + :raises MaasStorageError: if no suitable block device is found + """ + for block_dev in self.node_info: + if ( + block_dev["type"] == "physical" + and block_dev["id"] not in self.block_ids.values() + and partition_size <= int(block_dev["size"]) + ): + return block_dev["id"] + + raise MaasStorageError( + "No suitable block-device found for partition " + f"{partition_id} with size {partition_size} bytes" + ) + + def parse_block_devices(self): + """Find appropriate node block-device for use in layout.""" + for partition_id, partition_size in self.partition_sizes.items(): + self._logger_debug(f"Comparing size: Partition: {partition_size}") + block_device_id = self._select_block_dev( + partition_id, partition_size + ) + + # map partition id to block device id + self.block_ids[partition_id] = block_device_id + + def map_block_ids(self): + """Map parent disks to actual node block-devices. + + Updates self.device_list with "parent_disk_blkid". + """ + for dev in self.device_list: + block_id = self.block_ids.get(dev["parent_disk"]) + if block_id is not None: + dev["parent_disk_blkid"] = str(block_id) + + def _validate_alloc_pct_values(self): + """Sanity check partition allocation percentages.""" + alloc_pct_values = collections.defaultdict(int) + + for dev in self.device_list: + if dev["type"] == "partition": + # add pct together (default to 0 if unnused) + alloc_pct_values[dev["parent_disk"]] += dev.get("alloc_pct", 0) + + for dev_id, alloc_pct in alloc_pct_values.items(): + if alloc_pct > 100: + raise MaasStorageError( + "The total percentage of the partitions on disk " + f"'{dev_id}' exceeds 100." + ) + + def create_partition_sizes(self): + """Calculate actual partition size to write to disk.""" + self._validate_alloc_pct_values() + + for dev in self.device_list: + if dev["type"] == "partition": + # find corresponding block device + for block_dev in self.node_info: + if block_dev["id"] == self.block_ids[dev["parent_disk"]]: + # get the total size of the block device in bytes + total_size = int(block_dev["size"]) + break + + if "alloc_pct" in dev: + # avoid under-allocating space + dev["size"] = str( + math.ceil((total_size * dev.get("alloc_pct", 0)) / 100) + ) + else: + if "size" not in dev: + raise ValueError( + f"Partition '{str(dev['id'])}' does not have an " + "alloc_pct or size value." + ) + + # default to minimum required partition size + dev["size"] = self.convert_size_to_bytes(dev["size"]) + + def group_by_type(self): + """Group storage devices by type for processing. + + :return: dict with device types as keys and lists of devices as values + """ + devs_by_type = collections.defaultdict(list) + + for dev in self.device_list: + devs_by_type[dev["type"]].append(dev) + + return devs_by_type + + def process_by_dev_type(self, devs_by_type): + """Process each storage type together in sequence. + + :param devs_by_type: dict with device types as keys and + lists of devices as values + :raises MaasStorageError: if an error occurs during device processing + """ + # order in which storage types are processed + dev_type_order = ["disk", "partition", "format", "mount"] + # maps the device type to the method that processes it + dev_type_to_method = { + "disk": self.process_disk, + "partition": self.process_partition, + "mount": self.process_mount, + "format": self.process_format, + } + partn_data = {} + + for dev_type in dev_type_order: + devices = devs_by_type.get(dev_type) + if devices: + self._logger_debug(f"Processing type '{dev_type}':") + for dev in devices: + try: + if dev_type == "partition": + partn_data[dev["id"]] = dev_type_to_method[ + dev_type + ](dev) + else: + dev_type_to_method[dev_type](dev) + # do not proceed to subsequent/child types + except MaasStorageError as error: + raise MaasStorageError( + f"Unable to process device: {dev} " + f"of type: {dev_type}" + ) from error + + def _set_boot_disk(self, block_id): + """Mark a node block-device as the boot disk. + + :param block_id: ID of the block-device + """ + self._logger_debug(f"Setting boot disk {block_id}") + # self.call_cmd( + self.call_cmd( + [ + "maas", + self.maas_user, + "block-device", + "set-boot-disk", + self.node_id, + block_id, + ] + ) + + def _get_child_device(self, parent_device): + """Get the children devices from a parent device. + + :param parent_device: the parent device + :return: list of children devices + """ + children = [] + for dev in self.device_list: + if dev.get("parent_disk") == parent_device["id"]: + children.append(dev) + return children + + def process_disk(self, device): + """Process block-level storage (disks). + + :param device: the disk device to process + """ + self._logger_debug( + { + "device_id": device["id"], + "number": device.get("number"), + "block-id": device["parent_disk_blkid"], + } + ) + + # find boot mounts on child types + children = self._get_child_device(device) + + for child in children: + if child["type"] == "mount" and "/boot" in child["path"]: + self._logger_debug( + f"Disk {device['id']} has a mount with " + f"'boot' in its path: {child['path']}" + ) + self._set_boot_disk(device["parent_disk_blkid"]) + break + # apply disk name + if device.get("name"): + self._logger_debug({"name": device["name"]}) + self.call_cmd( + [ + "maas", + self.maas_user, + "block-device", + "update", + self.node_id, + device["parent_disk_blkid"], + f"name={device['name']}", + ] + ) + + def _create_partition(self, device): + """Create a partition on a disk and return the resulting partition ID. + + :param device: the partition device + :return: the resulting node partition ID + """ + cmd = [ + "maas", + self.maas_user, + "partitions", + "create", + self.node_id, + device["parent_disk_blkid"], + f"size={device['size']}", + ] + + return self.call_cmd(cmd, output_json=True) + + def process_partition(self, device): + """Process a partition from the storage layout config. + + :param device: the partition device to process + """ + self._logger_debug( + { + "device_id": device["id"], + "size": device["size"], + "number": device.get("number"), + "parent disk": device["parent_disk"], + "parent disk block-id": device["parent_disk_blkid"], + } + ) + partition_data = self._create_partition(device) + device["partition_id"] = str(partition_data["id"]) + + def _get_format_partition_id(self, volume): + """Get the partition ID from the specified format. + + :param volume: the volume ID + :return: the node partition ID + """ + for dev in self.device_list: + # sanitize comparison to accomidate user defined types + if dev["type"] == "partition" and str(volume) in [ + str(dev["id"]), + str(dev["device"]), + str(dev["number"]), + ]: + return dev["partition_id"] + + def process_format(self, device): + """Process a partition format from the storage layout config. + + :param device: the format device to process + """ + self._logger_debug( + { + "device_id": device["id"], + "fstype": device["fstype"], + "label": device["label"], + "parent disk": device["parent_disk"], + "parent disk block-id": device["parent_disk_blkid"], + } + ) + if device.get("volume"): + partition_id = self._get_format_partition_id(device["volume"]) + # make sure we can fetch the newly created parent partition_id + if partition_id is None: + raise MaasStorageError( + "Unable to find partition ID for volume" + f" {device['volume']}" + ) + self.call_cmd( + [ + "maas", + self.maas_user, + "partition", + "format", + self.node_id, + device["parent_disk_blkid"], + partition_id, + f"fstype={device['fstype']}", + f"label={device['label']}", + ] + ) + return + + # if the device does not have a 'volume' key, it's a block device + self.call_cmd( + [ + "maas", + self.maas_user, + "partition", + "format", + self.node_id, + device["parent_disk_blkid"], + f"fstype={device['fstype']}", + f"label={device['label']}", + ] + ) + + def _get_mount_partition_id(self, device): + """Get the partition ID from the specified mount path. + + :param device: the mount device + :return: the partition ID + """ + for dev in self.device_list: + if device == dev["id"]: + return self._get_format_partition_id(dev["volume"]) + + def process_mount(self, device): + """Process a mount path from the storage layout config. + + :param device: the mount device to process + """ + self._logger_debug( + { + "device_id": device["id"], + "path": device["path"], + "parent disk": device["parent_disk"], + "parent disk block-id": device["parent_disk_blkid"], + } + ) + partition_id = self._get_mount_partition_id(device["device"]) + # mount on partition + if partition_id: + self._logger_debug(f" on partition_id: {partition_id}") + self.call_cmd( + [ + "maas", + self.maas_user, + "partition", + "mount", + self.node_id, + device["parent_disk_blkid"], + partition_id, + f"mount_point={device['path']}", + ] + ) + # mount on block-device + else: + self._logger_debug(f" on disk: {device['parent_disk']}") + self.call_cmd( + [ + "maas", + self.maas_user, + "block-device", + "mount", + self.node_id, + device["parent_disk_blkid"], + f"mount_point={device['path']}", + ] + ) diff --git a/device-connectors/src/testflinger_device_connectors/devices/maas2/tests/test_maas_storage.py b/device-connectors/src/testflinger_device_connectors/devices/maas2/tests/test_maas_storage.py new file mode 100644 index 00000000..24a0138d --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/maas2/tests/test_maas_storage.py @@ -0,0 +1,535 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Maas2 agent storage module unit tests.""" + + +import pytest +import json +from unittest.mock import Mock, MagicMock, call +from testflinger_device_connectors.devices.maas2.maas_storage import ( + MaasStorage, + MaasStorageError, +) + + +class MockMaasStorage(MaasStorage): + """Enable mock subprocess calls.""" + + def __init__(self, maas_user, node_id): + super().__init__(maas_user, node_id) + self.call_cmd_mock = Mock() + + def call_cmd(self, cmd, output_json=False): + """Mock method to simulate the call_cmd method in the + parent MaasStorage class. + """ + + if output_json: + return json.loads(TestMaasStorage.node_info) + else: + # monkeypatch + return self.call_cmd_mock(cmd) + + +class TestMaasStorage: + """Test maas device connector storage module.""" + + node_info = json.dumps( + [ + { + "id": 1, + "name": "sda", + "type": "physical", + "size": "300000000000", + "path": "/dev/disk/by-dname/sda", + "filesystem": None, + "partitions": [ + { + "id": 10, + "type": "partition", + "size": "1000000000", + "parent_disk": 1, + "bootable": "true", + "filesystem": { + "mount_point": "/boot", + "fstype": "ext4", + }, + } + ], + }, + { + "id": 2, + "name": "sdb", + "type": "physical", + "size": "400000000000", + "path": "/dev/disk/by-dname/sdb", + "filesystem": None, + "partitions": [ + { + "id": 20, + "type": "partition", + "size": "20000000000", + "parent_disk": 2, + "filesystem": { + "mount_point": "/data", + "fstype": "ext4", + }, + } + ], + }, + { + "id": 3, + "name": "sdc", + "type": "physical", + "size": "900000000000", + "path": "/dev/disk/by-dname/sdc", + "filesystem": {"mount_point": "/backup", "fstype": "ext4"}, + "partitions": [], + }, + { + "id": 4, + "type": "virtual", + }, + ] + ) + + @pytest.fixture + def maas_storage(self): + """Provides a MockMaasStorage instance for testing.""" + maas_user = "maas_user" + node_id = "node_id" + yield MockMaasStorage(maas_user, node_id) + + def test_node_read(self, maas_storage): + """Checks if 'node_read' correctly returns node block-devices.""" + node_info = maas_storage._node_read() + assert node_info == json.loads(self.node_info) + + def test_call_cmd_output_json(self, maas_storage): + """Test 'call_cmd' when output_json is True. + Checks if the method correctly returns json data. + """ + result = maas_storage.call_cmd( + ["maas", "maas_user", "block-devices", "read", "node_id"], + output_json=True, + ) + assert result == json.loads(self.node_info) + + def test_convert_size_to_bytes(self, maas_storage): + """Check if 'convert_size_to_bytes' correctly + converts sizes from string format to byte values. + """ + assert maas_storage.convert_size_to_bytes("1G") == 1000000000 + assert maas_storage.convert_size_to_bytes("500M") == 500000000 + assert maas_storage.convert_size_to_bytes("10K") == 10000 + assert maas_storage.convert_size_to_bytes("1000") == 1000 + + with pytest.raises(MaasStorageError): + maas_storage.convert_size_to_bytes("1Tb") + maas_storage.convert_size_to_bytes("abc") + + def test_clear_storage_config(self, maas_storage): + """Checks if 'clear_storage_config' correctly clears the + storage configuration. + """ + maas_storage.clear_storage_config() + + maas_storage.call_cmd_mock.assert_has_calls( + [ + call( + [ + "maas", + maas_storage.maas_user, + "partition", + "delete", + maas_storage.node_id, + "1", # parent_block_id + "10", # partition_id + ] + ), + call( + [ + "maas", + maas_storage.maas_user, + "partition", + "delete", + maas_storage.node_id, + "2", + "20", + ] + ), + call( + [ + "maas", + maas_storage.maas_user, + "block-device", + "unmount", + maas_storage.node_id, + "3", # parent_block_id + ] + ), + call( + [ + "maas", + maas_storage.maas_user, + "block-device", + "unformat", + maas_storage.node_id, + "3", # parent_block_id + ] + ), + ] + ) + + def test_assign_parent_disk(self, maas_storage): + """Checks if 'assign_parent_disk' correctly assigns a parent disk + to a storage device. + """ + maas_storage.device_list = [ + {"id": 1, "type": "disk"}, + {"id": 10, "type": "partition", "device": 1}, + ] + + maas_storage.assign_parent_disk() + + assert maas_storage.device_list == [ + {"id": 1, "type": "disk", "parent_disk": 1}, + {"id": 10, "type": "partition", "device": 1, "parent_disk": 1}, + ] + + def test_gather_partitions(self, maas_storage): + """Checks if 'gather_partitions' correctly gathers partition sizes.""" + maas_storage.device_list = [ + {"id": 10, "type": "partition", "parent_disk": 1, "size": "500M"}, + {"id": 20, "type": "partition", "parent_disk": 1, "size": "1G"}, + {"id": 30, "type": "partition", "parent_disk": 2, "size": "2G"}, + ] + + maas_storage.gather_partitions() + + assert maas_storage.partition_sizes == {1: 1500000000, 2: 2000000000} + + def test_select_block_dev(self, maas_storage): + """Checks if 'select_block_dev' correctly selects a block + device based on id and size. + """ + maas_storage.device_list = [ + {"id": 10, "type": "partition", "parent_disk": 1, "size": "500M"}, + {"id": 20, "type": "partition", "parent_disk": 1, "size": "1G"}, + {"id": 30, "type": "partition", "parent_disk": 2, "size": "2G"}, + ] + + block_device_id = maas_storage._select_block_dev(10, 1500000000) + assert block_device_id == 1 + + block_device_id = maas_storage._select_block_dev(20, 2000000000) + assert block_device_id == 1 + + with pytest.raises(MaasStorageError): + maas_storage._select_block_dev(30, 50000000000000) + + def test_parse_block_devices(self, maas_storage): + """Checks if 'parse_block_devices' correctly choses the most + appropriate node block-id for the per-disk summed partition size. + """ + maas_storage.partition_sizes = { + 1: 1500000000, + 2: 2000000000, + } + maas_storage.device_list = [ + {"id": 10, "type": "partition", "parent_disk": 1, "size": "500M"}, + {"id": 20, "type": "partition", "parent_disk": 1, "size": "1G"}, + {"id": 30, "type": "partition", "parent_disk": 2, "size": "2G"}, + ] + + maas_storage.block_ids = {} + + maas_storage.parse_block_devices() + + assert maas_storage.block_ids == {1: 1, 2: 2} + + def test_map_block_ids(self, maas_storage): + """Checks if 'map_block_ids' correctly maps each partition to + the appropriate block-device id. + """ + maas_storage.block_ids = {1: 1, 2: 2, 3: 3} + + maas_storage.device_list = [ + {"id": 10, "type": "partition", "parent_disk": 1}, + {"id": 20, "type": "partition", "parent_disk": 2}, + {"id": 30, "type": "partition", "parent_disk": 3}, + ] + + maas_storage.map_block_ids() + + assert maas_storage.device_list == [ + { + "id": 10, + "type": "partition", + "parent_disk": 1, + "parent_disk_blkid": "1", + }, + { + "id": 20, + "type": "partition", + "parent_disk": 2, + "parent_disk_blkid": "2", + }, + { + "id": 30, + "type": "partition", + "parent_disk": 3, + "parent_disk_blkid": "3", + }, + ] + + def test_validate_alloc_pct_values(self, maas_storage): + """Checks if 'validate_alloc_pct_values' correctly validates total + per-disk allocation percentages do not exceed 100. + """ + maas_storage.device_list = [ + {"id": 10, "type": "partition", "parent_disk": 1, "alloc_pct": 50}, + {"id": 20, "type": "partition", "parent_disk": 1, "alloc_pct": 60}, + {"id": 30, "type": "partition", "parent_disk": 2, "alloc_pct": 90}, + ] + + with pytest.raises(MaasStorageError): + maas_storage._validate_alloc_pct_values() + + def test_create_partition_sizes(self, maas_storage): + """Checks if 'create_partition_sizes' correctly creates each partition + based on the given parameters. + """ + maas_storage.device_list = [ + {"id": 10, "type": "partition", "parent_disk": 1, "size": "1G"}, + {"id": 20, "type": "partition", "parent_disk": 2, "alloc_pct": 40}, + {"id": 30, "type": "partition", "parent_disk": 2, "alloc_pct": 60}, + ] + maas_storage.block_ids = {1: 1, 2: 2} + maas_storage.create_partition_sizes() + + assert maas_storage.device_list == [ + { + "id": 10, + "type": "partition", + "parent_disk": 1, + "size": 1000000000, + }, + { + "id": 20, + "type": "partition", + "parent_disk": 2, + "alloc_pct": 40, + "size": "160000000000", + }, + { + "id": 30, + "type": "partition", + "parent_disk": 2, + "alloc_pct": 60, + "size": "240000000000", + }, + ] + + def test_group_by_type(self, maas_storage): + """Checks if 'group_by_type' correctly groups each device by their + storage device type, into a list per that type. + """ + maas_storage.device_list = [ + {"id": 1, "type": "disk"}, + {"id": 10, "type": "partition"}, + {"id": 20, "type": "partition"}, + {"id": 40, "type": "format"}, + {"id": 50, "type": "format"}, + {"id": 60, "type": "mount"}, + ] + + result = maas_storage.group_by_type() + + assert result == { + "disk": [{"id": 1, "type": "disk"}], + "partition": [ + {"id": 10, "type": "partition"}, + {"id": 20, "type": "partition"}, + ], + "format": [ + {"id": 40, "type": "format"}, + {"id": 50, "type": "format"}, + ], + "mount": [{"id": 60, "type": "mount"}], + } + + def test_process_by_dev_type(self, maas_storage): + """Checks if 'process_by_dev_type' correctly batch-processes devices + based on their type-grouping list. + """ + devs_by_type = { + "disk": [{"id": 1, "type": "disk"}], + "partition": [{"id": 20, "type": "partition"}], + "format": [{"id": 40, "type": "format"}], + "mount": [{"id": 60, "type": "mount"}], + } + + mock_methods = { + "disk": "process_disk", + "partition": "process_partition", + "format": "process_format", + "mount": "process_mount", + } + + for dev_type, devices in devs_by_type.items(): + for device in devices: + setattr(maas_storage, mock_methods[dev_type], MagicMock()) + + setattr( + maas_storage, + f"_get_child_device_{dev_type}", + MagicMock(return_value=devices), + ) + + maas_storage.process_by_dev_type(devs_by_type) + + for dev_type, devices in devs_by_type.items(): + for device in devices: + mock_method = getattr(maas_storage, mock_methods[dev_type]) + mock_method.assert_called_once_with(device) + + def test_process_disk(self, maas_storage): + """Checks if 'process_disk' correctly processes a 'disk' + device type. + """ + maas_storage.device_list = [ + {"id": 1, "type": "disk", "parent_disk": 1} + ] + device = { + "id": 1, + "type": "disk", + "name": "sda", + "parent_disk_blkid": 1, + } + + maas_storage.process_disk(device) + + maas_storage.call_cmd_mock.assert_called_with( + [ + "maas", + maas_storage.maas_user, + "block-device", + "update", + maas_storage.node_id, + device["parent_disk_blkid"], + f"name={device['name']}", + ] + ) + + def test_process_partition(self, maas_storage): + """Checks if 'process_partition' correctly processes a 'partition' + device type. + """ + device = { + "id": 10, + "type": "partition", + "parent_disk": "sda", + "parent_disk_blkid": 1, + "size": "1G", + } + + maas_storage._create_partition = MagicMock(return_value={"id": 3}) + + maas_storage.process_partition(device) + + maas_storage._create_partition.assert_called_with(device) + + assert device["partition_id"] == "3" + + @pytest.mark.parametrize( + "partition_id, expected_error", + [ + (2, None), # Valid partition ID + (None, MaasStorageError), # Invalid partition ID + ], + ) + def test_process_format(self, maas_storage, partition_id, expected_error): + """Checks if 'process_format' correctly processes a 'format' + device type, with and without a valid 'volume' attribute. + """ + device = { + "id": 4, + "type": "format", + "fstype": "ext4", + "label": "root", + "parent_disk": 1, + "parent_disk_blkid": "sda", + "volume": "volume", + } + + maas_storage._get_format_partition_id = MagicMock( + return_value=partition_id + ) + + if expected_error: + with pytest.raises( + expected_error, match=r"Unable to find partition ID for volume" + ): + maas_storage.process_format(device) + else: + maas_storage.process_format(device) + maas_storage.call_cmd_mock.assert_called_with( + [ + "maas", + maas_storage.maas_user, + "partition", + "format", + maas_storage.node_id, + device["parent_disk_blkid"], + partition_id, + f"fstype={device['fstype']}", + f"label={device['label']}", + ] + ) + + def test_process_mount(self, maas_storage): + """Checks if 'process_mount' correctly processes a 'mount' + device type. + """ + device = { + "id": 6, + "type": "mount", + "path": "/mnt/data", + "parent_disk": 1, + "parent_disk_blkid": "sda", + "device": "device", + } + + maas_storage._get_mount_partition_id = MagicMock(return_value=2) + + maas_storage.process_mount(device) + + maas_storage._get_mount_partition_id.assert_called_with( + device["device"] + ) + + maas_storage.call_cmd_mock.assert_called_with( + [ + "maas", + maas_storage.maas_user, + "partition", + "mount", + maas_storage.node_id, + device["parent_disk_blkid"], + 2, + f"mount_point={device['path']}", + ] + ) diff --git a/device-connectors/src/testflinger_device_connectors/devices/multi/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/multi/__init__.py new file mode 100644 index 00000000..3ed76d70 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/multi/__init__.py @@ -0,0 +1,127 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu multi-device support code.""" + +import json +import logging +import os +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + SerialLogger, +) +from testflinger_device_connectors.devices.multi.multi import Multi +from testflinger_device_connectors.devices.multi.tfclient import TFClient + +device_name = "multi" + + +class DeviceConnector(DefaultDevice): + + """Device Connector for provisioning multiple devices at the same time""" + + def init_device(self, args): + """Read config data and initialize the device object.""" + with open(args.config, encoding="utf-8") as configfile: + self.config = yaml.safe_load(configfile) + self.job_data = testflinger_device_connectors.get_test_opportunity( + args.job_data + ) + testflinger_device_connectors.configure_logging(self.config) + testflinger_server = self.config.get("testflinger_server") + tfclient = TFClient(testflinger_server) + self.device = Multi(self.config, self.job_data, tfclient) + + def provision(self, args): + """Method called when the command is invoked.""" + self.init_device(args) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + self.device.provision() + logmsg(logging.INFO, "END provision") + + def runtest(self, args): + """ + The runtest method for multi-device connectors + + This is slightly different from the generic one because we also need + to import the job_list.json data and inject the device_ip for each + device into the environment + """ + self.init_device(args) + + logmsg(logging.INFO, "BEGIN testrun") + + test_cmds = self.job_data.get("test_data").get("test_cmds") + serial_host = self.config.get("serial_host") + serial_port = self.config.get("serial_port") + serial_proc = SerialLogger(serial_host, serial_port, "test-serial.log") + serial_proc.start() + + # Inject the IPs for each device into the environment + extra_env = self.get_device_ip_dict() + if "env" not in self.config: + self.config["env"] = {} + self.config["env"].update(extra_env) + + try: + exitcode = testflinger_device_connectors.run_test_cmds( + test_cmds, self.config + ) + except Exception as e: + raise e + finally: + serial_proc.stop() + testflinger_device_connectors.logmsg(logging.INFO, "END testrun") + return exitcode + + def get_job_list_data(self, job_list_file: str = "job_list.json") -> list: + """Read job_list.json and return the list data""" + if not os.path.exists(job_list_file): + logmsg( + logging.ERROR, + "Unable to find multi-job data file, job_list.json not found", + ) + return [] + with open(job_list_file) as job_list_file: + job_list_data = json.load(job_list_file) + return job_list_data + + def get_device_ip_dict(self): + """ + Read job_list.json and return a dict of device IPs like this that + can be used in the environment for the test commands: + { + "DEVICE_IP_1": "10.1.1.1", + "DEVICE_IP_2": "10.1.1.2" + } + """ + job_list_data = self.get_job_list_data() + device_ip_dict = {} + for i, job in enumerate(job_list_data): + key = "DEVICE_IP_{}".format(i + 1) + value = job.get("device_info", {}).get("device_ip") + device_ip_dict[key] = value + return device_ip_dict + + def cleanup(self, args): + """Cancel all subordinates jobs before finishing the multi-agent job""" + self.init_device(args) + job_list_data = self.get_job_list_data() + job_id_list = [job.get("job_id") for job in job_list_data] + self.device.cancel_jobs(job_id_list) diff --git a/device-connectors/src/testflinger_device_connectors/devices/multi/multi.py b/device-connectors/src/testflinger_device_connectors/devices/multi/multi.py new file mode 100644 index 00000000..6d65012f --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/multi/multi.py @@ -0,0 +1,184 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu multi-device support code.""" + +import json +import logging +import os +import time + +from testflinger_device_connectors.devices import ProvisioningError + +logger = logging.getLogger() + + +class Multi: + + """Device Connector for multi-device""" + + def __init__(self, config, job_data, client): + """Initialize the multi-device connector. + + :param config: path to the config file + :param job_data: path to the job data file + :param client: client object for talking to the Testflinger server + """ + self.config = config + self.job_data = job_data + self.agent_name = self.config.get("agent_name") + self.mount_point = os.path.join("/mnt", self.agent_name) + self.client = client + self.jobs = [] + + def provision(self): + """Provision multi-device connector by creating the specified jobs""" + self.create_jobs() + + # Wait for all jobs to reach the "allocated" state + unallocated = self.jobs.copy() + + # Set default timeout to allocate all devices to 2 hours + allocation_timeout = self.job_data.get( + "allocation_timeout", 2 * 60 * 60 + ) + start_time = time.time() + + while unallocated: + time.sleep(10) + self.terminate_if_parent_completed() + for job in unallocated: + state = self.client.get_status(job) + if state == "allocated": + unallocated.remove(job) + break + if state in ("cancelled", "complete", "completed"): + logger.error( + "Job %s failed to allocate, cancelling remaining jobs", + job, + ) + self.cancel_jobs(self.jobs) + raise ProvisioningError("Unable to allocate all devices") + # Timeout if we've been waiting too long for devices to allocate + if time.time() - start_time > allocation_timeout: + self.cancel_jobs(self.jobs) + raise ProvisioningError( + "Timed out waiting for devices to allocate" + ) + + self.save_job_list_file() + + def terminate_if_parent_completed(self): + """If parent job is completed or cancelled, cancel sub jobs""" + if self.this_job_completed(): + self.cancel_jobs(self.jobs) + raise ProvisioningError("Job cancelled or completed") + + def this_job_completed(self): + """ + If the job is completed, or cancelled, then we need to exit the + provision phase, and cleanup the subordinate jobs + """ + + job_id = self.job_data.get("job_id") + status = self.client.get_status(job_id) + if status in ("cancelled", "completed"): + return True + return False + + def save_job_list_file(self): + """ + Retrieve results for each job from the server, and look at the + "device_info" in each one to get the IP of the device, then + create a job_list.json file with a list of jobs that looks like: + [ + { + "job_id": "1234", + "device_info": { + "device_ip": "10.1.1.1" + } + }, + ... + ] + This file gets used by other steps that also need this information + """ + job_list = [] + for job in self.jobs: + device_info = self.client.get_results(job).get("device_info") + job_list.append( + { + "job_id": job, + "device_info": device_info, + } + ) + with open("job_list.json", "w") as json_file: + json.dump(job_list, json_file) + + def create_jobs(self): + """Create the jobs for the multi-device connector""" + jobs_list = self.job_data.get("provision_data", {}).get("jobs") + if not jobs_list: + raise ProvisioningError( + "You must specify a list of 'jobs' in " + "the 'provision_data' section of " + "your job." + ) + + logger.info("Creating test jobs") + for job in jobs_list: + if not isinstance(job, dict): + logger.error("Job is not a dict: %s", job) + continue + job = self.inject_allocate_data(job) + job = self.inject_parent_jobid(job) + + try: + job_id = self.client.submit_job(job) + except OSError as exc: + logger.exception("Unable to create job: %s", job_id) + self.cancel_jobs(self.jobs) + raise ProvisioningError( + f"Unable to create job: {job_id}" + ) from exc + + logger.info("Created job %s", job_id) + self.jobs.append(job_id) + + def inject_allocate_data(self, job): + """Inject the allocate_data section into the job + + :param job: the job to inject the allocate data into + :returns: the job with the allocate_data injected + """ + allocate_data = {"allocate_data": {"allocate": True}} + job.update(allocate_data) + return job + + def inject_parent_jobid(self, job): + """Inject the parent job_id into the job + + :param job: the job to inject the parent job_id into + :returns: the job with parent_job_id added to it + """ + parent_job_id = {"parent_job_id": self.job_data.get("job_id")} + job.update(parent_job_id) + return job + + def cancel_jobs(self, jobs): + """Cancel all jobs in the specified list of job_ids""" + for job in jobs: + try: + self.client.cancel_job(job) + except OSError: + logger.exception("Unable to cancel job: %s", job) diff --git a/device-connectors/src/testflinger_device_connectors/devices/multi/tests/test_multi.py b/device-connectors/src/testflinger_device_connectors/devices/multi/tests/test_multi.py new file mode 100644 index 00000000..b67aa960 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/multi/tests/test_multi.py @@ -0,0 +1,99 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Unit tests for multi-device support code.""" + +from uuid import uuid4 + +import pytest +from testflinger_device_connectors.devices.multi.multi import Multi +from testflinger_device_connectors.devices.multi.tfclient import TFClient + + +class MockTFClient(TFClient): + """Mock TFClient object""" + + def submit_job(self, job_data): + """Return a fake job id""" + return str(uuid4()) + + +def test_bad_tfclient_url(): + """Test that Multi raises an exception when TFClient URL is bad""" + with pytest.raises(ValueError): + TFClient(None) + with pytest.raises(ValueError): + TFClient("foo.com") + + +def test_inject_allocate_data(): + """Test that allocate_data section is injected into job""" + test_config = {"agent_name": "test_agent"} + job_data = { + "provision_data": { + "jobs": [ + {"job_id": "1"}, + {"job_id": "2"}, + ] + } + } + test_agent = Multi(test_config, job_data, MockTFClient("http://localhost")) + test_agent.create_jobs() + for job in test_agent.job_data["provision_data"]["jobs"]: + assert job["allocate_data"]["allocate"] is True + + +def test_inject_parent_jobid(): + """Test that parent_jobid is injected into job""" + test_config = {"agent_name": "test_agent"} + parent_job_id = "11111111-1111-1111-1111-111111111111" + job_data = { + "job_id": parent_job_id, + "provision_data": { + "jobs": [ + {"job_id": "1"}, + {"job_id": "2"}, + ] + }, + } + test_agent = Multi(test_config, job_data, MockTFClient("http://localhost")) + test_agent.create_jobs() + for job in test_agent.job_data["provision_data"]["jobs"]: + assert job["parent_job_id"] == parent_job_id + + +def test_this_job_completed(): + """Test this_job_completed() returns True only when the job is completed""" + test_config = {"agent_name": "test_agent"} + job_data = { + "job_id": "11111111-1111-1111-1111-111111111111", + } + + # completed state is detected as completed + completed_client = MockTFClient("http://localhost") + completed_client.get_status = lambda job_id: "completed" + test_agent = Multi(test_config, job_data, completed_client) + assert test_agent.this_job_completed() is True + + # cancelled state is detected as completed + cancelled_client = MockTFClient("http://localhost") + cancelled_client.get_status = lambda job_id: "cancelled" + test_agent = Multi(test_config, job_data, cancelled_client) + assert test_agent.this_job_completed() is True + + # anything else is not completed + incomplete_client = MockTFClient("http://localhost") + incomplete_client.get_status = lambda job_id: "something else" + test_agent = Multi(test_config, job_data, incomplete_client) + assert test_agent.this_job_completed() is False diff --git a/device-connectors/src/testflinger_device_connectors/devices/multi/tfclient.py b/device-connectors/src/testflinger_device_connectors/devices/multi/tfclient.py new file mode 100644 index 00000000..afa43f31 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/multi/tfclient.py @@ -0,0 +1,157 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Client for talking to Testflinger Server""" + +import json +import logging +import urllib.parse +import requests + +logger = logging.getLogger() + + +class TFClient: + """Testflinger connection class""" + + def __init__(self, url): + """Initialize the client with the url of the server + + :param url: URL of the Testflinger server + """ + if not url or not url.startswith("http"): + raise ValueError( + "Config item testflinger_server URL for multi-device " + "connectors must be specified and must start with http or " + "https!" + ) + self.server = url + + def get(self, uri_frag, timeout=15): + """Submit a GET request to the server + :param uri_frag: + endpoint for the GET request + :return: + String containing the response from the server + """ + uri = urllib.parse.urljoin(self.server, uri_frag) + try: + req = requests.get(uri, timeout=timeout) + except requests.exceptions.ConnectionError: + logger.error("Unable to communicate with specified server.") + raise + except IOError: + # This should catch all other timeout cases + logger.error( + "Timeout while trying to communicate with the server." + ) + raise + + try: + # If anything else went wrong, raise the proper exception + req.raise_for_status() + except OSError: + logger.error( + "Received status code %s from server.", req.status_code + ) + raise + return req.text + + def post(self, uri_frag, data, timeout=15): + """Submit a POST request to the server + :param uri_frag: + endpoint for the POST request + :return: + String containing the response from the server + """ + uri = urllib.parse.urljoin(self.server, uri_frag) + try: + req = requests.post(uri, json=data, timeout=timeout) + except requests.exceptions.ConnectTimeout: + logger.error( + "Timeout while trying to communicate with the server." + ) + raise + except requests.exceptions.ConnectionError: + logger.error("Unable to communicate with specified server.") + raise + + try: + # If anything else went wrong, raise the proper exception + req.raise_for_status() + except OSError: + logger.error( + "Received status code %s from server.", req.status_code + ) + raise + return req.text + + def get_status(self, job_id): + """Get the status of a test job + + :param job_id: + ID for the test job + :return: + String containing the job_state for the specified job_id + (waiting, setup, provision, test, reserved, released, + cancelled, complete) + """ + try: + endpoint = f"/v1/result/{job_id}" + data = json.loads(self.get(endpoint)) + state = data.get("job_state") + except OSError: + logger.error("Unable to get status for job %s", job_id) + state = "unknown" + return state + + def get_results(self, job_id): + """Get the results of a test job + + :param job_id: + ID for the test job + :return: + dict containing the results for the specified job_id + """ + try: + endpoint = f"/v1/result/{job_id}" + data = json.loads(self.get(endpoint)) + except OSError: + logger.error("Unable to get results for job %s", job_id) + data = {} + return data + + def submit_job(self, job_data): + """Submit a test job to the testflinger server + + :param job_data: + dict of data for the job to submit + :return: + ID for the test job + """ + endpoint = "/v1/job" + response = self.post(endpoint, job_data) + return json.loads(response).get("job_id") + + def cancel_job(self, job_id): + """Tell the server to cancel a specified job_id""" + try: + self.post(f"/v1/job/{job_id}/action", {"action": "cancel"}) + except requests.exceptions.HTTPError as exc: + # Ignore it if the job is already cancelled or completed + if exc.response.status_code != 400: + raise + except OSError: + logger.error("Unable to cancel job %s", job_id) + raise diff --git a/device-connectors/src/testflinger_device_connectors/devices/muxpi/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/muxpi/__init__.py new file mode 100644 index 00000000..fff51a48 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/muxpi/__init__.py @@ -0,0 +1,59 @@ +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu Raspberry PI muxpi support code.""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + SerialLogger, + catch, +) +from testflinger_device_connectors.devices.muxpi.muxpi import MuxPi + +device_name = "muxpi" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = MuxPi(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + serial_proc = SerialLogger( + serial_host, serial_port, "provision-serial.log" + ) + serial_proc.start() + try: + device.provision() + except Exception as e: + raise e + finally: + serial_proc.stop() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/muxpi/muxpi.py b/device-connectors/src/testflinger_device_connectors/devices/muxpi/muxpi.py new file mode 100644 index 00000000..1d0ea719 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/muxpi/muxpi.py @@ -0,0 +1,454 @@ +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu Raspberry PI muxpi support code.""" + +import json +import logging +import subprocess +import time +from contextlib import contextmanager +from pathlib import Path + +import yaml + +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class MuxPi: + + """Device Connector for MuxPi.""" + + IMAGE_PATH_IDS = { + "writable/usr/bin/firefox": "pi-desktop", + "writable/etc": "ubuntu", + "writable/system-data": "core", + "ubuntu-seed/snaps": "core20", + "cloudimg-rootfs/etc/cloud/cloud.cfg": "ubuntu-cpc", + } + + def __init__(self, config=None, job_data=None): + if config and job_data: + with open(config) as configfile: + self.config = yaml.safe_load(configfile) + with open(job_data) as j: + self.job_data = json.load(j) + else: + # For testing + self.config = {"agent_name": "test"} + self.job_data = {} + self.agent_name = self.config.get("agent_name") + self.mount_point = Path("/mnt") / self.agent_name + + def _run_control(self, cmd, timeout=60): + """ + Run a command on the control host over ssh + + :param cmd: + Command to run + :param timeout: + Timeout (default 60) + :returns: + Return output from the command, if any + """ + control_host = self.config.get("control_host") + control_user = self.config.get("control_user", "ubuntu") + ssh_cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(control_user, control_host), + cmd, + ] + try: + output = subprocess.check_output( + ssh_cmd, stderr=subprocess.STDOUT, timeout=timeout + ) + except subprocess.CalledProcessError as e: + raise ProvisioningError(e.output) + return output + + def _copy_to_control(self, local_file, remote_file): + """ + Copy a file to the control host over ssh + + :param local_file: + Local filename + :param remote_file: + Remote filename + """ + control_host = self.config.get("control_host") + control_user = self.config.get("control_user", "ubuntu") + ssh_cmd = [ + "scp", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + local_file, + "{}@{}:{}".format(control_user, control_host, remote_file), + ] + try: + output = subprocess.check_output(ssh_cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + raise ProvisioningError(e.output) + return output + + def provision(self): + try: + url = self.job_data["provision_data"]["url"] + except KeyError: + raise ProvisioningError( + 'You must specify a "url" value in ' + 'the "provision_data" section of ' + "your job_data" + ) + cmd = self.config.get("control_switch_local_cmd", "stm -ts") + self._run_control(cmd) + time.sleep(5) + logger.info("Flashing Test image") + try: + self.flash_test_image(url) + with self.remote_mount(): + image_type = self.get_image_type() + logger.info("Image type detected: {}".format(image_type)) + logger.info("Creating Test User") + self.create_user(image_type) + self.run_post_provision_script() + logger.info("Booting Test Image") + cmd = self.config.get("control_switch_device_cmd", "stm -dut") + self._run_control(cmd) + self.hardreset() + self.check_test_image_booted() + except Exception: + raise + + def flash_test_image(self, url): + """ + Flash the image at :image_url to the sd card. + + :param url: + URL to download the image from + :raises ProvisioningError: + If the command times out or anything else fails. + """ + # First unmount, just in case + self.unmount_writable_partition() + + test_device = self.config["test_device"] + cmd = ( + f"(set -o pipefail; curl -sf {url} | zstdcat| " + f"sudo dd of={test_device} bs=16M)" + ) + logger.info("Running: %s", cmd) + try: + # XXX: I hope 30 min is enough? but maybe not! + self._run_control(cmd, timeout=1800) + except Exception: + raise ProvisioningError("timeout reached while flashing image!") + try: + self._run_control("sync") + except Exception: + # Nothing should go wrong here, but let's sleep if it does + logger.warn("Something went wrong with the sync, sleeping...") + time.sleep(30) + try: + self._run_control( + "sudo hdparm -z {}".format(self.config["test_device"]), + timeout=30, + ) + except Exception: + raise ProvisioningError( + "Unable to run hdparm to rescan " "partitions" + ) + + def _get_part_labels(self): + test_device = self.config["test_device"] + lsblk_data = self._run_control( + "lsblk -o NAME,LABEL -J {}".format(test_device) + ) + lsblk_json = json.loads(lsblk_data.decode()) + # List of (name, label) pairs + return [ + (x.get("name"), self.mount_point / x.get("label")) + for x in lsblk_json["blockdevices"][0]["children"] + if x.get("name") and x.get("label") + ] + + @contextmanager + def remote_mount(self): + mount_list = self._get_part_labels() + # Sometimes the labels don't show up to lsblk right away + if not mount_list: + print("No valid partitions found, retrying...") + time.sleep(10) + mount_list = self._get_part_labels() + for dev, mount in mount_list: + try: + self._run_control("sudo mkdir -p {}".format(mount)) + self._run_control("sudo mount /dev/{} {}".format(dev, mount)) + except Exception: + # If unmountable or any other error, go on to the next one + mount_list.remove((dev, mount)) + continue + try: + yield self.mount_point + finally: + for _, mount in mount_list: + self._run_control("sudo umount {}".format(mount)) + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config.get("reboot_script", []): + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except Exception: + raise RecoveryError("timeout reaching control host!") + + def get_image_type(self): + """ + Figure out which kind of image is on the configured block device + + :returns: + image type as a string + """ + + def check_path(dir): + self._run_control("test -e {}".format(dir)) + + # First check if this is a ce-oem-iot image + if self.check_ce_oem_iot_image(): + return "ce-oem-iot" + + try: + disk_info_path = ( + self.mount_point / "writable/lib/firmware/*-tegra*/" + ) + self._run_control(f"ls {disk_info_path} &>/dev/null") + return "tegra" + except ProvisioningError: + # Not a tegra image + pass + + for path, img_type in self.IMAGE_PATH_IDS.items(): + try: + path = self.mount_point / path + check_path(path) + return img_type + except Exception: + # Path was not found, continue trying others + continue + # We have no idea what kind of image this is + return "unknown" + + def check_ce_oem_iot_image(self) -> bool: + """ + Determine if this is a ce-oem-iot image + + These images will have a .disk/info file with a buildstamp in it + that looks like: + iot-$project-$series-classic-(server|desktop)-$buildId + """ + try: + disk_info_path = self.mount_point / "writable/.disk/info" + buildstamp = '"iot-[a-z]+-[a-z-]*(classic-(server|desktop)-[0-9]+' + buildstamp += '|core-[0-9]+)"' + self._run_control(f"grep -E {buildstamp} {disk_info_path}") + return True + except ProvisioningError: + return False + + def unmount_writable_partition(self): + try: + self._run_control( + "sudo umount {}*".format(self.config["test_device"]), + timeout=30, + ) + except KeyError: + raise RecoveryError("Device config missing test_device") + except Exception: + # We might not be mounted, so expect this to fail sometimes + pass + + def create_user(self, image_type): + """Create user account for default ubuntu user""" + base = self.mount_point + remote_tmp = Path("/tmp") / self.agent_name + try: + data_path = Path(__file__).parent / "../../data/muxpi" + if image_type == "ce-oem-iot": + self._run_control("mkdir -p {}".format(remote_tmp)) + self._copy_to_control( + data_path / "ce-oem-iot/user-data", remote_tmp + ) + cmd = f"sudo cp {remote_tmp}/user-data {base}/system-boot/" + self._run_control(cmd) + self._configure_sudo() + if image_type == "tegra": + base = self.mount_point / "writable" + ci_path = base / "var/lib/cloud/seed/nocloud" + self._run_control(f"sudo mkdir -p {ci_path}") + self._run_control(f"mkdir -p {remote_tmp}") + self._copy_to_control( + data_path / "classic/user-data", remote_tmp + ) + cmd = f"sudo cp {remote_tmp}/user-data {ci_path}" + self._run_control(cmd) + self._configure_sudo() + return + if image_type == "pi-desktop": + # make a spot to scp files to + self._run_control("mkdir -p {}".format(remote_tmp)) + + # Override oem-config so that it uses the preseed + self._copy_to_control( + data_path / "pi-desktop/oem-config.service", remote_tmp + ) + cmd = ( + "sudo cp {}/oem-config.service " + "{}/writable/lib/systemd/system/" + "oem-config.service".format(remote_tmp, self.mount_point) + ) + self._run_control(cmd) + + # Copy the preseed + self._copy_to_control( + data_path / "pi-desktop/preseed.cfg", remote_tmp + ) + cmd = "sudo cp {}/preseed.cfg {}/writable/preseed.cfg".format( + remote_tmp, self.mount_point + ) + self._run_control(cmd) + + # Make sure NetworkManager is started + cmd = ( + "sudo cp -a " + "{}/writable/etc/systemd/system/multi-user.target.wants" + "/NetworkManager.service " + "{}/writable/etc/systemd/system/" + "oem-config.target.wants".format( + self.mount_point, self.mount_point + ) + ) + self._run_control(cmd) + + self._configure_sudo() + return + if image_type == "core20": + base = self.mount_point / "ubuntu-seed" + ci_path = base / "data/etc/cloud/cloud.cfg.d" + self._run_control(f"sudo mkdir -p {ci_path}") + self._run_control("mkdir -p {}".format(remote_tmp)) + self._copy_to_control( + data_path / "uc20/99_nocloud.cfg", remote_tmp + ) + cmd = f"sudo cp {remote_tmp}/99_nocloud.cfg {ci_path}" + self._run_control(cmd) + else: + # For core or ubuntu classic images + base = self.mount_point / "writable" + if image_type == "core": + base = base / "system-data" + if image_type == "ubuntu-cpc": + base = self.mount_point / "cloudimg-rootfs" + ci_path = base / "var/lib/cloud/seed/nocloud-net" + self._run_control(f"sudo mkdir -p {ci_path}") + self._run_control("mkdir -p {}".format(remote_tmp)) + self._copy_to_control( + data_path / "classic/meta-data", remote_tmp + ) + cmd = f"sudo cp {remote_tmp}/meta-data {ci_path}" + self._run_control(cmd) + self._copy_to_control( + data_path / "classic/user-data", remote_tmp + ) + cmd = f"sudo cp {remote_tmp}/user-data {ci_path}" + self._run_control(cmd) + if image_type == "ubuntu": + # This needs to be removed on classic for rpi, else + # cloud-init won't find the user-data we give it + rm_cmd = "sudo rm -f {}".format( + base / "etc/cloud/cloud.cfg.d/99-fake?cloud.cfg" + ) + self._run_control(rm_cmd) + except Exception: + raise ProvisioningError("Error creating user files") + + def _configure_sudo(self): + # Setup sudoers data + sudo_data = "ubuntu ALL=(ALL) NOPASSWD:ALL" + sudo_path = "{}/writable/etc/sudoers.d/ubuntu".format(self.mount_point) + self._run_control( + "sudo bash -c \"echo '{}' > {}\"".format(sudo_data, sudo_path) + ) + + def check_test_image_booted(self): + logger.info("Checking if test image booted.") + started = time.time() + # Retry for a while since we might still be rebooting + test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + test_password = self.job_data.get("test_data", {}).get( + "test_password", "ubuntu" + ) + while time.time() - started < 1200: + try: + time.sleep(10) + cmd = [ + "sshpass", + "-p", + test_password, + "ssh-copy-id", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, self.config["device_ip"]), + ] + subprocess.check_output( + cmd, stderr=subprocess.STDOUT, timeout=60 + ) + return True + except Exception: + pass + # If we get here, then we didn't boot in time + raise ProvisioningError("Failed to boot test image!") + + def run_post_provision_script(self): + # Run post provision commands on control host if there are any, but + # don't fail the provisioning step if any of them don't work + for cmd in self.config.get("post_provision_script", []): + logger.info("Running %s", cmd) + try: + self._run_control(cmd) + except Exception: + logger.warn("Error running %s", cmd) diff --git a/device-connectors/src/testflinger_device_connectors/devices/muxpi/tests/test_muxpi.py b/device-connectors/src/testflinger_device_connectors/devices/muxpi/tests/test_muxpi.py new file mode 100644 index 00000000..f4a17acc --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/muxpi/tests/test_muxpi.py @@ -0,0 +1,50 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Unit tests for muxpi device connector""" + +from subprocess import CalledProcessError +from testflinger_device_connectors.devices.muxpi.muxpi import MuxPi + + +def test_check_ce_oem_iot_image(mocker): + """Test check_ce_oem_iot_image.""" + buildstamp = "iot-limerick-kria-classic-desktop-2204-x07-20230302-63" + mocker.patch( + "subprocess.check_output", + return_value=buildstamp.encode(), + ) + muxpi = MuxPi() + assert muxpi.check_ce_oem_iot_image() is True + + buildstamp = "iot-baoshan-classic-server-2204-x04-20230807-149" + mocker.patch( + "subprocess.check_output", + return_value=buildstamp.encode(), + ) + muxpi = MuxPi() + assert muxpi.check_ce_oem_iot_image() is True + + buildstamp = "iot-havana-core-20-ptz-gm3-uc20-20230911-2" + mocker.patch( + "subprocess.check_output", + return_value=buildstamp.encode(), + ) + muxpi = MuxPi() + assert muxpi.check_ce_oem_iot_image() is True + + mocker.patch( + "subprocess.check_output", + side_effect=CalledProcessError(1, "cmd"), + ) + assert muxpi.check_ce_oem_iot_image() is False diff --git a/device-connectors/src/testflinger_device_connectors/devices/netboot/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/netboot/__init__.py new file mode 100644 index 00000000..df52a3c9 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/netboot/__init__.py @@ -0,0 +1,100 @@ +# Copyright (C) 2016-2019 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Netboot support code.""" + +import logging +import multiprocessing + +import yaml +from testflinger_device_connectors.devices import ( + DefaultDevice, + ProvisioningError, + RecoveryError, + SerialLogger, + catch, +) + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices.netboot.netboot import Netboot + +device_name = "netboot" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = Netboot(args.config) + image = testflinger_device_connectors.get_image(args.job_data) + if not image: + raise ProvisioningError("Error downloading image") + server_ip = testflinger_device_connectors.get_local_ip_addr() + # Ideally the default user/pass should be metadata about an image, + # but we don't currently have any concept of that stored. For now, + # we can give a reasonable guess based on the provisioning method. + test_username = testflinger_device_connectors.get_test_username( + job_data=args.job_data, default="admin" + ) + test_password = testflinger_device_connectors.get_test_password( + job_data=args.job_data, default="admin" + ) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Booting Master Image") + """Initial recovery process + If the netboot (master) image is already booted and we can get to then + URL for it, then just continue with provisioning. Otherwise, try to + force it into the test image first, recopy the ssh keys if necessary, + reboot if necessary, and get it into the netboot image before going on + """ + if not device.is_master_image_booted(): + try: + device.ensure_test_image(test_username, test_password) + device.ensure_master_image() + except ProvisioningError: + raise RecoveryError("Unable to put system in a usable state!") + q = multiprocessing.Queue() + file_server = multiprocessing.Process( + target=testflinger_device_connectors.serve_file, + args=( + q, + image, + ), + ) + file_server.start() + server_port = q.get() + logmsg(logging.INFO, "Flashing Test Image") + serial_host = config.get("serial_host") + serial_port = config.get("serial_port") + serial_proc = SerialLogger( + serial_host, serial_port, "provision-serial.log" + ) + serial_proc.start() + try: + device.flash_test_image(server_ip, server_port) + logmsg(logging.INFO, "Booting Test Image") + device.ensure_test_image(test_username, test_password) + except Exception as e: + raise e + finally: + file_server.terminate() + serial_proc.stop() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/netboot/netboot.py b/device-connectors/src/testflinger_device_connectors/devices/netboot/netboot.py new file mode 100644 index 00000000..e70f41b0 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/netboot/netboot.py @@ -0,0 +1,259 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Netboot support code.""" + +import logging +import subprocess +import time +import urllib.request + +import yaml + +from testflinger_device_connectors import CmdTimeoutError, runcmd +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class Netboot: + + """Testflinger Device Connector for Netboot.""" + + def __init__(self, config): + with open(config) as configfile: + self.config = yaml.safe_load(configfile) + + def setboot(self, mode): + """ + Set the boot mode of the device. + + :param mode: + One of 'master' or 'test' + :raises ProvisioningError: + If the command times out or anything else fails. + + This method sets the boot method to the specified value. + """ + if mode == "master": + setboot_script = self.config.get("select_master_script") + elif mode == "test": + setboot_script = self.config.get("select_test_script") + else: + raise ProvisioningError( + "Attempted to set boot mode to '{}' - " + "only 'master' or 'test' are supported " + "modes!".format(mode) + ) + self._run_cmd_list(setboot_script) + + def _run_cmd_list(self, cmdlist): + """ + Run a list of commands + + :param cmdlist: + List of commands to run + """ + if not cmdlist: + return + for cmd in cmdlist: + logger.info("Running %s", cmd) + try: + rc = runcmd(cmd, timeout=60) + except CmdTimeoutError: + raise ProvisioningError("timeout reaching control host!") + if rc: + raise ProvisioningError( + "Error running {} (rc={})".format(cmd, rc) + ) + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config["reboot_script"]: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except Exception: + raise RecoveryError("timeout reaching control host!") + + def ensure_test_image(self, test_username, test_password): + """ + Actively switch the device to boot the test image. + + :param test_username: + Username of the default user in the test image + :param test_password: + Password of the default user in the test image + :raises ProvisioningError: + If the command times out or anything else fails. + """ + logger.info("Booting the test image") + if self.is_test_image_booted(test_username, test_password): + return + self.setboot("test") + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, self.config["device_ip"]), + "sudo /sbin/reboot", + ] + try: + subprocess.check_call(cmd, timeout=60) + except Exception: + self.hardreset() + time.sleep(60) + + started = time.time() + # Retry for a while since we might still be rebooting + while time.time() - started < 900: + time.sleep(10) + if self.is_test_image_booted(test_username, test_password): + return + # If we got here, the test image never became available + raise ProvisioningError("Failed to boot test image!") + + def is_test_image_booted(self, test_username, test_password): + """ + Check if the test image is booted. + + :returns: + True if the test image is currently booted, False otherwise. + :param test_username: + Username of the default user in the test image + :param test_password: + Password of the default user in the test image + :returns: + True if the test image is currently booted, False otherwise. + """ + cmd = [ + "sshpass", + "-p", + test_password, + "ssh-copy-id", + "-f", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, self.config["device_ip"]), + ] + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=60) + except Exception: + return False + # If we get here, the above command proved we are in the test image + return True + + def is_master_image_booted(self): + """ + Check if the master image is booted. + + :returns: + True if the master image is currently booted, False otherwise. + + .. note:: + The master image is used for writing a new image to local media + """ + check_url = "http://{}:8989/check".format(self.config["device_ip"]) + data = "" + try: + logger.info("Checking if master image booted: %s", check_url) + with urllib.request.urlopen(check_url) as url: + data = url.read() + except Exception: + # Any connection error will fail through the normal path + pass + if "Testflinger Test Device Imager" in str(data): + return True + else: + return False + + def ensure_master_image(self): + """ + Actively switch the device to boot the test image. + + :raises RecoveryError: + If the command times out or anything else fails. + """ + logger.info("Making sure the master image is booted") + if self.is_master_image_booted(): + return + + self.setboot("master") + self.hardreset() + + started = time.time() + while time.time() - started < 600: + time.sleep(10) + master_is_booted = self.is_master_image_booted() + if master_is_booted: + break + # Check again if we are in the master image + if not master_is_booted: + raise RecoveryError("Could not reboot to master image!") + + def flash_test_image(self, server_ip, server_port): + """ + Flash the image at :image_url to the sd card. + + :param server_ip: + IP address of the image server. The image will be downloaded and + uncompressed over the SD card. + :param server_port: + TCP port to connect to on server_ip for downloading the image + :raises ProvisioningError: + If the command times out or anything else fails. + """ + url = r"http://{}:8989/writeimage?server={}:{}\&dev={}".format( + self.config["device_ip"], + server_ip, + server_port, + self.config["test_device"], + ) + logger.info("Triggering: %s", url) + try: + # XXX: I hope 30 min is enough? but maybe not! + req = urllib.request.urlopen(url, timeout=1800) + logger.info("Image write output:") + logger.info(str(req.read())) + except Exception: + raise ProvisioningError("Error while flashing image!") + + # Run post-flash hooks + post_flash_cmds = self.config.get("post_flash_cmds") + self._run_cmd_list(post_flash_cmds) + + # Now reboot the target system + url = "http://{}:8989/reboot".format(self.config["device_ip"]) + try: + logger.info("Rebooting target device: %s", url) + urllib.request.urlopen(url, timeout=10) + except Exception: + # FIXME: This could fail to return right now due to a bug + pass diff --git a/device-connectors/src/testflinger_device_connectors/devices/noprovision/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/noprovision/__init__.py new file mode 100644 index 00000000..465c8d06 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/noprovision/__init__.py @@ -0,0 +1,47 @@ +# Copyright (C) 2017-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Noprovision support code.""" + +import logging + +import yaml +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + catch, +) + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices.noprovision.noprovision import ( + Noprovision, +) + +device_name = "noprovision" + + +class DeviceConnector(DefaultDevice): + @catch(RecoveryError, 46) + def provision(self, args): + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = Noprovision(args.config) + test_username = testflinger_device_connectors.get_test_username( + args.job_data + ) + logmsg(logging.INFO, "BEGIN provision") + device.ensure_test_image(test_username) + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/noprovision/noprovision.py b/device-connectors/src/testflinger_device_connectors/devices/noprovision/noprovision.py new file mode 100644 index 00000000..463cd1d6 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/noprovision/noprovision.py @@ -0,0 +1,107 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Noprovision support code.""" + +import logging +import subprocess +import time + +import yaml + +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class Noprovision: + + """Testflinger Device Connector for Noprovision.""" + + def __init__(self, config): + with open(config) as configfile: + self.config = yaml.safe_load(configfile) + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config["reboot_script"]: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except subprocess.TimeoutExpired: + raise RecoveryError("timeout reaching control host!") + + def ensure_test_image(self, test_username): + """ + Actively switch the device to boot the test image. + + :param test_username: + Username of the default user in the test image + :param test_password: + Password of the default user in the test image + :raises ProvisioningError: + If the command times out or anything else fails. + """ + logger.info("Booting the test image") + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, self.config["device_ip"]), + "/bin/true", + ] + try: + subprocess.check_call(cmd) + return + except subprocess.SubprocessError: + pass + + self.hardreset() + time.sleep(60) + + started = time.time() + # Retry for a while since we might still be rebooting + while time.time() - started < 300: + try: + time.sleep(10) + cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "{}@{}".format(test_username, self.config["device_ip"]), + "/bin/true", + ] + subprocess.check_call(cmd) + return + except subprocess.SubprocessError: + # keep going if we aren't booted yet + pass + # If we got here, then it never booted to the test image + raise ProvisioningError("Failed to boot test image!") diff --git a/device-connectors/src/testflinger_device_connectors/devices/oemrecovery/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/oemrecovery/__init__.py new file mode 100644 index 00000000..331106a5 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/oemrecovery/__init__.py @@ -0,0 +1,49 @@ +# Copyright (C) 2018-2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu OEM Recovery provisioner support code.""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + catch, +) +from testflinger_device_connectors.devices.oemrecovery.oemrecovery import ( + OemRecovery, +) + +device_name = "oemrecovery" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = OemRecovery(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + device.provision() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/oemrecovery/oemrecovery.py b/device-connectors/src/testflinger_device_connectors/devices/oemrecovery/oemrecovery.py new file mode 100644 index 00000000..2bb56a79 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/oemrecovery/oemrecovery.py @@ -0,0 +1,175 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu OEM Recovery Provisioner support code.""" + +import json +import logging +import subprocess +import time + +import yaml + +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class OemRecovery: + + """Device Connector for OEM Recovery.""" + + def __init__(self, config, job_data): + with open(config, encoding="utf-8") as configfile: + self.config = yaml.safe_load(configfile) + with open(job_data, encoding="utf-8") as job_json: + self.job_data = json.load(job_json) + + def run_on_control_host(self, cmd, timeout=60): + """ + Run a command on the control host over ssh + + :param cmd: + Command to run + :param timeout: + Timeout (default 60) + :returns: + returncode, stdout + """ + try: + test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + except AttributeError: + test_username = "ubuntu" + ssh_cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + f"{test_username}@{self.config['device_ip']}", + cmd, + ] + proc = subprocess.run( + ssh_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + return proc.returncode, proc.stdout + + def provision(self): + """Provision the device""" + + # First, ensure the device is online and reachable + try: + self.copy_ssh_id() + except subprocess.CalledProcessError: + self.hardreset() + self.check_device_booted() + + logger.info("Recovering OEM image") + recovery_cmds = self.config.get("recovery_cmds") + self._run_cmd_list(recovery_cmds) + self.check_device_booted() + + def copy_ssh_id(self): + """Copy the ssh id to the device""" + try: + test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + test_password = self.job_data.get("test_data", {}).get( + "test_password", "ubuntu" + ) + except AttributeError: + test_username = "ubuntu" + test_password = "ubuntu" + cmd = [ + "sshpass", + "-p", + test_password, + "ssh-copy-id", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + f"{test_username}@{self.config['device_ip']}", + ] + subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=60) + + def check_device_booted(self): + """Check to see if the device is booted and reachable with ssh""" + logger.info("Checking to see if the device is available.") + started = time.time() + # Wait for provisioning to complete - can take a very long time + while time.time() - started < 3600: + try: + time.sleep(90) + self.copy_ssh_id() + return True + except subprocess.SubprocessError: + pass + # If we get here, then we didn't boot in time + agent_name = self.config.get("agent_name") + logger.error( + "Device %s unreachable, provisioning" "failed!", agent_name + ) + raise ProvisioningError("Failed to boot test image!") + + def _run_cmd_list(self, cmdlist): + """ + Run a list of commands + + :param cmdlist: + List of commands to run + """ + if not cmdlist: + return + for cmd in cmdlist: + logger.info("Running %s", cmd) + try: + return_code, output = self.run_on_control_host( + cmd, timeout=600 + ) + except subprocess.TimeoutExpired as exc: + raise ProvisioningError( + "timeout reaching control host!" + ) from exc + if return_code: + raise ProvisioningError(output) + logger.info(output) + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config["reboot_script"]: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except subprocess.SubprocessError as exc: + raise RecoveryError("Error running reboot script!") from exc diff --git a/device-connectors/src/testflinger_device_connectors/devices/oemscript/__init__.py b/device-connectors/src/testflinger_device_connectors/devices/oemscript/__init__.py new file mode 100644 index 00000000..766141e5 --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/oemscript/__init__.py @@ -0,0 +1,47 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu OEM Recovery provisioner support code.""" + +import logging + +import yaml + +import testflinger_device_connectors +from testflinger_device_connectors import logmsg +from testflinger_device_connectors.devices import ( + DefaultDevice, + RecoveryError, + catch, +) +from testflinger_device_connectors.devices.oemscript.oemscript import OemScript + +device_name = "oemscript" + + +class DeviceConnector(DefaultDevice): + + """Tool for provisioning baremetal with a given image.""" + + @catch(RecoveryError, 46) + def provision(self, args): + """Method called when the command is invoked.""" + with open(args.config) as configfile: + config = yaml.safe_load(configfile) + testflinger_device_connectors.configure_logging(config) + device = OemScript(args.config, args.job_data) + logmsg(logging.INFO, "BEGIN provision") + logmsg(logging.INFO, "Provisioning device") + device.provision() + logmsg(logging.INFO, "END provision") diff --git a/device-connectors/src/testflinger_device_connectors/devices/oemscript/oemscript.py b/device-connectors/src/testflinger_device_connectors/devices/oemscript/oemscript.py new file mode 100644 index 00000000..45267fba --- /dev/null +++ b/device-connectors/src/testflinger_device_connectors/devices/oemscript/oemscript.py @@ -0,0 +1,219 @@ +# Copyright (C) 2023 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Ubuntu OEM Script Provisioner support code.""" + +import json +import logging +import os +from pathlib import Path +import subprocess +import time +import yaml + +from testflinger_device_connectors import download +from testflinger_device_connectors.devices import ( + ProvisioningError, + RecoveryError, +) + +logger = logging.getLogger() + + +class OemScript: + """Device Connector for OEM Script.""" + + # Extra arguments to pass to the OEM script + extra_script_args = [] + + def __init__(self, config, job_data): + with open(config, encoding="utf-8") as configfile: + self.config = yaml.safe_load(configfile) + with open(job_data, encoding="utf-8") as job_json: + self.job_data = json.load(job_json) + + def run_on_control_host(self, cmd, timeout=60): + """ + Run a command on the control host over ssh + + :param cmd: + Command to run + :param timeout: + Timeout (default 60) + :returns: + returncode, stdout + """ + try: + test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + except AttributeError: + test_username = "ubuntu" + ssh_cmd = [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + f"{test_username}@{self.config['device_ip']}", + cmd, + ] + proc = subprocess.run( + ssh_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + return proc.returncode, proc.stdout + + def provision(self): + """Provision the device""" + + # First, ensure the device is online and reachable + try: + self.copy_ssh_id() + except subprocess.CalledProcessError: + self.hardreset() + self.check_device_booted() + + provision_data = self.job_data.get("provision_data", {}) + image_url = provision_data.get("url") + + # Download the .iso image from image_url + if not image_url: + logger.error( + "Please provide an image 'url' in the provision_data section" + ) + raise ProvisioningError("No image url provided") + image_file = download(image_url) + + self.run_recovery_script(image_file) + + self.check_device_booted() + + def run_recovery_script(self, image_file): + """Download and run the OEM recovery script""" + device_ip = self.config["device_ip"] + + data_path = Path(__file__).parent / "../../data/muxpi/oemscript" + recovery_script = data_path / "recovery-from-iso.sh" + + # Run the recovery script + logger.info("Running recovery script") + cmd = [ + recovery_script, + *self.extra_script_args, + "--local-iso", + image_file, + "--inject-ssh-key", + os.path.expanduser("~/.ssh/id_rsa.pub"), + "-t", + device_ip, + ] + proc = subprocess.run( + cmd, + timeout=60 * 60, # 1 hour - just in case + check=False, + ) + if proc.returncode: + logger.error( + "Recovery script failed with return code %s", proc.returncode + ) + raise ProvisioningError("Recovery script failed") + + def copy_ssh_id(self): + """Copy the ssh id to the device""" + try: + test_username = self.job_data.get("test_data", {}).get( + "test_username", "ubuntu" + ) + test_password = self.job_data.get("test_data", {}).get( + "test_password", "ubuntu" + ) + except AttributeError: + test_username = "ubuntu" + test_password = "ubuntu" + cmd = [ + "sshpass", + "-p", + test_password, + "ssh-copy-id", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + f"{test_username}@{self.config['device_ip']}", + ] + subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=60) + + def check_device_booted(self): + """Check to see if the device is booted and reachable with ssh""" + logger.info("Checking to see if the device is available.") + started = time.time() + # Wait for provisioning to complete - can take a very long time + while time.time() - started < 3600: + try: + time.sleep(90) + self.copy_ssh_id() + return True + except subprocess.SubprocessError: + pass + # If we get here, then we didn't boot in time + agent_name = self.config.get("agent_name") + logger.error( + "Device %s unreachable, provisioning" "failed!", agent_name + ) + raise ProvisioningError("Failed to boot test image!") + + def _run_cmd_list(self, cmdlist): + """ + Run a list of commands + + :param cmdlist: + List of commands to run + """ + if not cmdlist: + return + for cmd in cmdlist: + logger.info("Running %s", cmd) + try: + return_code, output = self.run_on_control_host( + cmd, timeout=600 + ) + except subprocess.TimeoutExpired as exc: + raise ProvisioningError( + "timeout reaching control host!" + ) from exc + if return_code: + raise ProvisioningError(output) + logger.info(output) + + def hardreset(self): + """ + Reboot the device. + + :raises RecoveryError: + If the command times out or anything else fails. + + .. note:: + This function runs the commands specified in 'reboot_script' + in the config yaml. + """ + for cmd in self.config["reboot_script"]: + logger.info("Running %s", cmd) + try: + subprocess.check_call(cmd.split(), timeout=120) + except subprocess.SubprocessError as exc: + raise RecoveryError("Error running reboot script!") from exc diff --git a/device-connectors/src/tests/__init__.py b/device-connectors/src/tests/__init__.py new file mode 100644 index 00000000..336bdc25 --- /dev/null +++ b/device-connectors/src/tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2019 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/device-connectors/src/tests/test_snappy_device_agents.py b/device-connectors/src/tests/test_snappy_device_agents.py new file mode 100644 index 00000000..3a8593f9 --- /dev/null +++ b/device-connectors/src/tests/test_snappy_device_agents.py @@ -0,0 +1,56 @@ +# Copyright (C) 2019 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import testflinger_device_connectors + + +class TestCommandsTemplate: + """Tests to ensure test_cmds templating works properly""" + + def test_known_config_items(self): + """Known config items should fill in the expected value""" + cmds = "test {item}" + config = {"item": "foo"} + expected = "test foo" + assert ( + testflinger_device_connectors._process_cmds_template_vars( + cmds, config + ) + == expected + ) + + def test_unknown_config_items(self): + """Unknown config items should not cause an error""" + cmds = "test {unknown_item}" + config = {} + assert ( + testflinger_device_connectors._process_cmds_template_vars( + cmds, config + ) + == cmds + ) + + def test_escaped_braces(self): + """Escaped braces should be unescaped, not interpreted""" + cmds = "test {{item}}" + config = {"item": "foo"} + expected = "test {item}" + assert ( + testflinger_device_connectors._process_cmds_template_vars( + cmds, config + ) + == expected + ) diff --git a/device-connectors/tox.ini b/device-connectors/tox.ini new file mode 100644 index 00000000..ac28e296 --- /dev/null +++ b/device-connectors/tox.ini @@ -0,0 +1,18 @@ +[tox] +envlist = py +skipsdist = true + +[testenv] +deps = + black + flake8 + pytest + pylint + pytest-cov + pytest-mock +commands = + {envbindir}/pip3 install . + {envbindir}/python -m black --check src + {envbindir}/python -m flake8 src + #{envbindir}/python -m pylint src + {envbindir}/python -m pytest --doctest-modules --cov=src diff --git a/.coveragerc b/server/.coveragerc similarity index 100% rename from .coveragerc rename to server/.coveragerc diff --git a/.gitignore b/server/.gitignore similarity index 100% rename from .gitignore rename to server/.gitignore diff --git a/.pylintrc b/server/.pylintrc similarity index 100% rename from .pylintrc rename to server/.pylintrc diff --git a/server/COPYING b/server/COPYING new file mode 100644 index 00000000..94a9ed02 --- /dev/null +++ b/server/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/Dockerfile b/server/Dockerfile similarity index 100% rename from Dockerfile rename to server/Dockerfile diff --git a/HACKING.md b/server/HACKING.md similarity index 100% rename from HACKING.md rename to server/HACKING.md diff --git a/README.rst b/server/README.rst similarity index 100% rename from README.rst rename to server/README.rst diff --git a/charm/Makefile b/server/charm/Makefile similarity index 100% rename from charm/Makefile rename to server/charm/Makefile diff --git a/charm/README.md b/server/charm/README.md similarity index 100% rename from charm/README.md rename to server/charm/README.md diff --git a/charm/charmcraft.yaml b/server/charm/charmcraft.yaml similarity index 100% rename from charm/charmcraft.yaml rename to server/charm/charmcraft.yaml diff --git a/charm/config.yaml b/server/charm/config.yaml similarity index 100% rename from charm/config.yaml rename to server/charm/config.yaml diff --git a/charm/lib/charms/data_platform_libs/v0/data_interfaces.py b/server/charm/lib/charms/data_platform_libs/v0/data_interfaces.py similarity index 100% rename from charm/lib/charms/data_platform_libs/v0/data_interfaces.py rename to server/charm/lib/charms/data_platform_libs/v0/data_interfaces.py diff --git a/charm/lib/charms/nginx_ingress_integrator/v0/nginx_route.py b/server/charm/lib/charms/nginx_ingress_integrator/v0/nginx_route.py similarity index 100% rename from charm/lib/charms/nginx_ingress_integrator/v0/nginx_route.py rename to server/charm/lib/charms/nginx_ingress_integrator/v0/nginx_route.py diff --git a/charm/metadata.yaml b/server/charm/metadata.yaml similarity index 100% rename from charm/metadata.yaml rename to server/charm/metadata.yaml diff --git a/charm/requirements.txt b/server/charm/requirements.txt similarity index 100% rename from charm/requirements.txt rename to server/charm/requirements.txt diff --git a/charm/src/charm.py b/server/charm/src/charm.py similarity index 100% rename from charm/src/charm.py rename to server/charm/src/charm.py diff --git a/charm/tests/unit/test_charm.py b/server/charm/tests/unit/test_charm.py similarity index 100% rename from charm/tests/unit/test_charm.py rename to server/charm/tests/unit/test_charm.py diff --git a/devel/docker-compose.override.yml b/server/devel/docker-compose.override.yml similarity index 100% rename from devel/docker-compose.override.yml rename to server/devel/docker-compose.override.yml diff --git a/devel/testflinger.yaml b/server/devel/testflinger.yaml similarity index 100% rename from devel/testflinger.yaml rename to server/devel/testflinger.yaml diff --git a/docker-compose.yml b/server/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to server/docker-compose.yml diff --git a/extras/README.md b/server/extras/README.md similarity index 100% rename from extras/README.md rename to server/extras/README.md diff --git a/extras/devices/LVFS/LVFS.py b/server/extras/devices/LVFS/LVFS.py similarity index 100% rename from extras/devices/LVFS/LVFS.py rename to server/extras/devices/LVFS/LVFS.py diff --git a/extras/devices/LVFS/tests/fwupd_data.py b/server/extras/devices/LVFS/tests/fwupd_data.py similarity index 100% rename from extras/devices/LVFS/tests/fwupd_data.py rename to server/extras/devices/LVFS/tests/fwupd_data.py diff --git a/extras/devices/LVFS/tests/test_LVFS.py b/server/extras/devices/LVFS/tests/test_LVFS.py similarity index 100% rename from extras/devices/LVFS/tests/test_LVFS.py rename to server/extras/devices/LVFS/tests/test_LVFS.py diff --git a/extras/devices/OEM/OEM.py b/server/extras/devices/OEM/OEM.py similarity index 100% rename from extras/devices/OEM/OEM.py rename to server/extras/devices/OEM/OEM.py diff --git a/extras/devices/__init__.py b/server/extras/devices/__init__.py similarity index 100% rename from extras/devices/__init__.py rename to server/extras/devices/__init__.py diff --git a/extras/devices/base.py b/server/extras/devices/base.py similarity index 100% rename from extras/devices/base.py rename to server/extras/devices/base.py diff --git a/extras/dmi.py b/server/extras/dmi.py similarity index 100% rename from extras/dmi.py rename to server/extras/dmi.py diff --git a/extras/tests/test_upgrade_fw.py b/server/extras/tests/test_upgrade_fw.py similarity index 100% rename from extras/tests/test_upgrade_fw.py rename to server/extras/tests/test_upgrade_fw.py diff --git a/extras/upgrade_fw.py b/server/extras/upgrade_fw.py similarity index 100% rename from extras/upgrade_fw.py rename to server/extras/upgrade_fw.py diff --git a/pyproject.toml b/server/pyproject.toml similarity index 100% rename from pyproject.toml rename to server/pyproject.toml diff --git a/server/renovate.json b/server/renovate.json new file mode 100644 index 00000000..39a2b6e9 --- /dev/null +++ b/server/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} diff --git a/setup.py b/server/setup.py similarity index 100% rename from setup.py rename to server/setup.py diff --git a/src/__init__.py b/server/src/__init__.py similarity index 100% rename from src/__init__.py rename to server/src/__init__.py diff --git a/server/src/api/__init__.py b/server/src/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/schemas.py b/server/src/api/schemas.py similarity index 100% rename from src/api/schemas.py rename to server/src/api/schemas.py diff --git a/src/api/v1.py b/server/src/api/v1.py similarity index 100% rename from src/api/v1.py rename to server/src/api/v1.py diff --git a/src/database.py b/server/src/database.py similarity index 100% rename from src/database.py rename to server/src/database.py diff --git a/src/static/assets/css/testflinger.css b/server/src/static/assets/css/testflinger.css similarity index 100% rename from src/static/assets/css/testflinger.css rename to server/src/static/assets/css/testflinger.css diff --git a/src/static/assets/js/filter.js b/server/src/static/assets/js/filter.js similarity index 100% rename from src/static/assets/js/filter.js rename to server/src/static/assets/js/filter.js diff --git a/src/templates/agent_detail.html b/server/src/templates/agent_detail.html similarity index 100% rename from src/templates/agent_detail.html rename to server/src/templates/agent_detail.html diff --git a/src/templates/agents.html b/server/src/templates/agents.html similarity index 100% rename from src/templates/agents.html rename to server/src/templates/agents.html diff --git a/src/templates/base.html b/server/src/templates/base.html similarity index 100% rename from src/templates/base.html rename to server/src/templates/base.html diff --git a/src/templates/job_detail.html b/server/src/templates/job_detail.html similarity index 100% rename from src/templates/job_detail.html rename to server/src/templates/job_detail.html diff --git a/src/templates/jobs.html b/server/src/templates/jobs.html similarity index 100% rename from src/templates/jobs.html rename to server/src/templates/jobs.html diff --git a/src/templates/queue_detail.html b/server/src/templates/queue_detail.html similarity index 100% rename from src/templates/queue_detail.html rename to server/src/templates/queue_detail.html diff --git a/src/templates/queues.html b/server/src/templates/queues.html similarity index 100% rename from src/templates/queues.html rename to server/src/templates/queues.html diff --git a/src/views.py b/server/src/views.py similarity index 100% rename from src/views.py rename to server/src/views.py diff --git a/terraform/README.md b/server/terraform/README.md similarity index 100% rename from terraform/README.md rename to server/terraform/README.md diff --git a/terraform/main.tf b/server/terraform/main.tf similarity index 100% rename from terraform/main.tf rename to server/terraform/main.tf diff --git a/terraform/variables.tf b/server/terraform/variables.tf similarity index 100% rename from terraform/variables.tf rename to server/terraform/variables.tf diff --git a/terraform/versions.tf b/server/terraform/versions.tf similarity index 100% rename from terraform/versions.tf rename to server/terraform/versions.tf diff --git a/testflinger.conf.example b/server/testflinger.conf.example similarity index 100% rename from testflinger.conf.example rename to server/testflinger.conf.example diff --git a/testflinger.env b/server/testflinger.env similarity index 100% rename from testflinger.env rename to server/testflinger.env diff --git a/testflinger.py b/server/testflinger.py similarity index 100% rename from testflinger.py rename to server/testflinger.py diff --git a/server/tests/__init__.py b/server/tests/__init__.py new file mode 100644 index 00000000..d16f4571 --- /dev/null +++ b/server/tests/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 2016 Canonical +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/tests/conftest.py b/server/tests/conftest.py similarity index 100% rename from tests/conftest.py rename to server/tests/conftest.py diff --git a/tests/test_app.py b/server/tests/test_app.py similarity index 100% rename from tests/test_app.py rename to server/tests/test_app.py diff --git a/tests/test_v1.py b/server/tests/test_v1.py similarity index 100% rename from tests/test_v1.py rename to server/tests/test_v1.py diff --git a/tox.ini b/server/tox.ini similarity index 100% rename from tox.ini rename to server/tox.ini