Add VPX reporting tools

Signed-off-by: Travis F. Collins <travis.collins@analog.com>

pyproject.toml

@@ -29,6 +29,7 @@ dependencies = [
     "tqdm",
     "requests",
     "pygithub",
+    "minio",
 ]
 
 [tool.setuptools.dynamic]
@@ -40,6 +41,9 @@ packages = ["telemetry", "telemetry.gparser", "telemetry.prod", "telemetry.repor
 [project.scripts]
 telemetry = "telemetry.cli:cli"
 
+[project.entry-points.pytest11]
+pybench = "telemetry.plugin"
+
 [project.urls]
 homepage = "https://sdgtt.github.io/telemetry/"
 documentation = "https://sdgtt.github.io/telemetry/"

requirements.txt

@@ -2,4 +2,5 @@ bs4==0.0.1
 tqdm==4.62.3
 junitparser==2.4.2
 elasticsearch==7.16.0
-pygithub==2.3.0
+pygithub==2.3.0
+minio

telemetry/__init__.py

@@ -6,6 +6,7 @@
 from telemetry.gparser import parser
 from telemetry.gparser import grabber
 from telemetry.report import gist, markdown
+from telemetry.dev.vpx import VPX
 
 import telemetry.prod as prod
 

telemetry/dev/__init__.py

@@ -0,0 +1 @@
+"""Artifact capture for development board work."""

telemetry/dev/core.py

@@ -0,0 +1,103 @@
+"""Core features for development board telemetry."""
+
+import datetime
+import os
+from typing import List
+
+import pymongo
+from minio import Minio
+import yaml
+
+
+class Core:
+    def __init__(
+        self,
+        project_type,
+        configfilename=None,
+        configs: dict = {},
+    ):
+
+        extracted_config = {"mongo": None, "minio": None}
+        if configfilename:
+            if not os.path.isfile(configfilename):
+                raise Exception(f"Config file {configfilename} does not exist")
+
+            with open(configfilename) as f:
+                config = yaml.load(f, Loader=yaml.FullLoader)
+
+            for k in config:
+                project = config[k]
+                project_name = project["project"]
+                if project_name == project_type:
+                    servers = project["servers"]
+                    for server in servers:
+                        if "type" not in server:
+                            continue
+                        if server["type"] == "mongo":
+                            extracted_config["mongo"] = server
+                        elif server["type"] == "minio":
+                            extracted_config["minio"] = server
+
+        # Merge configs where configs input takes precedence
+        for k in extracted_config:
+            if k in configs:
+                extracted_config[k] = {**extracted_config[k], **configs[k]}
+
+        if "mongo" in extracted_config:
+            self.mongo = self.setup_mongo(extracted_config["mongo"])
+        else:
+            ...
+            # raise Exception("No MongoDB configuration found")
+
+        if "minio" in extracted_config:
+            self.minio = self.setup_minio(extracted_config["minio"])
+        else:
+            ...
+            # raise Exception("No Minio configuration found")
+
+    def setup_mongo(self, config):
+        """Setup MongoDB connection."""
+        # Check config
+        required_fields = [
+            "username",
+            "password",
+            "address",
+            "port",
+            "database",
+            "collection",
+        ]
+        for f in required_fields:
+            if f not in config:
+                raise Exception(f"Mongo config missing field {f}")
+
+        cmd = f"mongodb://{config['username']}:{config['password']}@{config['address']}:{config['port']}/"
+
+        try:
+            self.client = pymongo.MongoClient(cmd)
+        except Exception as e1:
+            try:
+                cmd = cmd.replace("mongodb+srv://", "mongodb://")
+                self.client = pymongo.MongoClient(cmd)
+            except Exception as e2:
+                print(e1, e2)
+                raise Exception("Unable to connect to MongoDB")
+
+        db = self.client[config["database"]]
+        self.collection = db[config["collection"]]
+
+        return self.client
+
+    def setup_minio(self, config):
+        """Setup Minio connection."""
+        # Check config
+        required_fields = ["address", "port", "access_key", "secret_key"]
+        for f in required_fields:
+            if f not in config:
+                raise Exception(f"Minio config missing field {f}")
+        address = f"{config['address']}:{config['port']}"
+        return Minio(
+            address,
+            access_key=config["access_key"],
+            secret_key=config["secret_key"],
+            secure=False,
+        )

telemetry/dev/vpx.py

@@ -0,0 +1,107 @@
+import os
+
+from telemetry.dev.core import Core
+
+
+class VPX(Core):
+    """VPX Card telemetry functions."""
+
+    _mongo_database = "devboards"
+    _mongo_collection = "vpx_washington"
+    _minio_bucket = "devboards"
+    _minio_subfolder = "vpx_washington"
+
+    def __init__(self, configfilename=None, configs={}):
+        project = "vpx"
+        configs["mongo"] = {
+            "database": self._mongo_database,
+            "collection": self._mongo_collection,
+        }
+        configs["minio"] = {
+            "bucket": self._minio_bucket,
+            "subfolder": self._minio_subfolder,
+        }
+        super().__init__(project, configfilename, configs)
+
+    def _check_mongo_and_minio(self):
+        if not hasattr(self, "mongo"):
+            raise Exception("MongoDB not configured")
+        if not hasattr(self, "minio"):
+            raise Exception("Minio not configured")
+
+        # Check if bucket exists
+        if not self.minio.bucket_exists(self._minio_bucket):
+            # Create the bucket
+            print(f"Creating bucket {self._minio_bucket}")
+            self.minio.make_bucket(self._minio_bucket)
+
+    def submit_test_data(self, job_id: str, metadata: dict, artifact_files: list):
+        """Submit test data to MongoDB and Minio.
+
+        Data organization:
+        - MongoDB:
+            - Overview: Database/Collection/Document
+            - Namings:
+                - Jenkins test: devboards/vpx_washington/j<job_id>-<hdl_hash>-<linux_hash>
+                - Manual test: devboards/vpx_washington/m<test_start_time>-<hdl_hash>-<linux_hash>
+        - Minio:
+            - Overview: Bucket/Subfolder/File
+            - Namings:
+                - Jenkins test: devboards/vpx_washington/j<job_id>-h<hdl_hash>-s<linux_hash>/<artifact_files>
+                - Manual test: devboards/vpx_washington/m<test_start_time>-h<hdl_hash>-s<linux_hash>/<artifact_files>
+
+        Args:
+            job_id (str): Jenkins job ID. If manual use form "m<test_start_time>".
+            metadata (dict): Test metadata.
+            artifact_files (list): List of artifact files.
+        """
+        ## Checks
+        # Check if metadata has required fields
+        required_fields = ["junit_xml", "hdl_hash", "linux_hash", "test_date"]
+        for field in required_fields:
+            if field not in metadata:
+                raise Exception(f"metadata missing field {field}")
+
+        # Check if files exist
+        for file in artifact_files:
+            if not os.path.isfile(file):
+                raise Exception(f"File {file} does not exist")
+
+        # Check job ID format
+        if not job_id.startswith("j") and not job_id.startswith("m"):
+            raise Exception(
+                "Job ID must start with 'j' for Jenkins job or 'm' for manual test"
+            )
+        if job_id.startswith("j") and not job_id[1:].isdigit():
+            raise Exception("Job ID must have a numerical value after the 'j'")
+        if job_id.startswith("m"):
+            date_str = job_id[1:]
+            # Must be a valid date of form YYYYMMDD_HHMMSS
+            if len(date_str) != 15:
+                raise Exception(f"Manual test job ID must be of form 'mYYYYMMDD_HHMMSS', got {date_str}")
+            for i, c in enumerate(date_str):
+                if i == 8 and c != "_" or i != 8 and not c.isdigit():
+                    raise Exception(
+                        f"Manual test job ID must be of form 'mYYYYMMDD_HHMMSS', got {date_str}"
+                    )
+        # Generate extra metadata to link mongo and minio
+        metadata["job_id"] = job_id
+        metadata["artifact_files"] = artifact_files
+        metadata["hdl_hash"] = metadata["hdl_hash"].upper()
+        metadata["linux_hash"] = metadata["linux_hash"].upper()
+        metadata[
+            "miniopath"
+        ] = f"{self._minio_bucket}/{self._minio_subfolder}/{job_id}-h{metadata['hdl_hash']}-s{metadata['linux_hash']}"
+
+        # DB checks
+        self._check_mongo_and_minio()
+
+        # Perform operations
+        print(
+            f"Inserting test data to MongoDB collection {self._mongo_database}/{self._mongo_collection}"
+        )
+        self.collection.insert_one(metadata)
+        for file in artifact_files:
+            target = f"{self._minio_subfolder}/{job_id}-h{metadata['hdl_hash']}-s{metadata['linux_hash']}/{os.path.basename(file)}"
+            print(f"Uploading {file} to Minio bucket {self._minio_bucket} at {target}")
+            self.minio.fput_object(self._minio_bucket, target, file)

telemetry/plugin.py

@@ -0,0 +1,116 @@
+import os
+import time
+
+import pytest
+
+import telemetry
+
+required_metadata = {"vpx": {"hdl_hash": None, "linux_hash": None}}
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("telemetry")
+    group.addoption(
+        "--telemetry-enable",
+        action="store_true",
+        default=False,
+        help="Enable telemetry upload",
+    )
+    group.addoption(
+        "--telemetry-configpath",
+        action="store",
+        default=None,
+        help="Path to telemetry configuration file",
+    )
+    group.addoption(
+        "--telemetry-jenkins-job",
+        action="store",
+        default=None,
+        help="Path to junit xml report",
+    )
+    for field in required_metadata["vpx"]:
+        group.addoption(
+            f"--telemetry-{field}",
+            action="store",
+            default=None,
+            help=f"Metadata field {field}",
+        )
+
+
+# Run at the start of the test session
+def pytest_configure(config):
+    # Check if telemetry is enabled
+    if not config.option.telemetry_enable:
+        return
+
+    # Check dependencies
+    path = config.option.xmlpath
+    if not path:
+        raise Exception(
+            "junit report generation not enabled. Needed for telemetry upload. \nAdd --junitxml=path/to/report.xml to pytest command line"
+        )
+
+    # Check if databases are reachable
+    configfilename = config.option.telemetry_configpath
+    try:
+        res = telemetry.VPX(configfilename=configfilename)
+    except Exception as e:
+        raise Exception("Unable to connect to databases") from e
+
+    # Make sure we have necessary metadata before starting tests
+    project = "vpx"
+
+    # These can exist as environment variables or from the CLI
+    metadata = required_metadata[project]
+    for field in required_metadata[project]:
+        metadata[field] = os.getenv(field.upper(), "NA")
+    # Overwrite with CLI values
+    print(dir(config.option))
+    for field in required_metadata[project]:
+        field = field.replace("-", "_")
+        if hasattr(config.option, f"telemetry_{field}"):
+            val = getattr(config.option, f"telemetry_{field}")
+            if val:
+                metadata[field] = val
+
+    metadata["test_date"] = str(time.strftime("%Y%m%d_%H%M%S"))
+
+    # Save res to config for later use
+    config._telemetry = res
+    config._telemetry_metadata = metadata
+
+
+# Hook to run after all tests are done
+def pytest_sessionfinish(session, exitstatus):
+    # Get XML data
+    xmlpath = session.config.option.xmlpath
+    with open(xmlpath, "r") as f:
+        xml = f.read()
+    session.config._telemetry_metadata["junit_xml"] = xml
+
+    res = session.config._telemetry
+    print("res", res)
+    print("Uploading data")
+
+    # Create job ID
+    if session.config.option.telemetry_jenkins_job:
+        job_id = session.config.option.telemetry_jenkins_job
+    else:
+        job_id = "m" + str(time.strftime("%Y%m%d_%H%M%S"))
+    print(job_id)
+
+    # Get files
+    telemetry_files = session.config.stash.get("telemetry_files", None)
+    print(telemetry_files)
+
+    res.submit_test_data(
+        job_id, session.config._telemetry_metadata, [xmlpath] + telemetry_files
+    )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def telemetry_files(pytestconfig):
+    """Fixture to store files for telemetry upload."""
+    files = []
+    yield files
+    pytestconfig.stash["telemetry_files"] = files

tests/dev_test_config.yaml

@@ -0,0 +1,20 @@
+devboard_telemetry:
+  project: vpx
+  servers:
+    - name: minio
+      type: minio
+      address: localhost
+      port: 9000
+      # access_key: minio
+      # secret_key: analoganalog
+      access_key: KMKQ5o93bSp5VVVxdI5X
+      secret_key: zCLIpPPXKm61gUsKGjN1BzRHPHysqtZhJUwVPquS
+      # bucket: vpx_washington
+    - name: mongo
+      type: mongo
+      address: localhost
+      port: 27017
+      username: root
+      password: analog
+      # database: devboards
+      # collection: vpx_washington