Add charm family to test observer (#237)

* Add Charm family via migration

* Update backend with charms

* Add charms on the frontend

* Update migration to not use ORM

* Split charm unit test

* Fix linting

backend/migrations/versions/2024_12_13_1702-0e09759a33c3_add_charm_family.py

@@ -0,0 +1,84 @@
+"""Add charm family
+
+Revision ID: 0e09759a33c3
+Revises: 4c2e5946358c
+Create Date: 2024-12-13 17:02:14.136283+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+from test_observer.data_access.models_enums import FamilyName
+
+new_families = {
+    FamilyName.CHARM: ["edge", "beta", "candidate", "stable"],
+}
+
+
+# revision identifiers, used by Alembic.
+revision = "0e09759a33c3"
+down_revision = "4c2e5946358c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+
+    family_table = sa.table(
+        "family",
+        sa.column("id", sa.Integer()),
+        sa.column("name", sa.String()),
+        sa.column("created_at", sa.DateTime()),
+        sa.column("updated_at", sa.DateTime()),
+    )
+    stage_table = sa.table(
+        "stage",
+        sa.column("name", sa.String()),
+        sa.column("family_id", sa.Integer()),
+        sa.column("position", sa.Integer()),
+        sa.column("created_at", sa.DateTime()),
+        sa.column("updated_at", sa.DateTime()),
+    )
+
+    for family_name, stage_names in new_families.items():
+        # Create family
+        (inserted_family,) = conn.execute(
+            sa.insert(family_table)
+            .values(
+                name=family_name,
+                created_at=sa.func.now(),
+                updated_at=sa.func.now(),
+            )
+            .returning(family_table.c.id)
+        )
+
+        # Add stages
+        stage_position = 10
+        for stage_name in stage_names:
+            conn.execute(
+                sa.insert(stage_table).values(
+                    name=stage_name,
+                    family_id=inserted_family.id,
+                    position=stage_position,
+                    created_at=sa.func.now(),
+                    updated_at=sa.func.now(),
+                )
+            )
+            stage_position += 10
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+
+    family_table = sa.table(
+        "family",
+        sa.column("id", sa.Integer()),
+        sa.column("name", sa.String()),
+        sa.column("created_at", sa.DateTime()),
+        sa.column("updated_at", sa.DateTime()),
+    )
+
+    # Delete each family, stage deletes on cascade
+    for family_name in new_families:
+        conn.execute(sa.delete(family_table).where(family_table.c.name == family_name))

backend/scripts/seed_data.py

@@ -21,6 +21,7 @@
     EndTestExecutionRequest,
     StartDebTestExecutionRequest,
     StartSnapTestExecutionRequest,
+    StartCharmTestExecutionRequest,
 )
 from test_observer.data_access.models import Artefact
 from test_observer.data_access.models_enums import FamilyName
@@ -251,6 +252,18 @@
         ci_link="http://example17",
         test_plan="com.canonical.certification::sru-server",
     ),
+    StartCharmTestExecutionRequest(
+        family=FamilyName.CHARM,
+        name="postgresql-k8s",
+        version="123",
+        revision=123,
+        track="14",
+        arch="arm64",
+        execution_stage="candidate",
+        environment="juju=3.5 ubuntu=22.04 cloud=k8s",
+        ci_link="http://example13",
+        test_plan="com.canonical.solutions-qa::tbd",
+    ),
 ]
 
 END_TEST_EXECUTION_REQUESTS = [

backend/test_observer/controllers/test_executions/models.py

@@ -70,6 +70,12 @@ class StartDebTestExecutionRequest(_StartTestExecutionRequest):
     repo: str
 
 
+class StartCharmTestExecutionRequest(_StartTestExecutionRequest):
+    family: Literal[FamilyName.CHARM]
+    revision: int
+    track: str
+
+
 class C3TestResultStatus(str, Enum):
     PASS = "pass"
     FAIL = "fail"

backend/test_observer/controllers/test_executions/start_test.py

@@ -34,15 +34,20 @@
 from test_observer.data_access.repository import get_or_create
 from test_observer.data_access.setup import get_db
 
-from .models import StartDebTestExecutionRequest, StartSnapTestExecutionRequest
+from .models import (
+    StartDebTestExecutionRequest,
+    StartSnapTestExecutionRequest,
+    StartCharmTestExecutionRequest,
+)
 
 router = APIRouter()
 
 
 @router.put("/start-test")
 def start_test_execution(
     request: StartSnapTestExecutionRequest
-    | StartDebTestExecutionRequest = Body(discriminator="family"),
+    | StartDebTestExecutionRequest
+    | StartCharmTestExecutionRequest = Body(discriminator="family"),
     db: Session = Depends(get_db),
 ):
     stage = (
@@ -59,12 +64,15 @@ def start_test_execution(
             "name": request.name,
             "version": request.version,
         }
-        if isinstance(request, StartSnapTestExecutionRequest):
-            artefact_filter_kwargs["store"] = request.store
-            artefact_filter_kwargs["track"] = request.track
-        else:
-            artefact_filter_kwargs["series"] = request.series
-            artefact_filter_kwargs["repo"] = request.repo
+        match request:
+            case StartSnapTestExecutionRequest():
+                artefact_filter_kwargs["store"] = request.store
+                artefact_filter_kwargs["track"] = request.track
+            case StartDebTestExecutionRequest():
+                artefact_filter_kwargs["series"] = request.series
+                artefact_filter_kwargs["repo"] = request.repo
+            case StartCharmTestExecutionRequest():
+                artefact_filter_kwargs["track"] = request.track
 
         artefact = get_or_create(
             db,

backend/test_observer/data_access/models_enums.py

@@ -25,6 +25,7 @@
 class FamilyName(str, Enum):
     SNAP = "snap"
     DEB = "deb"
+    CHARM = "charm"
 
 
 class TestExecutionStatus(str, Enum):

backend/test_observer/data_access/repository.py

@@ -82,39 +82,65 @@ def get_artefacts_by_family(
             .group_by(Artefact.stage_id, Artefact.name)
         )
 
-        if family_name == FamilyName.SNAP:
-            subquery = (
-                base_query.add_columns(Artefact.track)
-                .group_by(Artefact.track)
-                .subquery()
-            )
-
-            query = session.query(Artefact).join(
-                subquery,
-                and_(
-                    Artefact.stage_id == subquery.c.stage_id,
-                    Artefact.name == subquery.c.name,
-                    Artefact.created_at == subquery.c.max_created,
-                    Artefact.track == subquery.c.track,
-                ),
-            )
-        else:
-            subquery = (
-                base_query.add_columns(Artefact.repo, Artefact.series)
-                .group_by(Artefact.repo, Artefact.series)
-                .subquery()
-            )
-
-            query = session.query(Artefact).join(
-                subquery,
-                and_(
-                    Artefact.stage_id == subquery.c.stage_id,
-                    Artefact.name == subquery.c.name,
-                    Artefact.created_at == subquery.c.max_created,
-                    Artefact.repo == subquery.c.repo,
-                    Artefact.series == subquery.c.series,
-                ),
-            )
+        match family_name:
+            case FamilyName.SNAP:
+                subquery = (
+                    base_query.add_columns(Artefact.track)
+                    .group_by(Artefact.track)
+                    .subquery()
+                )
+
+                query = session.query(Artefact).join(
+                    subquery,
+                    and_(
+                        Artefact.stage_id == subquery.c.stage_id,
+                        Artefact.name == subquery.c.name,
+                        Artefact.created_at == subquery.c.max_created,
+                        Artefact.track == subquery.c.track,
+                    ),
+                )
+
+            case FamilyName.DEB:
+                subquery = (
+                    base_query.add_columns(Artefact.repo, Artefact.series)
+                    .group_by(Artefact.repo, Artefact.series)
+                    .subquery()
+                )
+
+                query = session.query(Artefact).join(
+                    subquery,
+                    and_(
+                        Artefact.stage_id == subquery.c.stage_id,
+                        Artefact.name == subquery.c.name,
+                        Artefact.created_at == subquery.c.max_created,
+                        Artefact.repo == subquery.c.repo,
+                        Artefact.series == subquery.c.series,
+                    ),
+                )
+
+            case FamilyName.CHARM:
+                subquery = (
+                    base_query.join(ArtefactBuild)
+                    .add_columns(Artefact.track, ArtefactBuild.architecture)
+                    .group_by(Artefact.track, ArtefactBuild.architecture)
+                    .subquery()
+                )
+
+                query = (
+                    session.query(Artefact)
+                    .join(ArtefactBuild)
+                    .join(
+                        subquery,
+                        and_(
+                            Artefact.stage_id == subquery.c.stage_id,
+                            Artefact.name == subquery.c.name,
+                            Artefact.created_at == subquery.c.max_created,
+                            Artefact.track == subquery.c.track,
+                            ArtefactBuild.architecture == subquery.c.architecture,
+                        ),
+                    )
+                    .distinct()
+                )
 
     else:
         query = (

backend/tests/controllers/artefacts/test_artefacts.py

@@ -42,7 +42,7 @@ def test_get_latest_artefacts_by_family(
 
     old_timestamp = relevant_artefact.created_at - timedelta(days=1)
     generator.gen_artefact("edge", created_at=old_timestamp, version="1")
-    generator.gen_artefact("proposed")
+    generator.gen_artefact("proposed", family_name="deb")
 
     response = test_client.get("/v1/artefacts", params={"family": "snap"})
 

backend/tests/data_access/test_repository.py

@@ -20,7 +20,7 @@
 """Test services functions"""
 
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from sqlalchemy.orm import Session
 
@@ -87,16 +87,17 @@ def test_get_artefacts_by_family_name_latest(
     """We should get a only latest artefacts in each stage for the specified family"""
     # Arrange
     artefact_tuple = [
-        ("core20", "edge", datetime.utcnow(), "1"),
-        ("oem-jammy", "proposed", datetime.utcnow(), "1"),
-        ("core20", "edge", datetime.utcnow() - timedelta(days=10), "2"),
-        ("core20", "beta", datetime.utcnow() - timedelta(days=20), "3"),
+        ("core20", "snap", "edge", datetime.utcnow(), "1"),
+        ("oem-jammy", "deb", "proposed", datetime.utcnow(), "1"),
+        ("core20", "snap", "edge", datetime.utcnow() - timedelta(days=10), "2"),
+        ("core20", "snap", "beta", datetime.utcnow() - timedelta(days=20), "3"),
     ]
     expected_artefacts = {artefact_tuple[0], artefact_tuple[-1]}
 
-    for name, stage, created_at, version in artefact_tuple:
+    for name, family, stage, created_at, version in artefact_tuple:
         generator.gen_artefact(
             stage,
+            family_name=family,
             name=name,
             created_at=created_at,
             version=version,
@@ -108,6 +109,76 @@ def test_get_artefacts_by_family_name_latest(
     # Assert
     assert len(artefacts) == len(expected_artefacts)
     assert {
-        (artefact.name, artefact.stage.name, artefact.created_at, artefact.version)
+        (
+            artefact.name,
+            artefact.stage.family.name,
+            artefact.stage.name,
+            artefact.created_at,
+            artefact.version,
+        )
         for artefact in artefacts
     } == expected_artefacts
+
+
+def test_get_artefacts_by_family_charm_unique(
+    db_session: Session, generator: DataGenerator
+):
+    """For latest charms, artefacts should be unique on name, track, and version"""
+    # Arrange
+    specs = [
+        ("name-1", "track-1", "version-1"),
+        ("name-2", "track-1", "version-1"),
+        ("name-1", "track-2", "version-1"),
+        ("name-1", "track-1", "version-2"),
+    ]
+    for name, track, version in specs:
+        artefact = generator.gen_artefact(
+            "edge",
+            family_name="charm",
+            name=name,
+            version=version,
+            track=track,
+            created_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
+        )
+        generator.gen_artefact_build(
+            artefact,
+            "arch-1",
+        )
+
+    # Act
+    artefacts = get_artefacts_by_family(db_session, FamilyName.CHARM)
+
+    # Assert
+    assert len(artefacts) == 4
+
+
+def test_get_artefacts_by_family_charm_all_architectures(
+    db_session: Session, generator: DataGenerator
+):
+    """For latest charms, artefacts should return all known architectures"""
+    # Arrange
+    specs = [
+        ("version-3", "arch-1", 0),
+        ("version-2", "arch-1", -1),
+        ("version-1", "arch-2", -2),
+    ]
+    for version, arch, day in specs:
+        artefact = generator.gen_artefact(
+            "edge",
+            family_name="charm",
+            name="name",
+            version=version,
+            track="track",
+            created_at=datetime(2024, 1, 1, tzinfo=timezone.utc) + timedelta(days=day),
+        )
+        generator.gen_artefact_build(
+            artefact,
+            arch,
+        )
+
+    # Act
+    artefacts = get_artefacts_by_family(db_session, FamilyName.CHARM)
+
+    # Assert
+    assert len(artefacts) == 2
+    assert {artefact.version for artefact in artefacts} == {"version-3", "version-1"}