Merge pull request #2 from EPFL-ENAC/dev

feat: experiment files upload, visualization and extraction

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.6-alpine3.18
+FROM python:3.11.7-bullseye
 ENV POETRY_VERSION=1.7.1
 RUN pip install "poetry==$POETRY_VERSION"
 ENV PYTHONPATH="$PYTHONPATH:/app"
@@ -8,7 +8,7 @@ WORKDIR /app
 COPY poetry.lock pyproject.toml /app/
 RUN poetry config installer.max-workers 10
 RUN poetry config virtualenvs.create false
-RUN apk add --no-cache g++ postgresql-dev
+RUN apt-get update && apt-get install -y g++ libpq-dev libgl1
 RUN poetry install --no-interaction --without dev
 
 COPY alembic.ini prestart.sh /app/

backend/app/services/experiments/models.py

@@ -1,13 +1,16 @@
 from sqlmodel import SQLModel, Field, ARRAY, UniqueConstraint, Column, Integer, String, JSON
 from typing import List, Dict, Optional
 
+
 class ExperimentBase(SQLModel):
     scheme: Dict | None = Field(sa_column=Column(JSON))
+    files: Dict | None = Field(sa_column=Column(JSON))
     description: str | None
     experiment_id: str | None
     test_scale: float | None
     simultaneous_excitations_nb: int | None
-    applied_excitation_directions: List[str] | None = Field(sa_column=Column(ARRAY(String)))
+    applied_excitation_directions: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
     storeys_nb: int | None
     building_height: int | None
     total_building_height: int | None
@@ -17,32 +20,41 @@ class ExperimentBase(SQLModel):
     masonry_unit_material: str | None
     mortar_type: str | None
     masonry_compressive_strength: int | None
-    masonry_wall_thickness: List[int] | None = Field(sa_column=Column(ARRAY(Integer)))
+    masonry_wall_thickness: List[int] | None = Field(
+        sa_column=Column(ARRAY(Integer)))
     wall_leaves_nb: int | None
     internal_walls: bool | None
     mechanical_connectors: str | None
     connectors_activation: str | None
     retrofitted: bool | None
     retrofitting_application: str | None
-    retrofitting_type: List[str] | None = Field(sa_column=Column(ARRAY(String)))
+    retrofitting_type: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
     first_estimated_fundamental_period: float | None
     last_estimated_fundamental_period: float | None
     max_horizontal_pga: float | None
     max_estimated_dg: float | None
-    material_characterizations: List[str] | None = Field(sa_column=Column(ARRAY(String)))
-    associated_test_types: List[str] | None = Field(sa_column=Column(ARRAY(String)))
-    material_characterization_refs: List[str] | None = Field(sa_column=Column(ARRAY(String)))
+    material_characterizations: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
+    associated_test_types: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
+    material_characterization_refs: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
     digitalized_data: bool = Field(default=False)
-    experimental_results_reported: List[str] | None = Field(sa_column=Column(ARRAY(String)))
+    experimental_results_reported: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
     open_measured_data: bool = Field(default=False)
     link_to_open_measured_data: str | None
-    crack_types_observed: List[str] | None = Field(sa_column=Column(ARRAY(String)))
+    crack_types_observed: List[str] | None = Field(
+        sa_column=Column(ARRAY(String)))
     experimental_campaign_motivation: str | None
+    publication_year: int | None
 
     reference_id: int = Field(
         foreign_key="reference.id", index=True
     )
 
+
 class Experiment(ExperimentBase, table=True):
     __table_args__ = (UniqueConstraint("id"),)
     id: int = Field(
@@ -52,13 +64,15 @@ class Experiment(ExperimentBase, table=True):
         index=True,
     )
 
+
 class ExperimentRead(ExperimentBase):
     id: int
-    reference: Optional[str] = None
+    reference: Optional[Dict] = None
+
 
 class ExperimentCreate(ExperimentBase):
     pass
 
+
 class ExperimentUpdate(ExperimentBase):
     pass
-

backend/app/services/experiments/service.py

@@ -1,13 +1,13 @@
 from app.services.experiments.models import Experiment, ExperimentRead, ExperimentUpdate
-from app.services.references.models import Reference
-from app.services.runresults.models import RunResult
+from app.services.references.models import Reference, ReferenceRead
 from app.services.runresults.service import RunResultsService
 from app.services.files.s3client import s3_client
 from app.utils.query import QueryBuilder
 from sqlmodel import select
 from fastapi import HTTPException
 from app.db import AsyncSession
 
+
 class ExperimentsService:
 
     def __init__(self, session: AsyncSession):
@@ -25,15 +25,16 @@ async def get(self, experiment_id: int) -> ExperimentRead:
 
         # Get reference
         res = await self.session.exec(
-            select(Reference.id, Reference.reference).where(Reference.id == experiment.reference_id)
+            select(Reference).where(
+                Reference.id == experiment.reference_id)
         )
         reference = res.one_or_none()
-        reference_reference = reference.reference if reference else None
+        reference_reference = ReferenceRead.from_orm(reference)
 
         experiment.reference = reference_reference
 
         return experiment
-    
+
     async def find(self, filter: dict | str, sort: list | str, range: list | str) -> list[int, int, int, ExperimentRead]:
         """Get all experiments matching filter and range"""
         builder = QueryBuilder(Experiment, filter, sort, range)
@@ -50,22 +51,25 @@ async def find(self, filter: dict | str, sort: list | str, range: list | str) ->
         results = await self.session.exec(query)
         experiments = results.all()
         # Cast to ExperimentRead
-        experiments = [ExperimentRead.from_orm(experiment) for experiment in experiments]
+        experiments = [ExperimentRead.from_orm(
+            experiment) for experiment in experiments]
 
         # Reference IDs
         reference_ids = [experiment.reference_id for experiment in experiments]
         res = await self.session.exec(
-            select(Reference.id, Reference.reference).where(Reference.id.in_(reference_ids))
+            select(Reference).where(
+                Reference.id.in_(reference_ids))
         )
         references = res.all()
-        reference_dict = {reference.id: reference.reference for reference in references}
+        reference_dict = {
+            reference.id: reference for reference in references}
 
         # Add reference short name to each experiment
         for experiment in experiments:
-            experiment.reference = reference_dict[experiment.reference_id]
+            experiment.reference = ReferenceRead.from_orm(
+                reference_dict[experiment.reference_id])
 
         return [start, end, total_count, experiments]
-
 
     async def create(self, experiment: Experiment) -> Experiment:
         """Creates an experiment"""
@@ -74,7 +78,6 @@ async def create(self, experiment: Experiment) -> Experiment:
         await self.session.refresh(experiment)
         return experiment
 
-
     async def patch(self, experiment_id: int, experiment: ExperimentUpdate) -> ExperimentRead:
         """Updates an experiment"""
         res = await self.session.exec(
@@ -93,30 +96,49 @@ async def patch(self, experiment_id: int, experiment: ExperimentUpdate) -> Exper
 
         return experiment_db
 
-
     async def delete(self, experiment_id: int, recursive: bool = False) -> Experiment:
         """Delete an experiment by id"""
         res = await self.session.exec(
             select(Experiment).where(Experiment.id == experiment_id)
         )
         experiment = res.one_or_none()
-        
+
         if experiment:
             # Delete associated files
             if experiment.scheme:
                 await s3_client.delete_file(experiment.scheme['path'])
+            if experiment.files:
+                key = experiment.files['name']
+                s3_folder = f"experiments/{experiment_id}/{key}"
+                await s3_client.delete_files(s3_folder)
             # Delete run results
             if recursive:
                 run_results_service = RunResultsService(self.session)
                 await run_results_service.delete_by_experiment(experiment_id)
             # Delete experiment
             await self.session.delete(experiment)
             await self.session.commit()
-        
+
         return experiment
-
+
+    async def delete_files(self, experiment_id: int) -> None:
+        """Delete files of an experiment by id"""
+        res = await self.session.exec(
+            select(Experiment).where(Experiment.id == experiment_id)
+        )
+        experiment = res.one_or_none()
+
+        if experiment and experiment.files:
+            if experiment.files:
+                key = experiment.files['name']
+                s3_folder = f"experiments/{experiment_id}/{key}"
+                deleted = await s3_client.delete_files(s3_folder)
+                if deleted:
+                    experiment.files = None
+                    await self.session.commit()
+
     async def delete_by_reference(self, reference_id: int, recursive: bool = False) -> None:
         """Delete all experiments by reference id"""
-        start, stop, total_count, experiments = await self.find(filter={ "reference_id": reference_id }, sort=None, range=None)
+        start, stop, total_count, experiments = await self.find(filter={"reference_id": reference_id}, sort=None, range=None)
         if experiments:
-            [await self.delete(experiment.id, recursive) for experiment in experiments]
+            [await self.delete(experiment.id, recursive) for experiment in experiments]