From 98e1848a83ce6fc3660722813d0b3217689e287b Mon Sep 17 00:00:00 2001 From: Marco Cucchi Date: Tue, 12 Nov 2024 12:17:36 +0100 Subject: [PATCH 1/4] add origin to job status metadata --- cads_processing_api_service/models.py | 1 + cads_processing_api_service/utils.py | 18 ++++++++---------- tests/test_30_utils.py | 2 ++ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cads_processing_api_service/models.py b/cads_processing_api_service/models.py index 3836b5f..e3476bc 100644 --- a/cads_processing_api_service/models.py +++ b/cads_processing_api_service/models.py @@ -43,6 +43,7 @@ class StatusInfoMetadata(pydantic.BaseModel): datasetMetadata: dict[str, Any] | None = None qos: dict[str, Any] | None = None log: list[tuple[str, str]] | None = None + origin: str | None = None class StatusInfo(ogc_api_processes_fastapi.models.StatusInfo): diff --git a/cads_processing_api_service/utils.py b/cads_processing_api_service/utils.py index 4e780f7..f697712 100644 --- a/cads_processing_api_service/utils.py +++ b/cads_processing_api_service/utils.py @@ -641,14 +641,12 @@ def make_status_info( finished=job["finished_at"], updated=job["updated_at"], ) - if any( - field is not None for field in [request, results, dataset_metadata, qos, log] - ): - status_info.metadata = models.StatusInfoMetadata( - request=request, - results=results, - datasetMetadata=dataset_metadata, - qos=qos, - log=log, - ) + status_info.metadata = models.StatusInfoMetadata( + origin=job.get("origin", None), + request=request, + results=results, + datasetMetadata=dataset_metadata, + qos=qos, + log=log, + ) return status_info diff --git a/tests/test_30_utils.py b/tests/test_30_utils.py index 4153944..b783dcc 100644 --- a/tests/test_30_utils.py +++ b/tests/test_30_utils.py @@ -322,6 +322,7 @@ def test_make_status_info() -> None: "finished_at": "2023-01-01T16:20:12.175021", "updated_at": "2023-01-01T16:20:12.175021", "request_body": {"request": {"product_type": ["reanalysis"]}}, + "origin": "api", } status_info = utils.make_status_info(job) exp_status_info = models.StatusInfo( @@ -333,5 +334,6 @@ def test_make_status_info() -> None: started=job["started_at"], finished=job["finished_at"], updated=job["updated_at"], + metadata={"origin": "api"}, ) assert status_info == exp_status_info From 6b4c8bdf8e0c12c4fb0da24c31a46ce0ae99efb3 Mon Sep 17 00:00:00 2001 From: Marco Cucchi Date: Tue, 12 Nov 2024 14:44:38 +0100 Subject: [PATCH 2/4] filter out dataset not tagged with request portal --- cads_processing_api_service/clients.py | 6 ++++++ cads_processing_api_service/utils.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/cads_processing_api_service/clients.py b/cads_processing_api_service/clients.py index d4cae30..eb3c8cf 100644 --- a/cads_processing_api_service/clients.py +++ b/cads_processing_api_service/clients.py @@ -219,6 +219,11 @@ def post_process_execution( ) structlog.contextvars.bind_contextvars(user_uid=auth_info.user_uid) request_body = execution_content.model_dump() + portals = ( + [p.strip() for p in auth_info.portal_header.split(",")] + if auth_info.portal_header + else None + ) catalogue_sessionmaker = db_utils.get_catalogue_sessionmaker( db_utils.ConnectionMode.read ) @@ -228,6 +233,7 @@ def post_process_execution( table=self.process_table, session=catalogue_session, load_messages=True, + portals=portals, ) auth.verify_if_disabled(dataset.disabled_reason, auth_info.user_role) adaptor_properties = adaptors.get_adaptor_properties(dataset) diff --git a/cads_processing_api_service/utils.py b/cads_processing_api_service/utils.py index 4e780f7..ab4837c 100644 --- a/cads_processing_api_service/utils.py +++ b/cads_processing_api_service/utils.py @@ -69,6 +69,7 @@ def lookup_resource_by_id( table: type[cads_catalogue.database.Resource], session: sqlalchemy.orm.Session, load_messages: bool = False, + portals: list[str] | None = None, ) -> cads_catalogue.database.Resource: """Look for the resource identified by `id` into the Catalogue database. @@ -82,6 +83,8 @@ def lookup_resource_by_id( Catalogue database session. load_messages : bool, optional If True, load resource messages, by default False. + portals: list[str] | None, optional + List of portal names, by default None. Returns ------- @@ -100,6 +103,8 @@ def lookup_resource_by_id( ) if load_messages: statement = statement.options(sqlalchemy.orm.joinedload(table.messages)) + if portals: + statement = statement.filter(table.portal.in_(portals)) statement = statement.filter(table.resource_uid == resource_id) try: row: cads_catalogue.database.Resource = ( From f40d3e0e7d2ea5d2ecc855698545b8c1d09862d5 Mon Sep 17 00:00:00 2001 From: Marco Cucchi Date: Wed, 13 Nov 2024 11:53:07 +0100 Subject: [PATCH 3/4] include portals in cache key computation --- cads_processing_api_service/clients.py | 2 +- cads_processing_api_service/utils.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cads_processing_api_service/clients.py b/cads_processing_api_service/clients.py index eb3c8cf..1d4e461 100644 --- a/cads_processing_api_service/clients.py +++ b/cads_processing_api_service/clients.py @@ -233,7 +233,7 @@ def post_process_execution( table=self.process_table, session=catalogue_session, load_messages=True, - portals=portals, + portals=tuple(portals), ) auth.verify_if_disabled(dataset.disabled_reason, auth_info.user_role) adaptor_properties = adaptors.get_adaptor_properties(dataset) diff --git a/cads_processing_api_service/utils.py b/cads_processing_api_service/utils.py index ab4837c..80ee0ed 100644 --- a/cads_processing_api_service/utils.py +++ b/cads_processing_api_service/utils.py @@ -62,14 +62,15 @@ class JobSortCriterion(str, enum.Enum): key=lambda resource_id, table, session, - load_messages=False: cachetools.keys.hashkey(resource_id, table, load_messages), + load_messages=False, + portals=None: cachetools.keys.hashkey(resource_id, table, load_messages, portals), ) def lookup_resource_by_id( resource_id: str, table: type[cads_catalogue.database.Resource], session: sqlalchemy.orm.Session, load_messages: bool = False, - portals: list[str] | None = None, + portals: tuple[str] | None = None, ) -> cads_catalogue.database.Resource: """Look for the resource identified by `id` into the Catalogue database. @@ -83,8 +84,8 @@ def lookup_resource_by_id( Catalogue database session. load_messages : bool, optional If True, load resource messages, by default False. - portals: list[str] | None, optional - List of portal names, by default None. + portals: tuple[str] | None, optional + Portals to filter resources by, by default None. Returns ------- From 74c9f7b5c1a3a8dcd8cb8ba238aede513a674cd9 Mon Sep 17 00:00:00 2001 From: Marco Cucchi Date: Wed, 13 Nov 2024 13:08:53 +0100 Subject: [PATCH 4/4] filter out dataset messages for which live==False --- cads_processing_api_service/clients.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cads_processing_api_service/clients.py b/cads_processing_api_service/clients.py index 1d4e461..4a6e3a4 100644 --- a/cads_processing_api_service/clients.py +++ b/cads_processing_api_service/clients.py @@ -301,6 +301,7 @@ def post_process_execution( content=message.content, ) for message in dataset.messages + if message.live ] url = str(request.url) if url.rstrip("/").endswith("execute"):