Merge pull request #211 from ecmwf-projects/COPDS-2006-format-api-req…

…uest Format API request
ecmwf-projects · Sep 18, 2024 · cf8e05f · cf8e05f
2 parents 9ec2197 + 0e80610
commit cf8e05f
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 87 deletions.
diff --git a/cads_processing_api_service/config.py b/cads_processing_api_service/config.py
@@ -31,6 +31,15 @@
 client.retrieve(dataset, request).download()
 """
 
+API_REQUEST_MAX_LIST_LENGTH: dict[str, int] = {
+    "year": 3,
+    "month": 3,
+    "day": 3,
+    "time": 3,
+    "area": 4,
+    "pressure_level": 3,
+}
+
 ANONYMOUS_LICENCES_MESSAGE = (
     "The job has been submitted as an anonymous user. "
     "Please consider the following licences implicitly accepted: "
@@ -71,6 +80,7 @@ class Settings(pydantic_settings.BaseSettings):
     cache_resources_ttl: int = 10
 
     api_request_template: str = API_REQUEST_TEMPLATE
+    api_request_max_list_length: dict[str, int] = API_REQUEST_MAX_LIST_LENGTH
     missing_dataset_title: str = "Dataset not available"
     anonymous_licences_message: str = ANONYMOUS_LICENCES_MESSAGE
     deprecation_warning_message: str = DEPRECATION_WARNING_MESSAGE

diff --git a/cads_processing_api_service/translators.py b/cads_processing_api_service/translators.py
@@ -249,11 +249,38 @@ def translate_request_ids_into_labels(
     return request_labels
 
 
+def format_list(
+    value_list: list[int | float | str], max_items_per_line: int = 1
+) -> str:
+    if len(value_list) > max_items_per_line:
+        formatted = "[\n"
+        for i in range(0, len(value_list), max_items_per_line):
+            line = ", ".join(
+                f"'{item}'" if isinstance(item, str) else f"{item}"
+                for item in value_list[i : i + max_items_per_line]
+            )
+            formatted += f"        {line},\n"
+        formatted = formatted.rstrip(",\n") + "\n    ]"
+    else:
+        formatted = str(value_list)
+    return formatted
+
+
 def format_request_value(
-    request_value: str | list[str],
+    request_value: int | float | str | list[int | float | str],
+    key: str | None = None,
 ) -> str:
-    if isinstance(request_value, str):
-        formatted_request_value = f"'{request_value}'"
+    if isinstance(request_value, list):
+        if key is None:
+            formatted_request_value = format_list(request_value)
+        else:
+            api_request_max_list_length = (
+                config.ensure_settings().api_request_max_list_length
+            )
+            max_items_per_line = api_request_max_list_length.get(key, 1)
+            formatted_request_value = format_list(request_value, max_items_per_line)
+    elif isinstance(request_value, str):
+        formatted_request_value = f'"{request_value}"'
     else:
         formatted_request_value = str(request_value)
     return formatted_request_value
@@ -285,15 +312,15 @@ def format_api_request(
         "{"
         + ",".join(
             [
-                f"\n    '{key}': {format_request_value(value)}"
+                f'\n    "{key}": {format_request_value(value, key)}'
                 for key, value in request_inputs.items()
             ]
         )
         + "\n}"
     )
     api_request = api_request_template.format(
         process_id=process_id, api_request_kwargs=api_request_kwargs
-    )
+    ).replace("'", '"')
 
     return api_request
 

diff --git a/environment.yml b/environment.yml
@@ -10,6 +10,7 @@ channels:
 dependencies:
 - attrs
 - asyncpg
+- black
 - cachetools
 - fastapi>=0.109.0
 - pip

diff --git a/pyproject.toml b/pyproject.toml
@@ -15,6 +15,7 @@ classifiers = [
 ]
 dependencies = [
   "attrs",
+  "black",
   "cacholote",
   "cads-adaptors@git+https://github.com/ecmwf-projects/cads-adaptors.git",
   "cads-broker@git+https://github.com/ecmwf-projects/cads-broker.git",

diff --git a/tests/test_10_translators.py b/tests/test_10_translators.py
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# mypy: ignore-errors
+
 from typing import Any
 
-import cads_processing_api_service.translators
+from cads_processing_api_service import config, translators
 
 TEST_INPUT_CDS_SCHEMAS: dict[str, Any] = {
     "string_list": {
@@ -115,16 +117,12 @@ def test_extract_groups_labels() -> None:
         "val2": "Val2",
         "val3": "Val3",
     }
-    res_output = cads_processing_api_service.translators.extract_groups_labels(
-        test_groups, test_values
-    )
+    res_output = translators.extract_groups_labels(test_groups, test_values)
     assert res_output == exp_output
 
     test_groups = TEST_INPUT_CDS_SCHEMAS["string_list_array"]["details"]["groups"]
     exp_output = {"val1": "Val1", "val2": "Val2", "val3": "Val3"}
-    res_output = cads_processing_api_service.translators.extract_groups_labels(
-        test_groups
-    )
+    res_output = translators.extract_groups_labels(test_groups)
     assert res_output == exp_output
 
     test_groups = TEST_INPUT_CDS_SCHEMAS["string_list_array_groups"]["details"][
@@ -138,32 +136,24 @@ def test_extract_groups_labels() -> None:
         "val5": "Val5",
         "val6": "Val6",
     }
-    res_output = cads_processing_api_service.translators.extract_groups_labels(
-        test_groups
-    )
+    res_output = translators.extract_groups_labels(test_groups)
     assert res_output == exp_output
 
 
 def test_extract_labels() -> None:
     test_inputs_cds_schema = TEST_INPUT_CDS_SCHEMAS["string_list_array"]
     exp_output = {"val1": "Val1", "val2": "Val2", "val3": "Val3"}
-    res_output = cads_processing_api_service.translators.extract_labels(
-        test_inputs_cds_schema
-    )
+    res_output = translators.extract_labels(test_inputs_cds_schema)
     assert res_output == exp_output
 
     test_inputs_cds_schema = TEST_INPUT_CDS_SCHEMAS["string_list"]
     exp_output = {"val1": "Val1", "val2": "Val2", "val3": "Val3"}
-    res_output = cads_processing_api_service.translators.extract_labels(
-        test_inputs_cds_schema
-    )
+    res_output = translators.extract_labels(test_inputs_cds_schema)
     assert res_output == exp_output
 
     test_inputs_cds_schema = TEST_INPUT_CDS_SCHEMAS["free_edition_widget"]
     exp_output = {}
-    res_output = cads_processing_api_service.translators.extract_labels(
-        test_inputs_cds_schema
-    )
+    res_output = translators.extract_labels(test_inputs_cds_schema)
     assert res_output == exp_output
 
 
@@ -173,9 +163,7 @@ def test_translate_string_list() -> None:
         "type": "array",
         "items": {"type": "string", "enum": ["val1", "val2", "val3"]},
     }
-    res_output = cads_processing_api_service.translators.translate_string_list(
-        test_input
-    )
+    res_output = translators.translate_string_list(test_input)
     assert res_output == exp_ouput
 
 
@@ -185,9 +173,7 @@ def test_translate_string_list_array() -> None:
         "type": "array",
         "items": {"type": "string", "enum": ["val1", "val2", "val3"]},
     }
-    res_output = cads_processing_api_service.translators.translate_string_list_array(
-        test_input
-    )
+    res_output = translators.translate_string_list_array(test_input)
     assert res_output == exp_ouput
 
     test_input = TEST_INPUT_CDS_SCHEMAS["string_list_array_groups"]
@@ -198,18 +184,14 @@ def test_translate_string_list_array() -> None:
             "enum": ["val1", "val2", "val3", "val4", "val5", "val6"],
         },
     }
-    res_output = cads_processing_api_service.translators.translate_string_list_array(
-        test_input
-    )
+    res_output = translators.translate_string_list_array(test_input)
     assert res_output == exp_ouput
 
 
 def test_translate_string_choice() -> None:
     test_input = TEST_INPUT_CDS_SCHEMAS["string_choice"]
     exp_ouput = {"type": "string", "enum": ["val1", "val2", "val3"], "default": "val1"}
-    res_output = cads_processing_api_service.translators.translate_string_choice(
-        test_input
-    )
+    res_output = translators.translate_string_choice(test_input)
     assert res_output == exp_ouput
 
 
@@ -222,43 +204,39 @@ def test_translate_geographic_extent_map() -> None:
         "items": {"type": "number"},
         "default": [1, 2, 3, 4],
     }
-    res_output = (
-        cads_processing_api_service.translators.translate_geographic_extent_map(
-            test_input
-        )
-    )
+    res_output = translators.translate_geographic_extent_map(test_input)
     assert res_output == exp_ouput
 
 
 def test_make_request_labels() -> None:
     test_input_value_ids = ["1", "1", "1", "1"]
     test_input_cds_schema = TEST_INPUT_CDS_SCHEMAS["geographic_extent_map"]
     exp_output = ["North: 1°", "West: 1°", "South: 1°", "East: 1°"]
-    res_output = cads_processing_api_service.translators.make_request_labels(
+    res_output = translators.make_request_labels(
         test_input_value_ids, test_input_cds_schema
     )
     assert res_output == exp_output
 
     test_input_value_ids = [{"latitude": 10, "longitude": 10}]
     test_input_cds_schema = TEST_INPUT_CDS_SCHEMAS["geographic_location"]
     exp_output = ["Latitude: 10°", "Longitude: 10°"]
-    res_output = cads_processing_api_service.translators.make_request_labels(
+    res_output = translators.make_request_labels(
         test_input_value_ids, test_input_cds_schema
     )
     assert res_output == exp_output
 
     test_input_value_ids = ["val1", "val2"]
     test_input_cds_schema = TEST_INPUT_CDS_SCHEMAS["string_list"]
     exp_output = ["Val1", "Val2"]
-    res_output = cads_processing_api_service.translators.make_request_labels(
+    res_output = translators.make_request_labels(
         test_input_value_ids, test_input_cds_schema
     )
     assert res_output == exp_output
 
     test_input_value_ids = ["val1", "val4"]
     test_input_cds_schema = TEST_INPUT_CDS_SCHEMAS["string_list"]
     exp_output = ["Val1", "val4"]
-    res_output = cads_processing_api_service.translators.make_request_labels(
+    res_output = translators.make_request_labels(
         test_input_value_ids, test_input_cds_schema
     )
     assert res_output == exp_output
@@ -268,11 +246,7 @@ def test_translate_request_ids_into_labels() -> None:
     request = {"key1": "val1", "key2": "val2"}
     cds_schema = None
     exp_output = {"key1": "val1", "key2": "val2"}
-    res_output = (
-        cads_processing_api_service.translators.translate_request_ids_into_labels(
-            request, cds_schema
-        )
-    )
+    res_output = translators.translate_request_ids_into_labels(request, cds_schema)
     assert res_output == exp_output
 
     request = {
@@ -289,11 +263,7 @@ def test_translate_request_ids_into_labels() -> None:
         "String Choice": ["Val1"],
         "unknown_key": "unknown_value",
     }
-    res_output = (
-        cads_processing_api_service.translators.translate_request_ids_into_labels(
-            request, cds_schema
-        )
-    )
+    res_output = translators.translate_request_ids_into_labels(request, cds_schema)
     assert res_output == exp_output
 
     request = {}
@@ -309,52 +279,58 @@ def test_translate_request_ids_into_labels() -> None:
     }
 
 
+def test_format_list() -> None:
+    value_list = ["test_value_1", "test_value_2"]
+    max_items_per_line = 1
+    exp_output = "[\n        'test_value_1',\n        'test_value_2'\n    ]"
+    res_output = translators.format_list(value_list, max_items_per_line)
+    assert res_output == exp_output
+
+    max_items_per_line = 2
+    exp_output = "['test_value_1', 'test_value_2']"
+    res_output = translators.format_list(value_list, max_items_per_line)
+    assert res_output == exp_output
+
+
 def test_format_request_value() -> None:
-    test_value_1 = "test_value"
-    exp_output_1 = "'test_value'"
-    res_output_1 = cads_processing_api_service.translators.format_request_value(
-        test_value_1
-    )
-    assert res_output_1 == exp_output_1
+    test_value = "test_value"
+    exp_output = '"test_value"'
+    res_output = translators.format_request_value(test_value)
+    assert res_output == exp_output
 
-    test_value_2 = ["test_value_1", "test_value_2"]
-    exp_output_2 = "['test_value_1', 'test_value_2']"
-    res_output_2 = cads_processing_api_service.translators.format_request_value(
-        test_value_2
-    )
-    assert res_output_2 == exp_output_2
+    test_value = 1
+    exp_output = "1"
+    res_output = translators.format_request_value(test_value)
+    assert res_output == exp_output
+
+    test_value = ["test_value_1", "test_value_2"]
+    exp_output = "[\n        'test_value_1',\n        'test_value_2'\n    ]"
+    res_output = translators.format_request_value(test_value)
+    assert res_output == exp_output
 
 
 def test_format_api_request() -> None:
-    test_api_request_template = (
-        "import cads_api_client\n\n"
-        "request = {api_request_kwargs}\n\n"
-        "client = cads_api_client.ApiClient()\n"
-        "client.retrieve(\n\t"
-        "collection_id='{process_id}',\n\t"
-        "**request\n"
-        ")\n"
-    )
+    test_api_request_template = config.API_REQUEST_TEMPLATE
     test_process_id = "test_process_id"
     test_request = {
         "inputs": {
-            "variable": "test_variable_1",
-            "year": ["2000", "2001"],
+            "variable_1": "value_1",
+            "variable_2": ["value_1", "value_2"],
+            "variable_3": 1,
         }
     }
     exp_output = (
-        "import cads_api_client\n\n"
+        "import cdsapi\n\n"
+        'dataset = "test_process_id"\n'
         "request = {\n"
-        "    'variable': 'test_variable_1',\n"
-        "    'year': ['2000', '2001']\n"
+        '    "variable_1": "value_1",\n'
+        '    "variable_2": [\n        "value_1",\n        "value_2"\n    ],\n'
+        '    "variable_3": 1\n'
         "}\n\n"
-        "client = cads_api_client.ApiClient()\n"
-        "client.retrieve(\n\t"
-        "collection_id='test_process_id',\n\t"
-        "**request\n"
-        ")\n"
+        "client = cdsapi.Client()\n"
+        "client.retrieve(dataset, request).download()\n"
     )
-    res_output = cads_processing_api_service.translators.format_api_request(
+    res_output = translators.format_api_request(
         test_api_request_template, test_process_id, test_request
     )
     assert res_output == exp_output
-Original file line number
+Diff line change
@@ Expand Up / @@ -10,6 +10,7 @@ channels: @@
     dependencies:
     - attrs
     - asyncpg
+    - black
     - cachetools
     - fastapi>=0.109.0
     - pip
@@ Expand Down @@