Merge pull request #992 from dandi/validation_errors

Logging validation warnings and errors

dandi/bids_validator_xs.py

@@ -513,7 +513,7 @@ def write_report(
         for regex_entry in validation_result["schema_listing"]:
             f.write(f'\n\t- `{regex_entry["regex"]}`')
         f.write("\n")
-        if len(validation_result["path_tracking"]) > 0:
+        if validation_result["path_tracking"]:
             f.write("The following files were not matched by any regex schema entry:")
             f.write("\n\t* `")
             f.write("`\n\t* `".join(validation_result["path_tracking"]))
@@ -652,6 +652,32 @@ def select_schema_dir(
         )
 
 
+def log_errors(validation_result):
+    """
+    Raise errors for validation result.
+
+    Parameters
+    ----------
+    validation_result : dict
+        A dictionary as returned by `validate_all()` with keys including "schema_tracking",
+        "path_tracking", "path_listing", and, optionally "itemwise".
+        The "itemwise" value, if present, should be a list of dictionaries, with keys including
+        "path", "regex", and "match".
+    """
+    total_file_count = len(validation_result["path_listing"])
+    validated_files_count = total_file_count - len(validation_result["path_tracking"])
+    if validated_files_count == 0:
+        lgr.error("No valid BIDS files were found.")
+    for entry in validation_result["schema_tracking"]:
+        if entry["mandatory"]:
+            lgr.error(
+                "The `%s` regex pattern file required by BIDS was not found.",
+                entry["regex"],
+            )
+    for i in validation_result["path_tracking"]:
+        lgr.warning("The `%s` file was not matched by any regex schema entry.", i)
+
+
 def validate_bids(
     bids_paths,
     schema_reference_root="{module_path}/support/bids/schemadata/",
@@ -718,4 +744,6 @@ def validate_bids(
         else:
             write_report(validation_result)
 
+    log_errors(validation_result)
+
     return validation_result

dandi/cli/cmd_validate.py

@@ -4,6 +4,7 @@
 import click
 
 from .base import devel_debug_option, devel_option, lgr, map_to_click_exceptions
+from ..utils import pluralize
 
 
 @click.command()
@@ -30,12 +31,39 @@ def validate_bids(
     if report_flag and not report:
         report = report_flag
 
-    validate_bids_(
+    validation_result = validate_bids_(
         *paths,
         report=report,
         schema_version=schema,
         devel_debug=devel_debug,
     )
+    missing_files = [
+        pattern["regex"]
+        for pattern in validation_result["schema_tracking"]
+        if pattern["mandatory"]
+    ]
+    error_list = []
+    if missing_files:
+        error_substring = (
+            f"{pluralize(len(missing_files), 'filename pattern')} required "
+            "by BIDS could not be found"
+        )
+        error_list.append(error_substring)
+    if validation_result["path_tracking"]:
+        error_substring = (
+            f"{pluralize(len(validation_result['path_tracking']), 'filename')} "
+            "did not match any pattern known to BIDS"
+        )
+        error_list.append(error_substring)
+    if error_list:
+        error_string = " and ".join(error_list)
+        error_string = f"Summary: {error_string}."
+        click.secho(
+            error_string,
+            bold=True,
+            fg="red",
+        )
+        raise SystemExit(1)
 
 
 @click.command()

dandi/cli/tests/test_cmd_validate.py

@@ -0,0 +1,19 @@
+import os
+
+from click.testing import CliRunner
+
+
+def test_validate_bids_error(bids_examples):
+    from ..cmd_validate import validate_bids
+
+    expected_error = (
+        "Summary: 2 filename patterns required by BIDS could not be found "
+        "and 1 filename did not match any pattern known to BIDS.\n"
+    )
+    broken_dataset = os.path.join(bids_examples, "invalid_pet001")
+    r = CliRunner().invoke(validate_bids, [broken_dataset])
+    # Does it break?
+    assert r.exit_code == 1
+
+    # Does it report the issue correctly?
+    assert r.output == expected_error

dandi/tests/test_bids_validator_xs.py

@@ -1,5 +1,8 @@
 import os
 
+BIDS_EXAMPLES_BLACKLIST = [
+    "invalid_pet001",
+]
 BIDS_EXAMPLES_WHITELIST = [
     "asl003",
     "eeg_cbm",
@@ -317,6 +320,19 @@ def test_bids_datasets(bids_examples):
             assert len(result["path_tracking"]) == 0
 
 
+def test_error_datasets(bids_examples):
+    from dandi.bids_validator_xs import validate_bids
+
+    # Validate per dataset, with automatic schema selection:
+    for i in os.listdir(bids_examples):
+        if i in BIDS_EXAMPLES_BLACKLIST:
+            result = validate_bids(
+                os.path.join(bids_examples, i),
+            )
+            # Are there non-validated files?
+            assert len(result["path_tracking"]) != 0
+
+
 def test_bids_datasets_selected_paths(bids_examples, tmp_path):
     from dandi.bids_validator_xs import validate_bids