Show non-compliant models by default

matbench_discovery/enums.py

@@ -418,8 +418,11 @@ def geo_opt_path(self) -> str | None:
         """File path associated with the file URL if it exists, otherwise
         download the file first, then return the path.
         """
-        geo_opt_metrics = self.metrics.get("geo_opt", {})
-        if geo_opt_metrics in ("not available", "not applicable"):
+        geo_opt_metrics = self.metrics.get("geo_opt")
+        if geo_opt_metrics is None or geo_opt_metrics in (
+            "not available",
+            "not applicable",
+        ):
             return None
         rel_path = geo_opt_metrics.get("pred_file")
         file_url = geo_opt_metrics.get("pred_file_url")

matbench_discovery/metrics/geo_opt.py

@@ -93,7 +93,7 @@ def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> dict[str, float]:
     n_structs = len(spg_diff.dropna())
 
     # Fill NaN values with 1.0 (the stol value we set in StructureMatcher)
-    mean_rmsd = rmsd_vals.fillna(1.0).mean()
+    mean_rmsd = rmsd_vals.infer_objects(copy=False).fillna(1.0).mean()
     sym_ops_mae = n_sym_ops_diff.abs().mean()
 
     # Count cases where spacegroup changed

models/eSEN/eSEN-30m-mp.yml

@@ -1,5 +1,5 @@
 model_name: eSEN-30M-MP
-model_key: esen-30m-mp
+model_key: eSEN-30m-mp
 model_version: v2025.03.17
 matbench_discovery_version: 1.3.1
 date_added: "2025-03-17"

models/eSEN/eSEN-30m-oam.yml

@@ -1,5 +1,5 @@
 model_name: eSEN-30M-OAM
-model_key: esen-30m-oam
+model_key: eSEN-30m-oam
 model_version: v2025.03.17
 matbench_discovery_version: 1.3.1
 date_added: "2025-03-17"

models/grace/grace-2l-mptrj.yml

@@ -59,7 +59,7 @@ metrics:
   geo_opt:
     pred_file: models/grace/grace-2l-mptrj/2024-11-11-relaxed-structures.json.gz
     pred_file_url: https://figshare.com/files/52062590
-    struct_col: grace-2L-mp_structure
+    struct_col: grace2l_r6_structure
     symprec=1e-5:
       rmsd: 0.0186 # Å
       n_sym_ops_mae: 1.8703 # unitless

models/mattersim/mattersim-v1-5m.yml

@@ -130,7 +130,25 @@ metrics:
   geo_opt:
     pred_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.jsonl.gz
     pred_file_url: https://figshare.com/files/52062176
-    struct_col: mattersim-v1-5m_structure
+    struct_col: mattersim_5M_structure
+    symprec=1e-2:
+      rmsd: 0.0733 # Å
+      n_sym_ops_mae: 6.08 # unitless
+      symmetry_decrease: 0.03 # fraction
+      symmetry_match: 0.78 # fraction
+      symmetry_increase: 0.19 # fraction
+      n_structures: 100 # count
+      analysis_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.jsonl.gz-symprec=1e-2-moyo=0.4.2.csv.gz
+      analysis_file_url: https://figshare.com/files/53167865
+    symprec=1e-5:
+      rmsd: 0.0733 # Å
+      n_sym_ops_mae: 8.83 # unitless
+      symmetry_decrease: 0.05 # fraction
+      symmetry_match: 0.66 # fraction
+      symmetry_increase: 0.27 # fraction
+      n_structures: 100 # count
+      analysis_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.jsonl.gz-symprec=1e-5-moyo=0.4.2.csv.gz
+      analysis_file_url: https://figshare.com/files/53167868
   discovery:
     pred_file: models/mattersim/mattersim-v1-5M/2024-12-16-wbm-IS2RE.csv.gz
     # the original Graphormer-based replaced the M3GNet-based MatterSim on the leaderboard

scripts/analyze_geo_opt.py

@@ -70,13 +70,7 @@ def analyze_model_symprec(
 
     # Load model structures
     try:
-        if ml_relaxed_structs_path.endswith((".json", ".json.gz", ".json.xz")):
-            df_ml_structs = pd.read_json(ml_relaxed_structs_path)
-        else:
-            raise ValueError(
-                "Relaxed structure analysis currently only supports pymatgen JSON, "
-                f"got {ml_relaxed_structs_path}"
-            )
+        df_ml_structs = pd.read_json(ml_relaxed_structs_path, lines=True)
     except Exception as exc:
         exc.add_note(f"{model.label=} {ml_relaxed_structs_path=}")
         raise
@@ -187,7 +181,9 @@ def analyze_model_symprec(
     # %%
     print("Loading WBM PBE structures...")
     wbm_cse_path = DataFiles.wbm_computed_structure_entries.path
-    df_wbm_structs: pd.DataFrame = pd.read_json(wbm_cse_path).set_index(Key.mat_id)
+    df_wbm_structs: pd.DataFrame = pd.read_json(
+        wbm_cse_path, lines=True, orient="records"
+    ).set_index(Key.mat_id)
 
     if debug_mode:
         df_wbm_structs = df_wbm_structs.head(debug_mode)