diff --git a/matbench_discovery/enums.py b/matbench_discovery/enums.py index 9188b9b0..9d911bde 100644 --- a/matbench_discovery/enums.py +++ b/matbench_discovery/enums.py @@ -418,8 +418,11 @@ def geo_opt_path(self) -> str | None: """File path associated with the file URL if it exists, otherwise download the file first, then return the path. """ - geo_opt_metrics = self.metrics.get("geo_opt", {}) - if geo_opt_metrics in ("not available", "not applicable"): + geo_opt_metrics = self.metrics.get("geo_opt") + if geo_opt_metrics is None or geo_opt_metrics in ( + "not available", + "not applicable", + ): return None rel_path = geo_opt_metrics.get("pred_file") file_url = geo_opt_metrics.get("pred_file_url") diff --git a/matbench_discovery/metrics/geo_opt.py b/matbench_discovery/metrics/geo_opt.py index 299b5b36..36364b0a 100644 --- a/matbench_discovery/metrics/geo_opt.py +++ b/matbench_discovery/metrics/geo_opt.py @@ -93,7 +93,7 @@ def calc_geo_opt_metrics(df_model_analysis: pd.DataFrame) -> dict[str, float]: n_structs = len(spg_diff.dropna()) # Fill NaN values with 1.0 (the stol value we set in StructureMatcher) - mean_rmsd = rmsd_vals.fillna(1.0).mean() + mean_rmsd = rmsd_vals.infer_objects(copy=False).fillna(1.0).mean() sym_ops_mae = n_sym_ops_diff.abs().mean() # Count cases where spacegroup changed diff --git a/models/eSEN/eSEN-30m-mp.yml b/models/eSEN/eSEN-30m-mp.yml index b1765729..7fab86d0 100644 --- a/models/eSEN/eSEN-30m-mp.yml +++ b/models/eSEN/eSEN-30m-mp.yml @@ -1,5 +1,5 @@ model_name: eSEN-30M-MP -model_key: esen-30m-mp +model_key: eSEN-30m-mp model_version: v2025.03.17 matbench_discovery_version: 1.3.1 date_added: "2025-03-17" diff --git a/models/eSEN/eSEN-30m-oam.yml b/models/eSEN/eSEN-30m-oam.yml index 55c4c506..821f3e71 100644 --- a/models/eSEN/eSEN-30m-oam.yml +++ b/models/eSEN/eSEN-30m-oam.yml @@ -1,5 +1,5 @@ model_name: eSEN-30M-OAM -model_key: esen-30m-oam +model_key: eSEN-30m-oam model_version: v2025.03.17 matbench_discovery_version: 1.3.1 date_added: "2025-03-17" diff --git a/models/grace/grace-2l-mptrj.yml b/models/grace/grace-2l-mptrj.yml index b565cd6b..7ca072e3 100644 --- a/models/grace/grace-2l-mptrj.yml +++ b/models/grace/grace-2l-mptrj.yml @@ -59,7 +59,7 @@ metrics: geo_opt: pred_file: models/grace/grace-2l-mptrj/2024-11-11-relaxed-structures.json.gz pred_file_url: https://figshare.com/files/52062590 - struct_col: grace-2L-mp_structure + struct_col: grace2l_r6_structure symprec=1e-5: rmsd: 0.0186 # Å n_sym_ops_mae: 1.8703 # unitless diff --git a/models/mattersim/mattersim-v1-5m.yml b/models/mattersim/mattersim-v1-5m.yml index 710b5ef0..b8f3efa8 100644 --- a/models/mattersim/mattersim-v1-5m.yml +++ b/models/mattersim/mattersim-v1-5m.yml @@ -130,7 +130,25 @@ metrics: geo_opt: pred_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.jsonl.gz pred_file_url: https://figshare.com/files/52062176 - struct_col: mattersim-v1-5m_structure + struct_col: mattersim_5M_structure + symprec=1e-2: + rmsd: 0.0733 # Å + n_sym_ops_mae: 6.08 # unitless + symmetry_decrease: 0.03 # fraction + symmetry_match: 0.78 # fraction + symmetry_increase: 0.19 # fraction + n_structures: 100 # count + analysis_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.jsonl.gz-symprec=1e-2-moyo=0.4.2.csv.gz + analysis_file_url: https://figshare.com/files/53167865 + symprec=1e-5: + rmsd: 0.0733 # Å + n_sym_ops_mae: 8.83 # unitless + symmetry_decrease: 0.05 # fraction + symmetry_match: 0.66 # fraction + symmetry_increase: 0.27 # fraction + n_structures: 100 # count + analysis_file: models/mattersim/mattersim-v1-5M/2024-12-19-wbm-geo-opt.jsonl.gz-symprec=1e-5-moyo=0.4.2.csv.gz + analysis_file_url: https://figshare.com/files/53167868 discovery: pred_file: models/mattersim/mattersim-v1-5M/2024-12-16-wbm-IS2RE.csv.gz # the original Graphormer-based replaced the M3GNet-based MatterSim on the leaderboard diff --git a/scripts/analyze_geo_opt.py b/scripts/analyze_geo_opt.py index aa0d92ae..03a809f6 100644 --- a/scripts/analyze_geo_opt.py +++ b/scripts/analyze_geo_opt.py @@ -70,13 +70,7 @@ def analyze_model_symprec( # Load model structures try: - if ml_relaxed_structs_path.endswith((".json", ".json.gz", ".json.xz")): - df_ml_structs = pd.read_json(ml_relaxed_structs_path) - else: - raise ValueError( - "Relaxed structure analysis currently only supports pymatgen JSON, " - f"got {ml_relaxed_structs_path}" - ) + df_ml_structs = pd.read_json(ml_relaxed_structs_path, lines=True) except Exception as exc: exc.add_note(f"{model.label=} {ml_relaxed_structs_path=}") raise @@ -187,7 +181,9 @@ def analyze_model_symprec( # %% print("Loading WBM PBE structures...") wbm_cse_path = DataFiles.wbm_computed_structure_entries.path - df_wbm_structs: pd.DataFrame = pd.read_json(wbm_cse_path).set_index(Key.mat_id) + df_wbm_structs: pd.DataFrame = pd.read_json( + wbm_cse_path, lines=True, orient="records" + ).set_index(Key.mat_id) if debug_mode: df_wbm_structs = df_wbm_structs.head(debug_mode) diff --git a/site/src/figs/metrics-table-uip-megnet-combos.svelte b/site/src/figs/metrics-table-uip-megnet-combos.svelte deleted file mode 100644 index 7b21911a..00000000 --- a/site/src/figs/metrics-table-uip-megnet-combos.svelte +++ /dev/null @@ -1,235 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelF1 ↑DAF ↑Prec ↑Acc ↑TPR ↑TNR ↑MAE ↓RMSE ↓R2Training SizeModel Type
CHGNet0.613.040.520.840.740.860.060.100.691.6M (145.9K)UIP-GNN
M3GNet0.582.650.450.800.790.800.070.120.58188.3K (62.8K)UIP-GNN
M3GNet→MEGNet0.552.540.430.790.750.800.090.140.36188.3K (62.8K)UIP-GNN
CHGNet→MEGNet0.552.560.430.790.740.800.090.150.341.6M (145.9K)UIP-GNN
MEGNet0.512.700.460.810.570.860.130.20-0.28133.4KGNN
diff --git a/site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte b/site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte deleted file mode 100644 index a777a0d2..00000000 --- a/site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte +++ /dev/null @@ -1,231 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ModelF1 ↑DAF ↑Prec ↑Acc ↑TPR ↑TNR ↑MAE ↓RMSE ↓R2Training SizeModel Type
CHGNet0.613.320.510.850.760.870.060.100.691.6M (145.9K)UIP-GNN
M3GNet0.572.840.440.810.800.810.070.120.58188.3K (62.8K)UIP-GNN
M3GNet→MEGNet0.542.740.420.800.760.810.090.140.38188.3K (62.8K)UIP-GNN
CHGNet→MEGNet0.542.760.420.800.750.810.090.150.361.6M (145.9K)UIP-GNN
MEGNet0.512.920.450.830.580.870.130.21-0.25133.4KGNN
diff --git a/site/src/lib/HeatmapTable.svelte b/site/src/lib/HeatmapTable.svelte index 190696d8..a0122d8d 100644 --- a/site/src/lib/HeatmapTable.svelte +++ b/site/src/lib/HeatmapTable.svelte @@ -226,7 +226,7 @@ {#each clean_data as row (JSON.stringify(row))} - + {#each visible_columns as col (col.label + col.group)} {@const val = row[get_col_id(col)]} {@const color = calc_color(val, col)} diff --git a/site/src/lib/MetricsTable.svelte b/site/src/lib/MetricsTable.svelte index fdfefdaf..feb225b1 100644 --- a/site/src/lib/MetricsTable.svelte +++ b/site/src/lib/MetricsTable.svelte @@ -50,6 +50,7 @@ // Make metric_config reactive with $state to properly handle updates let metric_config = $state({ ...config }) + const [compliant_clr, noncompliant_clr] = [`#4caf50`, `#4682b4`] // Update metric_config when config prop changes $effect(() => { @@ -245,6 +246,7 @@ ) .map((model) => { const metrics = model.metrics?.discovery?.[discovery_set] + const is_compliant = model_is_compliant(model) // Get RMSD from geo_opt metrics if available, using the first symprec value const geo_opt_metrics = model.metrics?.geo_opt @@ -311,6 +313,9 @@ pr_url: { url: model.pr_url, title: `View pull request`, icon: `🔗` }, pred_files: { files: get_pred_file_urls(model), name: model.model_name }, } as LinkData, + row_style: show_noncompliant + ? `border-left: 3px solid ${is_compliant ? compliant_clr : noncompliant_clr};` + : null, } }) // Sort by combined score (descending) @@ -372,14 +377,26 @@ {...rest} > {#snippet controls()} - { - handle_filter_change(show_energy, show_noncomp) - }} - /> +
+
+ {#if show_noncompliant} + {#each [[compliant_clr, `Compliant`], [noncompliant_clr, `Non-compliant`]] as [clr, label]} +
+ + {label} +
+ {/each} + {/if} + { + handle_filter_change(show_energy, show_noncomp) + }} + /> +
+
{/snippet} {#snippet cell({ col, val })} @@ -448,13 +465,6 @@ diff --git a/site/src/lib/TableControls.svelte b/site/src/lib/TableControls.svelte index 9dcdbcb6..2fb1b0e4 100644 --- a/site/src/lib/TableControls.svelte +++ b/site/src/lib/TableControls.svelte @@ -89,26 +89,16 @@ gap: 1rem; flex-wrap: wrap; } - .info-icon-small { - font-size: 0.8em; - margin-left: 0.2em; - opacity: 0.7; - cursor: help; + font-size: 0.9em; } - .filter-option { display: flex; align-items: center; - font-size: 0.85em; gap: 0.3em; - cursor: pointer; - white-space: nowrap; } - /* Fix for sub and sup tags */ - :global(sub), - :global(sup) { + :global(:is(sub, sup)) { font-size: 0.7em; } diff --git a/site/src/lib/html-to-img.ts b/site/src/lib/html-to-img.ts index 40bebdf8..51c6f0d6 100644 --- a/site/src/lib/html-to-img.ts +++ b/site/src/lib/html-to-img.ts @@ -10,10 +10,7 @@ export async function generate_svg({ }) { try { // Find the metrics table - const table_el = - document.querySelector(`.heatmap-table`) || - document.querySelector(`.metrics-table table`) || - document.querySelector(`.metrics-table .heatmap-table-container table`) + const table_el = document.querySelector(`.heatmap-table`) if (!table_el) { console.error(`Table element not found for SVG export`) @@ -121,10 +118,7 @@ export async function generate_png({ }) { try { // Find the metrics table - const table_el = - document.querySelector(`.heatmap-table`) || - document.querySelector(`.metrics-table table`) || - document.querySelector(`.metrics-table .heatmap-table-container table`) + const table_el = document.querySelector(`.heatmap-table`) if (!table_el) { console.error(`Table element not found for PNG export`) diff --git a/site/src/lib/types.ts b/site/src/lib/types.ts index ec416704..cb85e481 100644 --- a/site/src/lib/types.ts +++ b/site/src/lib/types.ts @@ -142,5 +142,9 @@ export type CellVal = | null | Record | LinkData -export type RowData = Record + +export type RowData = Record & { + row_attributes?: Record +} + export type TableData = RowData[] diff --git a/site/src/routes/+page.svelte b/site/src/routes/+page.svelte index 2ad9c42e..844dfa48 100644 --- a/site/src/routes/+page.svelte +++ b/site/src/routes/+page.svelte @@ -14,13 +14,12 @@ import Readme from '$root/readme.md' import KappaNote from '$site/src/routes/kappa-note.md' import { pretty_num } from 'elementari' - import 'iconify-icon' import { Tooltip } from 'svelte-zoo' let n_wbm_stable_uniq_protos = 32_942 let n_wbm_uniq_protos = 215_488 - let show_non_compliant: boolean = $state(false) + let show_non_compliant: boolean = $state(true) let show_energy_only: boolean = $state(false) let show_combined_controls: boolean = $state(true) let export_error: string | null = $state(null) @@ -86,7 +85,7 @@ {#snippet metrics_table()} -
+
{#each Object.entries(DISCOVERY_SET_LABELS) as [key, { title, tooltip, link }] (key)} @@ -101,8 +100,9 @@ target="_blank" rel="noopener noreferrer" aria-label="Info" + style="line-height: 1;" > - + ⓘ {/if} @@ -110,16 +110,20 @@ {/each}
- visible_cols[col.label] ?? true} - model_filter={() => true} - {discovery_set} - {show_combined_controls} - {show_energy_only} - show_noncompliant={show_non_compliant} - config={metric_config} - style="width: 100%;" - /> +
+
+ visible_cols[col.label] ?? true} + model_filter={() => true} + {discovery_set} + {show_combined_controls} + {show_energy_only} + show_noncompliant={show_non_compliant} + config={metric_config} + style="width: 100%;" + /> +
+
Download table as @@ -152,7 +156,7 @@ adjust the importance of each metric component.

Training size is the number of materials used to train the model. For models trained - on DFT relaxations, we show the number of distinct frames in parentheses. In cases + on DFT relaxations, we show the number of distinct frames in parentheses). In cases where only the number of frames is known, we report the number of frames as the training set size. (N=x) in the Model Params column shows the number of estimators if an ensemble was used. DAF = Discovery Acceleration Factor measures @@ -231,6 +235,14 @@ display: grid; gap: 1ex; } + /* Table wrapper for full-width placement */ + .table-wrapper { + /* Use negative margin technique for full width */ + width: calc(100vw - 20px); + margin-left: calc(-50vw + 50% + 10px); + display: flex; + justify-content: center; + } figcaption { font-size: 0.9em; padding: 2pt 6pt; @@ -273,9 +285,6 @@ margin-top: 0.5em; flex-basis: 100%; } - div.export-error iconify-icon { - color: #ff6b6b; - } /* Caption Radar Container Styles */ figcaption.caption-radar-container { diff --git a/site/tests/contribute-page.test.ts b/site/tests/ContributePage.test.ts similarity index 100% rename from site/tests/contribute-page.test.ts rename to site/tests/ContributePage.test.ts diff --git a/site/tests/data-page.test.ts b/site/tests/DataPage.test.ts similarity index 100% rename from site/tests/data-page.test.ts rename to site/tests/DataPage.test.ts diff --git a/site/tests/diatomic-page.test.ts b/site/tests/DiatomicPage.test.ts similarity index 100% rename from site/tests/diatomic-page.test.ts rename to site/tests/DiatomicPage.test.ts diff --git a/site/tests/heatmap-table.test.svelte.ts b/site/tests/HeatmapTable.test.svelte.ts similarity index 100% rename from site/tests/heatmap-table.test.svelte.ts rename to site/tests/HeatmapTable.test.svelte.ts diff --git a/site/tests/landing-page.test.ts b/site/tests/LandingPage.test.ts similarity index 98% rename from site/tests/landing-page.test.ts rename to site/tests/LandingPage.test.ts index 0cc863bb..c32b4f9b 100644 --- a/site/tests/landing-page.test.ts +++ b/site/tests/LandingPage.test.ts @@ -13,7 +13,7 @@ describe(`Landing Page`, () => { const button_texts = Array.from(buttons).map((btn) => btn.textContent?.trim()) expect(button_texts).toContain(`Full Test Set`) - expect(button_texts).toContain(`Unique Prototypes`) + expect(button_texts).toContain(`Unique Prototypes ⓘ`) expect(button_texts).toContain(`10k Most Stable`) }) @@ -56,16 +56,16 @@ describe(`Landing Page`, () => { expect(toggle).toBeDefined() // Should be unchecked by default - expect(toggle?.checked).toBe(false) + expect(toggle?.checked).toBe(true) // get number of table rows const n_models_on_load = document.body.querySelectorAll(`tbody tr`).length // Click to show non-compliant models toggle?.click() - expect(toggle?.checked).toBe(true) + expect(toggle?.checked).toBe(false) await tick() const n_all_models = document.body.querySelectorAll(`tbody tr`).length - expect(n_all_models).toBeGreaterThan(n_models_on_load) + expect(n_all_models).toBeLessThan(n_models_on_load) }) it(`updates column visibility when toggling checkboxes`, async () => { diff --git a/site/tests/metrics-table.test.svelte.ts b/site/tests/MetricsTable.test.svelte.ts similarity index 100% rename from site/tests/metrics-table.test.svelte.ts rename to site/tests/MetricsTable.test.svelte.ts diff --git a/site/tests/model-card.test.svelte.ts b/site/tests/ModelCard.test.svelte.ts similarity index 100% rename from site/tests/model-card.test.svelte.ts rename to site/tests/ModelCard.test.svelte.ts diff --git a/site/tests/model-page.test.ts b/site/tests/ModelPage.test.ts similarity index 100% rename from site/tests/model-page.test.ts rename to site/tests/ModelPage.test.ts diff --git a/site/tests/models-page.test.ts b/site/tests/ModelsPage.test.ts similarity index 100% rename from site/tests/models-page.test.ts rename to site/tests/ModelsPage.test.ts diff --git a/site/tests/radar-chart.test.svelte.ts b/site/tests/RadarChart.test.svelte.ts similarity index 100% rename from site/tests/radar-chart.test.svelte.ts rename to site/tests/RadarChart.test.svelte.ts diff --git a/site/tests/table-column-toggle-menu.test.ts b/site/tests/TableColumnToggleMenu.test.ts similarity index 100% rename from site/tests/table-column-toggle-menu.test.ts rename to site/tests/TableColumnToggleMenu.test.ts diff --git a/site/tests/table-controls.test.svelte.ts b/site/tests/TableControls.test.svelte.ts similarity index 100% rename from site/tests/table-controls.test.svelte.ts rename to site/tests/TableControls.test.svelte.ts