Orb v3

contributing.md

@@ -130,7 +130,7 @@ To submit a new model to this benchmark and add it to our leaderboard, please cr
    url: https://<model-docs-or-similar>.org
    doi: https://doi.org/10.5281/zenodo.0000000
    preprint: https://arxiv.org/abs/xxxx.xxxxx
-   checkpoint_url: https://github.com/<org>/<repo>/releases/download/<tag>/<model-file> # checkpoint URL can be any URL but GitHub release URLs are encouraged
+   checkpoint_url: https://url.of/model-checkpoint-file # should be the direct download URL of the model checkpoint file. If there's a page documenting the checkpoint, append that as a YAML comment.
 
    openness: OSOD # see `Open` enum in matbench_discovery/enums.py
    train_task: S2EFS # see `Task` enum in matbench_discovery/enums.py
@@ -141,6 +141,13 @@ To submit a new model to this benchmark and add it to our leaderboard, please cr
    trained_for_benchmark: true
    n_estimators: 1
 
+   license:
+     code: MIT
+     code_url: https://url.of/code-license
+     checkpoint: CC-BY-4.0
+     # URL that points to the license file for the model checkpoint, not the checkpoint file itself.
+     checkpoint_url: https://url.of/model-checkpoint-license
+
    hyperparams: # strongly recommended to list relaxation hyperparams
       max_force: 0.05
       max_steps: 500

matbench_discovery/enums.py

@@ -344,6 +344,7 @@ class Model(Files, base_dir=f"{ROOT}/models"):
     # ORB
     orb_v2 = auto(), "orb/orb-v2.yml"
     orb_v2_mptrj = auto(), "orb/orb-v2-mptrj.yml"
+    orb_v3 = auto(), "orb/orb-v3.yml"
 
     # SevenNet trained on MPtrj
     # sevennet_0 = auto(), "sevennet/sevennet-0.yml"

models/orb/orb-v2-mptrj.yml

@@ -72,7 +72,7 @@ hyperparams:
   radial_cutoff: 10.0 # Å
   max_num_neighbors: 20
 
-requirements: # strongly recommended
+requirements:
   orb-models: "0.4.0"
   pynanoflann: "pynanoflann@git+https://github.com/dwastberg/pynanoflann#egg=af434039ae14bedcbb838a7808924d6689274168"
 

models/orb/orb-v2.yml

@@ -72,7 +72,7 @@ hyperparams:
   radial_cutoff: 10.0 # Å
   max_num_neighbors: 20
 
-requirements: # strongly recommended
+requirements:
   orb-models: "0.4.0"
   pynanoflann: "pynanoflann@git+https://github.com/dwastberg/pynanoflann#egg=af434039ae14bedcbb838a7808924d6689274168"
 

models/orb/orb-v3.yml

@@ -0,0 +1,170 @@
+model_name: ORB v3
+model_key: orb-v3
+model_version: v3
+matbench_discovery_version: 1.3.1
+date_added: "2025-04-05"
+date_published: "2025-04-05"
+authors:
+  - name: Ben Rhodes
+    affiliation: Orbital Materials
+    corresponding: true
+  - name: Sander Vandenhaute
+    affiliation: Orbital Materials
+  - name: Vaidotas Šimkus
+    affiliation: Orbital Materials
+  - name: James Gin-Pollock
+    affiliation: Orbital Materials
+  - name: Tim Duignan
+    affiliation: Orbital Materials
+  - name: Jonathan Godwin
+    affiliation: Orbital Materials
+    email: jonathan@orbitalmaterials.com
+  - name: Mark Neumann
+    affiliation: Orbital Materials
+    email: mark@orbitalmaterials.com
+    corresponding: true
+
+repo: https://github.com/orbital-materials/orb-models
+url: https://github.com/orbital-materials/orb-models
+doi: https://github.com/orbital-materials/orb-models
+paper: https://github.com/orbital-materials/orb-models
+pr_url: https://github.com/janosh/matbench-discovery/pull/240
+checkpoint_url: https://orbitalmaterials-public-models.s3.us-west-1.amazonaws.com/forcefields/orb-v3/orb-v3-conservative-inf-mpa-20250404.ckpt
+
+license:
+  code: Apache-2.0
+  code_url: https://github.com/orbital-materials/orb-models/blob/3e9580809/LICENSE
+  checkpoint: Apache-2.0
+  checkpoint_url: https://github.com/orbital-materials/orb-models/blob/3e9580809/LICENSE
+
+openness: OSCD
+trained_for_benchmark: true
+train_task: S2EFS
+test_task: IS2RE-SR
+targets: EFS_G
+model_type: UIP
+model_params: 25_510_582
+n_estimators: 1
+
+hyperparams:
+  max_force: 0.02
+  max_steps: 500
+  ase_optimizer: FIRE
+  cell_filter: FrechetCellFilter
+  optimizer: Adam
+  loss: MAE
+  loss_weights:
+    energy: 10.0
+    force: 1.0
+    stress: 1.0
+  batch_size: 300 # (avg, as batch size is dynamic)
+  initial_learning_rate: 0.0003
+  learning_rate_schedule: CosineAnnealingLR(T_max=100)
+  gradient_clip: 0.5
+  ema_decay: 0.999
+  n_layers: 5
+  features: 512 latent dim, 1024 MLP dim
+  num_rbfs: 8
+  radial_cutoff: 6.0 # Å
+  max_num_neighbors: 120
+
+requirements:
+  orb-models: "0.5.0"
+
+training_set: [MPtrj, Alex, OMat24]
+training_cost:
+  Nvidia H100: { amount: 8, hours: 125, cost: nan }
+
+notes:
+  Description: |
+    ORB-v3 is a pretrained model for atomic simulations. ORB-v3 uses a pretrained diffusion model backbone which is fine-tuned on MPtrj + Alexandria + OMAT.
+
+metrics:
+  phonons:
+    kappa_103:
+      κ_SRME: 0.21
+      pred_file: models/orb/orb-v3/2025-04-04-kappa-103-disp0.03-float64-orb-v3-con-inf-mpa.json.gz
+      pred_file_url: https://figshare.com/files/53461061
+  geo_opt:
+    pred_file: models/orb/orb-v3/2025-04-04-wbm-geo-opt-orb-v3-con-inf-mpa.jsonl.gz
+    pred_file_url: https://figshare.com/files/53461040
+    struct_col: orb_structure
+    symprec=1e-5:
+      rmsd: 0.075 # unitless
+      n_sym_ops_mae: 10.169 # unitless
+      symmetry_decrease: 0.869 # fraction
+      symmetry_match: 0.130 # fraction
+      symmetry_increase: 0.0001 # fraction
+      n_structures: 256963 # count
+      analysis_file: models/orb/orb-v3/2025-04-04-wbm-geo-opt-symprec=1e-5-moyo=0.4.2-orb-v3-con-inf-mpa.csv.gz
+      analysis_file_url: https://figshare.com/files/53461055
+    symprec=1e-2:
+      rmsd: 0.075 # unitless
+      n_sym_ops_mae: 2.609 # unitless
+      symmetry_decrease: 0.159 # fraction
+      symmetry_match: 0.728 # fraction
+      symmetry_increase: 0.101 # fraction
+      n_structures: 256963 # count
+      analysis_file: models/orb/orb-v3/2025-04-04-wbm-geo-opt-symprec=1e-2-moyo=0.4.2-orb-v3-con-inf-mpa.csv.gz
+      analysis_file_url: https://figshare.com/files/53461058
+  discovery:
+    pred_file: models/orb/orb-v3/2025-04-04-wbm-IS2RE-orb-v3-con-inf-mpa.csv.gz
+    pred_file_url: https://figshare.com/files/53461037
+    pred_col: e_form_per_atom_orb
+    full_test_set:
+      F1: 0.887 # fraction
+      DAF: 5.284 # dimensionless
+      Precision: 0.881 # fraction
+      Recall: 0.894 # fraction
+      Accuracy: 0.962 # fraction
+      TPR: 0.894 # fraction
+      FPR: 0.024 # fraction
+      TNR: 0.976 # fraction
+      FNR: 0.106 # fraction
+      TP: 38305.0 # count
+      FP: 5193.0 # count
+      TN: 208945.0 # count
+      FN: 4520.0 # count
+      MAE: 0.023 # eV/atom
+      RMSE: 0.078 # eV/atom
+      R2: 0.815 # dimensionless
+      missing_preds: 2 # count
+      missing_percent: 0.00% # fraction
+    most_stable_10k:
+      F1: 0.982 # fraction
+      DAF: 6.307 # dimensionless
+      Precision: 0.964 # fraction
+      Recall: 1.0 # fraction
+      Accuracy: 0.964 # fraction
+      TPR: 1.0 # fraction
+      FPR: 1.0 # fraction
+      TNR: 0.0 # fraction
+      FNR: 0.0 # fraction
+      TP: 9641.0 # count
+      FP: 359.0 # count
+      TN: 0.0 # count
+      FN: 0.0 # count
+      MAE: 0.021 # eV/atom
+      RMSE: 0.080 # eV/atom
+      R2: 0.860 # dimensionless
+      missing_preds: 0 # count
+      missing_percent: 0.00% # fraction
+    unique_prototypes:
+      F1: 0.906 # fraction
+      DAF: 5.906 # dimensionless
+      Precision: 0.903 # fraction
+      Recall: 0.908 # fraction
+      Accuracy: 0.971 # fraction
+      TPR: 0.908 # fraction
+      FPR: 0.018 # fraction
+      TNR: 0.982 # fraction
+      FNR: 0.092 # fraction
+      TP: 29916.0 # count
+      FP: 3216.0 # count
+      TN: 179330.0 # count
+      FN: 3026.0 # count
+      MAE: 0.024 # eV/atom
+      RMSE: 0.078 # eV/atom
+      R2: 0.821 # dimensionless
+      missing_preds: 0 # count
+      missing_percent: 0.00% # fraction