Add token 128011 to the Llama tokenizer (#2428)

pytorch · Mar 3, 2025 · 80da6a5 · 80da6a5
1 parent 67a8706
commit 80da6a5
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/torchtune/models/llama3/_tokenizer.py b/torchtune/models/llama3/_tokenizer.py
@@ -31,9 +31,10 @@
     "<|python_tag|>": 128010,
     "<|image|>": 128256,
     "<|video|>": 128012,
+    "<|reserved_special_token_244|>": 128011,
 }
 
-NUM_RESERVED_SPECIAL_TOKENS = 256
+NUM_RESERVED_SPECIAL_TOKENS = 257
 
 RESERVED_TOKENS = {
     f"<|reserved_special_token_{2 + i}|>": 128013 + i