Update model_prices_and_context_window.json #13804
+169
−0
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2295,6 +2295,38 @@ | |
| "supports_native_streaming": true, | ||
| "supports_reasoning": true | ||
| }, | ||
| "azure/gpt-5-dzs": { | ||
| "max_tokens": 128000, | ||
| "max_input_tokens": 272000, | ||
| "max_output_tokens": 128000, | ||
| "input_cost_per_token": 1.25e-06, | ||
| "output_cost_per_token": 1e-05, | ||
| "cache_read_input_token_cost": 1.25e-07, | ||
| "litellm_provider": "azure", | ||
| "mode": "chat", | ||
| "supported_endpoints": [ | ||
| "/v1/chat/completions", | ||
| "/v1/batch", | ||
| "/v1/responses" | ||
| ], | ||
| "supported_modalities": [ | ||
| "text", | ||
| "image" | ||
| ], | ||
| "supported_output_modalities": [ | ||
| "text" | ||
| ], | ||
| "supports_pdf_input": true, | ||
| "supports_function_calling": true, | ||
| "supports_parallel_function_calling": true, | ||
| "supports_response_schema": true, | ||
| "supports_vision": true, | ||
| "supports_prompt_caching": true, | ||
| "supports_system_messages": true, | ||
| "supports_tool_choice": true, | ||
| "supports_native_streaming": true, | ||
| "supports_reasoning": true | ||
| }, | ||
| "azure/gpt-5-2025-08-07": { | ||
| "max_tokens": 128000, | ||
| "max_input_tokens": 272000, | ||
|
|
@@ -2359,6 +2391,38 @@ | |
| "supports_native_streaming": true, | ||
| "supports_reasoning": true | ||
| }, | ||
| "azure/gpt-5-mini-dzs": { | ||
| "max_tokens": 128000, | ||
| "max_input_tokens": 272000, | ||
| "max_output_tokens": 128000, | ||
| "input_cost_per_token": 2.5e-07, | ||
| "output_cost_per_token": 2e-06, | ||
| "cache_read_input_token_cost": 2.5e-08, | ||
| "litellm_provider": "azure", | ||
| "mode": "chat", | ||
| "supported_endpoints": [ | ||
| "/v1/chat/completions", | ||
| "/v1/batch", | ||
| "/v1/responses" | ||
| ], | ||
| "supported_modalities": [ | ||
| "text", | ||
| "image" | ||
| ], | ||
| "supported_output_modalities": [ | ||
| "text" | ||
| ], | ||
| "supports_pdf_input": true, | ||
| "supports_function_calling": true, | ||
| "supports_parallel_function_calling": true, | ||
| "supports_response_schema": true, | ||
| "supports_vision": true, | ||
| "supports_prompt_caching": true, | ||
| "supports_system_messages": true, | ||
| "supports_tool_choice": true, | ||
| "supports_native_streaming": true, | ||
| "supports_reasoning": true | ||
| }, | ||
| "azure/gpt-5-mini-2025-08-07": { | ||
| "max_tokens": 128000, | ||
| "max_input_tokens": 272000, | ||
|
|
@@ -2703,6 +2767,44 @@ | |
| "search_context_size_high": 0.05 | ||
| } | ||
| }, | ||
| "azure/gpt-41-gs": { | ||
|
There was a problem hiding this comment. @MohammedSamirMahmoud the name change would break cost tracking based on returned azure model i recommend having global standard, be the 'default' pricing (no special |
||
| "max_tokens": 32768, | ||
| "max_input_tokens": 1047576, | ||
| "max_output_tokens": 32768, | ||
| "input_cost_per_token": 2e-06, | ||
| "output_cost_per_token": 8e-06, | ||
| "input_cost_per_token_batches": 1e-06, | ||
| "output_cost_per_token_batches": 4e-06, | ||
| "cache_read_input_token_cost": 5e-07, | ||
| "litellm_provider": "azure", | ||
| "mode": "chat", | ||
| "supported_endpoints": [ | ||
| "/v1/chat/completions", | ||
| "/v1/batch", | ||
| "/v1/responses" | ||
| ], | ||
| "supported_modalities": [ | ||
| "text", | ||
| "image" | ||
| ], | ||
| "supported_output_modalities": [ | ||
| "text" | ||
| ], | ||
| "supports_function_calling": true, | ||
| "supports_parallel_function_calling": true, | ||
| "supports_response_schema": true, | ||
| "supports_vision": true, | ||
| "supports_prompt_caching": true, | ||
| "supports_system_messages": true, | ||
| "supports_tool_choice": true, | ||
| "supports_native_streaming": true, | ||
| "supports_web_search": true, | ||
| "search_context_cost_per_query": { | ||
| "search_context_size_low": 0.03, | ||
| "search_context_size_medium": 0.035, | ||
| "search_context_size_high": 0.05 | ||
| } | ||
| }, | ||
| "azure/gpt-4.1-mini": { | ||
| "max_tokens": 32768, | ||
| "max_input_tokens": 1047576, | ||
|
|
@@ -2741,6 +2843,44 @@ | |
| "search_context_size_high": 0.03 | ||
| } | ||
| }, | ||
| "azure/gpt-41-mini-gs": { | ||
| "max_tokens": 32768, | ||
| "max_input_tokens": 1047576, | ||
| "max_output_tokens": 32768, | ||
| "input_cost_per_token": 4e-07, | ||
| "output_cost_per_token": 1.6e-06, | ||
| "input_cost_per_token_batches": 2e-07, | ||
| "output_cost_per_token_batches": 8e-07, | ||
| "cache_read_input_token_cost": 1e-07, | ||
| "litellm_provider": "azure", | ||
| "mode": "chat", | ||
| "supported_endpoints": [ | ||
| "/v1/chat/completions", | ||
| "/v1/batch", | ||
| "/v1/responses" | ||
| ], | ||
| "supported_modalities": [ | ||
| "text", | ||
| "image" | ||
| ], | ||
| "supported_output_modalities": [ | ||
| "text" | ||
| ], | ||
| "supports_function_calling": true, | ||
| "supports_parallel_function_calling": true, | ||
| "supports_response_schema": true, | ||
| "supports_vision": true, | ||
| "supports_prompt_caching": true, | ||
| "supports_system_messages": true, | ||
| "supports_tool_choice": true, | ||
| "supports_native_streaming": true, | ||
| "supports_web_search": true, | ||
| "search_context_cost_per_query": { | ||
| "search_context_size_low": 0.025, | ||
| "search_context_size_medium": 0.0275, | ||
| "search_context_size_high": 0.03 | ||
| } | ||
| }, | ||
| "azure/gpt-4.1-mini-2025-04-14": { | ||
| "max_tokens": 32768, | ||
| "max_input_tokens": 1047576, | ||
|
|
@@ -3022,6 +3162,35 @@ | |
| "supports_reasoning": true, | ||
| "supports_tool_choice": true | ||
| }, | ||
| "azure/o4-mini-gs": { | ||
| "max_tokens": 100000, | ||
| "max_input_tokens": 200000, | ||
| "max_output_tokens": 100000, | ||
| "input_cost_per_token": 1.1e-06, | ||
| "output_cost_per_token": 4.4e-06, | ||
| "cache_read_input_token_cost": 2.75e-07, | ||
| "litellm_provider": "azure", | ||
| "mode": "chat", | ||
| "supported_endpoints": [ | ||
| "/v1/chat/completions", | ||
| "/v1/batch", | ||
| "/v1/responses" | ||
| ], | ||
| "supported_modalities": [ | ||
| "text", | ||
| "image" | ||
| ], | ||
| "supported_output_modalities": [ | ||
| "text" | ||
| ], | ||
| "supports_function_calling": true, | ||
| "supports_parallel_function_calling": false, | ||
| "supports_vision": true, | ||
| "supports_prompt_caching": true, | ||
| "supports_response_schema": true, | ||
| "supports_reasoning": true, | ||
| "supports_tool_choice": true | ||
| }, | ||
| "azure/gpt-4o-mini-realtime-preview-2024-12-17": { | ||
| "max_tokens": 4096, | ||
| "max_input_tokens": 128000, | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is the pricing the same across all data zones?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
if not - what does
-dzsmean? and how would it account for regional differencesThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hey @krrishdholakia ,
So during the usage of litellm as a proxy server.
For Azure models we are setting up the following naming convention as a standard
azure/gpt-4.1 models:
gpt-41-gs --> Stands for Azure GPT 4.1 models deployed as Global standard
gpt-41-dzs --> Stands for Azure GPT 4.1 models deployed as Data Zone Standard (DZS)
gpt-41--> Stands for Azure GPT 4.1 models deployed as Global standard
gpt-5-dzs --> Stands for Azure GPT 5 models deployed as Data Zone Standard (DZS)
gpt-5-mini-dzs--> Stands for Azure GPT 5 models deployed as Data Zone Standard (DZS)
Now while calling these models as completions--> we get alot of informative headers from LiteLLM including model_region, model_tokens, model_cost, etc.
However when doing this using the responses endpoint, no headers are retrieved at all!
Typical error when starting LiteLLM Proxy Server:
{"message": "checking potential_model_names in litellm.model_cost: {'split_model': 'gpt-5-dzs', 'combined_model_name': 'azure/gpt-5-dzs', 'stripped_model_name': 'azure/gpt-5-dzs', 'combined_stripped_model_name': 'azure/gpt-5-dzs', 'custom_llm_provider': 'azure'}", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.045934"}
{"message": "Error getting model info: This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.045948"}
{"message": "added/updated model=azure/gpt-5-dzs in litellm.model_cost: azure/gpt-5-dzs", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.045969"}
{"message": "checking potential_model_names in litellm.model_cost: {'split_model': '69260937c7da004ba35663898e26638c0074cdfac2bc2a437189a78a2562c3bc', 'combined_model_name': '69260937c7da004ba35663898e26638c0074cdfac2bc2a437189a78a2562c3bc', 'stripped_model_name': '69260937c7da004ba35663898e26638c0074cdfac2bc2a437189a78a2562c3bc', 'combined_stripped_model_name': '69260937c7da004ba35663898e26638c0074cdfac2bc2a437189a78a2562c3bc', 'custom_llm_provider': None}", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046139"}
{"message": "Error getting model info: This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046157"}
{"message": "added/updated model=69260937c7da004ba35663898e26638c0074cdfac2bc2a437189a78a2562c3bc in litellm.model_cost: 69260937c7da004ba35663898e26638c0074cdfac2bc2a437189a78a2562c3bc", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046170"}
{"message": "checking potential_model_names in litellm.model_cost: {'split_model': 'responses/gpt-5-dzs', 'combined_model_name': 'azure/responses/gpt-5-dzs', 'stripped_model_name': 'azure/responses/gpt-5-dzs', 'combined_stripped_model_name': 'azure/responses/gpt-5-dzs', 'custom_llm_provider': 'azure'}", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046202"}
{"message": "Error getting model info: This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046215"}
{"message": "added/updated model=azure/responses/gpt-5-dzs in litellm.model_cost: azure/responses/gpt-5-dzs", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046227"}
{"message": "checking potential_model_names in litellm.model_cost: {'split_model': 'ce064fe813df870a53d7eaccbc5bac1fd09ea19673047cbfae522d57a33c7278', 'combined_model_name': 'ce064fe813df870a53d7eaccbc5bac1fd09ea19673047cbfae522d57a33c7278', 'stripped_model_name': 'ce064fe813df870a53d7eaccbc5bac1fd09ea19673047cbfae522d57a33c7278', 'combined_stripped_model_name': 'ce064fe813df870a53d7eaccbc5bac1fd09ea19673047cbfae522d57a33c7278', 'custom_llm_provider': None}", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046398"}
{"message": "Error getting model info: This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046415"}
{"message": "added/updated model=ce064fe813df870a53d7eaccbc5bac1fd09ea19673047cbfae522d57a33c7278 in litellm.model_cost: ce064fe813df870a53d7eaccbc5bac1fd09ea19673047cbfae522d57a33c7278", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046429"}
{"message": "checking potential_model_names in litellm.model_cost: {'split_model': 'gpt-5-mini-dzs', 'combined_model_name': 'azure/gpt-5-mini-dzs', 'stripped_model_name': 'azure/gpt-5-mini-dzs', 'combined_stripped_model_name': 'azure/gpt-5-mini-dzs', 'custom_llm_provider': 'azure'}", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046459"}
{"message": "Error getting model info: This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046473"}
{"message": "added/updated model=azure/gpt-5-mini-dzs in litellm.model_cost: azure/gpt-5-mini-dzs", "level": "DEBUG", "timestamp": "2025-08-21T13:19:18.046484"}
This typical behavior occurs only with responses endpoint, whenever the deployment-name is not exactly as your model maps, when i tried to create a new deployment following just whatever inside your model_maps --> azure/gpt-5, it worked fine! and retrieved at least one header which is the model_cost, that's why i was trying to add it inside the file,
might be from your logic, there is a certain place in responses api that is not acting as the chat completion when retrieving or mapping deployment-name.