From 3f3117fcb49956b4d6898580c7451a04c4b1cb44 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 00:29:46 +0000 Subject: [PATCH 01/14] add systeminformation package for gpu detection --- packages/core/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/package.json b/packages/core/package.json index be8d9c6a107..edb3c128abb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -70,6 +70,7 @@ "langchain": "0.3.6", "ollama-ai-provider": "0.16.1", "openai": "4.73.0", + "systeminformation": "5.23.5", "tinyld": "1.3.4", "together-ai": "0.7.0", "unique-names-generator": "4.7.1", From b01a9390c6ff9873258fb61204b46b8bc42cbc76 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 00:30:24 +0000 Subject: [PATCH 02/14] rearchitect models --- packages/core/src/models.ts | 999 ++++++++++++++++++------------------ 1 file changed, 509 insertions(+), 490 deletions(-) diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index 53d3b8be244..b76854cd5d5 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -1,497 +1,516 @@ import settings from "./settings.ts"; import { Models, ModelProviderName, ModelClass } from "./types.ts"; +import { elizaLogger } from "./index.ts"; +import si from "systeminformation"; -export const models: Models = { - [ModelProviderName.OPENAI]: { - endpoint: "https://api.openai.com/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.0, - presence_penalty: 0.0, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: settings.SMALL_OPENAI_MODEL || "gpt-4o-mini", - [ModelClass.MEDIUM]: settings.MEDIUM_OPENAI_MODEL || "gpt-4o", - [ModelClass.LARGE]: settings.LARGE_OPENAI_MODEL || "gpt-4o", - [ModelClass.EMBEDDING]: settings.EMBEDDING_OPENAI_MODEL || "text-embedding-3-small", - [ModelClass.IMAGE]: settings.IMAGE_OPENAI_MODEL || "dall-e-3", - }, - }, - [ModelProviderName.ETERNALAI]: { - endpoint: settings.ETERNALAI_URL, - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.0, - presence_penalty: 0.0, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: - settings.ETERNALAI_MODEL || - "neuralmagic/Meta-Llama-3.1-405B-Instruct-quantized.w4a16", - [ModelClass.MEDIUM]: - settings.ETERNALAI_MODEL || - "neuralmagic/Meta-Llama-3.1-405B-Instruct-quantized.w4a16", - [ModelClass.LARGE]: - settings.ETERNALAI_MODEL || - "neuralmagic/Meta-Llama-3.1-405B-Instruct-quantized.w4a16", - [ModelClass.EMBEDDING]: "", - [ModelClass.IMAGE]: "", - }, - }, - [ModelProviderName.ANTHROPIC]: { - settings: { - stop: [], - maxInputTokens: 200000, - maxOutputTokens: 4096, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - endpoint: "https://api.anthropic.com/v1", - model: { - [ModelClass.SMALL]: settings.SMALL_ANTHROPIC_MODEL || "claude-3-haiku-20240307", - [ModelClass.MEDIUM]: settings.MEDIUM_ANTHROPIC_MODEL || "claude-3-5-sonnet-20241022", - [ModelClass.LARGE]: settings.LARGE_ANTHROPIC_MODEL || "claude-3-5-sonnet-20241022", - }, - }, - [ModelProviderName.CLAUDE_VERTEX]: { - settings: { - stop: [], - maxInputTokens: 200000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - endpoint: "https://api.anthropic.com/v1", // TODO: check - model: { - [ModelClass.SMALL]: "claude-3-5-sonnet-20241022", - [ModelClass.MEDIUM]: "claude-3-5-sonnet-20241022", - [ModelClass.LARGE]: "claude-3-opus-20240229", - }, - }, - [ModelProviderName.GROK]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - endpoint: "https://api.x.ai/v1", - model: { - [ModelClass.SMALL]: settings.SMALL_GROK_MODEL || "grok-2-1212", - [ModelClass.MEDIUM]: settings.MEDIUM_GROK_MODEL || "grok-2-1212", - [ModelClass.LARGE]: settings.LARGE_GROK_MODEL || "grok-2-1212", - [ModelClass.EMBEDDING]: settings.EMBEDDING_GROK_MODEL || "grok-2-1212", // not sure about this one - }, - }, - [ModelProviderName.GROQ]: { - endpoint: "https://api.groq.com/openai/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8000, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - model: { - [ModelClass.SMALL]: - settings.SMALL_GROQ_MODEL || "llama-3.1-8b-instant", - [ModelClass.MEDIUM]: - settings.MEDIUM_GROQ_MODEL || "llama-3.3-70b-versatile", - [ModelClass.LARGE]: - settings.LARGE_GROQ_MODEL || "llama-3.2-90b-vision-preview", - [ModelClass.EMBEDDING]: - settings.EMBEDDING_GROQ_MODEL || "llama-3.1-8b-instant", - }, - }, - [ModelProviderName.LLAMACLOUD]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - repetition_penalty: 0.4, - temperature: 0.7, - }, - imageSettings: { - steps: 4, - }, - endpoint: "https://api.llamacloud.com/v1", - model: { - [ModelClass.SMALL]: "meta-llama/Llama-3.2-3B-Instruct-Turbo", - [ModelClass.MEDIUM]: "meta-llama-3.1-8b-instruct", - [ModelClass.LARGE]: "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - [ModelClass.EMBEDDING]: - "togethercomputer/m2-bert-80M-32k-retrieval", - [ModelClass.IMAGE]: "black-forest-labs/FLUX.1-schnell", - }, - }, - [ModelProviderName.TOGETHER]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - repetition_penalty: 0.4, - temperature: 0.7, - }, - imageSettings: { - steps: 4, - }, - endpoint: "https://api.together.ai/v1", - model: { - [ModelClass.SMALL]: "meta-llama/Llama-3.2-3B-Instruct-Turbo", - [ModelClass.MEDIUM]: "meta-llama-3.1-8b-instruct", - [ModelClass.LARGE]: "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - [ModelClass.EMBEDDING]: - "togethercomputer/m2-bert-80M-32k-retrieval", - [ModelClass.IMAGE]: "black-forest-labs/FLUX.1-schnell", - }, - }, - [ModelProviderName.LLAMALOCAL]: { - settings: { - stop: ["<|eot_id|>", "<|eom_id|>"], - maxInputTokens: 32768, - maxOutputTokens: 8192, - repetition_penalty: 0.4, - temperature: 0.7, - }, - model: { - [ModelClass.SMALL]: - "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true", - [ModelClass.MEDIUM]: - "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true", // TODO: ?download=true - [ModelClass.LARGE]: - "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true", - // "RichardErkhov/NousResearch_-_Meta-Llama-3.1-70B-gguf", // TODO: - [ModelClass.EMBEDDING]: - "togethercomputer/m2-bert-80M-32k-retrieval", - }, - }, - [ModelProviderName.GOOGLE]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - model: { - [ModelClass.SMALL]: - settings.SMALL_GOOGLE_MODEL || - settings.GOOGLE_MODEL || - "gemini-1.5-flash-latest", - [ModelClass.MEDIUM]: - settings.MEDIUM_GOOGLE_MODEL || - settings.GOOGLE_MODEL || - "gemini-1.5-flash-latest", - [ModelClass.LARGE]: - settings.LARGE_GOOGLE_MODEL || - settings.GOOGLE_MODEL || - "gemini-1.5-pro-latest", - [ModelClass.EMBEDDING]: - settings.EMBEDDING_GOOGLE_MODEL || - settings.GOOGLE_MODEL || - "text-embedding-004", - }, - }, - [ModelProviderName.REDPILL]: { - endpoint: "https://api.red-pill.ai/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.0, - presence_penalty: 0.0, - temperature: 0.6, - }, - // Available models: https://docs.red-pill.ai/get-started/supported-models - // To test other models, change the models below - model: { - [ModelClass.SMALL]: - settings.SMALL_REDPILL_MODEL || - settings.REDPILL_MODEL || - "gpt-4o-mini", - [ModelClass.MEDIUM]: - settings.MEDIUM_REDPILL_MODEL || - settings.REDPILL_MODEL || - "gpt-4o", - [ModelClass.LARGE]: - settings.LARGE_REDPILL_MODEL || - settings.REDPILL_MODEL || - "gpt-4o", - [ModelClass.EMBEDDING]: "text-embedding-3-small", - }, - }, - [ModelProviderName.OPENROUTER]: { - endpoint: "https://openrouter.ai/api/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - // Available models: https://openrouter.ai/models - // To test other models, change the models below - model: { - [ModelClass.SMALL]: - settings.SMALL_OPENROUTER_MODEL || - settings.OPENROUTER_MODEL || - "nousresearch/hermes-3-llama-3.1-405b", - [ModelClass.MEDIUM]: - settings.MEDIUM_OPENROUTER_MODEL || - settings.OPENROUTER_MODEL || - "nousresearch/hermes-3-llama-3.1-405b", - [ModelClass.LARGE]: - settings.LARGE_OPENROUTER_MODEL || - settings.OPENROUTER_MODEL || - "nousresearch/hermes-3-llama-3.1-405b", - [ModelClass.EMBEDDING]: "text-embedding-3-small", - }, - }, - [ModelProviderName.OLLAMA]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.7, - }, - endpoint: settings.OLLAMA_SERVER_URL || "http://localhost:11434", - model: { - [ModelClass.SMALL]: - settings.SMALL_OLLAMA_MODEL || - settings.OLLAMA_MODEL || - "llama3.2", - [ModelClass.MEDIUM]: - settings.MEDIUM_OLLAMA_MODEL || - settings.OLLAMA_MODEL || - "hermes3", - [ModelClass.LARGE]: - settings.LARGE_OLLAMA_MODEL || - settings.OLLAMA_MODEL || - "hermes3:70b", - [ModelClass.EMBEDDING]: - settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large", - }, - }, - [ModelProviderName.HEURIST]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - repetition_penalty: 0.4, - temperature: 0.7, - }, - imageSettings: { - steps: 20, - }, - endpoint: "https://llm-gateway.heurist.xyz", - model: { - [ModelClass.SMALL]: - settings.SMALL_HEURIST_MODEL || - "meta-llama/llama-3-70b-instruct", - [ModelClass.MEDIUM]: - settings.MEDIUM_HEURIST_MODEL || - "meta-llama/llama-3-70b-instruct", - [ModelClass.LARGE]: - settings.LARGE_HEURIST_MODEL || - "meta-llama/llama-3.1-405b-instruct", - [ModelClass.EMBEDDING]: "", //Add later, - [ModelClass.IMAGE]: settings.HEURIST_IMAGE_MODEL || "PepeXL", - }, - }, - [ModelProviderName.GALADRIEL]: { - endpoint: "https://api.galadriel.com/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.5, - presence_penalty: 0.5, - temperature: 0.8, - }, - model: { - [ModelClass.SMALL]: "llama3.1:70b", - [ModelClass.MEDIUM]: "llama3.1:70b", - [ModelClass.LARGE]: "llama3.1:405b", - [ModelClass.EMBEDDING]: "gte-large-en-v1.5", - [ModelClass.IMAGE]: "stabilityai/stable-diffusion-xl-base-1.0", - }, - }, - [ModelProviderName.FAL]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - repetition_penalty: 0.4, - temperature: 0.7, - }, - imageSettings: { - steps: 28, - }, - endpoint: "https://api.fal.ai/v1", - model: { - [ModelClass.SMALL]: "", // FAL doesn't provide text models - [ModelClass.MEDIUM]: "", - [ModelClass.LARGE]: "", - [ModelClass.EMBEDDING]: "", - [ModelClass.IMAGE]: "fal-ai/flux-lora", - }, - }, - [ModelProviderName.GAIANET]: { - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - repetition_penalty: 0.4, - temperature: 0.7, - }, - endpoint: settings.GAIANET_SERVER_URL, - model: { - [ModelClass.SMALL]: - settings.GAIANET_MODEL || - settings.SMALL_GAIANET_MODEL || - "llama3b", - [ModelClass.MEDIUM]: - settings.GAIANET_MODEL || - settings.MEDIUM_GAIANET_MODEL || - "llama", - [ModelClass.LARGE]: - settings.GAIANET_MODEL || - settings.LARGE_GAIANET_MODEL || - "qwen72b", - [ModelClass.EMBEDDING]: - settings.GAIANET_EMBEDDING_MODEL || "nomic-embed", - }, - }, - [ModelProviderName.ALI_BAILIAN]: { - endpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: "qwen-turbo", - [ModelClass.MEDIUM]: "qwen-plus", - [ModelClass.LARGE]: "qwen-max", - [ModelClass.IMAGE]: "wanx-v1", - }, - }, - [ModelProviderName.VOLENGINE]: { - endpoint: "https://open.volcengineapi.com/api/v3/", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.4, - presence_penalty: 0.4, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: "doubao-lite-128k", - [ModelClass.MEDIUM]: "doubao-pro-128k", - [ModelClass.LARGE]: "doubao-pro-128k", - [ModelClass.EMBEDDING]: "doubao-embedding", - }, - }, - [ModelProviderName.NANOGPT]: { - endpoint: "https://nano-gpt.com/api/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - frequency_penalty: 0.0, - presence_penalty: 0.0, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: settings.SMALL_NANOGPT_MODEL || "gpt-4o-mini", - [ModelClass.MEDIUM]: settings.MEDIUM_NANOGPT_MODEL || "gpt-4o", - [ModelClass.LARGE]: settings.LARGE_NANOGPT_MODEL || "gpt-4o", - } - }, - [ModelProviderName.HYPERBOLIC]: { - endpoint: "https://api.hyperbolic.xyz/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: - settings.SMALL_HYPERBOLIC_MODEL || - settings.HYPERBOLIC_MODEL || - "meta-llama/Llama-3.2-3B-Instruct", - [ModelClass.MEDIUM]: - settings.MEDIUM_HYPERBOLIC_MODEL || - settings.HYPERBOLIC_MODEL || - "meta-llama/Meta-Llama-3.1-70B-Instruct", - [ModelClass.LARGE]: - settings.LARGE_HYPERBOLIC_MODEL || - settings.HYPERBOLIC_MODEL || - "meta-llama/Meta-Llama-3.1-405-Instruct", - [ModelClass.IMAGE]: settings.IMAGE_HYPERBOLIC_MODEL || "FLUX.1-dev", - }, - }, - [ModelProviderName.VENICE]: { - endpoint: "https://api.venice.ai/api/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: settings.SMALL_VENICE_MODEL || "llama-3.3-70b", - [ModelClass.MEDIUM]: settings.MEDIUM_VENICE_MODEL || "llama-3.3-70b", - [ModelClass.LARGE]: settings.LARGE_VENICE_MODEL || "llama-3.1-405b", - [ModelClass.IMAGE]: settings.IMAGE_VENICE_MODEL || "fluently-xl", - }, - }, - [ModelProviderName.AKASH_CHAT_API]: { - endpoint: "https://chatapi.akash.network/api/v1", - settings: { - stop: [], - maxInputTokens: 128000, - maxOutputTokens: 8192, - temperature: 0.6, - }, - model: { - [ModelClass.SMALL]: - settings.SMALL_AKASH_CHAT_API_MODEL || - "Meta-Llama-3-2-3B-Instruct", - [ModelClass.MEDIUM]: - settings.MEDIUM_AKASH_CHAT_API_MODEL || - "Meta-Llama-3-3-70B-Instruct", - [ModelClass.LARGE]: - settings.LARGE_AKASH_CHAT_API_MODEL || - "Meta-Llama-3-1-405B-Instruct-FP8", - }, - }, -}; +function generateModels(hasGPU:Boolean) { + elizaLogger.debug('generateModels') + // image and text models mixed + const models: Models = { + [ModelProviderName.OPENAI]: { + endpoint: "https://api.openai.com/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: settings.SMALL_OPENAI_MODEL || "gpt-4o-mini", + [ModelClass.MEDIUM]: settings.MEDIUM_OPENAI_MODEL || "gpt-4o", + [ModelClass.LARGE]: settings.LARGE_OPENAI_MODEL || "gpt-4o", + [ModelClass.EMBEDDING]: settings.EMBEDDING_OPENAI_MODEL || "text-embedding-3-small", + [ModelClass.IMAGE]: settings.IMAGE_OPENAI_MODEL || "dall-e-3", + }, + }, + [ModelProviderName.ETERNALAI]: { + endpoint: settings.ETERNALAI_URL, + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: + settings.ETERNALAI_MODEL || + "neuralmagic/Meta-Llama-3.1-405B-Instruct-quantized.w4a16", + [ModelClass.MEDIUM]: + settings.ETERNALAI_MODEL || + "neuralmagic/Meta-Llama-3.1-405B-Instruct-quantized.w4a16", + [ModelClass.LARGE]: + settings.ETERNALAI_MODEL || + "neuralmagic/Meta-Llama-3.1-405B-Instruct-quantized.w4a16", + [ModelClass.EMBEDDING]: "", + [ModelClass.IMAGE]: "", + }, + }, + [ModelProviderName.ANTHROPIC]: { + settings: { + stop: [], + maxInputTokens: 200000, + maxOutputTokens: 4096, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + endpoint: "https://api.anthropic.com/v1", + model: { + [ModelClass.SMALL]: settings.SMALL_ANTHROPIC_MODEL || "claude-3-haiku-20240307", + [ModelClass.MEDIUM]: settings.MEDIUM_ANTHROPIC_MODEL || "claude-3-5-sonnet-20241022", + [ModelClass.LARGE]: settings.LARGE_ANTHROPIC_MODEL || "claude-3-5-sonnet-20241022", + }, + }, + [ModelProviderName.CLAUDE_VERTEX]: { + settings: { + stop: [], + maxInputTokens: 200000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + endpoint: "https://api.anthropic.com/v1", // TODO: check + model: { + [ModelClass.SMALL]: "claude-3-5-sonnet-20241022", + [ModelClass.MEDIUM]: "claude-3-5-sonnet-20241022", + [ModelClass.LARGE]: "claude-3-opus-20240229", + }, + }, + [ModelProviderName.GROK]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + endpoint: "https://api.x.ai/v1", + model: { + [ModelClass.SMALL]: settings.SMALL_GROK_MODEL || "grok-2-1212", + [ModelClass.MEDIUM]: settings.MEDIUM_GROK_MODEL || "grok-2-1212", + [ModelClass.LARGE]: settings.LARGE_GROK_MODEL || "grok-2-1212", + [ModelClass.EMBEDDING]: settings.EMBEDDING_GROK_MODEL || "grok-2-1212", // not sure about this one + }, + }, + [ModelProviderName.GROQ]: { + endpoint: "https://api.groq.com/openai/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8000, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + model: { + [ModelClass.SMALL]: + settings.SMALL_GROQ_MODEL || "llama-3.1-8b-instant", + [ModelClass.MEDIUM]: + settings.MEDIUM_GROQ_MODEL || "llama-3.3-70b-versatile", + [ModelClass.LARGE]: + settings.LARGE_GROQ_MODEL || "llama-3.2-90b-vision-preview", + [ModelClass.EMBEDDING]: + settings.EMBEDDING_GROQ_MODEL || "llama-3.1-8b-instant", + }, + }, + [ModelProviderName.LLAMACLOUD]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + imageSettings: { + steps: 4, + }, + endpoint: "https://api.llamacloud.com/v1", + model: { + [ModelClass.SMALL]: "meta-llama/Llama-3.2-3B-Instruct-Turbo", + [ModelClass.MEDIUM]: "meta-llama-3.1-8b-instruct", + [ModelClass.LARGE]: "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", + [ModelClass.EMBEDDING]: + "togethercomputer/m2-bert-80M-32k-retrieval", + [ModelClass.IMAGE]: "black-forest-labs/FLUX.1-schnell", + }, + }, + [ModelProviderName.TOGETHER]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + imageSettings: { + steps: 4, + }, + endpoint: "https://api.together.ai/v1", + model: { + [ModelClass.SMALL]: "meta-llama/Llama-3.2-3B-Instruct-Turbo", + [ModelClass.MEDIUM]: "meta-llama-3.1-8b-instruct", + [ModelClass.LARGE]: "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", + [ModelClass.EMBEDDING]: + "togethercomputer/m2-bert-80M-32k-retrieval", + [ModelClass.IMAGE]: "black-forest-labs/FLUX.1-schnell", + }, + }, + [ModelProviderName.LLAMALOCAL]: { + settings: { + stop: ["<|eot_id|>", "<|eom_id|>"], + maxInputTokens: 32768, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + model: { + [ModelClass.SMALL]: + "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true", + [ModelClass.MEDIUM]: + "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true", // TODO: ?download=true + [ModelClass.LARGE]: + "NousResearch/Hermes-3-Llama-3.1-8B-GGUF/resolve/main/Hermes-3-Llama-3.1-8B.Q8_0.gguf?download=true", + // "RichardErkhov/NousResearch_-_Meta-Llama-3.1-70B-gguf", // TODO: + [ModelClass.EMBEDDING]: + "togethercomputer/m2-bert-80M-32k-retrieval", + }, + }, + [ModelProviderName.GOOGLE]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + model: { + [ModelClass.SMALL]: + settings.SMALL_GOOGLE_MODEL || + settings.GOOGLE_MODEL || + "gemini-1.5-flash-latest", + [ModelClass.MEDIUM]: + settings.MEDIUM_GOOGLE_MODEL || + settings.GOOGLE_MODEL || + "gemini-1.5-flash-latest", + [ModelClass.LARGE]: + settings.LARGE_GOOGLE_MODEL || + settings.GOOGLE_MODEL || + "gemini-1.5-pro-latest", + [ModelClass.EMBEDDING]: + settings.EMBEDDING_GOOGLE_MODEL || + settings.GOOGLE_MODEL || + "text-embedding-004", + }, + }, + [ModelProviderName.REDPILL]: { + endpoint: "https://api.red-pill.ai/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.6, + }, + // Available models: https://docs.red-pill.ai/get-started/supported-models + // To test other models, change the models below + model: { + [ModelClass.SMALL]: + settings.SMALL_REDPILL_MODEL || + settings.REDPILL_MODEL || + "gpt-4o-mini", + [ModelClass.MEDIUM]: + settings.MEDIUM_REDPILL_MODEL || + settings.REDPILL_MODEL || + "gpt-4o", + [ModelClass.LARGE]: + settings.LARGE_REDPILL_MODEL || + settings.REDPILL_MODEL || + "gpt-4o", + [ModelClass.EMBEDDING]: "text-embedding-3-small", + }, + }, + [ModelProviderName.OPENROUTER]: { + endpoint: "https://openrouter.ai/api/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + // Available models: https://openrouter.ai/models + // To test other models, change the models below + model: { + [ModelClass.SMALL]: + settings.SMALL_OPENROUTER_MODEL || + settings.OPENROUTER_MODEL || + "nousresearch/hermes-3-llama-3.1-405b", + [ModelClass.MEDIUM]: + settings.MEDIUM_OPENROUTER_MODEL || + settings.OPENROUTER_MODEL || + "nousresearch/hermes-3-llama-3.1-405b", + [ModelClass.LARGE]: + settings.LARGE_OPENROUTER_MODEL || + settings.OPENROUTER_MODEL || + "nousresearch/hermes-3-llama-3.1-405b", + [ModelClass.EMBEDDING]: "text-embedding-3-small", + }, + }, + [ModelProviderName.OLLAMA]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.7, + }, + endpoint: settings.OLLAMA_SERVER_URL || "http://localhost:11434", + model: { + [ModelClass.SMALL]: + settings.SMALL_OLLAMA_MODEL || + settings.OLLAMA_MODEL || + "llama3.2", + [ModelClass.MEDIUM]: + settings.MEDIUM_OLLAMA_MODEL || + settings.OLLAMA_MODEL || + "hermes3", + [ModelClass.LARGE]: + settings.LARGE_OLLAMA_MODEL || + settings.OLLAMA_MODEL || + "hermes3:70b", + [ModelClass.EMBEDDING]: + settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large", + }, + }, + [ModelProviderName.HEURIST]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + imageSettings: { + steps: 20, + }, + endpoint: "https://llm-gateway.heurist.xyz", + model: { + [ModelClass.SMALL]: + settings.SMALL_HEURIST_MODEL || + "meta-llama/llama-3-70b-instruct", + [ModelClass.MEDIUM]: + settings.MEDIUM_HEURIST_MODEL || + "meta-llama/llama-3-70b-instruct", + [ModelClass.LARGE]: + settings.LARGE_HEURIST_MODEL || + "meta-llama/llama-3.1-405b-instruct", + [ModelClass.EMBEDDING]: "", //Add later, + [ModelClass.IMAGE]: settings.HEURIST_IMAGE_MODEL || "PepeXL", + }, + }, + [ModelProviderName.GALADRIEL]: { + endpoint: "https://api.galadriel.com/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.5, + presence_penalty: 0.5, + temperature: 0.8, + }, + model: { + [ModelClass.SMALL]: "llama3.1:70b", + [ModelClass.MEDIUM]: "llama3.1:70b", + [ModelClass.LARGE]: "llama3.1:405b", + [ModelClass.EMBEDDING]: "gte-large-en-v1.5", + [ModelClass.IMAGE]: "stabilityai/stable-diffusion-xl-base-1.0", + }, + }, + [ModelProviderName.FAL]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + imageSettings: { + steps: 28, + }, + endpoint: "https://api.fal.ai/v1", + model: { + [ModelClass.SMALL]: "", // FAL doesn't provide text models + [ModelClass.MEDIUM]: "", + [ModelClass.LARGE]: "", + [ModelClass.EMBEDDING]: "", + [ModelClass.IMAGE]: "fal-ai/flux-lora", + }, + }, + [ModelProviderName.GAIANET]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + endpoint: settings.GAIANET_SERVER_URL, + model: { + [ModelClass.SMALL]: + settings.GAIANET_MODEL || + settings.SMALL_GAIANET_MODEL || + "llama3b", + [ModelClass.MEDIUM]: + settings.GAIANET_MODEL || + settings.MEDIUM_GAIANET_MODEL || + "llama", + [ModelClass.LARGE]: + settings.GAIANET_MODEL || + settings.LARGE_GAIANET_MODEL || + "qwen72b", + [ModelClass.EMBEDDING]: + settings.GAIANET_EMBEDDING_MODEL || "nomic-embed", + }, + }, + [ModelProviderName.ALI_BAILIAN]: { + endpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: "qwen-turbo", + [ModelClass.MEDIUM]: "qwen-plus", + [ModelClass.LARGE]: "qwen-max", + [ModelClass.IMAGE]: "wanx-v1", + }, + }, + [ModelProviderName.VOLENGINE]: { + endpoint: "https://open.volcengineapi.com/api/v3/", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.4, + presence_penalty: 0.4, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: "doubao-lite-128k", + [ModelClass.MEDIUM]: "doubao-pro-128k", + [ModelClass.LARGE]: "doubao-pro-128k", + [ModelClass.EMBEDDING]: "doubao-embedding", + }, + }, + [ModelProviderName.NANOGPT]: { + endpoint: "https://nano-gpt.com/api/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + frequency_penalty: 0.0, + presence_penalty: 0.0, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: settings.SMALL_NANOGPT_MODEL || "gpt-4o-mini", + [ModelClass.MEDIUM]: settings.MEDIUM_NANOGPT_MODEL || "gpt-4o", + [ModelClass.LARGE]: settings.LARGE_NANOGPT_MODEL || "gpt-4o", + } + }, + [ModelProviderName.HYPERBOLIC]: { + endpoint: "https://api.hyperbolic.xyz/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: + settings.SMALL_HYPERBOLIC_MODEL || + settings.HYPERBOLIC_MODEL || + "meta-llama/Llama-3.2-3B-Instruct", + [ModelClass.MEDIUM]: + settings.MEDIUM_HYPERBOLIC_MODEL || + settings.HYPERBOLIC_MODEL || + "meta-llama/Meta-Llama-3.1-70B-Instruct", + [ModelClass.LARGE]: + settings.LARGE_HYPERBOLIC_MODEL || + settings.HYPERBOLIC_MODEL || + "meta-llama/Meta-Llama-3.1-405-Instruct", + [ModelClass.IMAGE]: settings.IMAGE_HYPERBOLIC_MODEL || "FLUX.1-dev", + }, + }, + [ModelProviderName.VENICE]: { + endpoint: "https://api.venice.ai/api/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: settings.SMALL_VENICE_MODEL || "llama-3.3-70b", + [ModelClass.MEDIUM]: settings.MEDIUM_VENICE_MODEL || "llama-3.3-70b", + [ModelClass.LARGE]: settings.LARGE_VENICE_MODEL || "llama-3.1-405b", + [ModelClass.IMAGE]: settings.IMAGE_VENICE_MODEL || "fluently-xl", + }, + }, + [ModelProviderName.AKASH_CHAT_API]: { + endpoint: "https://chatapi.akash.network/api/v1", + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + temperature: 0.6, + }, + model: { + [ModelClass.SMALL]: + settings.SMALL_AKASH_CHAT_API_MODEL || + "Meta-Llama-3-2-3B-Instruct", + [ModelClass.MEDIUM]: + settings.MEDIUM_AKASH_CHAT_API_MODEL || + "Meta-Llama-3-3-70B-Instruct", + [ModelClass.LARGE]: + settings.LARGE_AKASH_CHAT_API_MODEL || + "Meta-Llama-3-1-405B-Instruct-FP8", + }, + }, + }; + return models; +} -export function getModel(provider: ModelProviderName, type: ModelClass) { - return models[provider].model[type]; +// maybe we can pass in provider to minimze set up above... +// memioized getter +let cacheModels +async function getModels() { + if (!cacheModels) { + const systemInfo = await si.graphics(); + const hasCUDA = systemInfo.controllers.some((controller) => + controller.vendor.toLowerCase().includes("nvidia") + ); + cacheModels = generateModels(hasCUDA); + } + return cacheModels } -export function getEndpoint(provider: ModelProviderName) { - return models[provider].endpoint; +export async function getModelProviderData(provider: ModelProviderName) { + elizaLogger.debug('getModelProviderData') + const models = await getModels() + return models[provider] } From 051b92866f820731582670b34cce796566561c08 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 00:31:33 +0000 Subject: [PATCH 03/14] use core.models getModelProviderData refactor --- .../src/actions/chat_with_attachments.ts | 6 +- .../src/actions/summarize_conversation.ts | 5 +- .../src/actions/chat_with_attachments.ts | 4 +- .../src/actions/summarize_conversation.ts | 4 +- packages/core/src/embedding.ts | 15 ++-- packages/core/src/generation.ts | 72 +++++++++++-------- packages/plugin-node/src/services/image.ts | 13 ++-- packages/plugin-node/src/services/llama.ts | 2 + 8 files changed, 72 insertions(+), 49 deletions(-) diff --git a/packages/client-discord/src/actions/chat_with_attachments.ts b/packages/client-discord/src/actions/chat_with_attachments.ts index c55aca1a935..2b83cd2a5b9 100644 --- a/packages/client-discord/src/actions/chat_with_attachments.ts +++ b/packages/client-discord/src/actions/chat_with_attachments.ts @@ -1,6 +1,6 @@ import { composeContext } from "@ai16z/eliza"; import { generateText, trimTokens } from "@ai16z/eliza"; -import { models } from "@ai16z/eliza"; +import { getModelProviderData } from "@ai16z/eliza"; import { parseJSONObjectFromText } from "@ai16z/eliza"; import { Action, @@ -22,7 +22,7 @@ Summarization objective: {{objective}} # Instructions: Summarize the attachments. Return the summary. Do not acknowledge this request, just summarize and continue the existing summary if there is one. Capture any important details based on the objective. Only respond with the new summary text.`; -export const attachmentIdsTemplate = `# Messages we are summarizing +export const attachmentIdsTemplate = `# Messages we are summarizing {{recentMessages}} # Instructions: {{senderName}} is requesting a summary of specific attachments. Your goal is to determine their objective, along with the list of attachment IDs to summarize. @@ -183,7 +183,7 @@ const summarizeAction = { let currentSummary = ""; - const model = models[runtime.character.modelProvider]; + const model = await getModelProviderData(runtime.character.modelProvider); const chunkSize = model.settings.maxOutputTokens; state.attachmentsWithText = attachmentsWithText; diff --git a/packages/client-discord/src/actions/summarize_conversation.ts b/packages/client-discord/src/actions/summarize_conversation.ts index 1d3a8ebdc7d..4f39954f0e6 100644 --- a/packages/client-discord/src/actions/summarize_conversation.ts +++ b/packages/client-discord/src/actions/summarize_conversation.ts @@ -1,7 +1,7 @@ import { composeContext } from "@ai16z/eliza"; import { generateText, splitChunks, trimTokens } from "@ai16z/eliza"; import { getActorDetails } from "@ai16z/eliza"; -import { models } from "@ai16z/eliza"; +import { getModelProviderData } from "@ai16z/eliza"; import { parseJSONObjectFromText } from "@ai16z/eliza"; import { Action, @@ -247,7 +247,8 @@ const summarizeAction = { let currentSummary = ""; - const model = models[runtime.character.settings.model]; + + const model = await getModelProviderData(runtime.character.settings.model); const chunkSize = model.settings.maxContextLength - 1000; const chunks = await splitChunks(formattedMemories, chunkSize, 0); diff --git a/packages/client-slack/src/actions/chat_with_attachments.ts b/packages/client-slack/src/actions/chat_with_attachments.ts index 1d2e2c3f565..efb9b473629 100644 --- a/packages/client-slack/src/actions/chat_with_attachments.ts +++ b/packages/client-slack/src/actions/chat_with_attachments.ts @@ -4,7 +4,7 @@ import { trimTokens, parseJSONObjectFromText, } from "@ai16z/eliza"; -import { models } from "@ai16z/eliza"; +import { getModelProviderData } from "@ai16z/eliza"; import { Action, ActionExample, @@ -194,7 +194,7 @@ const summarizeAction: Action = { let currentSummary = ""; - const model = models[runtime.character.modelProvider]; + const model = await getModelProviderData(runtime.character.modelProvider); const chunkSize = model.settings.maxOutputTokens; currentState.attachmentsWithText = attachmentsWithText; diff --git a/packages/client-slack/src/actions/summarize_conversation.ts b/packages/client-slack/src/actions/summarize_conversation.ts index f99a77b3a39..97f7e298196 100644 --- a/packages/client-slack/src/actions/summarize_conversation.ts +++ b/packages/client-slack/src/actions/summarize_conversation.ts @@ -5,7 +5,7 @@ import { trimTokens, parseJSONObjectFromText, } from "@ai16z/eliza"; -import { models } from "@ai16z/eliza"; +import { getModelProviderData } from "@ai16z/eliza"; import { getActorDetails } from "@ai16z/eliza"; import { Action, @@ -265,7 +265,7 @@ const summarizeAction: Action = { let currentSummary = ""; - const model = models[runtime.character.modelProvider]; + const model = await getModelProviderData(runtime.character.modelProvider); const chunkSize = model.settings.maxOutputTokens; const chunks = await splitChunks(formattedMemories, chunkSize, 0); diff --git a/packages/core/src/embedding.ts b/packages/core/src/embedding.ts index 49c1a4163c2..71416e138dc 100644 --- a/packages/core/src/embedding.ts +++ b/packages/core/src/embedding.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { models } from "./models.ts"; +import { getModelProviderData } from "./models.ts"; import { IAgentRuntime, ModelProviderName } from "./types.ts"; import settings from "./settings.ts"; import elizaLogger from "./logger.ts"; @@ -181,22 +181,24 @@ export async function embed(runtime: IAgentRuntime, input: string) { } if (config.provider === "Ollama") { + const ollamaSettings = await getModelProviderData(ModelProviderName.OLLAMA) return await getRemoteEmbedding(input, { model: config.model, endpoint: runtime.character.modelEndpointOverride || - models[ModelProviderName.OLLAMA].endpoint, + ollamaSettings.endpoint, isOllama: true, dimensions: config.dimensions, }); } if (config.provider=="GaiaNet") { + const gaianetSettings = await getModelProviderData(ModelProviderName.GAIANET) return await getRemoteEmbedding(input, { model: config.model, endpoint: runtime.character.modelEndpointOverride || - models[ModelProviderName.GAIANET].endpoint || + gaianetSettings.endpoint || settings.SMALL_GAIANET_SERVER_URL || settings.MEDIUM_GAIANET_SERVER_URL || settings.LARGE_GAIANET_SERVER_URL, @@ -218,11 +220,12 @@ export async function embed(runtime: IAgentRuntime, input: string) { } // Fallback to remote override + const modelSettings = await getModelProviderData(runtime.character.modelProvider) return await getRemoteEmbedding(input, { model: config.model, endpoint: runtime.character.modelEndpointOverride || - models[runtime.character.modelProvider].endpoint, + modelSettings.endpoint, apiKey: runtime.token, dimensions: config.dimensions, }); @@ -304,7 +307,9 @@ export async function embed(runtime: IAgentRuntime, input: string) { : "not an array", sample: Array.isArray(embedding) ? embedding.slice(0, 5) - : embedding, + : typeof embedding === 'object' && embedding !== null ? + Object.values(embedding).slice(0, 5) + : embedding, // not an array or object }); // Process the embedding into the correct format diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index 59b620d6481..1cba617d5b9 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -15,7 +15,7 @@ import { encodingForModel, TiktokenModel } from "js-tiktoken"; import Together from "together-ai"; import { ZodSchema } from "zod"; import { elizaLogger } from "./index.ts"; -import { getModel, models } from "./models.ts"; +import { getModelProviderData } from "./models.ts"; import { parseBooleanFromText, parseJsonArrayFromText, @@ -74,9 +74,10 @@ export async function generateText({ }); const provider = runtime.modelProvider; + const modelData = await getModelProviderData(provider); const endpoint = - runtime.character.modelEndpointOverride || models[provider].endpoint; - let model = models[provider].model[modelClass]; + runtime.character.modelEndpointOverride || modelData.endpoint; + let model = modelData.endpoint.model[modelClass]; // allow character.json settings => secrets to override models // FIXME: add MODEL_MEDIUM support @@ -146,11 +147,12 @@ export async function generateText({ elizaLogger.info("Selected model:", model); - const temperature = models[provider].settings.temperature; - const frequency_penalty = models[provider].settings.frequency_penalty; - const presence_penalty = models[provider].settings.presence_penalty; - const max_context_length = models[provider].settings.maxInputTokens; - const max_response_length = models[provider].settings.maxOutputTokens; + const modelSettings = modelData.settings + const temperature = modelSettings.temperature; + const frequency_penalty = modelSettings.frequency_penalty; + const presence_penalty = modelSettings.presence_penalty; + const max_context_length = modelSettings.maxInputTokens; + const max_response_length = modelSettings.maxOutputTokens; const apiKey = runtime.token; @@ -162,7 +164,7 @@ export async function generateText({ let response: string; - const _stop = stop || models[provider].settings.stop; + const _stop = stop || modelSettings.stop; elizaLogger.debug( `Using provider: ${provider}, model: ${model}, temperature: ${temperature}, max response length: ${max_response_length}` ); @@ -354,7 +356,7 @@ export async function generateText({ case ModelProviderName.REDPILL: { elizaLogger.debug("Initializing RedPill model."); - const serverUrl = models[provider].endpoint; + const serverUrl = modelData.endpoint; const openai = createOpenAI({ apiKey, baseURL: serverUrl, @@ -381,7 +383,7 @@ export async function generateText({ case ModelProviderName.OPENROUTER: { elizaLogger.debug("Initializing OpenRouter model."); - const serverUrl = models[provider].endpoint; + const serverUrl = modelData.endpoint; const openrouter = createOpenAI({ apiKey, baseURL: serverUrl, @@ -411,7 +413,7 @@ export async function generateText({ elizaLogger.debug("Initializing Ollama model."); const ollamaProvider = createOllama({ - baseURL: models[provider].endpoint + "/api", + baseURL: modelData.endpoint + "/api", fetch: runtime.fetch, }); const ollama = ollamaProvider(model); @@ -460,7 +462,7 @@ export async function generateText({ case ModelProviderName.GAIANET: { elizaLogger.debug("Initializing GAIANET model."); - var baseURL = models[provider].endpoint; + var baseURL = modelData.endpoint; if (!baseURL) { switch (modelClass) { case ModelClass.SMALL: @@ -716,10 +718,11 @@ export async function generateTrueOrFalse({ modelClass: string; }): Promise { let retryDelay = 1000; + const modelData = await getModelProviderData(runtime.modelProvider); const stop = Array.from( new Set([ - ...(models[runtime.modelProvider].settings.stop || []), + ...(modelData.settings.stop || []), ["\n"], ]) ) as string[]; @@ -890,8 +893,8 @@ export async function generateMessageResponse({ context: string; modelClass: string; }): Promise { - const max_context_length = - models[runtime.modelProvider].settings.maxInputTokens; + const modelData = await getModelProviderData(runtime.modelProvider); + const max_context_length = modelData.settings.maxInputTokens; context = trimTokens(context, max_context_length, "gpt-4o"); let retryLength = 1000; // exponential backoff while (true) { @@ -941,8 +944,9 @@ export const generateImage = async ( data?: string[]; error?: any; }> => { - const model = getModel(runtime.imageModelProvider, ModelClass.IMAGE); - const modelSettings = models[runtime.imageModelProvider].imageSettings; + const imageModelData = await getModelProviderData(runtime.imageModelProvider) + const model = imageModelData.model[ModelClass.IMAGE]; + const modelSettings = imageModelData.imageSettings; elizaLogger.info("Generating image with options:", { imageModelProvider: model, @@ -1280,15 +1284,17 @@ export const generateObject = async ({ } const provider = runtime.modelProvider; - const model = models[provider].model[modelClass] as TiktokenModel; + const modelData = await getModelProviderData(runtime.modelProvider); + const model = modelData.model[modelClass] as TiktokenModel; if (!model) { throw new Error(`Unsupported model class: ${modelClass}`); } - const temperature = models[provider].settings.temperature; - const frequency_penalty = models[provider].settings.frequency_penalty; - const presence_penalty = models[provider].settings.presence_penalty; - const max_context_length = models[provider].settings.maxInputTokens; - const max_response_length = models[provider].settings.maxOutputTokens; + const modelSettings = modelData.settings; + const temperature = modelSettings.temperature; + const frequency_penalty = modelSettings.frequency_penalty; + const presence_penalty = modelSettings.presence_penalty; + const max_context_length = modelSettings.maxInputTokens; + const max_response_length = modelSettings.maxOutputTokens; const apiKey = runtime.token; try { @@ -1300,7 +1306,7 @@ export const generateObject = async ({ maxTokens: max_response_length, frequencyPenalty: frequency_penalty, presencePenalty: presence_penalty, - stop: stop || models[provider].settings.stop, + stop: stop || modelSettings.stop, }; const response = await handleProvider({ @@ -1403,7 +1409,8 @@ async function handleOpenAI({ mode, modelOptions, }: ProviderOptions): Promise> { - const baseURL = models.openai.endpoint || undefined; + const openaiSettings = await getModelProviderData(ModelProviderName.OPENAI) + const baseURL = openaiSettings.endpoint || undefined; const openai = createOpenAI({ apiKey, baseURL }); return await aiGenerateObject({ model: openai.languageModel(model), @@ -1456,7 +1463,8 @@ async function handleGrok({ mode, modelOptions, }: ProviderOptions): Promise> { - const grok = createOpenAI({ apiKey, baseURL: models.grok.endpoint }); + const grokSettings = await getModelProviderData(ModelProviderName.GROK) + const grok = createOpenAI({ apiKey, baseURL: grokSettings.endpoint }); return await aiGenerateObject({ model: grok.languageModel(model, { parallelToolCalls: false }), schema, @@ -1534,7 +1542,8 @@ async function handleRedPill({ mode, modelOptions, }: ProviderOptions): Promise> { - const redPill = createOpenAI({ apiKey, baseURL: models.redpill.endpoint }); + const redpillSettings = await getModelProviderData(ModelProviderName.REDPILL); + const redPill = createOpenAI({ apiKey, baseURL: redpillSettings.endpoint }); return await aiGenerateObject({ model: redPill.languageModel(model), schema, @@ -1560,9 +1569,10 @@ async function handleOpenRouter({ mode, modelOptions, }: ProviderOptions): Promise> { + const openrouterSettings = await getModelProviderData(ModelProviderName.OPENROUTER); const openRouter = createOpenAI({ apiKey, - baseURL: models.openrouter.endpoint, + baseURL: openrouterSettings.endpoint, }); return await aiGenerateObject({ model: openRouter.languageModel(model), @@ -1589,8 +1599,10 @@ async function handleOllama({ modelOptions, provider, }: ProviderOptions): Promise> { + const ollamaSettings = await getModelProviderData(ModelProviderName.OLLAMA); const ollamaProvider = createOllama({ - baseURL: models[provider].endpoint + "/api", + // is it weird to append /api? not doing it anywhere else? + baseURL: ollamaSettings.endpoint + "/api", }); const ollama = ollamaProvider(model); return await aiGenerateObject({ diff --git a/packages/plugin-node/src/services/image.ts b/packages/plugin-node/src/services/image.ts index 540065a4aac..07afbe4316c 100644 --- a/packages/plugin-node/src/services/image.ts +++ b/packages/plugin-node/src/services/image.ts @@ -1,10 +1,11 @@ -import { elizaLogger, models } from "@ai16z/eliza"; -import { Service } from "@ai16z/eliza"; import { + elizaLogger, + Service, IAgentRuntime, ModelProviderName, ServiceType, IImageDescriptionService, + getModelProviderData, } from "@ai16z/eliza"; import { AutoProcessor, @@ -93,9 +94,10 @@ export class ImageDescriptionService imageUrl: string ): Promise<{ title: string; description: string }> { if (!this.initialized) { - const model = models[this.runtime?.character?.modelProvider]; + const provider = this.runtime?.character?.modelProvider; + const model = await getModelProviderData(provider); - if (model === models[ModelProviderName.LLAMALOCAL]) { + if (provider === ModelProviderName.LLAMALOCAL) { await this.initializeLocalModel(); } else { this.modelId = "gpt-4o-mini"; @@ -195,8 +197,9 @@ export class ImageDescriptionService }, ]; + const imageModelData = await getModelProviderData(this.runtime.imageModelProvider); const endpoint = - models[this.runtime.imageModelProvider].endpoint ?? + imageModelData.endpoint ?? "https://api.openai.com/v1"; const response = await fetch(endpoint + "/chat/completions", { diff --git a/packages/plugin-node/src/services/llama.ts b/packages/plugin-node/src/services/llama.ts index 20868ea53d8..8d53d4c5e8b 100644 --- a/packages/plugin-node/src/services/llama.ts +++ b/packages/plugin-node/src/services/llama.ts @@ -22,6 +22,8 @@ import path from "path"; import si from "systeminformation"; import { fileURLToPath } from "url"; +process.env.ONNXRUNTIME_LOG_LEVEL = '4'; // Error level + const wordsToPunish = [ " please", " feel", From 4e387c3aca2d13ad941fa7c84adea582dfb419e5 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:16:58 +0000 Subject: [PATCH 04/14] new files --- packages/client-whatsapp/eslint.config.mjs | 17 +++++++++++++++++ packages/client-whatsapp/tsconfig.json | 10 ++++++++++ packages/client-whatsapp/tsup.config.ts | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 packages/client-whatsapp/eslint.config.mjs create mode 100644 packages/client-whatsapp/tsconfig.json create mode 100644 packages/client-whatsapp/tsup.config.ts diff --git a/packages/client-whatsapp/eslint.config.mjs b/packages/client-whatsapp/eslint.config.mjs new file mode 100644 index 00000000000..67614d246e9 --- /dev/null +++ b/packages/client-whatsapp/eslint.config.mjs @@ -0,0 +1,17 @@ +import eslintGlobalConfig from "../../eslint.config.mjs"; + +export default [ + { + ignores: [ + "**/node_modules/*", + "**/coverage/*", + "**/dist/*", + "**/types/*", + "**/scripts/concatenated-output.ts", + "rollup.config.js", + "jest.config.js", + "docs/", + ], + }, + ...eslintGlobalConfig, +]; diff --git a/packages/client-whatsapp/tsconfig.json b/packages/client-whatsapp/tsconfig.json new file mode 100644 index 00000000000..73993deaaf7 --- /dev/null +++ b/packages/client-whatsapp/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../core/tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": [ + "src/**/*.ts" + ] +} \ No newline at end of file diff --git a/packages/client-whatsapp/tsup.config.ts b/packages/client-whatsapp/tsup.config.ts new file mode 100644 index 00000000000..e42bf4efeae --- /dev/null +++ b/packages/client-whatsapp/tsup.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + outDir: "dist", + sourcemap: true, + clean: true, + format: ["esm"], // Ensure you're targeting CommonJS + external: [ + "dotenv", // Externalize dotenv to prevent bundling + "fs", // Externalize fs to use Node.js built-in module + "path", // Externalize other built-ins if necessary + "@reflink/reflink", + "@node-llama-cpp", + "https", + "http", + "agentkeepalive", + // Add other modules you want to externalize + ], +}); From 08a563dd8fec23f7347ec8358345f0ec0cf5478f Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:17:32 +0000 Subject: [PATCH 05/14] remove unused var --- packages/client-discord/src/messages.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client-discord/src/messages.ts b/packages/client-discord/src/messages.ts index 36411e82b61..87fb3aa9f54 100644 --- a/packages/client-discord/src/messages.ts +++ b/packages/client-discord/src/messages.ts @@ -512,7 +512,6 @@ export class MessageManager { const guild = message.guild; const member = guild?.members.cache.get(this.client.user?.id as string); const nickname = member?.nickname; - const memberId = member?.id; // Don't consider role mentions as direct mentions const hasRoleMentionOnly = message.mentions.roles.size > 0 && !isMentioned; From 3fb8e6d4252bca198995d3b3a1b55dcad9d640a3 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:17:50 +0000 Subject: [PATCH 06/14] remove duplicate method --- packages/client-telegram/src/messageManager.ts | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/client-telegram/src/messageManager.ts b/packages/client-telegram/src/messageManager.ts index 8ee3937fa1b..6644feb2a89 100644 --- a/packages/client-telegram/src/messageManager.ts +++ b/packages/client-telegram/src/messageManager.ts @@ -314,20 +314,6 @@ export class MessageManager { return contextSimilarity >= similarityThreshold; } - private _isMessageForMe(message: Message): boolean { - const botUsername = this.bot.botInfo?.username; - if (!botUsername) return false; - - const messageText = 'text' in message ? message.text : - 'caption' in message ? (message as any).caption : ''; - if (!messageText) return false; - - const isMentioned = messageText.includes(`@${botUsername}`); - const hasUsername = messageText.toLowerCase().includes(botUsername.toLowerCase()); - - return isMentioned || (!this.runtime.character.clientConfig?.telegram?.shouldRespondOnlyToMentions && hasUsername); - } - private _checkInterest(chatId: string): boolean { const chatState = this.interestChats[chatId]; if (!chatState) return false; @@ -367,7 +353,7 @@ export class MessageManager { const messageText = 'text' in message ? message.text : 'caption' in message ? (message as any).caption : ''; if (!messageText) return false; - + const isReplyToBot = (message as any).reply_to_message?.from?.is_bot === true && (message as any).reply_to_message?.from?.username === botUsername; const isMentioned = messageText.includes(`@${botUsername}`); @@ -422,7 +408,7 @@ export class MessageManager { message: Message, state: State ): Promise { - + if (this.runtime.character.clientConfig?.telegram?.shouldRespondOnlyToMentions) { return this._isMessageForMe(message); } From 05d2a010ff608cbfc60a82af7d94ad0cd25b07ff Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:18:06 +0000 Subject: [PATCH 07/14] fix lint script --- packages/client-whatsapp/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client-whatsapp/package.json b/packages/client-whatsapp/package.json index 9a20800ecc6..65cdd7f8f71 100644 --- a/packages/client-whatsapp/package.json +++ b/packages/client-whatsapp/package.json @@ -8,7 +8,7 @@ "build": "tsc", "clean": "rimraf dist", "test": "jest", - "lint": "eslint src --ext .ts" + "lint": "eslint --fix --cache ." }, "dependencies": { "@ai16z/eliza": "workspace:*" From e3621e1cec95cfac6d4df6302ee567634a323397 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:18:27 +0000 Subject: [PATCH 08/14] use runtime in stop --- packages/client-whatsapp/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client-whatsapp/src/index.ts b/packages/client-whatsapp/src/index.ts index f38b9dae85d..f1214f4cd8a 100644 --- a/packages/client-whatsapp/src/index.ts +++ b/packages/client-whatsapp/src/index.ts @@ -21,8 +21,8 @@ export const WhatsAppClientInterface: Client = { return client; }, stop: async (runtime: IAgentRuntime) => { - elizaLogger.warn("WhatsApp client stopping..."); + elizaLogger.warn("WhatsApp client stopping... for", runtime.character.name); }, }; -export default WhatsAppClientInterface; \ No newline at end of file +export default WhatsAppClientInterface; \ No newline at end of file From efaa6e8c47e21435ffb9b543956aee28c96f9dae Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:18:50 +0000 Subject: [PATCH 09/14] remove unused includes, let => const --- .../plugin-nft-generation/src/provider/wallet/walletSolana.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/plugin-nft-generation/src/provider/wallet/walletSolana.ts b/packages/plugin-nft-generation/src/provider/wallet/walletSolana.ts index 98b2bee2330..f70b5505d20 100644 --- a/packages/plugin-nft-generation/src/provider/wallet/walletSolana.ts +++ b/packages/plugin-nft-generation/src/provider/wallet/walletSolana.ts @@ -19,12 +19,10 @@ import { keypairIdentity, percentAmount, publicKey, - sol, TransactionBuilder, Umi, } from "@metaplex-foundation/umi"; import { getExplorerLink } from "@solana-developers/helpers"; -import { transferSol } from "@metaplex-foundation/mpl-toolbox"; import bs58 from "bs58"; import { elizaLogger } from "@ai16z/eliza"; @@ -56,7 +54,7 @@ export class WalletSolana { } async getBalance() { - let balance = await this.connection.getBalance(this.walletPublicKey); + const balance = await this.connection.getBalance(this.walletPublicKey); return { value: balance, formater: `${balance / LAMPORTS_PER_SOL} SOL`, From 37b22e8208fea35832aaddf706a33bdd0f90e44c Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:19:05 +0000 Subject: [PATCH 10/14] remove unused model --- packages/plugin-node/src/services/image.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/plugin-node/src/services/image.ts b/packages/plugin-node/src/services/image.ts index 07afbe4316c..dd056ac674d 100644 --- a/packages/plugin-node/src/services/image.ts +++ b/packages/plugin-node/src/services/image.ts @@ -95,7 +95,6 @@ export class ImageDescriptionService ): Promise<{ title: string; description: string }> { if (!this.initialized) { const provider = this.runtime?.character?.modelProvider; - const model = await getModelProviderData(provider); if (provider === ModelProviderName.LLAMALOCAL) { await this.initializeLocalModel(); From 7b8a8a466deeb4c3de884a37bb3c63ecdbc6ac65 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:19:20 +0000 Subject: [PATCH 11/14] adjust logging --- packages/core/src/models.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index b76854cd5d5..3c8c7c908b9 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -3,8 +3,8 @@ import { Models, ModelProviderName, ModelClass } from "./types.ts"; import { elizaLogger } from "./index.ts"; import si from "systeminformation"; -function generateModels(hasGPU:Boolean) { - elizaLogger.debug('generateModels') +function generateModels(hasGPU:boolean) { + elizaLogger.debug('generateModels', hasGPU) // image and text models mixed const models: Models = { [ModelProviderName.OPENAI]: { @@ -510,7 +510,6 @@ async function getModels() { } export async function getModelProviderData(provider: ModelProviderName) { - elizaLogger.debug('getModelProviderData') const models = await getModels() return models[provider] } From aa6bf4aed222b1f95cfed50da16827e7cdd700b6 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 01:19:41 +0000 Subject: [PATCH 12/14] remove unused provider in ollama params --- packages/core/src/generation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index 1cba617d5b9..bbe959474e9 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -1597,7 +1597,8 @@ async function handleOllama({ schemaDescription, mode, modelOptions, - provider, + // not used? + //provider, }: ProviderOptions): Promise> { const ollamaSettings = await getModelProviderData(ModelProviderName.OLLAMA); const ollamaProvider = createOllama({ From 132d4ef5a205d812d8eb5e940d17abf2e32e4a18 Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 02:49:11 +0000 Subject: [PATCH 13/14] make throw as tests expect, style --- packages/core/src/models.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index 3c8c7c908b9..955c0e3cf3b 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -506,10 +506,13 @@ async function getModels() { ); cacheModels = generateModels(hasCUDA); } - return cacheModels + return cacheModels; } export async function getModelProviderData(provider: ModelProviderName) { - const models = await getModels() + const models = await getModels(); + if (!models[provider]) { + throw new Error(`getModelProviderData failed Unkonwn provider ${provider}`); + } return models[provider] } From 45eb65727715f8dcd722830dff859c9bf6e4acfb Mon Sep 17 00:00:00 2001 From: odilitime Date: Fri, 20 Dec 2024 02:49:48 +0000 Subject: [PATCH 14/14] update unit test for models' getModelProviderData update --- packages/core/src/tests/generation.test.ts | 8 +- packages/core/src/tests/models.test.ts | 96 ++++++++++------------ 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/packages/core/src/tests/generation.test.ts b/packages/core/src/tests/generation.test.ts index f1ec8f9bc69..f5e7741c40d 100644 --- a/packages/core/src/tests/generation.test.ts +++ b/packages/core/src/tests/generation.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import { ModelProviderName, IAgentRuntime } from "../types"; -import { models } from "../models"; +import { getModelProviderData } from "../models"; import { generateText, generateTrueOrFalse, @@ -15,6 +15,7 @@ vi.mock("../index.ts", () => ({ log: vi.fn(), info: vi.fn(), error: vi.fn(), + debug: vi.fn(), }, })); @@ -76,9 +77,10 @@ describe("Generation", () => { expect(result).toBe("mocked response"); }); - it("should use correct model settings from provider config", () => { + it("should use correct model settings from provider config", async () => { const modelProvider = mockRuntime.modelProvider; - const modelSettings = models[modelProvider].settings; + console.log('modelProvider', modelProvider) + const modelSettings = (await getModelProviderData(modelProvider)).settings; expect(modelSettings).toBeDefined(); expect(modelSettings.temperature).toBeDefined(); diff --git a/packages/core/src/tests/models.test.ts b/packages/core/src/tests/models.test.ts index f336093cfdd..21a232c8369 100644 --- a/packages/core/src/tests/models.test.ts +++ b/packages/core/src/tests/models.test.ts @@ -1,4 +1,4 @@ -import { getModel, getEndpoint, models } from "../models.ts"; +import { getModelProviderData } from "../models.ts"; import { ModelProviderName, ModelClass } from "../types.ts"; import { describe, test, expect, vi } from "vitest"; @@ -25,12 +25,12 @@ vi.mock("../settings", () => { describe("Model Provider Configuration", () => { describe("OpenAI Provider", () => { - test("should have correct endpoint", () => { - expect(models[ModelProviderName.OPENAI].endpoint).toBe("https://api.openai.com/v1"); + test("should have correct endpoint", async () => { + expect((await getModelProviderData(ModelProviderName.OPENAI)).endpoint).toBe("https://api.openai.com/v1"); }); - test("should have correct model mappings", () => { - const openAIModels = models[ModelProviderName.OPENAI].model; + test("should have correct model mappings", async () => { + const openAIModels = (await getModelProviderData(ModelProviderName.OPENAI)).model; expect(openAIModels[ModelClass.SMALL]).toBe("gpt-4o-mini"); expect(openAIModels[ModelClass.MEDIUM]).toBe("gpt-4o"); expect(openAIModels[ModelClass.LARGE]).toBe("gpt-4o"); @@ -38,8 +38,8 @@ describe("Model Provider Configuration", () => { expect(openAIModels[ModelClass.IMAGE]).toBe("dall-e-3"); }); - test("should have correct settings configuration", () => { - const settings = models[ModelProviderName.OPENAI].settings; + test("should have correct settings configuration", async () => { + const settings = (await getModelProviderData(ModelProviderName.OPENAI)).settings; expect(settings.maxInputTokens).toBe(128000); expect(settings.maxOutputTokens).toBe(8192); expect(settings.temperature).toBe(0.6); @@ -49,19 +49,19 @@ describe("Model Provider Configuration", () => { }); describe("Anthropic Provider", () => { - test("should have correct endpoint", () => { - expect(models[ModelProviderName.ANTHROPIC].endpoint).toBe("https://api.anthropic.com/v1"); + test("should have correct endpoint", async () => { + expect((await getModelProviderData(ModelProviderName.ANTHROPIC)).endpoint).toBe("https://api.anthropic.com/v1"); }); - test("should have correct model mappings", () => { - const anthropicModels = models[ModelProviderName.ANTHROPIC].model; + test("should have correct model mappings", async () => { + const anthropicModels = (await getModelProviderData(ModelProviderName.ANTHROPIC)).model; expect(anthropicModels[ModelClass.SMALL]).toBe("claude-3-haiku-20240307"); expect(anthropicModels[ModelClass.MEDIUM]).toBe("claude-3-5-sonnet-20241022"); expect(anthropicModels[ModelClass.LARGE]).toBe("claude-3-5-sonnet-20241022"); }); - test("should have correct settings configuration", () => { - const settings = models[ModelProviderName.ANTHROPIC].settings; + test("should have correct settings configuration", async () => { + const settings = (await getModelProviderData(ModelProviderName.ANTHROPIC)).settings; expect(settings.maxInputTokens).toBe(200000); expect(settings.maxOutputTokens).toBe(4096); expect(settings.temperature).toBe(0.7); @@ -71,12 +71,12 @@ describe("Model Provider Configuration", () => { }); describe("LlamaCloud Provider", () => { - test("should have correct endpoint", () => { - expect(models[ModelProviderName.LLAMACLOUD].endpoint).toBe("https://api.llamacloud.com/v1"); + test("should have correct endpoint", async () => { + expect((await getModelProviderData(ModelProviderName.LLAMACLOUD)).endpoint).toBe("https://api.llamacloud.com/v1"); }); - test("should have correct model mappings", () => { - const llamaCloudModels = models[ModelProviderName.LLAMACLOUD].model; + test("should have correct model mappings", async () => { + const llamaCloudModels = (await getModelProviderData(ModelProviderName.LLAMACLOUD)).model; expect(llamaCloudModels[ModelClass.SMALL]).toBe("meta-llama/Llama-3.2-3B-Instruct-Turbo"); expect(llamaCloudModels[ModelClass.MEDIUM]).toBe("meta-llama-3.1-8b-instruct"); expect(llamaCloudModels[ModelClass.LARGE]).toBe("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"); @@ -84,8 +84,8 @@ describe("Model Provider Configuration", () => { expect(llamaCloudModels[ModelClass.IMAGE]).toBe("black-forest-labs/FLUX.1-schnell"); }); - test("should have correct settings configuration", () => { - const settings = models[ModelProviderName.LLAMACLOUD].settings; + test("should have correct settings configuration", async () => { + const settings = (await getModelProviderData(ModelProviderName.LLAMACLOUD)).settings; expect(settings.maxInputTokens).toBe(128000); expect(settings.maxOutputTokens).toBe(8192); expect(settings.temperature).toBe(0.7); @@ -94,8 +94,8 @@ describe("Model Provider Configuration", () => { }); describe("Google Provider", () => { - test("should have correct model mappings", () => { - const googleModels = models[ModelProviderName.GOOGLE].model; + test("should have correct model mappings", async () => { + const googleModels = (await getModelProviderData(ModelProviderName.GOOGLE)).model; expect(googleModels[ModelClass.SMALL]).toBe("gemini-1.5-flash-latest"); expect(googleModels[ModelClass.MEDIUM]).toBe("gemini-1.5-flash-latest"); expect(googleModels[ModelClass.LARGE]).toBe("gemini-1.5-pro-latest"); @@ -104,42 +104,36 @@ describe("Model Provider Configuration", () => { }); describe("Model Retrieval Functions", () => { - describe("getModel function", () => { - test("should retrieve correct models for different providers and classes", () => { - expect(getModel(ModelProviderName.OPENAI, ModelClass.SMALL)).toBe("gpt-4o-mini"); - expect(getModel(ModelProviderName.ANTHROPIC, ModelClass.LARGE)).toBe("claude-3-5-sonnet-20241022"); - expect(getModel(ModelProviderName.LLAMACLOUD, ModelClass.MEDIUM)).toBe("meta-llama-3.1-8b-instruct"); + describe("getModelProviderData function", () => { + test("should retrieve correct models for different providers and classes", async () => { + expect((await getModelProviderData(ModelProviderName.OPENAI)).model[ModelClass.SMALL]).toBe("gpt-4o-mini"); + expect((await getModelProviderData(ModelProviderName.ANTHROPIC)).model[ModelClass.LARGE]).toBe("claude-3-5-sonnet-20241022"); + expect((await getModelProviderData(ModelProviderName.LLAMACLOUD)).model[ModelClass.MEDIUM]).toBe("meta-llama-3.1-8b-instruct"); }); - test("should handle environment variable overrides", () => { - expect(getModel(ModelProviderName.OPENROUTER, ModelClass.SMALL)).toBe("mock-small-model"); - expect(getModel(ModelProviderName.OPENROUTER, ModelClass.LARGE)).toBe("mock-large-model"); - expect(getModel(ModelProviderName.ETERNALAI, ModelClass.SMALL)).toBe("mock-eternal-model"); + test("should handle environment variable overrides", async () => { + expect((await getModelProviderData(ModelProviderName.OPENROUTER)).model[ModelClass.SMALL]).toBe("mock-small-model"); + expect((await getModelProviderData(ModelProviderName.OPENROUTER)).model[ModelClass.LARGE]).toBe("mock-large-model"); + expect((await getModelProviderData(ModelProviderName.ETERNALAI)).model[ModelClass.SMALL]).toBe("mock-eternal-model"); }); - test("should throw error for invalid model provider", () => { - expect(() => getModel("INVALID_PROVIDER" as any, ModelClass.SMALL)).toThrow(); - }); - }); - - describe("getEndpoint function", () => { - test("should retrieve correct endpoints for different providers", () => { - expect(getEndpoint(ModelProviderName.OPENAI)).toBe("https://api.openai.com/v1"); - expect(getEndpoint(ModelProviderName.ANTHROPIC)).toBe("https://api.anthropic.com/v1"); - expect(getEndpoint(ModelProviderName.LLAMACLOUD)).toBe("https://api.llamacloud.com/v1"); - expect(getEndpoint(ModelProviderName.ETERNALAI)).toBe("https://mock.eternal.ai"); + test("should throw error for invalid model provider", async () => { + await expect(getModelProviderData("INVALID_PROVIDER" as any)).rejects.toThrow(); }); - test("should throw error for invalid provider", () => { - expect(() => getEndpoint("INVALID_PROVIDER" as any)).toThrow(); + test("should retrieve correct endpoints for different providers", async () => { + expect((await getModelProviderData(ModelProviderName.OPENAI)).endpoint).toBe("https://api.openai.com/v1"); + expect((await getModelProviderData(ModelProviderName.ANTHROPIC)).endpoint).toBe("https://api.anthropic.com/v1"); + expect((await getModelProviderData(ModelProviderName.LLAMACLOUD)).endpoint).toBe("https://api.llamacloud.com/v1"); + expect((await getModelProviderData(ModelProviderName.ETERNALAI)).endpoint).toBe("https://mock.eternal.ai"); }); }); }); describe("Model Settings Validation", () => { test("all providers should have required settings", () => { - Object.values(ModelProviderName).forEach(provider => { - const providerConfig = models[provider]; + Object.values(ModelProviderName).forEach(async provider => { + const providerConfig = await getModelProviderData(provider); expect(providerConfig.settings).toBeDefined(); expect(providerConfig.settings.maxInputTokens).toBeGreaterThan(0); expect(providerConfig.settings.maxOutputTokens).toBeGreaterThan(0); @@ -148,8 +142,8 @@ describe("Model Settings Validation", () => { }); test("all providers should have model mappings for basic model classes", () => { - Object.values(ModelProviderName).forEach(provider => { - const providerConfig = models[provider]; + Object.values(ModelProviderName).forEach(async provider => { + const providerConfig = await getModelProviderData(provider); expect(providerConfig.model).toBeDefined(); expect(providerConfig.model[ModelClass.SMALL]).toBeDefined(); expect(providerConfig.model[ModelClass.MEDIUM]).toBeDefined(); @@ -159,14 +153,14 @@ describe("Model Settings Validation", () => { }); describe("Environment Variable Integration", () => { - test("should use environment variables for LlamaCloud models", () => { - const llamaConfig = models[ModelProviderName.LLAMACLOUD]; + test("should use environment variables for LlamaCloud models", async () => { + const llamaConfig = await getModelProviderData(ModelProviderName.LLAMACLOUD); expect(llamaConfig.model[ModelClass.SMALL]).toBe("meta-llama/Llama-3.2-3B-Instruct-Turbo"); expect(llamaConfig.model[ModelClass.LARGE]).toBe("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"); }); - test("should use environment variables for Together models", () => { - const togetherConfig = models[ModelProviderName.TOGETHER]; + test("should use environment variables for Together models", async () => { + const togetherConfig = await getModelProviderData(ModelProviderName.TOGETHER); expect(togetherConfig.model[ModelClass.SMALL]).toBe("meta-llama/Llama-3.2-3B-Instruct-Turbo"); expect(togetherConfig.model[ModelClass.LARGE]).toBe("meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo"); });