From a1f3323dd4436b8d58a9968b6bbfbc5cc0f53426 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:06:44 +0000 Subject: [PATCH 01/11] updated Characters.ts in core/types to include an optional imageModelProvider allowing for use of different models between textGeneration and imageGeneration. If imageModelProvider is not set, it defaults to modelProvider and as such, functionality should remain the same. Also added another Model and ModelProviderName for fal.ai --- packages/core/src/types.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index fd5f65b50cb..c1f9d58e34a 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -199,6 +199,7 @@ export type Models = { [ModelProviderName.OPENROUTER]: Model; [ModelProviderName.OLLAMA]: Model; [ModelProviderName.HEURIST]: Model; + [ModelProviderName.FAL]: Model; }; /** @@ -218,6 +219,7 @@ export enum ModelProviderName { OPENROUTER = "openrouter", OLLAMA = "ollama", HEURIST = "heurist", + FAL = "falai" } /** @@ -610,6 +612,9 @@ export type Character = { /** Model provider to use */ modelProvider: ModelProviderName; + /** Image model provider to use, if different from modelProvider */ + imageModelProvider?: ModelProviderName; + /** Optional model endpoint override */ modelEndpointOverride?: string; From 7d746e6cafe27526edf29985aa802e5dd11a5cdc Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:10:22 +0000 Subject: [PATCH 02/11] given the option for distinction between the model being used for generateText and generateImage, added a new property for the imageModelProvider on the runtime class. This property is set to the imageModelProvider on character if present, otherwise it defaults to the model provider hence, the same model will be used for bother generateText and generateImage --- packages/core/src/runtime.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 3a628d3dfe4..157b6823dc9 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -98,6 +98,11 @@ export class AgentRuntime implements IAgentRuntime { */ modelProvider: ModelProviderName; + /** + * The model to use for generateImage. + */ + imageModelProvider: ModelProviderName; + /** * Fetch function to use * Some environments may not have access to the global fetch function and need a custom fetch override. @@ -303,7 +308,12 @@ export class AgentRuntime implements IAgentRuntime { opts.modelProvider ?? this.modelProvider; + this.imageModelProvider = + this.character.imageModelProvider ?? + this.modelProvider; + elizaLogger.info("Selected model provider:", this.modelProvider); + elizaLogger.info("Selected image model provider:", this.imageModelProvider); // Validate model provider if (!Object.values(ModelProviderName).includes(this.modelProvider)) { From b09a795aa65c16a14acb041d38ff56776bdbb28c Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:11:36 +0000 Subject: [PATCH 03/11] updateds core models to include a new entry for fal.ai --- packages/core/src/models.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts index b98c4d5ced4..178de1b5a14 100644 --- a/packages/core/src/models.ts +++ b/packages/core/src/models.ts @@ -258,6 +258,26 @@ export const models: Models = { [ModelClass.IMAGE]: "PepeXL", }, }, + [ModelProviderName.FAL]: { + settings: { + stop: [], + maxInputTokens: 128000, + maxOutputTokens: 8192, + repetition_penalty: 0.4, + temperature: 0.7, + }, + imageSettings: { + steps: 28, + }, + endpoint: "https://api.fal.ai/v1", + model: { + [ModelClass.SMALL]: "", // FAL doesn't provide text models + [ModelClass.MEDIUM]: "", + [ModelClass.LARGE]: "", + [ModelClass.EMBEDDING]: "", + [ModelClass.IMAGE]: "fal-ai/flux-lora", + }, + }, }; export function getModel(provider: ModelProviderName, type: ModelClass) { From 4fc3c699bbf8175a997e8760ebd5ae7b5571af81 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:12:59 +0000 Subject: [PATCH 04/11] updated imageGeneration action validate function to include check for FAL_API_KEY not that it can be used for generateImage --- packages/plugin-image-generation/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/plugin-image-generation/src/index.ts b/packages/plugin-image-generation/src/index.ts index ab95d0c3f3d..b044f2e1eb7 100644 --- a/packages/plugin-image-generation/src/index.ts +++ b/packages/plugin-image-generation/src/index.ts @@ -82,10 +82,11 @@ const imageGeneration: Action = { const anthropicApiKeyOk = !!runtime.getSetting("ANTHROPIC_API_KEY"); const togetherApiKeyOk = !!runtime.getSetting("TOGETHER_API_KEY"); const heuristApiKeyOk = !!runtime.getSetting("HEURIST_API_KEY"); + const falApiKeyOk = !!runtime.getSetting("FAL_API_KEY"); // TODO: Add openai DALL-E generation as well - return anthropicApiKeyOk || togetherApiKeyOk || heuristApiKeyOk; + return anthropicApiKeyOk || togetherApiKeyOk || heuristApiKeyOk || falApiKeyOk; }, handler: async ( runtime: IAgentRuntime, From f9d2c44c4d5c3186b90dbba0c7599d84552e2a32 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:15:23 +0000 Subject: [PATCH 05/11] update core packages to include fal.ai client --- packages/core/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/package.json b/packages/core/package.json index b0b5b4dedd9..374bcb2870d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -58,6 +58,7 @@ "@ai-sdk/groq": "0.0.3", "@ai-sdk/openai": "1.0.4", "@anthropic-ai/sdk": "0.30.1", + "@fal-ai/client": "^1.2.0", "@types/uuid": "10.0.0", "ai": "3.4.33", "anthropic-vertex-ai": "1.0.2", From e1a98c582b94aea2b9ac8933dbf785ec448136e8 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:17:45 +0000 Subject: [PATCH 06/11] include image generation plugin if FAL_API_KEY is present by default --- agent/src/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/agent/src/index.ts b/agent/src/index.ts index 7b6fee17ee0..e95592c4033 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -30,6 +30,7 @@ import { coinbaseMassPaymentsPlugin, } from "@ai16z/plugin-coinbase"; import { confluxPlugin } from "@ai16z/plugin-conflux"; +import { imageGenerationPlugin } from "@ai16z/plugin-image-generation"; import { createNodePlugin, } from "@ai16z/plugin-node"; @@ -182,6 +183,11 @@ export function getTokenForProvider( character.settings?.secrets?.GROQ_API_KEY || settings.GROQ_API_KEY ); + case ModelProviderName.FAL: + return ( + character.settings?.secrets?.FAL_API_KEY || + settings.FAL_API_KEY + ); } } @@ -283,6 +289,10 @@ export function createAgent( ? coinbaseMassPaymentsPlugin : null, getSecret(character, "BUTTPLUG_API_KEY") ? buttplugPlugin : null, + getSecret(character, "FAL_API_KEY") && + getSecret(character, "FAL_API_KEY") + ? imageGenerationPlugin + : null, ].filter(Boolean), providers: [], actions: [], From 1f76c9b1901456a780c8171efa0826036541977b Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:32:00 +0000 Subject: [PATCH 07/11] updated runtime interfact to include property for imageModelProvider --- packages/core/src/generation.ts | 69 +++++++++++++++++++++++++++++---- packages/core/src/types.ts | 1 + 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index cc9fb965fae..68a9b12eaa5 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -32,6 +32,7 @@ import { ModelProviderName, ServiceType, } from "./types.ts"; +import { fal, } from "@fal-ai/client"; /** * Send a message to the model for a text generateText - receive a string back and parse how you'd like @@ -769,15 +770,22 @@ export const generateImage = async ( count = 1; } - const model = getModel(runtime.character.modelProvider, ModelClass.IMAGE); - const modelSettings = models[runtime.character.modelProvider].imageSettings; - const apiKey = - runtime.token ?? - runtime.getSetting("HEURIST_API_KEY") ?? + const model = getModel(runtime.character.imageModelProvider, ModelClass.IMAGE); + const modelSettings = models[runtime.character.imageModelProvider].imageSettings; + + elizaLogger.info("Generating image with options:", { + imageModelProvider: model, + }); + + const apiKey = runtime.character.imageModelProvider === runtime.character.modelProvider + ? runtime.token + : runtime.getSetting("HEURIST_API_KEY") ?? runtime.getSetting("TOGETHER_API_KEY") ?? + runtime.getSetting("FAL_API_KEY") ?? runtime.getSetting("OPENAI_API_KEY"); + try { - if (runtime.character.modelProvider === ModelProviderName.HEURIST) { + if (runtime.character.imageModelProvider === ModelProviderName.HEURIST) { const response = await fetch( "http://sequencer.heurist.xyz/submit_job", { @@ -815,7 +823,7 @@ export const generateImage = async ( const imageURL = await response.json(); return { success: true, data: [imageURL] }; } else if ( - runtime.character.modelProvider === ModelProviderName.LLAMACLOUD + runtime.character.imageModelProvider === ModelProviderName.LLAMACLOUD ) { const together = new Together({ apiKey: apiKey as string }); const response = await together.images.create({ @@ -844,6 +852,53 @@ export const generateImage = async ( }) ); return { success: true, data: base64s }; + } else if (runtime.character.imageModelProvider === ModelProviderName.FAL) { + fal.config({ + credentials: apiKey as string + }); + + // Prepare the input parameters according to their schema + const input = { + prompt: prompt, + image_size: "square" as const, + num_inference_steps: modelSettings?.steps ?? 50, + guidance_scale: 3.5, + num_images: count, + enable_safety_checker: true, + output_format: "png" as const, + seed: data.seed ?? 6252023, + ...(runtime.getSetting("FAL_AI_LORA_PATH") ? { + loras: [ + { + path: runtime.getSetting("FAL_AI_LORA_PATH"), + scale: 1 + } + ] + } : {}) + }; + + // Subscribe to the model + const result = await fal.subscribe(model, { + input, + logs: true, + onQueueUpdate: (update) => { + if (update.status === "IN_PROGRESS") { + console.log(update.logs.map((log) => log.message)); + } + } + }); + + // Convert the returned image URLs to base64 to match existing functionality + const base64Promises = result.data.images.map(async (image) => { + const response = await fetch(image.url); + const blob = await response.blob(); + const buffer = await blob.arrayBuffer(); + const base64 = Buffer.from(buffer).toString('base64'); + return `data:${image.content_type};base64,${base64}`; + }); + + const base64s = await Promise.all(base64Promises); + return { success: true, data: base64s }; } else { let targetSize = `${width}x${height}`; if ( diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index c1f9d58e34a..d023c8104ae 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -956,6 +956,7 @@ export interface IAgentRuntime { databaseAdapter: IDatabaseAdapter; token: string | null; modelProvider: ModelProviderName; + imageModelProvider: ModelProviderName; character: Character; providers: Provider[]; actions: Action[]; From 99ffdfd4cbee7f9ea37383e679fad1c23c37fe51 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 10:33:56 +0000 Subject: [PATCH 08/11] updated generation to include implementation for fal.ai image generation and leverage the new distinction between imageModelProvider and modelProvider. Have changed the checks from runtime.character.modelProvider to runtime.modelProvider as it should be set upon runtime intialisation for character --- packages/core/src/generation.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts index 68a9b12eaa5..eac189291be 100644 --- a/packages/core/src/generation.ts +++ b/packages/core/src/generation.ts @@ -770,14 +770,14 @@ export const generateImage = async ( count = 1; } - const model = getModel(runtime.character.imageModelProvider, ModelClass.IMAGE); - const modelSettings = models[runtime.character.imageModelProvider].imageSettings; + const model = getModel(runtime.imageModelProvider, ModelClass.IMAGE); + const modelSettings = models[runtime.imageModelProvider].imageSettings; elizaLogger.info("Generating image with options:", { imageModelProvider: model, }); - const apiKey = runtime.character.imageModelProvider === runtime.character.modelProvider + const apiKey = runtime.imageModelProvider === runtime.modelProvider ? runtime.token : runtime.getSetting("HEURIST_API_KEY") ?? runtime.getSetting("TOGETHER_API_KEY") ?? @@ -785,7 +785,7 @@ export const generateImage = async ( runtime.getSetting("OPENAI_API_KEY"); try { - if (runtime.character.imageModelProvider === ModelProviderName.HEURIST) { + if (runtime.imageModelProvider === ModelProviderName.HEURIST) { const response = await fetch( "http://sequencer.heurist.xyz/submit_job", { @@ -823,7 +823,7 @@ export const generateImage = async ( const imageURL = await response.json(); return { success: true, data: [imageURL] }; } else if ( - runtime.character.imageModelProvider === ModelProviderName.LLAMACLOUD + runtime.imageModelProvider === ModelProviderName.LLAMACLOUD ) { const together = new Together({ apiKey: apiKey as string }); const response = await together.images.create({ @@ -852,7 +852,7 @@ export const generateImage = async ( }) ); return { success: true, data: base64s }; - } else if (runtime.character.imageModelProvider === ModelProviderName.FAL) { + } else if (runtime.imageModelProvider === ModelProviderName.FAL) { fal.config({ credentials: apiKey as string }); From 01e8bb2a52d5185175b046ca1692a282cecba5a5 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 12:01:10 +0000 Subject: [PATCH 09/11] updated plugin-image-generation environment variable checks to include fal.ai --- packages/plugin-image-generation/src/enviroment.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/plugin-image-generation/src/enviroment.ts b/packages/plugin-image-generation/src/enviroment.ts index 9fea6d94a53..f867d1ed015 100644 --- a/packages/plugin-image-generation/src/enviroment.ts +++ b/packages/plugin-image-generation/src/enviroment.ts @@ -6,18 +6,20 @@ export const imageGenEnvSchema = z ANTHROPIC_API_KEY: z.string().optional(), TOGETHER_API_KEY: z.string().optional(), HEURIST_API_KEY: z.string().optional(), + FAL_API_KEY: z.string().optional(), }) .refine( (data) => { return !!( data.ANTHROPIC_API_KEY || data.TOGETHER_API_KEY || - data.HEURIST_API_KEY + data.HEURIST_API_KEY || + data.FAL_API_KEY ); }, { message: - "At least one of ANTHROPIC_API_KEY, TOGETHER_API_KEY, or HEURIST_API_KEY is required", + "At least one of ANTHROPIC_API_KEY, TOGETHER_API_KEY, HEURIST_API_KEY or FAL_API_KEY is required", } ); @@ -37,6 +39,9 @@ export async function validateImageGenConfig( HEURIST_API_KEY: runtime.getSetting("HEURIST_API_KEY") || process.env.HEURIST_API_KEY, + FAL_API_KEY: + runtime.getSetting("FAL_API_KEY") || + process.env.FAL_API_KEY, }; return imageGenEnvSchema.parse(config); From ab922fcf2b7b127cf78e000fe251bffb292c0366 Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 14:05:21 +0000 Subject: [PATCH 10/11] updated example env file for optional fal.ai vars --- .env.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env.example b/.env.example index 1360105b995..1f6ae15ed72 100644 --- a/.env.example +++ b/.env.example @@ -146,3 +146,5 @@ COINBASE_COMMERCE_KEY= DSTACK_SIMULATOR_ENDPOINT= WALLET_SECRET_SALT=secret_salt +FAL_API_KEY= +FAL_AI_LORA_PATH= \ No newline at end of file From 27542e73e9dc286229e219468385bf270c911fac Mon Sep 17 00:00:00 2001 From: yoniebans Date: Thu, 28 Nov 2024 15:35:51 +0000 Subject: [PATCH 11/11] added HEURIST and OPENAI as valid options for automatically enabling image generation --- agent/src/index.ts | 9 +++++---- packages/plugin-image-generation/src/enviroment.ts | 9 +++++++-- packages/plugin-image-generation/src/index.ts | 5 ++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/agent/src/index.ts b/agent/src/index.ts index 67346b8c8a1..680d279488f 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -93,7 +93,7 @@ export async function loadCharacters( for (const characterPath of characterPaths) { let content = null; let resolvedPath = ""; - + // Try different path resolutions in order const pathsToTry = [ characterPath, // exact path as specified @@ -320,12 +320,13 @@ export function createAgent( ? coinbaseCommercePlugin : null, getSecret(character, "COINBASE_API_KEY") && - getSecret(character, "COINBASE_PRIVATE_KEY") + getSecret(character, "COINBASE_PRIVATE_KEY") ? coinbaseMassPaymentsPlugin : null, getSecret(character, "BUTTPLUG_API_KEY") ? buttplugPlugin : null, - getSecret(character, "FAL_API_KEY") && - getSecret(character, "FAL_API_KEY") + getSecret(character, "FAL_API_KEY") || + getSecret(character, "OPENAI_API_KEY") || + getSecret(character, "HEURIST_API_KEY") ? imageGenerationPlugin : null, getSecret(character, "WALLET_SECRET_SALT") ? teePlugin : null, diff --git a/packages/plugin-image-generation/src/enviroment.ts b/packages/plugin-image-generation/src/enviroment.ts index f867d1ed015..6cd1be473cf 100644 --- a/packages/plugin-image-generation/src/enviroment.ts +++ b/packages/plugin-image-generation/src/enviroment.ts @@ -7,6 +7,7 @@ export const imageGenEnvSchema = z TOGETHER_API_KEY: z.string().optional(), HEURIST_API_KEY: z.string().optional(), FAL_API_KEY: z.string().optional(), + OPENAI_API_KEY: z.string().optional(), }) .refine( (data) => { @@ -14,12 +15,13 @@ export const imageGenEnvSchema = z data.ANTHROPIC_API_KEY || data.TOGETHER_API_KEY || data.HEURIST_API_KEY || - data.FAL_API_KEY + data.FAL_API_KEY || + data.OPENAI_API_KEY ); }, { message: - "At least one of ANTHROPIC_API_KEY, TOGETHER_API_KEY, HEURIST_API_KEY or FAL_API_KEY is required", + "At least one of ANTHROPIC_API_KEY, TOGETHER_API_KEY, HEURIST_API_KEY, FAL_API_KEY or OPENAI_API_KEY is required", } ); @@ -42,6 +44,9 @@ export async function validateImageGenConfig( FAL_API_KEY: runtime.getSetting("FAL_API_KEY") || process.env.FAL_API_KEY, + OPENAI_API_KEY: + runtime.getSetting("OPENAI_API_KEY") || + process.env.OPENAI_API_KEY, }; return imageGenEnvSchema.parse(config); diff --git a/packages/plugin-image-generation/src/index.ts b/packages/plugin-image-generation/src/index.ts index b044f2e1eb7..ef2b978dbac 100644 --- a/packages/plugin-image-generation/src/index.ts +++ b/packages/plugin-image-generation/src/index.ts @@ -83,10 +83,9 @@ const imageGeneration: Action = { const togetherApiKeyOk = !!runtime.getSetting("TOGETHER_API_KEY"); const heuristApiKeyOk = !!runtime.getSetting("HEURIST_API_KEY"); const falApiKeyOk = !!runtime.getSetting("FAL_API_KEY"); + const openAiApiKeyOk = !!runtime.getSetting("OPENAI_API_KEY"); - // TODO: Add openai DALL-E generation as well - - return anthropicApiKeyOk || togetherApiKeyOk || heuristApiKeyOk || falApiKeyOk; + return anthropicApiKeyOk || togetherApiKeyOk || heuristApiKeyOk || falApiKeyOk || openAiApiKeyOk; }, handler: async ( runtime: IAgentRuntime,