Skip to content

Commit e4d55a0

Browse files
committed
Merge branch 'add-image-provider' of https://github.com/AmberAGI/eliza into pr491
2 parents 4c658d7 + 066cc98 commit e4d55a0

File tree

8 files changed

+169
-61
lines changed

8 files changed

+169
-61
lines changed

.env.example

+5
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,10 @@ IMAGE_GEN= # Set to TRUE to enable image generation
8181
USE_OPENAI_EMBEDDING= # Set to TRUE for OpenAI/1536, leave blank for local
8282
USE_OLLAMA_EMBEDDING= # Set to TRUE for OLLAMA/1024, leave blank for local
8383

84+
#Generation Prompts
85+
SYSTEM_PROMPT= # Leave blank for empty system prompt or defined in character config
86+
IMAGE_GENERATION_PROMPT= # Leave blank for default image generation prompt or defined in character config
87+
8488
# OpenRouter Models
8589
OPENROUTER_MODEL= # Default: uses hermes 70b/405b
8690
SMALL_OPENROUTER_MODEL=
@@ -187,6 +191,7 @@ TELEGRAM_BOT_TOKEN=
187191

188192
# Together Configuration
189193
TOGETHER_API_KEY=
194+
TOGETHER_IMAGE_MODEL= #Leave blank for default black-forest-labs/FLUX.1-schnell
190195

191196
# Server Configuration
192197
SERVER_PORT=3000

.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ tweets/
2525
*.onnx
2626
*.wav
2727
*.mp3
28+
*.png
29+
*.jpg
30+
*.jpeg
31+
*.webp
2832

2933
logs/
3034

agent/src/index.ts

+3
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ import Database from "better-sqlite3";
5959
import fs from "fs";
6060
import path from "path";
6161
import { fileURLToPath } from "url";
62+
import { character } from "./character.ts";
63+
import { imageGenerationPlugin } from "@ai16z/plugin-image-generation";
64+
import type { DirectClient } from "@ai16z/client-direct";
6265
import yargs from "yargs";
6366

6467
const __filename = fileURLToPath(import.meta.url); // get the resolved path to the file

mise.toml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[tools]
2+
node = "23.1.0"
3+
pnpm = "latest"

packages/client-telegram/src/messageManager.ts

+13-9
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
11
import { Message } from "@telegraf/types";
22
import { Context, Telegraf } from "telegraf";
33

4-
import { composeContext, elizaLogger, ServiceType } from "@elizaos/core";
5-
import { getEmbeddingZeroVector } from "@elizaos/core";
64
import {
5+
composeContext,
6+
ServiceType,
77
Content,
88
HandlerCallback,
99
IAgentRuntime,
10+
getEmbeddingZeroVector,
1011
IImageDescriptionService,
1112
Memory,
1213
ModelClass,
1314
State,
1415
UUID,
1516
Media,
17+
elizaLogger,
18+
stringToUuid,
19+
generateMessageResponse,
20+
generateShouldRespond,
21+
messageCompletionFooter,
22+
shouldRespondFooter,
1623
} from "@elizaos/core";
17-
import { stringToUuid } from "@elizaos/core";
18-
19-
import { generateMessageResponse, generateShouldRespond } from "@elizaos/core";
20-
import { messageCompletionFooter, shouldRespondFooter } from "@elizaos/core";
21-
2224
import { cosineSimilarity } from "./utils";
2325
import {
2426
MESSAGE_CONSTANTS,
@@ -136,10 +138,11 @@ Note that {{agentName}} is capable of reading/seeing/hearing various forms of me
136138
137139
{{recentMessages}}
138140
139-
# Task: Generate a post/reply in the voice, style and perspective of {{agentName}} (@{{twitterUserName}}) while using the thread of tweets as additional context:
141+
# Task: Generate a reply in the voice and style of {{agentName}}, aka @{{twitterUserName}}
142+
Write a very short reply that is from the perspective of {{agentName}}. Try to write something totally different than previous posts. Do not add commentary or acknowledge this request, just write the reply. Use the thread of tweets as additional context:
140143
Current Post:
141144
{{currentPost}}
142-
Thread of Tweets You Are Replying To:
145+
Thread of messages you are replying to:
143146
144147
{{formattedConversation}}
145148
` + messageCompletionFooter;
@@ -682,6 +685,7 @@ export class MessageManager {
682685
content: Content,
683686
replyToMessageId?: number
684687
): Promise<Message.TextMessage[]> {
688+
685689
if (content.attachments && content.attachments.length > 0) {
686690
content.attachments.map(async (attachment: Media) => {
687691
if (attachment.contentType.startsWith("image")) {

packages/core/src/generation.ts

+26-13
Original file line numberDiff line numberDiff line change
@@ -942,22 +942,36 @@ export const generateImage = async (
942942
data?: string[];
943943
error?: any;
944944
}> => {
945-
const model = getModel(runtime.imageModelProvider, ModelClass.IMAGE);
946-
const modelSettings = models[runtime.imageModelProvider].imageSettings;
945+
const imageModelProvider =
946+
runtime.character.imageModelProvider ?? runtime.character.modelProvider;
947+
const model = getModel(imageModelProvider, ModelClass.IMAGE);
948+
const modelSettings = models[imageModelProvider].imageSettings;
947949

948950
elizaLogger.info("Generating image with options:", {
949951
imageModelProvider: model,
950952
});
951953

952-
const apiKey =
953-
runtime.imageModelProvider === runtime.modelProvider
954-
? runtime.token
955-
: (runtime.getSetting("HEURIST_API_KEY") ??
956-
runtime.getSetting("TOGETHER_API_KEY") ??
957-
runtime.getSetting("FAL_API_KEY") ??
958-
runtime.getSetting("OPENAI_API_KEY") ??
959-
runtime.getSetting("VENICE_API_KEY"));
960-
954+
let apiKey = runtime.token;
955+
switch (imageModelProvider) {
956+
case ModelProviderName.HEURIST:
957+
apiKey = runtime.getSetting("HEURIST_API_KEY");
958+
break;
959+
case ModelProviderName.TOGETHER:
960+
apiKey = runtime.getSetting("TOGETHER_API_KEY");
961+
break;
962+
case ModelProviderName.FAL:
963+
apiKey = runtime.getSetting("FAL_API_KEY");
964+
break;
965+
case ModelProviderName.LLAMACLOUD:
966+
apiKey = runtime.getSetting("LLAMACLOUD_API_KEY");
967+
break;
968+
case ModelProviderName.VENICE:
969+
apiKey = runtime.getSetting("VENICE_API_KEY");
970+
break;
971+
case ModelProviderName.OPENAI:
972+
apiKey = runtime.getSetting("OPENAI_API_KEY");
973+
break;
974+
}
961975
try {
962976
if (runtime.imageModelProvider === ModelProviderName.HEURIST) {
963977
const response = await fetch(
@@ -1003,7 +1017,7 @@ export const generateImage = async (
10031017
) {
10041018
const together = new Together({ apiKey: apiKey as string });
10051019
const response = await together.images.create({
1006-
model: "black-forest-labs/FLUX.1-schnell",
1020+
model: runtime.getSetting("TOGETHER_IMAGE_MODEL") ?? "black-forest-labs/FLUX.1-schnell",
10071021
prompt: data.prompt,
10081022
width: data.width,
10091023
height: data.height,
@@ -1022,7 +1036,6 @@ export const generateImage = async (
10221036
throw new Error("Invalid response format from Together AI");
10231037
}
10241038

1025-
// Rest of the code remains the same...
10261039
const base64s = await Promise.all(
10271040
togetherResponse.data.map(async (image) => {
10281041
if (!image.url) {

packages/core/src/types.ts

+4-1
Original file line numberDiff line numberDiff line change
@@ -638,11 +638,13 @@ export type Character = {
638638

639639
/** Optional system prompt */
640640
system?: string;
641+
/** Optional image generation prompt */
642+
imageGenerationPrompt?: string;
641643

642644
/** Model provider to use */
643645
modelProvider: ModelProviderName;
644646

645-
/** Image model provider to use, if different from modelProvider */
647+
/** Optional image model provider to use, if different from modelProvider */
646648
imageModelProvider?: ModelProviderName;
647649

648650
/** Optional model endpoint override */
@@ -655,6 +657,7 @@ export type Character = {
655657
messageHandlerTemplate?: string;
656658
shouldRespondTemplate?: string;
657659
continueMessageHandlerTemplate?: string;
660+
imagePromptTemplate? :string;
658661
evaluationTemplate?: string;
659662
twitterSearchTemplate?: string;
660663
twitterActionTemplate?: string;

0 commit comments

Comments
 (0)